diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml new file mode 100644 index 000000000..9cc05d23e --- /dev/null +++ b/.github/workflows/terminal-bench.yml @@ -0,0 +1,205 @@ +name: Terminal Bench + +on: + schedule: + - cron: '0 7 * * *' + workflow_dispatch: + inputs: + tb2_tasks: + description: Comma-separated Terminal Bench 2 task names for the Linux Harbor lane + default: fix-git,financial-document-processor,code-from-image,chess-best-move,git-multibranch,kv-store-grpc,large-scale-text-editing + required: true + tb2_dataset: + description: Harbor dataset for the TB2 lane + default: terminal-bench@2.0 + required: true + model: + description: Optional model passed to ug --model (blank uses ug's default) + default: '' + required: false + claude_version: + description: Exact Claude Code version + default: 2.1.290 + required: true + codex_version: + description: Exact Codex version + default: 0.154.0 + required: true + +permissions: + contents: read + +concurrency: + group: terminal-bench-${{ github.ref }} + cancel-in-progress: false + +env: + PYTHONUTF8: 1 + TB2_TASKS: ${{ inputs.tb2_tasks || 'fix-git,financial-document-processor,code-from-image,chess-best-move,git-multibranch,kv-store-grpc,large-scale-text-editing' }} + TB2_DATASET: ${{ inputs.tb2_dataset || 'terminal-bench@2.0' }} + MODEL: ${{ inputs.model }} + CLAUDE_VERSION: ${{ inputs.claude_version || '2.1.290' }} + CODEX_VERSION: ${{ inputs.codex_version || '0.154.0' }} + HARBOR_VERSION: 0.23.0 + WINDOWS_PACKAGE_INDEX: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + WINDOWS_NPM_REGISTRY: https://databricks.jfrog.io/artifactory/api/npm/db-npm/ + +jobs: + # Real TB2 tasks run in Harbor's Linux containers, with ug installed inside each one. + harbor: + name: Harbor · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} · Linux + runs-on: ubuntu-latest + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + agent: [claude, codex] + env: + UV_INDEX_URL: https://pypi.org/simple + # bench_auth.py mints a fresh token per task from the CUJ service principal when + # its secrets are set, and otherwise falls back to the pre-fetched e2e bearer. + UCODE_TEST_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }} + DATABRICKS_BEARER: ${{ secrets.DATABRICKS_BEARER }} + UG_BENCH_SP_WORKSPACE: ${{ secrets.UG_CUJ1_WORKSPACE }} + UG_BENCH_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + UG_BENCH_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + PYTHONPATH: ${{ github.workspace }}/bench/terminal_bench + AGENT_CLASS: ${{ matrix.agent == 'claude' && 'UgClaude' || 'UgCodex' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - name: Build the ug wheel installed into each task container + run: | + uv build --wheel --out-dir "$RUNNER_TEMP/dist" + echo "UG_BENCH_WHEEL=$(ls "$RUNNER_TEMP"/dist/*.whl)" >> "$GITHUB_ENV" + - name: Run TB2 tasks + env: + AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }} + run: | + includes=() + IFS=, read -ra tasks <<< "$TB2_TASKS" + for task in "${tasks[@]}"; do includes+=(-i "$task"); done + uvx --python 3.12 --from "harbor==$HARBOR_VERSION" harbor run -y \ + -d "$TB2_DATASET" "${includes[@]}" \ + -a "ug_agent:$AGENT_CLASS" --ak "version=$AGENT_VERSION" \ + ${MODEL:+-m "$MODEL"} -n 3 -o "$RUNNER_TEMP/harbor-jobs/tb2" + - name: Run ug's own tasks + if: ${{ !cancelled() }} + env: + AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }} + run: | + includes=() + while read -r task; do includes+=(-i "$task"); done \ + < <(python3 bench/terminal_bench/bench_tasks.py "${{ matrix.agent }}") + uvx --python 3.12 --from "harbor==$HARBOR_VERSION" harbor run -y \ + -p bench/terminal_bench/tasks "${includes[@]}" --resume-trajectory \ + -a "ug_agent:$AGENT_CLASS" --ak "version=$AGENT_VERSION" \ + ${MODEL:+-m "$MODEL"} -n 3 -o "$RUNNER_TEMP/harbor-jobs/ug" + - name: Summarize + if: ${{ !cancelled() }} + run: | + status=0 + python3 bench/terminal_bench/summarize_harbor.py "$RUNNER_TEMP/harbor-jobs/tb2" "TB2 · ${{ matrix.agent }}" || status=1 + python3 bench/terminal_bench/summarize_harbor.py "$RUNNER_TEMP/harbor-jobs/ug" "ug tasks in Harbor · ${{ matrix.agent }}" || status=1 + exit $status + - name: Scrub the bearer token from job output + if: ${{ !cancelled() }} + run: python3 bench/terminal_bench/scrub_secrets.py "$RUNNER_TEMP/harbor-jobs" + - name: Upload Harbor jobs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: terminal-bench-harbor-${{ matrix.agent }} + path: ${{ runner.temp }}/harbor-jobs/ + + # Windows runners can't run Linux containers, so ug's own tasks also run on the host. + native: + name: Native · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} · ${{ matrix.os == 'windows-server-latest' && 'Windows' || 'Linux' }} + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + permissions: + contents: read + id-token: write + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-server-latest] + agent: [claude, codex] + env: + UV_INDEX_URL: https://pypi.org/simple + # bench_auth.py mints a fresh token per task from the CUJ service principal when + # its secrets are set, and otherwise falls back to the pre-fetched e2e bearer. + UCODE_TEST_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }} + DATABRICKS_BEARER: ${{ secrets.DATABRICKS_BEARER }} + UG_BENCH_SP_WORKSPACE: ${{ secrets.UG_CUJ1_WORKSPACE }} + UG_BENCH_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + UG_BENCH_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + AGENT: ${{ matrix.agent }} + AGENT_PACKAGE: ${{ matrix.agent == 'claude' && '@anthropic-ai/claude-code' || '@openai/codex' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22.19.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - uses: databricks/setup-cli@bdb89f81c11a5bd647fd55b585b7c396ec68a25a # v1.0.0 + - name: Expose the Databricks CLI on Windows + if: ${{ runner.os == 'Windows' }} + run: | + test -f "$RUNNER_TEMP/.bin/databricks.exe" + cygpath -w "$RUNNER_TEMP/.bin" >> "$GITHUB_PATH" + - name: Authenticate to JFrog with GitHub OIDC + id: jfrog + if: ${{ runner.os == 'Windows' }} + uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 + env: + JF_URL: https://databricks.jfrog.io + with: + oidc-provider-name: github-actions + - name: Install the agent CLI + env: + NPM_TOKEN: ${{ steps.jfrog.outputs.oidc-token }} + AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }} + run: | + if [[ "$RUNNER_OS" == Windows ]]; then + npmrc="$RUNNER_TEMP/jfrog.npmrc" + printf 'registry=%s\n//%s:_authToken=%s\n' \ + "$WINDOWS_NPM_REGISTRY" "${WINDOWS_NPM_REGISTRY#https://}" "$NPM_TOKEN" > "$npmrc" + npm install -g --userconfig "$npmrc" "$AGENT_PACKAGE@$AGENT_VERSION" + rm "$npmrc" + else + npm install -g "$AGENT_PACKAGE@$AGENT_VERSION" + fi + - name: Install ug + env: + UV_INDEX_DATABRICKS_PYPI_USERNAME: ${{ steps.jfrog.outputs.oidc-user }} + UV_INDEX_DATABRICKS_PYPI_PASSWORD: ${{ steps.jfrog.outputs.oidc-token }} + run: | + if [[ "$RUNNER_OS" == Windows ]]; then + unset UV_INDEX_URL + export UV_DEFAULT_INDEX="databricks-pypi=$WINDOWS_PACKAGE_INDEX" + fi + uv lock + uv tool install . + uv sync + - name: Run ug's own tasks on the host + run: uv run --no-sync python bench/terminal_bench/run_native.py --agent "$AGENT" --output "$RUNNER_TEMP/tb-native" ${MODEL:+--model "$MODEL"} < /dev/null + - name: Scrub the bearer token from task output + if: ${{ !cancelled() }} + run: uv run --no-sync python bench/terminal_bench/scrub_secrets.py "$RUNNER_TEMP/tb-native" + - name: Upload native results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: terminal-bench-native-${{ matrix.agent }}-${{ matrix.os }} + include-hidden-files: true + path: | + ${{ runner.temp }}/tb-native/results.json + ${{ runner.temp }}/tb-native/logs/ diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md new file mode 100644 index 000000000..6cc4dce76 --- /dev/null +++ b/bench/terminal_bench/README.md @@ -0,0 +1,134 @@ +# Terminal Bench for ug + +```text + _ _ _ _ _ +| |_ ___ _ _ _ __ (_)_ _ __ _| | | |__ ___ _ _ __| |_ +| _/ -_) '_| ' \| | ' \/ _` | | | '_ \/ -_) ' \/ _| ' \ + \__\___|_| |_|_|_|_|_||_\__,_|_| |_.__/\___|_||_\__|_||_| +``` + +This directory runs Terminal Bench tasks through `ug claude` and `ug codex`. It checks that an agent keeps working when ug configures and launches it. The `Terminal Bench` workflow in `.github/workflows/terminal-bench.yml` runs every night at 07:00 UTC and on manual dispatch. + +The workflow runs tasks in two lanes. The Harbor lane runs a [Terminal Bench 2](https://www.tbench.ai/) subset and the tasks in `tasks/` in Docker containers on Linux. The native lane runs the tasks in `tasks/` directly on Linux and Windows runners. + +```mermaid +flowchart LR + workflow["Terminal Bench workflow"] --> harbor["Harbor lane
Linux, Docker"] + workflow --> native["Native lane
Linux and Windows"] + harbor --> tb2["TB2 subset"] + harbor --> own["tasks/"] + native --> own +``` + +Each lane runs its tasks for Claude and for Codex. [TASKS.md](TASKS.md) describes the task format and lists every task. + +## Why there are two lanes + +Each lane tests something the other can't. + +The Harbor lane exists for real TB2 tasks. Each TB2 task ships a Linux Docker image with the services, packages, and root access it needs. Harbor starts a fresh container per trial, so a task can't change the runner or another trial. `ug_agent.py` installs ug, the Databricks CLI, and the agent CLI in each container. + +The native lane exists for Windows. GitHub's Windows runners can't run Linux containers, and TB2 has no Windows images, so Harbor can't test ug on Windows. `run_native.py` runs the tasks in `tasks/` on the runner itself instead. Those tasks use only the Python standard library so they run on both operating systems. + +The native Linux job overlaps with Harbor. It adds one thing: ug running outside a container, on a machine with a normal user home. + +## How a task runs + +Both lanes run a task the same way. The agent gets the task's `instruction.md` once, at launch, and works on its own until it exits or hits the task's `timeout_sec`. Nothing sends it more instructions while it runs. That matches Terminal Bench and Harbor's built-in `claude-code` and `codex` agents, which also launch `claude --print` and `codex exec` with the instruction and wait. + +Claude reads the instruction on stdin in print mode (`-p`). Codex gets it as an argument to `codex exec`. The host mints the gateway token, so the agent never sees the service principal secret. The verifier runs after the agent exits and decides the result. + +```mermaid +sequenceDiagram + participant host as Runner host + participant env as Task directory + participant agent as ug and agent CLI + participant gateway as AI Gateway + host->>env: Copy environment/app/ and run environment/setup.py + host->>host: bench_auth.py mints a token + host->>agent: ug configure, then ug claude or ug codex with instruction.md + agent->>gateway: Model calls with the token + agent->>env: Read files, run commands, write output + host->>env: Run tests/test_outputs.py + env-->>host: Exit code 0 is a pass +``` + +Every launch runs with smart routing on (`ENABLE_SMART_ROUTING_V2=1`). ug routes Claude's prompt before launch, because print mode (`-p`) has no terminal UI to switch models in, and it routes Claude's subagents. Codex sends ug's router header on every model call. + +The Harbor lane runs `ug configure` in each container. The native lane runs it once at the start of the job. + +A multi-step task launches the agent again for each step, with that step's instruction and a new token. The new launch resumes the previous session with `--continue` for Claude or `exec resume --last` for Codex, and the verifier runs after each step. All TB2 tasks have one step. `resume-session` is the only multi-step task here. + +## What the bench doesn't test + +The bench treats ug as a black box. It installs ug from this checkout, runs `ug configure`, and launches the agent with `ug claude` or `ug codex`. Verifiers check only what the agent left behind. Some of ug runs differently from a user's setup: + +- Auth goes through `DATABRICKS_BEARER` with a token the host mints. ug's `databricks auth login` and OAuth flows don't run. +- Databricks AI Tools, the skills and plugins that `--enable-databricks-ai-tools` installs, aren't installed. That's ug's default. +- The CUJ1 workspace's managed config lists no MCP servers, so ug registers none. +- The harness installs a pinned Claude Code or Codex before ug starts, so ug's agent install doesn't run for that agent. It can still run for the other one. In a Claude job, ug tries to install Codex because the CUJ1 managed config enables both. +- Claude runs with `--dangerously-skip-permissions` and Codex with `--dangerously-bypass-approvals-and-sandbox`. Permission prompts and the Codex sandbox don't run. + +## Run the workflow with other tasks or a model + +To change the TB2 tasks or pin a model, dispatch the workflow with inputs: + +```sh +gh workflow run terminal-bench.yml -f tb2_tasks=fix-git,code-from-image -f model= +``` + +The workflow also takes `tb2_dataset`, `claude_version`, and `codex_version`. GitHub can dispatch the workflow only from a file that is already on `main`. + +## Read the results + +Each job writes a table of task results to its run summary. + +A Harbor job fails only when a trial hits a harness error, such as a failed agent install or an API error. A task with reward 0 doesn't fail the job, because TB2 tasks are meant to be hard. A native job fails when fewer than 75% of its tasks pass. Agents miss a different task or two from run to run, so one miss doesn't fail the job. Change the bar with `run_native.py --min-pass-rate`. + +Each job uploads its files as an artifact named `terminal-bench-harbor-` or `terminal-bench-native--`. `scrub_secrets.py` replaces bearer tokens in those files before upload. + +To tell a ug problem from an agent mistake, read the verifier output first and the agent log second: + +- In a Harbor artifact, read `//verifier/test-stdout.txt`, then `//agent/ug-.txt`. +- In a native artifact, read `logs//.verifier.log`, then `logs//.log`. A single-step task names its step `agent`. + +## Run tasks locally + +To check the tasks without a gateway, run the reference solutions. Every task passes. + +```sh +uv run python bench/terminal_bench/run_native.py --agent oracle --output /tmp/tb +``` + +Add `--task ` to run one task. + +To run the tasks through ug, export credentials as described in [Credentials](#credentials) and pick an agent. The runner calls `ug configure` for that agent, so it replaces your own configuration for it. + +```sh +uv run python bench/terminal_bench/run_native.py --agent claude --output /tmp/tb +``` + +To run in Harbor, you need Docker. Build a ug wheel, point `UG_BENCH_WHEEL` at it, and start a job: + +```sh +uv build --wheel --out-dir /tmp/dist +export UG_BENCH_WHEEL=$(ls /tmp/dist/*.whl) +PYTHONPATH=bench/terminal_bench uvx --python 3.12 --from harbor==0.23.0 harbor run -y \ + -d terminal-bench@2.0 -i fix-git -a ug_agent:UgClaude -o /tmp/harbor-jobs +``` + +For Codex, use `ug_agent:UgCodex`. To run ug's own tasks, replace `-d terminal-bench@2.0` with `-p bench/terminal_bench/tasks --resume-trajectory`. + +## Credentials + +`bench_auth.py` picks the credentials for each task. If `UG_BENCH_CLIENT_ID`, `UG_BENCH_CLIENT_SECRET`, and `UG_BENCH_SP_WORKSPACE` are set, it mints a new OAuth M2M token on the host before each task. The agent gets that token and never the service principal secret, because the agent has a full shell. If those variables aren't set, it uses `DATABRICKS_BEARER` against `UCODE_TEST_WORKSPACE`. + +CI uses the `UG_CUJ_SP_CLIENT_ID` service principal against `UG_CUJ1_WORKSPACE`. That workspace publishes a managed coding-agent config, so its model policy applies to every CI run. The policy doesn't allow haiku, for example. + +## Add a task + +1. Copy the task in `tasks/` that is closest to yours. [TASKS.md](TASKS.md) describes each file. +2. Write `tests/test_outputs.py` so it checks evidence that only the feature can produce, such as a hook log or an MCP call log, as well as the answer. An agent can read every file in the task, so the answer alone is easy to fake. +3. If only one agent supports the feature, set `agents` under `[metadata]` in `task.toml`. +4. Run `run_native.py --agent oracle --task ` and confirm the task passes. +5. Confirm the verifier fails when the agent does nothing. Copy `environment/app/` into an empty directory, or run `environment/setup.py `. Then run `tests/test_outputs.py` with `TASK_APP_DIR` set to that directory. diff --git a/bench/terminal_bench/TASKS.md b/bench/terminal_bench/TASKS.md new file mode 100644 index 000000000..c16ef5e75 --- /dev/null +++ b/bench/terminal_bench/TASKS.md @@ -0,0 +1,54 @@ +# Task reference + +Each directory in `tasks/` is one task in Harbor's task format. Harbor and `run_native.py` read the same files. `bench_tasks.py ` prints the tasks that one agent runs. + +## Files in a task + +| Path | Contents | +|------|----------| +| `task.toml` | Timeouts, resources, and metadata. `agents` under `[metadata]` lists the agents that run the task, and defaults to Claude and Codex. Each `[[steps]]` entry adds a step. | +| `instruction.md` | The prompt the agent gets. | +| `environment/Dockerfile` | The Harbor image. It copies `environment/app/` to `/app` and runs `environment/setup.py /app` if the task has them. | +| `environment/app/` | Fixture files. The runner copies them into the task directory. | +| `environment/setup.py` | Generates fixtures. It takes the task directory as its only argument. | +| `tests/test_outputs.py` | The verifier. Exit code 0 is a pass. | +| `tests/test.sh` | Runs the verifier in Harbor and writes the reward to `/logs/verifier/reward.txt`. | +| `solution/solve.py` | The reference solution that `--agent oracle` runs. | +| `solution/solve.sh` | Runs the reference solution in Harbor. | + +A multi-step task has `instruction.md`, `tests/`, and `solution/` under `steps//` for each step. Each step after the first resumes the previous session. Harbor resumes with `--resume-trajectory`. `run_native.py` passes `--continue` to Claude and `exec resume --last` to Codex. A failed step stops the task. + +Verifiers and solutions use only the Python standard library. They run on Windows and in `python:3.12-slim`. + +## Environment variables + +Verifiers and solutions read these variables. + +| Variable | Value | +|----------|-------| +| `TASK_APP_DIR` | The task directory. Defaults to `/app`, which is the task directory in Harbor. | +| `TASK_AGENT_LOG` | The agent's log for the current step. Only `run_native.py` sets it. In Harbor, read `/logs/agent/ug-.txt`. | +| `TASK_ORACLE` | `1` when the oracle runs. A check that reads the agent log skips itself. | + +## ug's tasks + +| Task | Agents | Passes when | +|------|--------|-------------| +| `user-hooks` | Claude | `release.txt` holds the codename from the project's `UserPromptSubmit` hook, and the `PostToolUse` hook logged the write. | +| `mcp-server` | Claude | `stock.json` matches the random values that the project's stdio MCP server logged for each lookup. | +| `project-instructions` | Claude, Codex | The output is under `out/`, and each new Python file starts with the license header that `CLAUDE.md` and `AGENTS.md` require. | +| `project-skill` | Claude | The release notes follow the project skill, and the skill's script logged a run. | +| `subagent-model` | Claude | Each file has a correct audit, and the transcript shows a subagent on a haiku model. If a workspace policy disallows haiku, the transcript must show the policy notice and the subagent on the fallback model it names. | +| `image-code` | Claude, Codex | `code.txt` holds the 8-digit code drawn in `code.png`. | +| `pdf-extract` | Claude, Codex | `total.txt` holds the total from `invoice.pdf`, whose text stream is compressed. | +| `web-fetch` | Claude, Codex | `answer.txt` holds a value from a file in this repo, pinned to a fixed commit. | +| `background-shell` | Claude, Codex | `result.txt` holds the value that a 150-second job prints when it finishes. Claude's default command timeout is 2 minutes. | +| `resume-session` | Claude, Codex | Step 1 deletes `vault.txt` without saving its passphrase to a file. Step 2 resumes the session and writes the passphrase to `answer.txt`. | +| `log-triage` | Claude, Codex | `report.json` has the correct counts for five independent service directories. | +| `fix-failing-tests` | Claude, Codex | The unit tests and extra hidden checks pass, and `tests/` is unchanged. | +| `scrub-git-secret` | Claude, Codex | Every copy of the token in history is replaced with `REDACTED`, and both branches keep their commit messages in order. | +| `background-service` | Claude, Codex | `token.txt` matches the token the server served, and nothing listens on the server's port afterward. | + +## Default TB2 subset + +The Harbor lane runs these Terminal Bench 2 tasks unless the `tb2_tasks` input overrides them: `fix-git`, `financial-document-processor`, `code-from-image`, `chess-best-move`, `git-multibranch`, `kv-store-grpc`, and `large-scale-text-editing`. diff --git a/bench/terminal_bench/bench_auth.py b/bench/terminal_bench/bench_auth.py new file mode 100644 index 000000000..74bb41fd4 --- /dev/null +++ b/bench/terminal_bench/bench_auth.py @@ -0,0 +1,60 @@ +"""Workspace auth for bench runs. + +With service-principal credentials set, mint a fresh bearer per task on the host so +long runs outlive a single token and the SP secret never reaches the agent. +Otherwise fall back to a pre-fetched DATABRICKS_BEARER. +""" + +from __future__ import annotations + +import base64 +import json +import os +import urllib.parse +import urllib.request + +SECRET_ENV = ("UG_BENCH_CLIENT_ID", "UG_BENCH_CLIENT_SECRET") + + +def _sp_credentials() -> tuple[str, str] | None: + client_id, secret = (os.environ.get(name, "").strip() for name in SECRET_ENV) + return (client_id, secret) if client_id and secret else None + + +def workspace() -> str: + if _sp_credentials() and (host := os.environ.get("UG_BENCH_SP_WORKSPACE", "").strip()): + return host + host = os.environ.get("UCODE_TEST_WORKSPACE", "").strip() + if not host: + raise RuntimeError("Set UCODE_TEST_WORKSPACE, or UG_BENCH_SP_WORKSPACE with SP credentials") + return host + + +def bearer(host: str) -> str: + credentials = _sp_credentials() + if credentials is None: + token = os.environ.get("DATABRICKS_BEARER", "").strip() + if not token: + raise RuntimeError( + "Set DATABRICKS_BEARER, or UG_BENCH_CLIENT_ID/UG_BENCH_CLIENT_SECRET" + ) + return token + basic = base64.b64encode(":".join(credentials).encode()).decode() + request = urllib.request.Request( + f"{host.rstrip('/')}/oidc/v1/token", + data=urllib.parse.urlencode( + {"grant_type": "client_credentials", "scope": "all-apis"} + ).encode(), + headers={"Authorization": f"Basic {basic}"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response)["access_token"] + + +def agent_env(base: dict[str, str]) -> dict[str, str]: + """Copy of `base` with a fresh bearer and without the SP secret.""" + env = {key: value for key, value in base.items() if key not in SECRET_ENV} + host = workspace() + env["DATABRICKS_HOST"] = host + env["DATABRICKS_BEARER"] = bearer(host) + return env diff --git a/bench/terminal_bench/bench_tasks.py b/bench/terminal_bench/bench_tasks.py new file mode 100644 index 000000000..8bb8cddb6 --- /dev/null +++ b/bench/terminal_bench/bench_tasks.py @@ -0,0 +1,24 @@ +"""List bench tasks for an agent. Tasks opt in with `[metadata] agents` in task.toml.""" + +import sys +import tomllib +from pathlib import Path + +TASKS_DIR = Path(__file__).resolve().parent / "tasks" + + +def task_agents(config: dict) -> list[str]: + return config.get("metadata", {}).get("agents", ["claude", "codex"]) + + +def names_for(agent: str) -> list[str]: + return [ + task.name + for task in sorted(TASKS_DIR.iterdir()) + if agent == "oracle" + or agent in task_agents(tomllib.loads((task / "task.toml").read_text())) + ] + + +if __name__ == "__main__": + print("\n".join(names_for(sys.argv[1]))) diff --git a/bench/terminal_bench/run_native.py b/bench/terminal_bench/run_native.py new file mode 100644 index 000000000..3ca1a9da5 --- /dev/null +++ b/bench/terminal_bench/run_native.py @@ -0,0 +1,230 @@ +"""Run Terminal Bench-format tasks directly on the host through ug. + +Windows runners can't run the Linux containers Harbor uses, so this runner +copies each task's fixtures into a temp dir, launches the agent headlessly +there, and runs the task's Python verifier against the result. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import signal +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +import bench_auth +from bench_tasks import TASKS_DIR, names_for + +from ucode.os_compatibility.subprocess_cross_os import popen, run + + +def agent_command( + agent: str, ug: str, model: str | None, step_dir: Path, instruction: str, resume: bool +) -> tuple[list[str], str | None]: + """Return argv plus the stdin to send. Codex takes the prompt as an argument.""" + model_args = ["--model", model] if model else [] + if agent == "claude": + argv = [ug, "claude", *model_args, "--", "-p", "--verbose", "--output-format"] + argv += ["stream-json", "--dangerously-skip-permissions"] + return argv + (["--continue"] if resume else []), instruction + if agent == "codex": + argv = [ug, "codex", *model_args, "--", "exec", *(["resume", "--last"] if resume else [])] + argv += ["--dangerously-bypass-approvals-and-sandbox", "--skip-git-repo-check", "--json"] + return argv + ["--enable", "unified_exec", "--", instruction], None + return [sys.executable, str(step_dir / "solution" / "solve.py")], None + + +def task_steps(task_dir: Path, config: dict) -> list[tuple[str, Path, float]]: + """(name, dir holding instruction.md/tests/solution, agent timeout) for each step.""" + default = config.get("agent", {}).get("timeout_sec", 600) + if not config.get("steps"): + return [("agent", task_dir, default)] + return [ + ( + step["name"], + task_dir / "steps" / step["name"], + step.get("agent", {}).get("timeout_sec", default), + ) + for step in config["steps"] + ] + + +def kill_tree(proc: subprocess.Popen) -> None: + if os.name == "nt": + run(["taskkill", "/T", "/F", "/PID", str(proc.pid)], capture_output=True) + else: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + + +def prepare(task_dir: Path, workdir: Path) -> None: + app = task_dir / "environment" / "app" + if app.is_dir(): + shutil.copytree(app, workdir) + workdir.mkdir(parents=True, exist_ok=True) + setup = task_dir / "environment" / "setup.py" + if setup.exists(): + run([sys.executable, str(setup), str(workdir)], check=True) + + +def run_agent( + argv: list[str], stdin: str | None, workdir: Path, env: dict, log_path: Path, timeout: float +) -> tuple[subprocess.Popen, bool]: + with log_path.open("w", encoding="utf-8") as log: + proc = popen( + argv, + cwd=workdir, + env=env, + stdin=subprocess.PIPE if stdin is not None else subprocess.DEVNULL, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + start_new_session=os.name != "nt", + ) + try: + proc.communicate(stdin, timeout=timeout) + except subprocess.TimeoutExpired: + kill_tree(proc) + return proc, True + return proc, False + + +def verify(step_dir: Path, workdir: Path, env: dict, timeout: float) -> tuple[bool, str]: + try: + verifier = run( + [sys.executable, str(step_dir / "tests" / "test_outputs.py")], + cwd=workdir, + env=env, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return False, f"verifier timed out after {timeout}s\n" + return verifier.returncode == 0, verifier.stdout + verifier.stderr + + +def run_task(args: argparse.Namespace, task_dir: Path, out: Path) -> dict: + config = tomllib.loads((task_dir / "task.toml").read_text()) + verifier_timeout = config.get("verifier", {}).get("timeout_sec", 120) + workdir = out / "work" / task_dir.name + logs = out / "logs" / task_dir.name + logs.mkdir(parents=True, exist_ok=True) + prepare(task_dir, workdir) + + env = {**os.environ, "TASK_APP_DIR": str(workdir), "IS_SANDBOX": "1"} + env["ENABLE_SMART_ROUTING_V2"] = "1" + if args.agent == "oracle": + # Checks that read the agent transcript have nothing to read for the oracle. + env["TASK_ORACLE"] = "1" + # Tasks say "run it with Python", so put this interpreter first on PATH. + env["PATH"] = os.pathsep.join([str(Path(sys.executable).parent), env.get("PATH", "")]) + started = time.monotonic() + result = {"task": task_dir.name, "passed": True, "agent_exit": None, "timed_out": False} + for index, (name, step_dir, agent_timeout) in enumerate(task_steps(task_dir, config)): + if args.agent != "oracle": + env = bench_auth.agent_env(env) + log_path = logs / f"{name}.log" + env["TASK_AGENT_LOG"] = str(log_path) + instruction = (step_dir / "instruction.md").read_text() + argv, stdin = agent_command( + args.agent, args.ug, args.model, step_dir, instruction, resume=index > 0 + ) + proc, timed_out = run_agent(argv, stdin, workdir, env, log_path, agent_timeout) + # Verify before cleanup so a server the agent left running still counts against it. + passed, verifier_log = verify(step_dir, workdir, env, verifier_timeout) + (logs / f"{name}.verifier.log").write_text(verifier_log, encoding="utf-8") + # ug's routing wrappers log under ~/.ucode; keep them as evidence that routing ran. + for ucode_log in (Path.home() / ".ucode").glob("*.log"): + (logs / f"{name}.ucode").mkdir(exist_ok=True) + shutil.copy2(ucode_log, logs / f"{name}.ucode" / ucode_log.name) + if not timed_out and os.name != "nt": + kill_tree(proc) + result.update(agent_exit=proc.returncode, timed_out=timed_out) + if not passed: + result.update(passed=False, failed_step=name) + break + result["duration_sec"] = round(time.monotonic() - started, 1) + return result + + +def write_summary(args: argparse.Namespace, results: list[dict], out: Path) -> None: + (out / "results.json").write_text(json.dumps(results, indent=2)) + passed = sum(r["passed"] for r in results) + lines = [ + f"### Native tasks · {args.agent} · {sys.platform}: {passed}/{len(results)} passed" + f" (needs {args.min_pass_rate:.0%})", + "", + "| Task | Result | Agent exit | Time (s) |", + "|------|--------|-----------|----------|", + ] + for r in results: + status = "pass" if r["passed"] else ("timeout" if r["timed_out"] else "fail") + lines.append(f"| {r['task']} | {status} | {r['agent_exit']} | {r['duration_sec']} |") + summary = "\n".join(lines) + "\n" + print(summary) + if step_summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with open(step_summary, "a", encoding="utf-8") as f: + f.write(summary) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", choices=["claude", "codex", "oracle"], required=True) + parser.add_argument("--task", action="append", help="Task name; repeatable. Default: all.") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--ug", default=shutil.which("ug"), help="Absolute path to ug.") + parser.add_argument("--model") + parser.add_argument( + "--min-pass-rate", + type=float, + default=0.75, + help="Fraction of tasks that must pass for exit code 0. Agents miss some tasks run to run.", + ) + args = parser.parse_args() + + names = args.task or names_for(args.agent) + out = args.output.resolve() + if (out / "work").exists(): + shutil.rmtree(out / "work") + out.mkdir(parents=True, exist_ok=True) + + if args.agent != "oracle": + if not args.ug: + parser.error("--ug is required when ug isn't on PATH") + args.ug = str(Path(args.ug).resolve()) + env = bench_auth.agent_env(dict(os.environ)) + run( + [ + args.ug, + "configure", + "--agents", + args.agent, + "--workspace", + env["DATABRICKS_HOST"], + "--skip-validate", + "--skip-upgrade", + "--disable-databricks-ai-tools", + ], + check=True, + env=env, + stdin=subprocess.DEVNULL, + ) + + results = [run_task(args, TASKS_DIR / name, out) for name in names] + write_summary(args, results, out) + passed = sum(r["passed"] for r in results) + return 0 if results and passed / len(results) >= args.min_pass_rate else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/terminal_bench/scrub_secrets.py b/bench/terminal_bench/scrub_secrets.py new file mode 100644 index 000000000..5991caa72 --- /dev/null +++ b/bench/terminal_bench/scrub_secrets.py @@ -0,0 +1,20 @@ +"""Redact Databricks bearers from every file under a directory before upload. + +Per-task tokens are minted on the fly, so match token shapes as well as the +DATABRICKS_BEARER value. +""" + +import os +import re +import sys +from pathlib import Path + +TOKEN = re.compile(rb"eyJ[\w-]+\.[\w-]+\.[\w-]+|dapi[0-9a-f]{32}(?:-\d+)?") +secret = os.environ.get("DATABRICKS_BEARER", "").strip().encode() + +for path in Path(sys.argv[1]).rglob("*"): + if path.is_file() and not path.is_symlink(): + data = path.read_bytes() + scrubbed = TOKEN.sub(b"***", data.replace(secret, b"***") if secret else data) + if scrubbed != data: + path.write_bytes(scrubbed) diff --git a/bench/terminal_bench/summarize_harbor.py b/bench/terminal_bench/summarize_harbor.py new file mode 100644 index 000000000..93b36c249 --- /dev/null +++ b/bench/terminal_bench/summarize_harbor.py @@ -0,0 +1,41 @@ +"""Summarize a Harbor jobs dir as a markdown table. + +Exits non-zero when a trial errored for a reason other than a timeout, since +that points at the ug harness rather than the agent's score. +""" + +import json +import os +import sys +from pathlib import Path + + +def main(jobs_dir: Path, title: str) -> int: + rows = [] + harness_errors = 0 + for path in sorted(jobs_dir.glob("*/*/result.json")): + result = json.loads(path.read_text()) + rewards = (result.get("verifier_result") or {}).get("rewards") or {} + error = (result.get("exception_info") or {}).get("exception_type", "") + if error and "Timeout" not in error: + harness_errors += 1 + rows.append((result["task_name"], rewards.get("reward", "-"), error or "-")) + + solved = sum(1 for _, reward, _ in rows if reward == 1) + lines = [ + f"### {title}: {solved}/{len(rows)} solved", + "", + "| Task | Reward | Error |", + "|------|--------|-------|", + *(f"| {task} | {reward} | {error} |" for task, reward, error in rows), + ] + summary = "\n".join(lines) + "\n" + print(summary) + if step_summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with open(step_summary, "a", encoding="utf-8") as f: + f.write(summary) + return 1 if harness_errors or not rows else 0 + + +if __name__ == "__main__": + sys.exit(main(Path(sys.argv[1]), sys.argv[2])) diff --git a/bench/terminal_bench/tasks/background-service/environment/Dockerfile b/bench/terminal_bench/tasks/background-service/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/background-service/environment/app/server.py b/bench/terminal_bench/tasks/background-service/environment/app/server.py new file mode 100644 index 000000000..b5bc1c6d9 --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/environment/app/server.py @@ -0,0 +1,39 @@ +"""Token service. Run it in the background; it stops itself after 15 minutes.""" + +import hashlib +import http.server +import secrets +import threading +from pathlib import Path + +STATE = Path(__file__).resolve().parent / ".server_state" +TOKEN = secrets.token_hex(16) + + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: + if self.path != "/token": + self.send_error(404) + elif self.headers.get("X-Bench-Client") != "ug": + self.send_error(403, "missing X-Bench-Client: ug header") + else: + body = TOKEN.encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def main() -> None: + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + STATE.mkdir(exist_ok=True) + (STATE / "token.sha256").write_text(hashlib.sha256(TOKEN.encode()).hexdigest()) + (STATE / "port").write_text(str(server.server_address[1])) + print(f"listening on 127.0.0.1:{server.server_address[1]}", flush=True) + threading.Timer(900, server.shutdown).start() + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/background-service/instruction.md b/bench/terminal_bench/tasks/background-service/instruction.md new file mode 100644 index 000000000..45919b376 --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/instruction.md @@ -0,0 +1,8 @@ +`server.py` in the current directory is a small token service. + +1. Start it in the background with Python. Once it's ready it writes its port to `.server_state/port`. +2. Fetch `GET http://127.0.0.1:/token`. The request needs the header `X-Bench-Client: ug`. +3. Write the token from the response body to `token.txt`. +4. Stop the server. Nothing should be listening on that port when you finish. + +Don't edit `server.py` or anything in `.server_state/`. diff --git a/bench/terminal_bench/tasks/background-service/solution/solve.py b/bench/terminal_bench/tasks/background-service/solution/solve.py new file mode 100644 index 000000000..ce19d1f5f --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/solution/solve.py @@ -0,0 +1,22 @@ +"""Reference solution for background-service.""" + +import os +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +proc = subprocess.Popen([sys.executable, "server.py"], cwd=app) +port_file = app / ".server_state" / "port" +for _ in range(100): + if port_file.exists() and port_file.read_text(): + break + time.sleep(0.1) +request = urllib.request.Request( + f"http://127.0.0.1:{port_file.read_text()}/token", headers={"X-Bench-Client": "ug"} +) +(app / "token.txt").write_text(urllib.request.urlopen(request).read().decode()) +proc.terminate() +proc.wait() diff --git a/bench/terminal_bench/tasks/background-service/solution/solve.sh b/bench/terminal_bench/tasks/background-service/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/background-service/task.toml b/bench/terminal_bench/tasks/background-service/task.toml new file mode 100644 index 000000000..9cad17abe --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "system-administration" +tags = ["background-process", "http", "cleanup"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/background-service/tests/test.sh b/bench/terminal_bench/tasks/background-service/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/background-service/tests/test_outputs.py b/bench/terminal_bench/tasks/background-service/tests/test_outputs.py new file mode 100644 index 000000000..243ee9956 --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/tests/test_outputs.py @@ -0,0 +1,26 @@ +"""Verify the agent fetched the live token and then stopped the server.""" + +import hashlib +import os +import socket +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + state = APP / ".server_state" + assert (state / "port").exists(), "server.py was never started" + token = (APP / "token.txt").read_text().strip() + want = (state / "token.sha256").read_text().strip() + assert hashlib.sha256(token.encode()).hexdigest() == want, "token.txt does not match" + + port = int((state / "port").read_text()) + with socket.socket() as sock: + sock.settimeout(2) + assert sock.connect_ex(("127.0.0.1", port)) != 0, "server is still running" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/background-shell/environment/Dockerfile b/bench/terminal_bench/tasks/background-shell/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py b/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py new file mode 100644 index 000000000..54fdafbfc --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py @@ -0,0 +1,14 @@ +"""A slow job. It prints its result to stdout only once it finishes.""" + +import hashlib +import secrets +import time +from pathlib import Path + +result = secrets.token_hex(8) +state = Path(__file__).resolve().parent / ".job_state" +state.mkdir(exist_ok=True) +(state / "started").write_text(str(time.time())) +time.sleep(150) +(state / "result.sha256").write_text(hashlib.sha256(result.encode()).hexdigest()) +print(f"RESULT={result}", flush=True) diff --git a/bench/terminal_bench/tasks/background-shell/instruction.md b/bench/terminal_bench/tasks/background-shell/instruction.md new file mode 100644 index 000000000..5c60f0fbb --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/instruction.md @@ -0,0 +1,3 @@ +Run `python slow_job.py`. It takes about two and a half minutes and prints a line `RESULT=` only when it finishes. + +Write the value to `result.txt`. The result is only printed to the job's output. diff --git a/bench/terminal_bench/tasks/background-shell/solution/solve.py b/bench/terminal_bench/tasks/background-shell/solution/solve.py new file mode 100644 index 000000000..d744f624c --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/solution/solve.py @@ -0,0 +1,12 @@ +"""Reference solution for background-shell.""" + +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +out = subprocess.run( + [sys.executable, "slow_job.py"], cwd=app, capture_output=True, text=True, check=True +).stdout +(app / "result.txt").write_text(out.strip().removeprefix("RESULT=") + "\n") diff --git a/bench/terminal_bench/tasks/background-shell/solution/solve.sh b/bench/terminal_bench/tasks/background-shell/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/background-shell/task.toml b/bench/terminal_bench/tasks/background-shell/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/background-shell/tests/test.sh b/bench/terminal_bench/tasks/background-shell/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py b/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py new file mode 100644 index 000000000..845536a6a --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py @@ -0,0 +1,19 @@ +"""Verify the agent waited for the long job and captured its output.""" + +import hashlib +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + state = APP / ".job_state" + assert (state / "result.sha256").exists(), "slow_job.py never finished" + value = (APP / "result.txt").read_text().strip().removeprefix("RESULT=") + assert hashlib.sha256(value.encode()).hexdigest() == (state / "result.sha256").read_text() + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile b/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/__init__.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py new file mode 100644 index 000000000..1cccbcb30 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py @@ -0,0 +1,34 @@ +"""Order pricing for the inventory service.""" + +from decimal import ROUND_HALF_UP, Decimal + +# (minimum quantity, discount fraction), checked from the largest tier down. +BULK_TIERS = [(100, Decimal("0.15")), (50, Decimal("0.10")), (10, Decimal("0.05"))] + +TAX_RATES = {"CA": Decimal("0.0725"), "NY": Decimal("0.04"), "OR": Decimal("0")} + + +def bulk_discount(quantity: int) -> Decimal: + for minimum, discount in BULK_TIERS: + if quantity > minimum: + return discount + return Decimal("0") + + +def round_cents(amount: Decimal) -> Decimal: + return amount.quantize(Decimal("0.01")) + + +def line_total(unit_price: str, quantity: int) -> Decimal: + if quantity < 0: + raise ValueError("quantity must be non-negative") + subtotal = Decimal(unit_price) * quantity + return round_cents(subtotal - subtotal * bulk_discount(quantity)) + + +def order_total(lines: list[tuple[str, int]], state: str, coupon: str | None = None) -> Decimal: + subtotal = sum((line_total(price, qty) for price, qty in lines), Decimal("0")) + if coupon == "SAVE10": + subtotal -= Decimal("10") + tax = subtotal * TAX_RATES[state] + return round_cents(subtotal + tax) diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py new file mode 100644 index 000000000..afc2a7982 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py @@ -0,0 +1,41 @@ +import unittest +from decimal import Decimal + +from inventory.pricing import bulk_discount, line_total, order_total + + +class BulkDiscountTest(unittest.TestCase): + def test_tier_boundaries_are_inclusive(self): + self.assertEqual(bulk_discount(9), Decimal("0")) + self.assertEqual(bulk_discount(10), Decimal("0.05")) + self.assertEqual(bulk_discount(50), Decimal("0.10")) + self.assertEqual(bulk_discount(100), Decimal("0.15")) + + +class LineTotalTest(unittest.TestCase): + def test_half_cents_round_up(self): + # 3 x 0.125 = 0.375, which rounds to 0.38. + self.assertEqual(line_total("0.125", 3), Decimal("0.38")) + + def test_bulk_discount_applies(self): + self.assertEqual(line_total("2.00", 10), Decimal("19.00")) + + def test_negative_quantity_rejected(self): + with self.assertRaises(ValueError): + line_total("1.00", -1) + + +class OrderTotalTest(unittest.TestCase): + def test_tax_is_added(self): + self.assertEqual(order_total([("10.00", 1)], "CA"), Decimal("10.73")) + + def test_coupon_never_makes_total_negative(self): + self.assertEqual(order_total([("4.00", 1)], "NY", coupon="SAVE10"), Decimal("0.00")) + + def test_unknown_state_is_a_value_error(self): + with self.assertRaises(ValueError): + order_total([("1.00", 1)], "ZZ") + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/terminal_bench/tasks/fix-failing-tests/instruction.md b/bench/terminal_bench/tasks/fix-failing-tests/instruction.md new file mode 100644 index 000000000..814aaa905 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/instruction.md @@ -0,0 +1,9 @@ +The `inventory` package in the current directory has a failing test suite. + +Run it with `python -m unittest discover -s tests` and fix the code in `inventory/` until every test passes. + +Rules: +- Do not modify anything under `tests/`. +- Keep the public functions in `inventory/pricing.py` and their signatures. +- Money values are `Decimal`s rounded to cents with half-up rounding. +- `order_total` must raise `ValueError` for a state with no tax rate, and a coupon can never make the total negative. diff --git a/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py new file mode 100644 index 000000000..c727ff660 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py @@ -0,0 +1,23 @@ +"""Reference fix for fix-failing-tests.""" + +import os +from pathlib import Path + +path = Path(os.environ.get("TASK_APP_DIR", "/app")) / "inventory" / "pricing.py" +source = path.read_text() +for old, new in [ + ("if quantity > minimum:", "if quantity >= minimum:"), + ( + 'amount.quantize(Decimal("0.01"))', + 'amount.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)', + ), + ( + ' subtotal -= Decimal("10")\n', + ' subtotal = max(subtotal - Decimal("10"), Decimal("0"))\n' + " if state not in TAX_RATES:\n" + ' raise ValueError(f"unknown state: {state}")\n', + ), +]: + assert old in source, old + source = source.replace(old, new) +path.write_text(source) diff --git a/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/fix-failing-tests/task.toml b/bench/terminal_bench/tasks/fix-failing-tests/task.toml new file mode 100644 index 000000000..6d2b95265 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "debugging" +tags = ["python", "testing", "edit-loop"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh b/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py b/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py new file mode 100644 index 000000000..4ae6aa386 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py @@ -0,0 +1,39 @@ +"""Verify the inventory fixes. Runs from Harbor (/app) or the native runner (TASK_APP_DIR).""" + +import hashlib +import os +import subprocess +import sys +from decimal import Decimal +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +VISIBLE_TESTS_SHA256 = "d169ead570f87b1aa2eb351c1c226f42bea4b438e5e24720a98df3a5beb96002" + + +def main() -> None: + tests = (APP / "tests" / "test_pricing.py").read_bytes().replace(b"\r\n", b"\n") + assert hashlib.sha256(tests).hexdigest() == VISIBLE_TESTS_SHA256, "tests/ was modified" + + result = subprocess.run( + [sys.executable, "-m", "unittest", "discover", "-s", "tests"], + cwd=APP, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + sys.path.insert(0, str(APP)) + from inventory.pricing import bulk_discount, line_total, order_total + + assert bulk_discount(49) == Decimal("0.05") + assert bulk_discount(99) == Decimal("0.10") + assert line_total("0.005", 1) == Decimal("0.01") + assert line_total("1.00", 100) == Decimal("85.00") + assert order_total([("20.00", 1)], "NY", coupon="SAVE10") == Decimal("10.40") + assert order_total([("3.00", 2)], "OR") == Decimal("6.00") + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/image-code/environment/Dockerfile b/bench/terminal_bench/tasks/image-code/environment/Dockerfile new file mode 100644 index 000000000..18ba6f9bd --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/image-code/environment/setup.py b/bench/terminal_bench/tasks/image-code/environment/setup.py new file mode 100644 index 000000000..65481d7bb --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/environment/setup.py @@ -0,0 +1,52 @@ +"""Draw a numeric code into code.png with a 5x7 bitmap font (standard library only).""" + +import struct +import sys +import zlib +from pathlib import Path + +CODE = "47193825" +FONT = { + "0": ["01110", "10001", "10011", "10101", "11001", "10001", "01110"], + "1": ["00100", "01100", "00100", "00100", "00100", "00100", "01110"], + "2": ["01110", "10001", "00001", "00010", "00100", "01000", "11111"], + "3": ["11110", "00001", "00001", "01110", "00001", "00001", "11110"], + "4": ["00010", "00110", "01010", "10010", "11111", "00010", "00010"], + "5": ["11111", "10000", "11110", "00001", "00001", "10001", "01110"], + "6": ["00110", "01000", "10000", "11110", "10001", "10001", "01110"], + "7": ["11111", "00001", "00010", "00100", "01000", "01000", "01000"], + "8": ["01110", "10001", "10001", "01110", "10001", "10001", "01110"], + "9": ["01110", "10001", "10001", "01111", "00001", "00010", "01100"], +} +SCALE, GAP, MARGIN = 12, 2, 3 + + +def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + +def main(app: Path) -> None: + cols = MARGIN * 2 + len(CODE) * (5 + GAP) - GAP + rows = MARGIN * 2 + 7 + grid = [[255] * cols for _ in range(rows)] + for i, digit in enumerate(CODE): + for y, row in enumerate(FONT[digit]): + for x, bit in enumerate(row): + if bit == "1": + grid[MARGIN + y][MARGIN + i * (5 + GAP) + x] = 0 + raw = b"".join( + b"\x00" + bytes(v for v in row for _ in range(SCALE)) for row in grid for _ in range(SCALE) + ) + header = struct.pack(">IIBBBBB", cols * SCALE, rows * SCALE, 8, 0, 0, 0, 0) + png = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(raw)) + + chunk(b"IEND", b"") + ) + app.mkdir(parents=True, exist_ok=True) + (app / "code.png").write_bytes(png) + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/bench/terminal_bench/tasks/image-code/instruction.md b/bench/terminal_bench/tasks/image-code/instruction.md new file mode 100644 index 000000000..ecc297105 --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/instruction.md @@ -0,0 +1 @@ +`code.png` shows an 8-digit code. Look at the image and write the code to `code.txt`. diff --git a/bench/terminal_bench/tasks/image-code/solution/solve.py b/bench/terminal_bench/tasks/image-code/solution/solve.py new file mode 100644 index 000000000..5381a4a5b --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/solution/solve.py @@ -0,0 +1,6 @@ +"""Reference solution for image-code.""" + +import os +from pathlib import Path + +Path(os.environ.get("TASK_APP_DIR", "/app"), "code.txt").write_text("47193825\n") diff --git a/bench/terminal_bench/tasks/image-code/solution/solve.sh b/bench/terminal_bench/tasks/image-code/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/image-code/task.toml b/bench/terminal_bench/tasks/image-code/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/image-code/tests/test.sh b/bench/terminal_bench/tasks/image-code/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/image-code/tests/test_outputs.py b/bench/terminal_bench/tasks/image-code/tests/test_outputs.py new file mode 100644 index 000000000..81b22af69 --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/tests/test_outputs.py @@ -0,0 +1,15 @@ +"""Verify the code read from the image.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + assert (APP / "code.txt").read_text().strip() == "47193825" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/log-triage/environment/Dockerfile b/bench/terminal_bench/tasks/log-triage/environment/Dockerfile new file mode 100644 index 000000000..18ba6f9bd --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/log-triage/environment/setup.py b/bench/terminal_bench/tasks/log-triage/environment/setup.py new file mode 100644 index 000000000..55140114d --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/environment/setup.py @@ -0,0 +1,39 @@ +"""Generate deterministic service logs under /logs.""" + +import random +import sys +from datetime import datetime, timedelta +from pathlib import Path + +SERVICES = ["auth", "billing", "gateway", "search", "storage"] +CODES = ["E101", "E202", "E303", "E404", "E505", "E606"] +MESSAGES = { + "INFO": ["request served", "cache warmed", "heartbeat ok", "config reloaded"], + "WARN": ["slow upstream", "retrying request", "queue depth high"], + "ERROR": ["upstream failed", "timeout", "bad payload", "disk full"], +} + + +def main(app: Path) -> None: + rng = random.Random(4242) + start = datetime(2026, 9, 1) + for service in SERVICES: + # Each service favors a different error code so the answer isn't uniform. + weights = [rng.randint(1, 4) for _ in CODES] + weights[rng.randrange(len(CODES))] += 8 + for day in range(4): + date = start + timedelta(days=day) + lines = [] + for i in range(rng.randint(250, 400)): + ts = date + timedelta(seconds=i * 37 + rng.randint(0, 30)) + level = rng.choices(["INFO", "WARN", "ERROR"], [80, 12, 8])[0] + message = rng.choice(MESSAGES[level]) + code = f" code={rng.choices(CODES, weights)[0]}" if level == "ERROR" else "" + lines.append(f"{ts.isoformat()} {level:<5} [{service}] {message}{code}") + out = app / "logs" / service / f"{date:%Y-%m-%d}.log" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines) + "\n", newline="\n") + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/bench/terminal_bench/tasks/log-triage/instruction.md b/bench/terminal_bench/tasks/log-triage/instruction.md new file mode 100644 index 000000000..2f9e2593f --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/instruction.md @@ -0,0 +1,22 @@ +`logs/` holds four days of logs for five services, one directory per service. +Each line looks like: + +``` +2026-09-01T00:00:12 ERROR [billing] upstream failed code=E303 +``` + +The level is `INFO`, `WARN`, or `ERROR`. Only `ERROR` lines carry a `code=`. + +Write `report.json` in the current directory with one entry per service: + +```json +{ + "auth": {"errors": 0, "warnings": 0, "top_error_code": "E101"}, + ... +} +``` + +- `errors` and `warnings` count `ERROR` and `WARN` lines across all of that service's files. +- `top_error_code` is the code seen most often in that service's `ERROR` lines. Break ties by the alphabetically smallest code. + +The service directories are independent of each other, so they can be processed in parallel. diff --git a/bench/terminal_bench/tasks/log-triage/solution/solve.py b/bench/terminal_bench/tasks/log-triage/solution/solve.py new file mode 100644 index 000000000..ad49e2135 --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/solution/solve.py @@ -0,0 +1,19 @@ +"""Reference solution for log-triage.""" + +import json +import os +import re +from collections import Counter +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +report = {} +for service_dir in sorted((app / "logs").iterdir()): + text = "".join(log.read_text() for log in service_dir.glob("*.log")) + codes = Counter(re.findall(r" ERROR .* code=(E\d+)$", text, re.MULTILINE)) + report[service_dir.name] = { + "errors": len(re.findall(r"^\S+ ERROR ", text, re.MULTILINE)), + "warnings": len(re.findall(r"^\S+ WARN ", text, re.MULTILINE)), + "top_error_code": min(codes, key=lambda code: (-codes[code], code)), + } +(app / "report.json").write_text(json.dumps(report, indent=2)) diff --git a/bench/terminal_bench/tasks/log-triage/solution/solve.sh b/bench/terminal_bench/tasks/log-triage/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/log-triage/task.toml b/bench/terminal_bench/tasks/log-triage/task.toml new file mode 100644 index 000000000..bf1050aa1 --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "data-processing" +tags = ["logs", "fan-out", "subagents"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/log-triage/tests/test.sh b/bench/terminal_bench/tasks/log-triage/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py b/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py new file mode 100644 index 000000000..aaebaff63 --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py @@ -0,0 +1,42 @@ +"""Verify report.json against counts recomputed from the logs.""" + +import json +import os +import re +from collections import Counter +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +LINE = re.compile(r"^\S+ (INFO|WARN|ERROR)\s+\[(\w+)\] .*?(?: code=(E\d+))?$") + + +def expected() -> dict: + report = {} + for service_dir in sorted((APP / "logs").iterdir()): + levels: Counter[str] = Counter() + codes: Counter[str] = Counter() + for log in service_dir.glob("*.log"): + for line in log.read_text().splitlines(): + match = LINE.match(line) + assert match, line + levels[match[1]] += 1 + if match[3]: + codes[match[3]] += 1 + top = max(sorted(codes), key=lambda code: codes[code]) + report[service_dir.name] = { + "errors": levels["ERROR"], + "warnings": levels["WARN"], + "top_error_code": top, + } + return report + + +def main() -> None: + actual = json.loads((APP / "report.json").read_text()) + want = expected() + assert actual == want, {"expected": want, "actual": actual} + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile b/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json b/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json new file mode 100644 index 000000000..2fc9d2f15 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json @@ -0,0 +1 @@ +{"enableAllProjectMcpServers": true} diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json b/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json new file mode 100644 index 000000000..e5413902e --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "inventory": {"command": "python", "args": ["mcp_server.py"]} + } +} diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py b/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py new file mode 100644 index 000000000..74704dc14 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py @@ -0,0 +1,51 @@ +"""Minimal stdio MCP server with one tool. Stock levels are random per server start.""" + +import json +import random +import sys +from pathlib import Path + +LOG = Path(__file__).resolve().parent / ".mcp_log" / "calls.jsonl" +STOCK: dict[str, int] = {} +TOOL = { + "name": "get_stock", + "description": "Return the current number of units in stock for a SKU.", + "inputSchema": { + "type": "object", + "properties": {"sku": {"type": "string"}}, + "required": ["sku"], + }, +} + + +def handle(method: str, params: dict) -> dict: + if method == "initialize": + return { + "protocolVersion": params.get("protocolVersion", "2025-06-18"), + "capabilities": {"tools": {}}, + "serverInfo": {"name": "inventory", "version": "1.0"}, + } + if method == "tools/list": + return {"tools": [TOOL]} + if method == "tools/call": + sku = params["arguments"]["sku"].strip().upper() + units = STOCK.setdefault(sku, random.randint(100, 999)) + LOG.parent.mkdir(exist_ok=True) + with LOG.open("a", encoding="utf-8") as f: + f.write(json.dumps({"sku": sku, "units": units}) + "\n") + return {"content": [{"type": "text", "text": f"{sku}: {units} units in stock"}]} + if method == "ping": + return {} + raise KeyError(method) + + +for line in sys.stdin: + message = json.loads(line) + if "id" not in message: + continue + try: + response = {"result": handle(message["method"], message.get("params") or {})} + except KeyError: + response = {"error": {"code": -32601, "message": f"unknown method {message['method']}"}} + sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": message["id"], **response}) + "\n") + sys.stdout.flush() diff --git a/bench/terminal_bench/tasks/mcp-server/instruction.md b/bench/terminal_bench/tasks/mcp-server/instruction.md new file mode 100644 index 000000000..9b5642bf9 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/instruction.md @@ -0,0 +1,5 @@ +This project has an `inventory` MCP server. Use its `get_stock` tool to look up the stock for SKUs `A-100`, `B-200`, and `C-300`. + +Write `stock.json` in the current directory mapping each SKU to its unit count as an integer, for example `{"A-100": 0, ...}`. + +Stock levels are only available from the MCP tool. diff --git a/bench/terminal_bench/tasks/mcp-server/solution/solve.py b/bench/terminal_bench/tasks/mcp-server/solution/solve.py new file mode 100644 index 000000000..dca647d48 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/solution/solve.py @@ -0,0 +1,35 @@ +"""Reference run: speak MCP to the server over stdio.""" + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +requests = [{"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {}}] +requests += [ + { + "jsonrpc": "2.0", + "id": i, + "method": "tools/call", + "params": {"name": "get_stock", "arguments": {"sku": sku}}, + } + for i, sku in enumerate(["A-100", "B-200", "C-300"], start=1) +] +out = subprocess.run( + [sys.executable, "mcp_server.py"], + cwd=app, + input="".join(json.dumps(r) + "\n" for r in requests), + capture_output=True, + text=True, + check=True, +).stdout +stock = {} +for line in out.splitlines()[1:]: + sku, units = re.match( + r"(\S+): (\d+)", json.loads(line)["result"]["content"][0]["text"] + ).groups() + stock[sku] = int(units) +(app / "stock.json").write_text(json.dumps(stock)) diff --git a/bench/terminal_bench/tasks/mcp-server/solution/solve.sh b/bench/terminal_bench/tasks/mcp-server/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/mcp-server/task.toml b/bench/terminal_bench/tasks/mcp-server/task.toml new file mode 100644 index 000000000..a699c38db --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/mcp-server/tests/test.sh b/bench/terminal_bench/tasks/mcp-server/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py b/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py new file mode 100644 index 000000000..cc3ff235f --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py @@ -0,0 +1,30 @@ +"""Verify stock.json matches what the MCP server actually returned.""" + +import json +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +SKUS = ["A-100", "B-200", "C-300"] + + +def main() -> None: + log = APP / ".mcp_log" / "calls.jsonl" + assert log.exists(), "the MCP tool was never called" + served = {} + for line in log.read_text().splitlines(): + call = json.loads(line) + served[call["sku"]] = call["units"] + actual = json.loads((APP / "stock.json").read_text()) + for sku in SKUS: + assert sku in served, f"get_stock was never called for {sku}" + assert actual.get(sku) == served[sku], { + "sku": sku, + "served": served[sku], + "got": actual.get(sku), + } + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile b/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile new file mode 100644 index 000000000..18ba6f9bd --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/pdf-extract/environment/setup.py b/bench/terminal_bench/tasks/pdf-extract/environment/setup.py new file mode 100644 index 000000000..86476b4ef --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/environment/setup.py @@ -0,0 +1,45 @@ +"""Write invoice.pdf with a Flate-compressed text stream (standard library only).""" + +import sys +import zlib +from pathlib import Path + +LINES = [ + "Northwind Supplies GmbH", + "Invoice INV-2026-0419", + "Consulting hours: 38", + "Subtotal: 6,169.91 EUR", + "VAT (19%): 1,172.28 EUR", + "Total due: 7,342.19 EUR", +] + + +def main(app: Path) -> None: + text = "BT /F1 14 Tf 72 720 Td 20 TL " + " ".join(f"({line}) Tj T*" for line in LINES) + " ET" + stream = zlib.compress(text.encode()) + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Length %d /Filter /FlateDecode >>\nstream\n" % len(stream) + stream + b"\nendstream", + ] + pdf = bytearray(b"%PDF-1.4\n") + offsets = [] + for number, body in enumerate(objects, start=1): + offsets.append(len(pdf)) + pdf += b"%d 0 obj\n" % number + body + b"\nendobj\n" + xref = len(pdf) + pdf += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) + pdf += b"".join(b"%010d 00000 n \n" % offset for offset in offsets) + pdf += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % ( + len(objects) + 1, + xref, + ) + app.mkdir(parents=True, exist_ok=True) + (app / "invoice.pdf").write_bytes(bytes(pdf)) + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/bench/terminal_bench/tasks/pdf-extract/instruction.md b/bench/terminal_bench/tasks/pdf-extract/instruction.md new file mode 100644 index 000000000..e931c3aea --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/instruction.md @@ -0,0 +1 @@ +Read `invoice.pdf` and write the invoice's total due to `total.txt` exactly as printed, for example `1,234.56 EUR`. diff --git a/bench/terminal_bench/tasks/pdf-extract/solution/solve.py b/bench/terminal_bench/tasks/pdf-extract/solution/solve.py new file mode 100644 index 000000000..b636f8c4b --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/solution/solve.py @@ -0,0 +1,6 @@ +"""Reference solution for pdf-extract.""" + +import os +from pathlib import Path + +Path(os.environ.get("TASK_APP_DIR", "/app"), "total.txt").write_text("7,342.19 EUR\n") diff --git a/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh b/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/pdf-extract/task.toml b/bench/terminal_bench/tasks/pdf-extract/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/pdf-extract/tests/test.sh b/bench/terminal_bench/tasks/pdf-extract/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py b/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py new file mode 100644 index 000000000..8306a656d --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py @@ -0,0 +1,16 @@ +"""Verify the total read from the PDF.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + total = (APP / "total.txt").read_text().strip() + assert total.removeprefix("Total due:").strip() == "7,342.19 EUR", total + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile b/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md b/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md new file mode 100644 index 000000000..22d1c0db1 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md @@ -0,0 +1,4 @@ +# Project conventions + +- Put every output file under `out/`. Never write output files to the project root. +- Every Python file you create must start with the line `# SPDX-License-Identifier: Apache-2.0`. diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md b/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md new file mode 100644 index 000000000..22d1c0db1 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md @@ -0,0 +1,4 @@ +# Project conventions + +- Put every output file under `out/`. Never write output files to the project root. +- Every Python file you create must start with the line `# SPDX-License-Identifier: Apache-2.0`. diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt b/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt new file mode 100644 index 000000000..f6b86b2b2 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt @@ -0,0 +1,40 @@ +model cache gateway token tool token route shell gateway tool +gateway token cache cache token agent token tool +gateway shell token agent shell gateway shell shell cache gateway agent +tool model harness cache model +token shell harness tool model token shell shell agent route token tool token +gateway shell agent prompt tool cache route prompt shell prompt route harness agent model +token shell harness tool prompt route prompt harness +token token tool cache model route model prompt cache gateway token tool shell route +route shell prompt shell prompt token token harness prompt token +harness shell prompt harness cache +gateway prompt route model shell token prompt gateway agent harness +agent cache cache prompt token model prompt +tool harness model cache tool harness cache route cache agent model +model model agent agent gateway prompt +model harness harness gateway model cache tool route shell shell route model tool shell +prompt tool cache cache cache +token prompt cache gateway agent token agent prompt model token route +gateway token gateway shell model tool token route shell gateway token agent shell cache +harness route shell route prompt token token +prompt prompt prompt harness token model token route harness prompt model tool +agent tool route model tool +tool harness token harness tool +model route agent tool tool tool route agent shell agent +cache agent agent tool prompt route gateway gateway +prompt harness agent shell route prompt route route token +token agent prompt agent route agent prompt shell +gateway prompt route token token cache agent prompt model cache route token cache prompt +token model model model gateway model shell prompt model shell shell +route model tool tool model gateway gateway token tool model cache agent +gateway harness agent harness tool agent shell route +tool cache model gateway route prompt shell tool cache +model tool model tool tool gateway prompt model shell gateway model model model +shell token tool gateway route tool tool tool prompt token tool gateway +agent harness gateway token tool prompt tool gateway +prompt route shell tool shell tool +harness prompt tool tool prompt tool agent tool +tool agent prompt model cache token cache prompt route +agent cache token agent harness token +route model harness model prompt agent token +prompt model agent model cache tool cache route cache agent route diff --git a/bench/terminal_bench/tasks/project-instructions/instruction.md b/bench/terminal_bench/tasks/project-instructions/instruction.md new file mode 100644 index 000000000..27e157f0f --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/instruction.md @@ -0,0 +1 @@ +Write a Python script that counts the words in `data.txt`, run it, and save the count to a file named `count.txt`. diff --git a/bench/terminal_bench/tasks/project-instructions/solution/solve.py b/bench/terminal_bench/tasks/project-instructions/solution/solve.py new file mode 100644 index 000000000..1f9c1deab --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/solution/solve.py @@ -0,0 +1,16 @@ +"""Reference solution for project-instructions.""" + +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +(app / "out").mkdir(exist_ok=True) +script = app / "out" / "count_words.py" +script.write_text( + "# SPDX-License-Identifier: Apache-2.0\n" + "from pathlib import Path\n" + "Path('out/count.txt').write_text(str(len(Path('data.txt').read_text().split())))\n" +) +subprocess.run([sys.executable, str(script)], cwd=app, check=True) diff --git a/bench/terminal_bench/tasks/project-instructions/solution/solve.sh b/bench/terminal_bench/tasks/project-instructions/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/project-instructions/task.toml b/bench/terminal_bench/tasks/project-instructions/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/project-instructions/tests/test.sh b/bench/terminal_bench/tasks/project-instructions/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py b/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py new file mode 100644 index 000000000..3eb0f9c76 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py @@ -0,0 +1,24 @@ +"""Verify the agent followed the project's instruction file.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +HEADER = "# SPDX-License-Identifier: Apache-2.0" + + +def main() -> None: + expected = len((APP / "data.txt").read_text().split()) + assert not (APP / "count.txt").exists(), "count.txt was written to the project root" + assert (APP / "out" / "count.txt").read_text().strip() == str(expected) + scripts = [p for p in APP.rglob("*.py") if ".claude" not in p.parts] + assert scripts, "no Python script was written" + for script in scripts: + assert script.read_text().splitlines()[0].strip() == HEADER, ( + f"{script.name} lacks the SPDX header" + ) + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/project-skill/environment/Dockerfile b/bench/terminal_bench/tasks/project-skill/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md new file mode 100644 index 000000000..f843dca9e --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md @@ -0,0 +1,10 @@ +--- +name: release-notes +description: Build release notes for this project. Use whenever someone asks for release notes or a changelog. +--- + +# Release notes + +1. Run `python .claude/skills/release-notes/build.py `. It reads `changes.txt` and prints a draft. +2. Save the draft to `dist/release-notes-.md`. +3. Add this as the last line of the file, exactly: `Signed-off-by: release-bot` diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py new file mode 100644 index 000000000..6188516e1 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py @@ -0,0 +1,20 @@ +"""Draft release notes from changes.txt, grouped by change type.""" + +import json +import sys +from pathlib import Path + +root = Path(__file__).resolve().parents[3] +version = sys.argv[1] +groups: dict[str, list[str]] = {} +for line in (root / "changes.txt").read_text().splitlines(): + kind, _, text = line.partition(": ") + groups.setdefault(kind.strip(), []).append(text.strip()) +print(f"# Release {version}\n") +for kind in sorted(groups): + print(f"## {kind.title()}\n") + print("\n".join(f"- {item}" for item in groups[kind]) + "\n") +log = root / ".skill_log" / "runs.jsonl" +log.parent.mkdir(exist_ok=True) +with log.open("a", encoding="utf-8") as f: + f.write(json.dumps({"version": version}) + "\n") diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt b/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt new file mode 100644 index 000000000..ffd3e31e6 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt @@ -0,0 +1,5 @@ +fix: Retry gateway calls on HTTP 429 +feature: Add a Codex model catalog +fix: Keep custom headers on reconfigure +docs: Document managed settings +feature: Smart routing for subagents diff --git a/bench/terminal_bench/tasks/project-skill/instruction.md b/bench/terminal_bench/tasks/project-skill/instruction.md new file mode 100644 index 000000000..ca7a222cd --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/instruction.md @@ -0,0 +1 @@ +Write the release notes for version 2.3.0. diff --git a/bench/terminal_bench/tasks/project-skill/solution/solve.py b/bench/terminal_bench/tasks/project-skill/solution/solve.py new file mode 100644 index 000000000..278a7430c --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/solution/solve.py @@ -0,0 +1,19 @@ +"""Reference solution: follow the skill.""" + +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +draft = subprocess.run( + [sys.executable, ".claude/skills/release-notes/build.py", "2.3.0"], + cwd=app, + capture_output=True, + text=True, + check=True, +).stdout +(app / "dist").mkdir(exist_ok=True) +(app / "dist" / "release-notes-2.3.0.md").write_text( + draft.rstrip() + "\n\nSigned-off-by: release-bot\n" +) diff --git a/bench/terminal_bench/tasks/project-skill/solution/solve.sh b/bench/terminal_bench/tasks/project-skill/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/project-skill/task.toml b/bench/terminal_bench/tasks/project-skill/task.toml new file mode 100644 index 000000000..a699c38db --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/project-skill/tests/test.sh b/bench/terminal_bench/tasks/project-skill/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py b/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py new file mode 100644 index 000000000..8c409b092 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py @@ -0,0 +1,30 @@ +"""Verify the release-notes skill's procedure was followed.""" + +import json +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + log = APP / ".skill_log" / "runs.jsonl" + assert log.exists(), "the skill's build script never ran" + assert any( + json.loads(line)["version"].lstrip("v") == "2.3.0" for line in log.read_text().splitlines() + ) + notes = (APP / "dist" / "release-notes-2.3.0.md").read_text().strip().splitlines() + assert notes[0].strip() == "# Release 2.3.0", notes[0] + assert notes[-1].strip() == "Signed-off-by: release-bot", notes[-1] + body = "\n".join(notes) + for item in [ + "Retry gateway calls on HTTP 429", + "Smart routing for subagents", + "Document managed settings", + ]: + assert item in body, item + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/resume-session/environment/Dockerfile b/bench/terminal_bench/tasks/resume-session/environment/Dockerfile new file mode 100644 index 000000000..18ba6f9bd --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/resume-session/environment/setup.py b/bench/terminal_bench/tasks/resume-session/environment/setup.py new file mode 100644 index 000000000..0e4ea09b5 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/environment/setup.py @@ -0,0 +1,7 @@ +"""Write the vault file the first step reads.""" + +import sys +from pathlib import Path + +Path(sys.argv[1]).mkdir(parents=True, exist_ok=True) +Path(sys.argv[1], "vault.txt").write_text("passphrase: amber-falcon-7342\n") diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md b/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md new file mode 100644 index 000000000..1bf7f3a62 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md @@ -0,0 +1,3 @@ +`vault.txt` holds a passphrase. Read it and remember it, then delete `vault.txt`. + +Don't write the passphrase to any file. You'll be asked for it later in this session. diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py new file mode 100644 index 000000000..fec3b5770 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py @@ -0,0 +1,9 @@ +"""Reference step 1: the oracle has no memory, so keep the value outside the project.""" + +import os +import tempfile +from pathlib import Path + +vault = Path(os.environ.get("TASK_APP_DIR", "/app"), "vault.txt") +Path(tempfile.gettempdir(), "ug-bench-passphrase").write_text(vault.read_text().split(": ", 1)[1]) +vault.unlink() diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py new file mode 100644 index 000000000..0134d2485 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py @@ -0,0 +1,18 @@ +"""Verify the vault is gone and the passphrase wasn't stashed in the project.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + assert not (APP / "vault.txt").exists(), "vault.txt still exists" + for path in APP.rglob("*"): + if path.is_file(): + assert b"amber-falcon-7342" not in path.read_bytes(), f"passphrase written to {path}" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md b/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md new file mode 100644 index 000000000..928e46bf9 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md @@ -0,0 +1 @@ +Earlier in this session you read a passphrase from `vault.txt`. Write it to `answer.txt`. diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py new file mode 100644 index 000000000..b53573d04 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py @@ -0,0 +1,8 @@ +"""Reference step 2.""" + +import os +import tempfile +from pathlib import Path + +value = Path(tempfile.gettempdir(), "ug-bench-passphrase").read_text() +Path(os.environ.get("TASK_APP_DIR", "/app"), "answer.txt").write_text(value) diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py new file mode 100644 index 000000000..f362c0f7e --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py @@ -0,0 +1,15 @@ +"""Verify the resumed session recalled the passphrase.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + assert "amber-falcon-7342" in (APP / "answer.txt").read_text() + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/resume-session/task.toml b/bench/terminal_bench/tasks/resume-session/task.toml new file mode 100644 index 000000000..84280a982 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 + +[[steps]] +name = "learn" +min_reward = 1.0 + +[[steps]] +name = "recall" +min_reward = 1.0 diff --git a/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile b/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile new file mode 100644 index 000000000..b23e2016a --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py b/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py new file mode 100644 index 000000000..397b36cf7 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py @@ -0,0 +1,57 @@ +"""Build a git repo at that leaks a token in history on two branches.""" + +import subprocess +import sys +from pathlib import Path + +TOKEN = "ugb_live_7f3a9c2e1d4b8a6f0e5d" + + +def git(app: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=app, check=True, capture_output=True) + + +def commit(app: Path, message: str, files: dict[str, str]) -> None: + for name, content in files.items(): + path = app / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, newline="\n") + git(app, "add", "-A") + git(app, "commit", "-m", message) + + +def main(app: Path) -> None: + app.mkdir(parents=True, exist_ok=True) + git(app, "init", "-q", "-b", "main") + for key, value in [ + ("user.name", "Bench Bot"), + ("user.email", "bench@example.com"), + ("core.autocrlf", "false"), + ("commit.gpgsign", "false"), + ]: + git(app, "config", key, value) + + commit(app, "Initial commit", {"README.md": "# report service\n", "app.py": "print('hi')\n"}) + commit(app, "Add settings", {"config/settings.py": f'API_TOKEN = "{TOKEN}"\nTIMEOUT = 30\n'}) + commit( + app, + "Use settings in app", + {"app.py": "from config.settings import API_TOKEN, TIMEOUT\n\nprint(TIMEOUT)\n"}, + ) + git(app, "branch", "feature/report") + commit( + app, + "Read token from the environment", + {"config/settings.py": 'import os\n\nAPI_TOKEN = os.environ["API_TOKEN"]\nTIMEOUT = 30\n'}, + ) + git(app, "checkout", "-q", "feature/report") + commit( + app, + "Document deploy", + {"docs/deploy.md": f"Deploy with:\n\n API_TOKEN={TOKEN} python app.py\n"}, + ) + git(app, "checkout", "-q", "main") + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/bench/terminal_bench/tasks/scrub-git-secret/instruction.md b/bench/terminal_bench/tasks/scrub-git-secret/instruction.md new file mode 100644 index 000000000..a6840a384 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/instruction.md @@ -0,0 +1,8 @@ +The git repository in the current directory leaked an API token (it starts with `ugb_live_`). It was committed on `main` and later removed, and it also shows up on the `feature/report` branch. + +Rewrite the history so the token is unreachable from every ref: + +- Replace every occurrence of the token, in every commit on every branch, with the literal text `REDACTED`. Keep the rest of each file as is. +- Keep both branches, the same number of commits on each, and the same commit messages in the same order. +- Don't leave backup refs (such as `refs/original/`) that still contain the token. +- Leave `main` checked out when you are done. diff --git a/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py new file mode 100644 index 000000000..8e530fae9 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py @@ -0,0 +1,32 @@ +"""Reference solution for scrub-git-secret using filter-branch.""" + +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +token = "ugb_live_7f3a9c2e1d4b8a6f0e5d" +# Python one-liner keeps the tree filter portable across Linux and Git Bash. +tree_filter = ( + f'"{Path(sys.executable).as_posix()}" -c "' + "import pathlib;" + "[p.write_text(p.read_text().replace('" + token + "','REDACTED'), newline='\\n')" + " for p in pathlib.Path('.').rglob('*') if p.is_file() and '" + token + "' in p.read_text()]\"" +) +env = {**os.environ, "FILTER_BRANCH_SQUELCH_WARNING": "1"} +subprocess.run( + ["git", "filter-branch", "-f", "--tree-filter", tree_filter, "--", "--all"], + cwd=app, + check=True, + env=env, +) +refs = subprocess.run( + ["git", "for-each-ref", "--format=%(refname)", "refs/original"], + cwd=app, + check=True, + capture_output=True, + text=True, +).stdout.split() +for ref in refs: + subprocess.run(["git", "update-ref", "-d", ref], cwd=app, check=True) diff --git a/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/scrub-git-secret/task.toml b/bench/terminal_bench/tasks/scrub-git-secret/task.toml new file mode 100644 index 000000000..452f8a548 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "medium" +category = "security" +tags = ["git", "history-rewrite", "destructive"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh b/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py b/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py new file mode 100644 index 000000000..a98c50519 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py @@ -0,0 +1,44 @@ +"""Verify the token is gone from every ref while history keeps its shape.""" + +import os +import subprocess +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +TOKEN = "ugb_live_7f3a9c2e1d4b8a6f0e5d" +EXPECTED = { + "main": [ + "Read token from the environment", + "Use settings in app", + "Add settings", + "Initial commit", + ], + "feature/report": ["Document deploy", "Use settings in app", "Add settings", "Initial commit"], +} + + +def git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=APP, check=True, capture_output=True, text=True + ).stdout + + +def main() -> None: + for branch, messages in EXPECTED.items(): + assert git("log", "--format=%s", branch).splitlines() == messages, branch + + for commit in git("rev-list", "--all").split(): + found = subprocess.run(["git", "grep", "-F", TOKEN, commit], cwd=APP, capture_output=True) + assert found.returncode == 1, f"token still in {commit}" + assert TOKEN not in git("show", "-s", "--format=%B", commit) + + settings = git("show", "main~2:config/settings.py") + assert "REDACTED" in settings and "TIMEOUT = 30" in settings, settings + assert "REDACTED" in git("show", "feature/report:docs/deploy.md") + assert "os.environ" in git("show", "main:config/settings.py") + assert git("rev-parse", "--abbrev-ref", "HEAD").strip() == "main" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile b/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile new file mode 100644 index 000000000..3a5c9c9b3 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md b/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md new file mode 100644 index 000000000..0088c6a87 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md @@ -0,0 +1,14 @@ +--- +name: checksum-auditor +description: Audits a single file by recording its SHA-256 checksum. Use it for every checksum audit. +model: haiku +tools: Read, Bash, Write +--- + +You audit exactly one file. Compute its SHA-256 with Python's `hashlib`, then write +`audit/.txt` with two lines: + +``` +auditor: checksum-auditor +sha256: +``` diff --git a/bench/terminal_bench/tasks/subagent-model/environment/setup.py b/bench/terminal_bench/tasks/subagent-model/environment/setup.py new file mode 100644 index 000000000..1a053edb4 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/environment/setup.py @@ -0,0 +1,10 @@ +"""Write random data files to audit.""" + +import os +import sys +from pathlib import Path + +data = Path(sys.argv[1], "data") +data.mkdir(parents=True, exist_ok=True) +for name in ["alpha.bin", "bravo.bin", "charlie.bin"]: + (data / name).write_bytes(os.urandom(4096)) diff --git a/bench/terminal_bench/tasks/subagent-model/instruction.md b/bench/terminal_bench/tasks/subagent-model/instruction.md new file mode 100644 index 000000000..31562d60a --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/instruction.md @@ -0,0 +1 @@ +Audit every file in `data/` with the `checksum-auditor` subagent. Run one subagent per file, and don't compute the checksums yourself. diff --git a/bench/terminal_bench/tasks/subagent-model/solution/solve.py b/bench/terminal_bench/tasks/subagent-model/solution/solve.py new file mode 100644 index 000000000..5a0cf9623 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/solution/solve.py @@ -0,0 +1,13 @@ +"""Reference solution for subagent-model (the file side only).""" + +import hashlib +import os +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +(app / "audit").mkdir(exist_ok=True) +for path in (app / "data").iterdir(): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + (app / "audit" / f"{path.name}.txt").write_text( + f"auditor: checksum-auditor\nsha256: {digest}\n" + ) diff --git a/bench/terminal_bench/tasks/subagent-model/solution/solve.sh b/bench/terminal_bench/tasks/subagent-model/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/subagent-model/task.toml b/bench/terminal_bench/tasks/subagent-model/task.toml new file mode 100644 index 000000000..a699c38db --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/subagent-model/tests/test.sh b/bench/terminal_bench/tasks/subagent-model/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py b/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py new file mode 100644 index 000000000..73cec0e47 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py @@ -0,0 +1,54 @@ +"""Verify the audits and that a subagent ran on the haiku model.""" + +import glob +import hashlib +import json +import os +import re +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def agent_log() -> Path: + paths = [os.environ.get("TASK_AGENT_LOG", "")] + glob.glob("/logs/agent/ug-claude.txt") + return next(Path(p) for p in paths if p and Path(p).exists()) + + +def read_transcript(log: Path) -> tuple[set[str], list[str]]: + """Models subagents ran on, and any notices that their requested model was restricted.""" + models, restricted = set(), [] + for line in log.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + event = json.loads(line) + except ValueError: + continue + if event.get("type") == "assistant" and event.get("parent_tool_use_id"): + models.add(event.get("message", {}).get("model", "")) + if str(event.get("key", "")).startswith("agent-model-restricted-checksum-auditor"): + restricted.append(event.get("text", "")) + return models, restricted + + +def main() -> None: + for path in sorted((APP / "data").iterdir()): + audit = (APP / "audit" / f"{path.name}.txt").read_text().splitlines() + assert audit[0].strip() == "auditor: checksum-auditor", audit + assert audit[1].strip() == f"sha256: {hashlib.sha256(path.read_bytes()).hexdigest()}", audit + if os.environ.get("TASK_ORACLE"): + print("ok (oracle: skipped subagent check)") + return + models, restricted = read_transcript(agent_log()) + assert models, "no subagent ran" + if any("haiku" in m for m in models): + print("ok", models) + return + # A workspace policy can disallow haiku; then the subagent must run on the named fallback. + assert restricted, f"subagent ignored model: haiku without a policy notice: {models}" + fallback = re.search(r"Using (\S+?) instead", restricted[0])[1] + assert all(fallback.endswith(m) for m in models), {"fallback": fallback, "models": models} + print("ok (haiku restricted by policy)", fallback, models) + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile b/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile new file mode 100644 index 000000000..3a5c9c9b3 --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py new file mode 100644 index 000000000..6035f9883 --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py @@ -0,0 +1,24 @@ +"""Project hook: share the release codename and log every hook call.""" + +import json +import sys +from pathlib import Path + +root = Path(__file__).resolve().parents[2] +payload = json.load(sys.stdin) +log = root / ".hook_log" / "events.jsonl" +log.parent.mkdir(exist_ok=True) +with log.open("a", encoding="utf-8") as f: + f.write( + json.dumps( + { + "event": sys.argv[1], + "tool": payload.get("tool_name"), + "file": (payload.get("tool_input") or {}).get("file_path"), + } + ) + + "\n" + ) +if sys.argv[1] == "prompt": + codename = (root / ".claude" / "hooks" / "codename").read_text().strip() + print(f"The release codename for this session is {codename}.") diff --git a/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json new file mode 100644 index 000000000..d8c1cd98c --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json @@ -0,0 +1,13 @@ +{ + "hooks": { + "UserPromptSubmit": [ + {"hooks": [{"type": "command", "command": "python \"$CLAUDE_PROJECT_DIR/.claude/hooks/hook.py\" prompt"}]} + ], + "PostToolUse": [ + { + "matcher": "Write|Edit", + "hooks": [{"type": "command", "command": "python \"$CLAUDE_PROJECT_DIR/.claude/hooks/hook.py\" tool"}] + } + ] + } +} diff --git a/bench/terminal_bench/tasks/user-hooks/environment/setup.py b/bench/terminal_bench/tasks/user-hooks/environment/setup.py new file mode 100644 index 000000000..9efc79370 --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/environment/setup.py @@ -0,0 +1,6 @@ +"""Store the codename the hook hands to the agent.""" + +import sys +from pathlib import Path + +Path(sys.argv[1], ".claude", "hooks", "codename").write_text("velvet-harbor-58\n") diff --git a/bench/terminal_bench/tasks/user-hooks/instruction.md b/bench/terminal_bench/tasks/user-hooks/instruction.md new file mode 100644 index 000000000..8b62f6606 --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/instruction.md @@ -0,0 +1,3 @@ +This project's hooks give you the release codename at the start of the session. + +Write a file `release.txt` in the current directory that contains only that codename. diff --git a/bench/terminal_bench/tasks/user-hooks/solution/solve.py b/bench/terminal_bench/tasks/user-hooks/solution/solve.py new file mode 100644 index 000000000..14ad8694f --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/solution/solve.py @@ -0,0 +1,15 @@ +"""Reference run: drive the hook the way Claude Code would.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +hook = [sys.executable, str(app / ".claude" / "hooks" / "hook.py")] +out = subprocess.run([*hook, "prompt"], input="{}", capture_output=True, text=True, check=True) +codename = out.stdout.strip().removesuffix(".").rsplit(" ", 1)[-1] +(app / "release.txt").write_text(codename + "\n") +payload = {"tool_name": "Write", "tool_input": {"file_path": str(app / "release.txt")}} +subprocess.run([*hook, "tool"], input=json.dumps(payload), text=True, check=True) diff --git a/bench/terminal_bench/tasks/user-hooks/solution/solve.sh b/bench/terminal_bench/tasks/user-hooks/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/user-hooks/task.toml b/bench/terminal_bench/tasks/user-hooks/task.toml new file mode 100644 index 000000000..a699c38db --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/user-hooks/tests/test.sh b/bench/terminal_bench/tasks/user-hooks/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py b/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py new file mode 100644 index 000000000..7968cb97c --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py @@ -0,0 +1,24 @@ +"""Verify the codename came from the prompt hook and the write went through the tool hook.""" + +import json +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + assert (APP / "release.txt").read_text().strip() == "velvet-harbor-58" + log = APP / ".hook_log" / "events.jsonl" + assert log.exists(), "no project hook ran" + events = [json.loads(line) for line in log.read_text().splitlines()] + assert any(e["event"] == "prompt" for e in events), "UserPromptSubmit hook didn't run" + assert any( + e["event"] == "tool" and (e["file"] or "").replace("\\", "/").endswith("release.txt") + for e in events + ), f"PostToolUse hook didn't see release.txt being written: {events}" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile b/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile new file mode 100644 index 000000000..1c0593216 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-slim +WORKDIR /app diff --git a/bench/terminal_bench/tasks/web-fetch/instruction.md b/bench/terminal_bench/tasks/web-fetch/instruction.md new file mode 100644 index 000000000..7b6a09943 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/instruction.md @@ -0,0 +1,5 @@ +Fetch this file from the web: + +https://raw.githubusercontent.com/databricks/unity-gateway/ba0d79ed9d429e08af3e7cf110e379c004a3a503/pyproject.toml + +Write the exact value of the `description` field under `[project]` to `answer.txt`, without quotes. diff --git a/bench/terminal_bench/tasks/web-fetch/solution/solve.py b/bench/terminal_bench/tasks/web-fetch/solution/solve.py new file mode 100644 index 000000000..8c4faece9 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/solution/solve.py @@ -0,0 +1,14 @@ +"""Reference solution for web-fetch.""" + +import os +import tomllib +import urllib.request +from pathlib import Path + +url = ( + "https://raw.githubusercontent.com/databricks/unity-gateway/" + "ba0d79ed9d429e08af3e7cf110e379c004a3a503/pyproject.toml" +) +with urllib.request.urlopen(url, timeout=30) as response: + description = tomllib.loads(response.read().decode())["project"]["description"] +Path(os.environ.get("TASK_APP_DIR", "/app"), "answer.txt").write_text(description + "\n") diff --git a/bench/terminal_bench/tasks/web-fetch/solution/solve.sh b/bench/terminal_bench/tasks/web-fetch/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/web-fetch/task.toml b/bench/terminal_bench/tasks/web-fetch/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/web-fetch/tests/test.sh b/bench/terminal_bench/tasks/web-fetch/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py b/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py new file mode 100644 index 000000000..9a9648d82 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py @@ -0,0 +1,16 @@ +"""Verify the value fetched from the pinned file.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + answer = (APP / "answer.txt").read_text().strip().strip("\"'") + assert answer == "Bootstrap Codex against a Databricks workspace with minimal setup.", answer + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/ug_agent.py b/bench/terminal_bench/ug_agent.py new file mode 100644 index 000000000..da146b9db --- /dev/null +++ b/bench/terminal_bench/ug_agent.py @@ -0,0 +1,155 @@ +"""Harbor agents that run Claude Code and Codex inside a task container through ug. + +Use with `harbor run -a ug_agent:UgClaude` (or `UgCodex`) and this directory on +PYTHONPATH. The host must export UG_BENCH_WHEEL (a built ug wheel) plus the +workspace auth that bench_auth.py reads. +""" + +from __future__ import annotations + +import os +import shlex +from pathlib import Path, PurePosixPath + +import bench_auth +from harbor.agents.capabilities import AgentCapabilities +from harbor.agents.installed.base import BaseInstalledAgent +from harbor.agents.installed.node_install import nvm_node_install_snippet +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +INSTALL_DIR = PurePosixPath("/installed-agent") +UG = '"$HOME/.local/bin/ug"' +SHELL_PREFIX = 'export PATH="$HOME/.local/bin:$PATH"; if [ -s "$HOME/.nvm/nvm.sh" ]; then . "$HOME/.nvm/nvm.sh"; fi; ' + + +def _required_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise RuntimeError(f"{name} must be set on the host running harbor") + return value + + +class _UgAgent(BaseInstalledAgent): + AGENT = "" + PROMPT_ON_STDIN = False + capabilities = AgentCapabilities(resume=True) + + async def _install_agent_cli(self, environment: BaseEnvironment) -> None: + raise NotImplementedError + + def _launch_args(self, resume: bool) -> str: + """Agent args after ug's `--`. Without PROMPT_ON_STDIN they pass $UG_BENCH_INSTRUCTION.""" + raise NotImplementedError + + async def install(self, environment: BaseEnvironment) -> None: + wheel = Path(_required_env("UG_BENCH_WHEEL")) + await self.ensure_system_dependencies( + environment, ("curl", "bash", "git", "ca_certificates", "procps", "unzip") + ) + await environment.upload_file(wheel, str(INSTALL_DIR / wheel.name)) + await self.exec_as_root( + environment, + command=( + "command -v databricks >/dev/null || curl -fsSL " + "https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh" + ), + ) + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; curl -LsSf https://astral.sh/uv/install.sh | sh && " + f'"$HOME/.local/bin/uv" tool install --python 3.12 {shlex.quote(str(INSTALL_DIR / wheel.name))}' + ), + ) + await self._install_agent_cli(environment) + + async def run( + self, instruction: str, environment: BaseEnvironment, context: AgentContext + ) -> None: + auth = bench_auth.agent_env({}) + workspace = auth["DATABRICKS_HOST"] + env = { + **auth, + # Lets Claude Code skip permissions as root inside the container. + "IS_SANDBOX": "1", + "ENABLE_SMART_ROUTING_V2": "1", + "UG_BENCH_INSTRUCTION": instruction, + } + if not self._resume: + await self.exec_as_agent( + environment, + command=( + f"{SHELL_PREFIX}{UG} configure --agents {self.AGENT} " + f"--workspace {shlex.quote(workspace)} --skip-validate --skip-upgrade " + "--disable-databricks-ai-tools &1 | tee -a {log.as_posix()}" + ), + env=env, + ) + finally: + # ug's routing wrappers log under ~/.ucode; keep them as evidence that routing ran. + ucode_logs = (self.environment_logs_dir / "ucode").as_posix() + await environment.exec( + command=f'mkdir -p {ucode_logs} && cp "$HOME"/.ucode/*.log {ucode_logs}/ 2>/dev/null || true' + ) + + +class UgClaude(_UgAgent): + AGENT = "claude" + PROMPT_ON_STDIN = True + + @staticmethod + def name() -> str: + return "ug-claude" + + async def _install_agent_cli(self, environment: BaseEnvironment) -> None: + version = f" {shlex.quote(self._version)}" if self._version else "" + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; curl -fsSL " + f"https://downloads.claude.ai/claude-code-releases/bootstrap.sh | bash -s --{version} && " + '"$HOME/.local/bin/claude" --version' + ), + ) + + def _launch_args(self, resume: bool) -> str: + args = "-p --verbose --output-format stream-json --dangerously-skip-permissions" + return f"{args} --continue" if resume else args + + +class UgCodex(_UgAgent): + AGENT = "codex" + + @staticmethod + def name() -> str: + return "ug-codex" + + async def _install_agent_cli(self, environment: BaseEnvironment) -> None: + version = f"@{self._version}" if self._version else "@latest" + await self.exec_as_agent( + environment, + command=( + f"set -euo pipefail; {nvm_node_install_snippet()} && " + f"npm install -g @openai/codex{shlex.quote(version)} && codex --version" + ), + env={"NVM_NODEJS_ORG_MIRROR": "https://nodejs.org/dist"}, + ) + + def _launch_args(self, resume: bool) -> str: + return ( + f"exec {'resume --last ' if resume else ''}--dangerously-bypass-approvals-and-sandbox " + '--skip-git-repo-check --json --enable unified_exec -- "$UG_BENCH_INSTRUCTION"