From b0a72966bd514900428b5642832f2a5103fe13bc Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:50:38 -0400 Subject: [PATCH 01/19] ci: Run Terminal Bench through ug on Linux and Windows Adds a nightly and manual Terminal Bench workflow. Linux runs a TB2 subset and ug's own tasks in Harbor containers through a custom ug agent. Linux and Windows also run ug's tasks natively on the host. Auth mints a per-task token from the CUJ service principal when its secrets are set, else uses DATABRICKS_BEARER. Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 198 +++++++++++++++++ bench/terminal_bench/README.md | 65 ++++++ bench/terminal_bench/bench_auth.py | 60 ++++++ bench/terminal_bench/run_native.py | 201 ++++++++++++++++++ bench/terminal_bench/scrub_secrets.py | 20 ++ bench/terminal_bench/summarize_harbor.py | 41 ++++ .../background-service/environment/Dockerfile | 3 + .../environment/app/server.py | 39 ++++ .../tasks/background-service/instruction.md | 8 + .../background-service/solution/solve.py | 22 ++ .../background-service/solution/solve.sh | 2 + .../tasks/background-service/task.toml | 18 ++ .../tasks/background-service/tests/test.sh | 3 + .../background-service/tests/test_outputs.py | 26 +++ .../fix-failing-tests/environment/Dockerfile | 3 + .../environment/app/inventory/__init__.py | 0 .../environment/app/inventory/pricing.py | 34 +++ .../environment/app/tests/test_pricing.py | 41 ++++ .../tasks/fix-failing-tests/instruction.md | 9 + .../tasks/fix-failing-tests/solution/solve.py | 23 ++ .../tasks/fix-failing-tests/solution/solve.sh | 2 + .../tasks/fix-failing-tests/task.toml | 18 ++ .../tasks/fix-failing-tests/tests/test.sh | 3 + .../fix-failing-tests/tests/test_outputs.py | 39 ++++ .../tasks/log-triage/environment/Dockerfile | 4 + .../tasks/log-triage/environment/setup.py | 39 ++++ .../tasks/log-triage/instruction.md | 22 ++ .../tasks/log-triage/solution/solve.py | 19 ++ .../tasks/log-triage/solution/solve.sh | 2 + .../terminal_bench/tasks/log-triage/task.toml | 18 ++ .../tasks/log-triage/tests/test.sh | 3 + .../tasks/log-triage/tests/test_outputs.py | 42 ++++ .../scrub-git-secret/environment/Dockerfile | 5 + .../scrub-git-secret/environment/setup.py | 57 +++++ .../tasks/scrub-git-secret/instruction.md | 8 + .../tasks/scrub-git-secret/solution/solve.py | 32 +++ .../tasks/scrub-git-secret/solution/solve.sh | 2 + .../tasks/scrub-git-secret/task.toml | 18 ++ .../tasks/scrub-git-secret/tests/test.sh | 3 + .../scrub-git-secret/tests/test_outputs.py | 44 ++++ bench/terminal_bench/ug_agent.py | 136 ++++++++++++ 41 files changed, 1332 insertions(+) create mode 100644 .github/workflows/terminal-bench.yml create mode 100644 bench/terminal_bench/README.md create mode 100644 bench/terminal_bench/bench_auth.py create mode 100644 bench/terminal_bench/run_native.py create mode 100644 bench/terminal_bench/scrub_secrets.py create mode 100644 bench/terminal_bench/summarize_harbor.py create mode 100644 bench/terminal_bench/tasks/background-service/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/background-service/environment/app/server.py create mode 100644 bench/terminal_bench/tasks/background-service/instruction.md create mode 100644 bench/terminal_bench/tasks/background-service/solution/solve.py create mode 100755 bench/terminal_bench/tasks/background-service/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/background-service/task.toml create mode 100755 bench/terminal_bench/tasks/background-service/tests/test.sh create mode 100644 bench/terminal_bench/tasks/background-service/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/__init__.py create mode 100644 bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py create mode 100644 bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py create mode 100644 bench/terminal_bench/tasks/fix-failing-tests/instruction.md create mode 100644 bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py create mode 100755 bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/fix-failing-tests/task.toml create mode 100755 bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh create mode 100644 bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/log-triage/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/log-triage/environment/setup.py create mode 100644 bench/terminal_bench/tasks/log-triage/instruction.md create mode 100644 bench/terminal_bench/tasks/log-triage/solution/solve.py create mode 100755 bench/terminal_bench/tasks/log-triage/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/log-triage/task.toml create mode 100755 bench/terminal_bench/tasks/log-triage/tests/test.sh create mode 100644 bench/terminal_bench/tasks/log-triage/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py create mode 100644 bench/terminal_bench/tasks/scrub-git-secret/instruction.md create mode 100644 bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py create mode 100755 bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/scrub-git-secret/task.toml create mode 100755 bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh create mode 100644 bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py create mode 100644 bench/terminal_bench/ug_agent.py diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml new file mode 100644 index 000000000..7d49ce11f --- /dev/null +++ b/.github/workflows/terminal-bench.yml @@ -0,0 +1,198 @@ +name: Terminal Bench + +on: + schedule: + - cron: '0 7 * * *' + workflow_dispatch: + inputs: + tb2_tasks: + description: Comma-separated Terminal Bench 2 task names for the Linux Harbor lane + default: git-multibranch,financial-document-processor,hf-model-inference,nginx-request-logging,sanitize-git-repo,git-leak-recovery,multi-source-data-merger,kv-store-grpc + required: true + tb2_dataset: + description: Harbor dataset for the TB2 lane + default: terminal-bench@2.0 + required: true + model: + description: Optional model passed to ug --model (blank uses ug's default) + default: '' + required: false + claude_version: + description: Exact Claude Code version + default: 2.1.290 + required: true + codex_version: + description: Exact Codex version + default: 0.154.0 + required: true + +permissions: + contents: read + +concurrency: + group: terminal-bench-${{ github.ref }} + cancel-in-progress: false + +env: + PYTHONUTF8: 1 + TB2_TASKS: ${{ inputs.tb2_tasks || 'git-multibranch,financial-document-processor,hf-model-inference,nginx-request-logging,sanitize-git-repo,git-leak-recovery,multi-source-data-merger,kv-store-grpc' }} + TB2_DATASET: ${{ inputs.tb2_dataset || 'terminal-bench@2.0' }} + MODEL: ${{ inputs.model }} + CLAUDE_VERSION: ${{ inputs.claude_version || '2.1.290' }} + CODEX_VERSION: ${{ inputs.codex_version || '0.154.0' }} + HARBOR_VERSION: 0.23.0 + WINDOWS_PACKAGE_INDEX: https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + WINDOWS_NPM_REGISTRY: https://databricks.jfrog.io/artifactory/api/npm/db-npm/ + +jobs: + # Real TB2 tasks run in Harbor's Linux containers, with ug installed inside each one. + harbor: + name: Harbor · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} · Linux + runs-on: ubuntu-latest + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + agent: [claude, codex] + env: + UV_INDEX_URL: https://pypi.org/simple + # bench_auth.py mints a fresh token per task from the CUJ service principal when + # its secrets are set, and otherwise falls back to the pre-fetched e2e bearer. + UCODE_TEST_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }} + DATABRICKS_BEARER: ${{ secrets.DATABRICKS_BEARER }} + UG_BENCH_SP_WORKSPACE: ${{ secrets.UG_CUJ1_WORKSPACE }} + UG_BENCH_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + UG_BENCH_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + PYTHONPATH: ${{ github.workspace }}/bench/terminal_bench + AGENT_CLASS: ${{ matrix.agent == 'claude' && 'UgClaude' || 'UgCodex' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - name: Build the ug wheel installed into each task container + run: | + uv build --wheel --out-dir "$RUNNER_TEMP/dist" + echo "UG_BENCH_WHEEL=$(ls "$RUNNER_TEMP"/dist/*.whl)" >> "$GITHUB_ENV" + - name: Run TB2 tasks + env: + AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }} + run: | + includes=() + IFS=, read -ra tasks <<< "$TB2_TASKS" + for task in "${tasks[@]}"; do includes+=(-i "$task"); done + uvx --python 3.12 --from "harbor==$HARBOR_VERSION" harbor run -y \ + -d "$TB2_DATASET" "${includes[@]}" \ + -a "ug_agent:$AGENT_CLASS" --ak "version=$AGENT_VERSION" \ + ${MODEL:+-m "$MODEL"} -n 3 -o "$RUNNER_TEMP/harbor-jobs/tb2" + - name: Run ug's own tasks + if: ${{ !cancelled() }} + env: + AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }} + run: | + uvx --python 3.12 --from "harbor==$HARBOR_VERSION" harbor run -y \ + -p bench/terminal_bench/tasks \ + -a "ug_agent:$AGENT_CLASS" --ak "version=$AGENT_VERSION" \ + ${MODEL:+-m "$MODEL"} -n 3 -o "$RUNNER_TEMP/harbor-jobs/ug" + - name: Summarize + if: ${{ !cancelled() }} + run: | + status=0 + python3 bench/terminal_bench/summarize_harbor.py "$RUNNER_TEMP/harbor-jobs/tb2" "TB2 · ${{ matrix.agent }}" || status=1 + python3 bench/terminal_bench/summarize_harbor.py "$RUNNER_TEMP/harbor-jobs/ug" "ug tasks in Harbor · ${{ matrix.agent }}" || status=1 + exit $status + - name: Scrub the bearer token from job output + if: ${{ !cancelled() }} + run: python3 bench/terminal_bench/scrub_secrets.py "$RUNNER_TEMP/harbor-jobs" + - name: Upload Harbor jobs + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: terminal-bench-harbor-${{ matrix.agent }} + path: ${{ runner.temp }}/harbor-jobs/ + + # Windows runners can't run Linux containers, so ug's own tasks also run on the host. + native: + name: Native · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }} · ${{ matrix.os == 'windows-server-latest' && 'Windows' || 'Linux' }} + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + permissions: + contents: read + id-token: write + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-server-latest] + agent: [claude, codex] + env: + UV_INDEX_URL: https://pypi.org/simple + # bench_auth.py mints a fresh token per task from the CUJ service principal when + # its secrets are set, and otherwise falls back to the pre-fetched e2e bearer. + UCODE_TEST_WORKSPACE: ${{ secrets.UCODE_TEST_WORKSPACE }} + DATABRICKS_BEARER: ${{ secrets.DATABRICKS_BEARER }} + UG_BENCH_SP_WORKSPACE: ${{ secrets.UG_CUJ1_WORKSPACE }} + UG_BENCH_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + UG_BENCH_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + AGENT: ${{ matrix.agent }} + AGENT_PACKAGE: ${{ matrix.agent == 'claude' && '@anthropic-ai/claude-code' || '@openai/codex' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22.19.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - uses: databricks/setup-cli@bdb89f81c11a5bd647fd55b585b7c396ec68a25a # v1.0.0 + - name: Expose the Databricks CLI on Windows + if: ${{ runner.os == 'Windows' }} + run: | + test -f "$RUNNER_TEMP/.bin/databricks.exe" + cygpath -w "$RUNNER_TEMP/.bin" >> "$GITHUB_PATH" + - name: Authenticate to JFrog with GitHub OIDC + id: jfrog + if: ${{ runner.os == 'Windows' }} + uses: jfrog/setup-jfrog-cli@279b1f629f43dd5bc658d8361ac4802a7ef8d2d5 # v4.9.1 + env: + JF_URL: https://databricks.jfrog.io + with: + oidc-provider-name: github-actions + - name: Install the agent CLI + env: + NPM_TOKEN: ${{ steps.jfrog.outputs.oidc-token }} + AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }} + run: | + if [[ "$RUNNER_OS" == Windows ]]; then + npmrc="$RUNNER_TEMP/jfrog.npmrc" + printf 'registry=%s\n//%s:_authToken=%s\n' \ + "$WINDOWS_NPM_REGISTRY" "${WINDOWS_NPM_REGISTRY#https://}" "$NPM_TOKEN" > "$npmrc" + npm install -g --userconfig "$npmrc" "$AGENT_PACKAGE@$AGENT_VERSION" + rm "$npmrc" + else + npm install -g "$AGENT_PACKAGE@$AGENT_VERSION" + fi + - name: Install ug + env: + UV_INDEX_DATABRICKS_PYPI_USERNAME: ${{ steps.jfrog.outputs.oidc-user }} + UV_INDEX_DATABRICKS_PYPI_PASSWORD: ${{ steps.jfrog.outputs.oidc-token }} + run: | + if [[ "$RUNNER_OS" == Windows ]]; then + unset UV_INDEX_URL + export UV_DEFAULT_INDEX="databricks-pypi=$WINDOWS_PACKAGE_INDEX" + fi + uv lock + uv tool install . + uv sync + - name: Run ug's own tasks on the host + run: uv run --no-sync python bench/terminal_bench/run_native.py --agent "$AGENT" --output "$RUNNER_TEMP/tb-native" ${MODEL:+--model "$MODEL"} < /dev/null + - name: Scrub the bearer token from task output + if: ${{ !cancelled() }} + run: uv run --no-sync python bench/terminal_bench/scrub_secrets.py "$RUNNER_TEMP/tb-native" + - name: Upload native results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: terminal-bench-native-${{ matrix.agent }}-${{ matrix.os }} + include-hidden-files: true + path: | + ${{ runner.temp }}/tb-native/results.json + ${{ runner.temp }}/tb-native/logs/ diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md new file mode 100644 index 000000000..dc217fcc6 --- /dev/null +++ b/bench/terminal_bench/README.md @@ -0,0 +1,65 @@ +# Terminal Bench + +Runs Terminal Bench-style tasks through `ug claude` and `ug codex`. The +`Terminal Bench` workflow runs nightly and on demand. + +There are two lanes: + +- Harbor (Linux). Runs real [Terminal Bench 2](https://www.tbench.ai/) tasks + plus the tasks in `tasks/` inside Harbor's Docker containers. + `ug_agent.py` installs the ug wheel, the Databricks CLI, and the agent CLI in + each container, then launches the agent headlessly through ug. +- Native (Linux and Windows). GitHub's Windows runners can't run Linux + containers, so `run_native.py` runs the tasks in `tasks/` on the host. + +The default TB2 subset was picked to make the agent spawn subagents, use a range +of tools, and touch system state (services, global installs, git history +rewrites). Change it with the workflow's `tb2_tasks` input. + +## Tasks + +Each task follows the Harbor layout: `task.toml`, `instruction.md`, +`environment/` (a Dockerfile, plus `app/` fixtures or a `setup.py`), +`tests/test_outputs.py`, and `solution/solve.py`. The verifier and solution read +the task dir from `TASK_APP_DIR` (default `/app`) so the same files work in +both lanes. Keep them standard-library Python so they run on Windows too. + +| Task | Exercises | +|------|-----------| +| `fix-failing-tests` | Run tests, edit, rerun | +| `log-triage` | Independent per-service work that can fan out to subagents | +| `scrub-git-secret` | Destructive git history rewrite across branches | +| `background-service` | Start a background process, query it, shut it down | + +## Running locally + +Check tasks with the reference solutions (no gateway needed): + +```sh +uv run python bench/terminal_bench/run_native.py --agent oracle --output /tmp/tb +``` + +## Auth + +`bench_auth.py` picks the credentials. With `UG_BENCH_CLIENT_ID`, +`UG_BENCH_CLIENT_SECRET`, and `UG_BENCH_SP_WORKSPACE` set, it mints a fresh +OAuth M2M token on the host before each task. Only that token reaches the agent, +never the SP secret. Otherwise it uses `DATABRICKS_BEARER` against +`UCODE_TEST_WORKSPACE`. CI uses the CUJ service principal and `UG_CUJ1_WORKSPACE`. + +Uploaded artifacts have tokens redacted by `scrub_secrets.py`. + +Run the tasks through ug with either set of credentials exported. This rewrites +your `ug configure` state for the agent: + +```sh +uv run python bench/terminal_bench/run_native.py --agent claude --output /tmp/tb +``` + +Harbor needs Docker, `UG_BENCH_WHEEL`, and the same credentials: + +```sh +uv build --wheel --out-dir /tmp/dist && export UG_BENCH_WHEEL=$(ls /tmp/dist/*.whl) +PYTHONPATH=bench/terminal_bench uvx --python 3.12 --from harbor==0.23.0 harbor run -y \ + -d terminal-bench@2.0 -i fix-git -a ug_agent:UgClaude -o /tmp/harbor-jobs +``` diff --git a/bench/terminal_bench/bench_auth.py b/bench/terminal_bench/bench_auth.py new file mode 100644 index 000000000..74bb41fd4 --- /dev/null +++ b/bench/terminal_bench/bench_auth.py @@ -0,0 +1,60 @@ +"""Workspace auth for bench runs. + +With service-principal credentials set, mint a fresh bearer per task on the host so +long runs outlive a single token and the SP secret never reaches the agent. +Otherwise fall back to a pre-fetched DATABRICKS_BEARER. +""" + +from __future__ import annotations + +import base64 +import json +import os +import urllib.parse +import urllib.request + +SECRET_ENV = ("UG_BENCH_CLIENT_ID", "UG_BENCH_CLIENT_SECRET") + + +def _sp_credentials() -> tuple[str, str] | None: + client_id, secret = (os.environ.get(name, "").strip() for name in SECRET_ENV) + return (client_id, secret) if client_id and secret else None + + +def workspace() -> str: + if _sp_credentials() and (host := os.environ.get("UG_BENCH_SP_WORKSPACE", "").strip()): + return host + host = os.environ.get("UCODE_TEST_WORKSPACE", "").strip() + if not host: + raise RuntimeError("Set UCODE_TEST_WORKSPACE, or UG_BENCH_SP_WORKSPACE with SP credentials") + return host + + +def bearer(host: str) -> str: + credentials = _sp_credentials() + if credentials is None: + token = os.environ.get("DATABRICKS_BEARER", "").strip() + if not token: + raise RuntimeError( + "Set DATABRICKS_BEARER, or UG_BENCH_CLIENT_ID/UG_BENCH_CLIENT_SECRET" + ) + return token + basic = base64.b64encode(":".join(credentials).encode()).decode() + request = urllib.request.Request( + f"{host.rstrip('/')}/oidc/v1/token", + data=urllib.parse.urlencode( + {"grant_type": "client_credentials", "scope": "all-apis"} + ).encode(), + headers={"Authorization": f"Basic {basic}"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response)["access_token"] + + +def agent_env(base: dict[str, str]) -> dict[str, str]: + """Copy of `base` with a fresh bearer and without the SP secret.""" + env = {key: value for key, value in base.items() if key not in SECRET_ENV} + host = workspace() + env["DATABRICKS_HOST"] = host + env["DATABRICKS_BEARER"] = bearer(host) + return env diff --git a/bench/terminal_bench/run_native.py b/bench/terminal_bench/run_native.py new file mode 100644 index 000000000..5c23950fc --- /dev/null +++ b/bench/terminal_bench/run_native.py @@ -0,0 +1,201 @@ +"""Run Terminal Bench-format tasks directly on the host through ug. + +Windows runners can't run the Linux containers Harbor uses, so this runner +copies each task's fixtures into a temp dir, launches the agent headlessly +there, and runs the task's Python verifier against the result. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import signal +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +import bench_auth + +from ucode.os_compatibility.subprocess_cross_os import popen, run + +TASKS_DIR = Path(__file__).resolve().parent / "tasks" + + +def agent_command(agent: str, ug: str, model: str | None, task_dir: Path) -> list[str]: + model_args = ["--model", model] if model else [] + if agent == "claude": + return [ + ug, + "claude", + *model_args, + "--", + "-p", + "--verbose", + "--output-format", + "stream-json", + "--dangerously-skip-permissions", + ] + if agent == "codex": + return [ + ug, + "codex", + *model_args, + "--", + "exec", + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "--json", + "-", + ] + return [sys.executable, str(task_dir / "solution" / "solve.py")] + + +def kill_tree(proc: subprocess.Popen) -> None: + if os.name == "nt": + run(["taskkill", "/T", "/F", "/PID", str(proc.pid)], capture_output=True) + else: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + + +def prepare(task_dir: Path, workdir: Path) -> None: + app = task_dir / "environment" / "app" + if app.is_dir(): + shutil.copytree(app, workdir) + workdir.mkdir(parents=True, exist_ok=True) + setup = task_dir / "environment" / "setup.py" + if setup.exists(): + run([sys.executable, str(setup), str(workdir)], check=True) + + +def run_task(args: argparse.Namespace, task_dir: Path, out: Path) -> dict: + config = tomllib.loads((task_dir / "task.toml").read_text()) + agent_timeout = config.get("agent", {}).get("timeout_sec", 600) + verifier_timeout = config.get("verifier", {}).get("timeout_sec", 120) + workdir = out / "work" / task_dir.name + logs = out / "logs" / task_dir.name + logs.mkdir(parents=True, exist_ok=True) + prepare(task_dir, workdir) + + env = {**os.environ, "TASK_APP_DIR": str(workdir), "IS_SANDBOX": "1"} + if args.agent != "oracle": + env = bench_auth.agent_env(env) + # Tasks say "run it with Python", so put this interpreter first on PATH. + env["PATH"] = os.pathsep.join([str(Path(sys.executable).parent), env.get("PATH", "")]) + instruction = (task_dir / "instruction.md").read_text() + started = time.monotonic() + timed_out = False + with (logs / "agent.log").open("w", encoding="utf-8") as log: + proc = popen( + agent_command(args.agent, args.ug, args.model, task_dir), + cwd=workdir, + env=env, + stdin=subprocess.PIPE, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + start_new_session=os.name != "nt", + ) + try: + proc.communicate(instruction, timeout=agent_timeout) + except subprocess.TimeoutExpired: + timed_out = True + kill_tree(proc) + duration = time.monotonic() - started + + # Verify before cleanup so a server the agent left running still counts against it. + try: + verifier = run( + [sys.executable, str(task_dir / "tests" / "test_outputs.py")], + cwd=workdir, + env=env, + capture_output=True, + text=True, + timeout=verifier_timeout, + ) + passed = verifier.returncode == 0 + verifier_log = verifier.stdout + verifier.stderr + except subprocess.TimeoutExpired: + passed, verifier_log = False, f"verifier timed out after {verifier_timeout}s\n" + (logs / "verifier.log").write_text(verifier_log, encoding="utf-8") + if not timed_out and os.name != "nt": + kill_tree(proc) + return { + "task": task_dir.name, + "passed": passed, + "agent_exit": proc.returncode, + "timed_out": timed_out, + "duration_sec": round(duration, 1), + } + + +def write_summary(args: argparse.Namespace, results: list[dict], out: Path) -> None: + (out / "results.json").write_text(json.dumps(results, indent=2)) + passed = sum(r["passed"] for r in results) + lines = [ + f"### Native tasks · {args.agent} · {sys.platform}: {passed}/{len(results)} passed", + "", + "| Task | Result | Agent exit | Time (s) |", + "|------|--------|-----------|----------|", + ] + for r in results: + status = "pass" if r["passed"] else ("timeout" if r["timed_out"] else "fail") + lines.append(f"| {r['task']} | {status} | {r['agent_exit']} | {r['duration_sec']} |") + summary = "\n".join(lines) + "\n" + print(summary) + if step_summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with open(step_summary, "a", encoding="utf-8") as f: + f.write(summary) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--agent", choices=["claude", "codex", "oracle"], required=True) + parser.add_argument("--task", action="append", help="Task name; repeatable. Default: all.") + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--ug", default=shutil.which("ug"), help="Absolute path to ug.") + parser.add_argument("--model") + args = parser.parse_args() + + names = args.task or sorted(p.name for p in TASKS_DIR.iterdir() if p.is_dir()) + out = args.output.resolve() + if (out / "work").exists(): + shutil.rmtree(out / "work") + out.mkdir(parents=True, exist_ok=True) + + if args.agent != "oracle": + if not args.ug: + parser.error("--ug is required when ug isn't on PATH") + args.ug = str(Path(args.ug).resolve()) + env = bench_auth.agent_env(dict(os.environ)) + run( + [ + args.ug, + "configure", + "--agents", + args.agent, + "--workspace", + env["DATABRICKS_HOST"], + "--skip-validate", + "--skip-upgrade", + "--disable-databricks-ai-tools", + ], + check=True, + env=env, + stdin=subprocess.DEVNULL, + ) + + results = [run_task(args, TASKS_DIR / name, out) for name in names] + write_summary(args, results, out) + return 0 if all(r["passed"] for r in results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/terminal_bench/scrub_secrets.py b/bench/terminal_bench/scrub_secrets.py new file mode 100644 index 000000000..5991caa72 --- /dev/null +++ b/bench/terminal_bench/scrub_secrets.py @@ -0,0 +1,20 @@ +"""Redact Databricks bearers from every file under a directory before upload. + +Per-task tokens are minted on the fly, so match token shapes as well as the +DATABRICKS_BEARER value. +""" + +import os +import re +import sys +from pathlib import Path + +TOKEN = re.compile(rb"eyJ[\w-]+\.[\w-]+\.[\w-]+|dapi[0-9a-f]{32}(?:-\d+)?") +secret = os.environ.get("DATABRICKS_BEARER", "").strip().encode() + +for path in Path(sys.argv[1]).rglob("*"): + if path.is_file() and not path.is_symlink(): + data = path.read_bytes() + scrubbed = TOKEN.sub(b"***", data.replace(secret, b"***") if secret else data) + if scrubbed != data: + path.write_bytes(scrubbed) diff --git a/bench/terminal_bench/summarize_harbor.py b/bench/terminal_bench/summarize_harbor.py new file mode 100644 index 000000000..93b36c249 --- /dev/null +++ b/bench/terminal_bench/summarize_harbor.py @@ -0,0 +1,41 @@ +"""Summarize a Harbor jobs dir as a markdown table. + +Exits non-zero when a trial errored for a reason other than a timeout, since +that points at the ug harness rather than the agent's score. +""" + +import json +import os +import sys +from pathlib import Path + + +def main(jobs_dir: Path, title: str) -> int: + rows = [] + harness_errors = 0 + for path in sorted(jobs_dir.glob("*/*/result.json")): + result = json.loads(path.read_text()) + rewards = (result.get("verifier_result") or {}).get("rewards") or {} + error = (result.get("exception_info") or {}).get("exception_type", "") + if error and "Timeout" not in error: + harness_errors += 1 + rows.append((result["task_name"], rewards.get("reward", "-"), error or "-")) + + solved = sum(1 for _, reward, _ in rows if reward == 1) + lines = [ + f"### {title}: {solved}/{len(rows)} solved", + "", + "| Task | Reward | Error |", + "|------|--------|-------|", + *(f"| {task} | {reward} | {error} |" for task, reward, error in rows), + ] + summary = "\n".join(lines) + "\n" + print(summary) + if step_summary := os.environ.get("GITHUB_STEP_SUMMARY"): + with open(step_summary, "a", encoding="utf-8") as f: + f.write(summary) + return 1 if harness_errors or not rows else 0 + + +if __name__ == "__main__": + sys.exit(main(Path(sys.argv[1]), sys.argv[2])) diff --git a/bench/terminal_bench/tasks/background-service/environment/Dockerfile b/bench/terminal_bench/tasks/background-service/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/background-service/environment/app/server.py b/bench/terminal_bench/tasks/background-service/environment/app/server.py new file mode 100644 index 000000000..b5bc1c6d9 --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/environment/app/server.py @@ -0,0 +1,39 @@ +"""Token service. Run it in the background; it stops itself after 15 minutes.""" + +import hashlib +import http.server +import secrets +import threading +from pathlib import Path + +STATE = Path(__file__).resolve().parent / ".server_state" +TOKEN = secrets.token_hex(16) + + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: + if self.path != "/token": + self.send_error(404) + elif self.headers.get("X-Bench-Client") != "ug": + self.send_error(403, "missing X-Bench-Client: ug header") + else: + body = TOKEN.encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def main() -> None: + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + STATE.mkdir(exist_ok=True) + (STATE / "token.sha256").write_text(hashlib.sha256(TOKEN.encode()).hexdigest()) + (STATE / "port").write_text(str(server.server_address[1])) + print(f"listening on 127.0.0.1:{server.server_address[1]}", flush=True) + threading.Timer(900, server.shutdown).start() + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/background-service/instruction.md b/bench/terminal_bench/tasks/background-service/instruction.md new file mode 100644 index 000000000..45919b376 --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/instruction.md @@ -0,0 +1,8 @@ +`server.py` in the current directory is a small token service. + +1. Start it in the background with Python. Once it's ready it writes its port to `.server_state/port`. +2. Fetch `GET http://127.0.0.1:/token`. The request needs the header `X-Bench-Client: ug`. +3. Write the token from the response body to `token.txt`. +4. Stop the server. Nothing should be listening on that port when you finish. + +Don't edit `server.py` or anything in `.server_state/`. diff --git a/bench/terminal_bench/tasks/background-service/solution/solve.py b/bench/terminal_bench/tasks/background-service/solution/solve.py new file mode 100644 index 000000000..ce19d1f5f --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/solution/solve.py @@ -0,0 +1,22 @@ +"""Reference solution for background-service.""" + +import os +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +proc = subprocess.Popen([sys.executable, "server.py"], cwd=app) +port_file = app / ".server_state" / "port" +for _ in range(100): + if port_file.exists() and port_file.read_text(): + break + time.sleep(0.1) +request = urllib.request.Request( + f"http://127.0.0.1:{port_file.read_text()}/token", headers={"X-Bench-Client": "ug"} +) +(app / "token.txt").write_text(urllib.request.urlopen(request).read().decode()) +proc.terminate() +proc.wait() diff --git a/bench/terminal_bench/tasks/background-service/solution/solve.sh b/bench/terminal_bench/tasks/background-service/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/background-service/task.toml b/bench/terminal_bench/tasks/background-service/task.toml new file mode 100644 index 000000000..9cad17abe --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "system-administration" +tags = ["background-process", "http", "cleanup"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/background-service/tests/test.sh b/bench/terminal_bench/tasks/background-service/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/background-service/tests/test_outputs.py b/bench/terminal_bench/tasks/background-service/tests/test_outputs.py new file mode 100644 index 000000000..243ee9956 --- /dev/null +++ b/bench/terminal_bench/tasks/background-service/tests/test_outputs.py @@ -0,0 +1,26 @@ +"""Verify the agent fetched the live token and then stopped the server.""" + +import hashlib +import os +import socket +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + state = APP / ".server_state" + assert (state / "port").exists(), "server.py was never started" + token = (APP / "token.txt").read_text().strip() + want = (state / "token.sha256").read_text().strip() + assert hashlib.sha256(token.encode()).hexdigest() == want, "token.txt does not match" + + port = int((state / "port").read_text()) + with socket.socket() as sock: + sock.settimeout(2) + assert sock.connect_ex(("127.0.0.1", port)) != 0, "server is still running" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile b/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/__init__.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py new file mode 100644 index 000000000..1cccbcb30 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/inventory/pricing.py @@ -0,0 +1,34 @@ +"""Order pricing for the inventory service.""" + +from decimal import ROUND_HALF_UP, Decimal + +# (minimum quantity, discount fraction), checked from the largest tier down. +BULK_TIERS = [(100, Decimal("0.15")), (50, Decimal("0.10")), (10, Decimal("0.05"))] + +TAX_RATES = {"CA": Decimal("0.0725"), "NY": Decimal("0.04"), "OR": Decimal("0")} + + +def bulk_discount(quantity: int) -> Decimal: + for minimum, discount in BULK_TIERS: + if quantity > minimum: + return discount + return Decimal("0") + + +def round_cents(amount: Decimal) -> Decimal: + return amount.quantize(Decimal("0.01")) + + +def line_total(unit_price: str, quantity: int) -> Decimal: + if quantity < 0: + raise ValueError("quantity must be non-negative") + subtotal = Decimal(unit_price) * quantity + return round_cents(subtotal - subtotal * bulk_discount(quantity)) + + +def order_total(lines: list[tuple[str, int]], state: str, coupon: str | None = None) -> Decimal: + subtotal = sum((line_total(price, qty) for price, qty in lines), Decimal("0")) + if coupon == "SAVE10": + subtotal -= Decimal("10") + tax = subtotal * TAX_RATES[state] + return round_cents(subtotal + tax) diff --git a/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py new file mode 100644 index 000000000..afc2a7982 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/environment/app/tests/test_pricing.py @@ -0,0 +1,41 @@ +import unittest +from decimal import Decimal + +from inventory.pricing import bulk_discount, line_total, order_total + + +class BulkDiscountTest(unittest.TestCase): + def test_tier_boundaries_are_inclusive(self): + self.assertEqual(bulk_discount(9), Decimal("0")) + self.assertEqual(bulk_discount(10), Decimal("0.05")) + self.assertEqual(bulk_discount(50), Decimal("0.10")) + self.assertEqual(bulk_discount(100), Decimal("0.15")) + + +class LineTotalTest(unittest.TestCase): + def test_half_cents_round_up(self): + # 3 x 0.125 = 0.375, which rounds to 0.38. + self.assertEqual(line_total("0.125", 3), Decimal("0.38")) + + def test_bulk_discount_applies(self): + self.assertEqual(line_total("2.00", 10), Decimal("19.00")) + + def test_negative_quantity_rejected(self): + with self.assertRaises(ValueError): + line_total("1.00", -1) + + +class OrderTotalTest(unittest.TestCase): + def test_tax_is_added(self): + self.assertEqual(order_total([("10.00", 1)], "CA"), Decimal("10.73")) + + def test_coupon_never_makes_total_negative(self): + self.assertEqual(order_total([("4.00", 1)], "NY", coupon="SAVE10"), Decimal("0.00")) + + def test_unknown_state_is_a_value_error(self): + with self.assertRaises(ValueError): + order_total([("1.00", 1)], "ZZ") + + +if __name__ == "__main__": + unittest.main() diff --git a/bench/terminal_bench/tasks/fix-failing-tests/instruction.md b/bench/terminal_bench/tasks/fix-failing-tests/instruction.md new file mode 100644 index 000000000..814aaa905 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/instruction.md @@ -0,0 +1,9 @@ +The `inventory` package in the current directory has a failing test suite. + +Run it with `python -m unittest discover -s tests` and fix the code in `inventory/` until every test passes. + +Rules: +- Do not modify anything under `tests/`. +- Keep the public functions in `inventory/pricing.py` and their signatures. +- Money values are `Decimal`s rounded to cents with half-up rounding. +- `order_total` must raise `ValueError` for a state with no tax rate, and a coupon can never make the total negative. diff --git a/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py new file mode 100644 index 000000000..c727ff660 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.py @@ -0,0 +1,23 @@ +"""Reference fix for fix-failing-tests.""" + +import os +from pathlib import Path + +path = Path(os.environ.get("TASK_APP_DIR", "/app")) / "inventory" / "pricing.py" +source = path.read_text() +for old, new in [ + ("if quantity > minimum:", "if quantity >= minimum:"), + ( + 'amount.quantize(Decimal("0.01"))', + 'amount.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)', + ), + ( + ' subtotal -= Decimal("10")\n', + ' subtotal = max(subtotal - Decimal("10"), Decimal("0"))\n' + " if state not in TAX_RATES:\n" + ' raise ValueError(f"unknown state: {state}")\n', + ), +]: + assert old in source, old + source = source.replace(old, new) +path.write_text(source) diff --git a/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/fix-failing-tests/task.toml b/bench/terminal_bench/tasks/fix-failing-tests/task.toml new file mode 100644 index 000000000..6d2b95265 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "debugging" +tags = ["python", "testing", "edit-loop"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh b/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py b/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py new file mode 100644 index 000000000..4ae6aa386 --- /dev/null +++ b/bench/terminal_bench/tasks/fix-failing-tests/tests/test_outputs.py @@ -0,0 +1,39 @@ +"""Verify the inventory fixes. Runs from Harbor (/app) or the native runner (TASK_APP_DIR).""" + +import hashlib +import os +import subprocess +import sys +from decimal import Decimal +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +VISIBLE_TESTS_SHA256 = "d169ead570f87b1aa2eb351c1c226f42bea4b438e5e24720a98df3a5beb96002" + + +def main() -> None: + tests = (APP / "tests" / "test_pricing.py").read_bytes().replace(b"\r\n", b"\n") + assert hashlib.sha256(tests).hexdigest() == VISIBLE_TESTS_SHA256, "tests/ was modified" + + result = subprocess.run( + [sys.executable, "-m", "unittest", "discover", "-s", "tests"], + cwd=APP, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + sys.path.insert(0, str(APP)) + from inventory.pricing import bulk_discount, line_total, order_total + + assert bulk_discount(49) == Decimal("0.05") + assert bulk_discount(99) == Decimal("0.10") + assert line_total("0.005", 1) == Decimal("0.01") + assert line_total("1.00", 100) == Decimal("85.00") + assert order_total([("20.00", 1)], "NY", coupon="SAVE10") == Decimal("10.40") + assert order_total([("3.00", 2)], "OR") == Decimal("6.00") + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/log-triage/environment/Dockerfile b/bench/terminal_bench/tasks/log-triage/environment/Dockerfile new file mode 100644 index 000000000..18ba6f9bd --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/log-triage/environment/setup.py b/bench/terminal_bench/tasks/log-triage/environment/setup.py new file mode 100644 index 000000000..55140114d --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/environment/setup.py @@ -0,0 +1,39 @@ +"""Generate deterministic service logs under /logs.""" + +import random +import sys +from datetime import datetime, timedelta +from pathlib import Path + +SERVICES = ["auth", "billing", "gateway", "search", "storage"] +CODES = ["E101", "E202", "E303", "E404", "E505", "E606"] +MESSAGES = { + "INFO": ["request served", "cache warmed", "heartbeat ok", "config reloaded"], + "WARN": ["slow upstream", "retrying request", "queue depth high"], + "ERROR": ["upstream failed", "timeout", "bad payload", "disk full"], +} + + +def main(app: Path) -> None: + rng = random.Random(4242) + start = datetime(2026, 9, 1) + for service in SERVICES: + # Each service favors a different error code so the answer isn't uniform. + weights = [rng.randint(1, 4) for _ in CODES] + weights[rng.randrange(len(CODES))] += 8 + for day in range(4): + date = start + timedelta(days=day) + lines = [] + for i in range(rng.randint(250, 400)): + ts = date + timedelta(seconds=i * 37 + rng.randint(0, 30)) + level = rng.choices(["INFO", "WARN", "ERROR"], [80, 12, 8])[0] + message = rng.choice(MESSAGES[level]) + code = f" code={rng.choices(CODES, weights)[0]}" if level == "ERROR" else "" + lines.append(f"{ts.isoformat()} {level:<5} [{service}] {message}{code}") + out = app / "logs" / service / f"{date:%Y-%m-%d}.log" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text("\n".join(lines) + "\n", newline="\n") + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/bench/terminal_bench/tasks/log-triage/instruction.md b/bench/terminal_bench/tasks/log-triage/instruction.md new file mode 100644 index 000000000..2f9e2593f --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/instruction.md @@ -0,0 +1,22 @@ +`logs/` holds four days of logs for five services, one directory per service. +Each line looks like: + +``` +2026-09-01T00:00:12 ERROR [billing] upstream failed code=E303 +``` + +The level is `INFO`, `WARN`, or `ERROR`. Only `ERROR` lines carry a `code=`. + +Write `report.json` in the current directory with one entry per service: + +```json +{ + "auth": {"errors": 0, "warnings": 0, "top_error_code": "E101"}, + ... +} +``` + +- `errors` and `warnings` count `ERROR` and `WARN` lines across all of that service's files. +- `top_error_code` is the code seen most often in that service's `ERROR` lines. Break ties by the alphabetically smallest code. + +The service directories are independent of each other, so they can be processed in parallel. diff --git a/bench/terminal_bench/tasks/log-triage/solution/solve.py b/bench/terminal_bench/tasks/log-triage/solution/solve.py new file mode 100644 index 000000000..ad49e2135 --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/solution/solve.py @@ -0,0 +1,19 @@ +"""Reference solution for log-triage.""" + +import json +import os +import re +from collections import Counter +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +report = {} +for service_dir in sorted((app / "logs").iterdir()): + text = "".join(log.read_text() for log in service_dir.glob("*.log")) + codes = Counter(re.findall(r" ERROR .* code=(E\d+)$", text, re.MULTILINE)) + report[service_dir.name] = { + "errors": len(re.findall(r"^\S+ ERROR ", text, re.MULTILINE)), + "warnings": len(re.findall(r"^\S+ WARN ", text, re.MULTILINE)), + "top_error_code": min(codes, key=lambda code: (-codes[code], code)), + } +(app / "report.json").write_text(json.dumps(report, indent=2)) diff --git a/bench/terminal_bench/tasks/log-triage/solution/solve.sh b/bench/terminal_bench/tasks/log-triage/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/log-triage/task.toml b/bench/terminal_bench/tasks/log-triage/task.toml new file mode 100644 index 000000000..bf1050aa1 --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "data-processing" +tags = ["logs", "fan-out", "subagents"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/log-triage/tests/test.sh b/bench/terminal_bench/tasks/log-triage/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py b/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py new file mode 100644 index 000000000..aaebaff63 --- /dev/null +++ b/bench/terminal_bench/tasks/log-triage/tests/test_outputs.py @@ -0,0 +1,42 @@ +"""Verify report.json against counts recomputed from the logs.""" + +import json +import os +import re +from collections import Counter +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +LINE = re.compile(r"^\S+ (INFO|WARN|ERROR)\s+\[(\w+)\] .*?(?: code=(E\d+))?$") + + +def expected() -> dict: + report = {} + for service_dir in sorted((APP / "logs").iterdir()): + levels: Counter[str] = Counter() + codes: Counter[str] = Counter() + for log in service_dir.glob("*.log"): + for line in log.read_text().splitlines(): + match = LINE.match(line) + assert match, line + levels[match[1]] += 1 + if match[3]: + codes[match[3]] += 1 + top = max(sorted(codes), key=lambda code: codes[code]) + report[service_dir.name] = { + "errors": levels["ERROR"], + "warnings": levels["WARN"], + "top_error_code": top, + } + return report + + +def main() -> None: + actual = json.loads((APP / "report.json").read_text()) + want = expected() + assert actual == want, {"expected": want, "actual": actual} + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile b/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile new file mode 100644 index 000000000..b23e2016a --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py b/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py new file mode 100644 index 000000000..397b36cf7 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/environment/setup.py @@ -0,0 +1,57 @@ +"""Build a git repo at that leaks a token in history on two branches.""" + +import subprocess +import sys +from pathlib import Path + +TOKEN = "ugb_live_7f3a9c2e1d4b8a6f0e5d" + + +def git(app: Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=app, check=True, capture_output=True) + + +def commit(app: Path, message: str, files: dict[str, str]) -> None: + for name, content in files.items(): + path = app / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, newline="\n") + git(app, "add", "-A") + git(app, "commit", "-m", message) + + +def main(app: Path) -> None: + app.mkdir(parents=True, exist_ok=True) + git(app, "init", "-q", "-b", "main") + for key, value in [ + ("user.name", "Bench Bot"), + ("user.email", "bench@example.com"), + ("core.autocrlf", "false"), + ("commit.gpgsign", "false"), + ]: + git(app, "config", key, value) + + commit(app, "Initial commit", {"README.md": "# report service\n", "app.py": "print('hi')\n"}) + commit(app, "Add settings", {"config/settings.py": f'API_TOKEN = "{TOKEN}"\nTIMEOUT = 30\n'}) + commit( + app, + "Use settings in app", + {"app.py": "from config.settings import API_TOKEN, TIMEOUT\n\nprint(TIMEOUT)\n"}, + ) + git(app, "branch", "feature/report") + commit( + app, + "Read token from the environment", + {"config/settings.py": 'import os\n\nAPI_TOKEN = os.environ["API_TOKEN"]\nTIMEOUT = 30\n'}, + ) + git(app, "checkout", "-q", "feature/report") + commit( + app, + "Document deploy", + {"docs/deploy.md": f"Deploy with:\n\n API_TOKEN={TOKEN} python app.py\n"}, + ) + git(app, "checkout", "-q", "main") + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/bench/terminal_bench/tasks/scrub-git-secret/instruction.md b/bench/terminal_bench/tasks/scrub-git-secret/instruction.md new file mode 100644 index 000000000..a6840a384 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/instruction.md @@ -0,0 +1,8 @@ +The git repository in the current directory leaked an API token (it starts with `ugb_live_`). It was committed on `main` and later removed, and it also shows up on the `feature/report` branch. + +Rewrite the history so the token is unreachable from every ref: + +- Replace every occurrence of the token, in every commit on every branch, with the literal text `REDACTED`. Keep the rest of each file as is. +- Keep both branches, the same number of commits on each, and the same commit messages in the same order. +- Don't leave backup refs (such as `refs/original/`) that still contain the token. +- Leave `main` checked out when you are done. diff --git a/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py new file mode 100644 index 000000000..8e530fae9 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.py @@ -0,0 +1,32 @@ +"""Reference solution for scrub-git-secret using filter-branch.""" + +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +token = "ugb_live_7f3a9c2e1d4b8a6f0e5d" +# Python one-liner keeps the tree filter portable across Linux and Git Bash. +tree_filter = ( + f'"{Path(sys.executable).as_posix()}" -c "' + "import pathlib;" + "[p.write_text(p.read_text().replace('" + token + "','REDACTED'), newline='\\n')" + " for p in pathlib.Path('.').rglob('*') if p.is_file() and '" + token + "' in p.read_text()]\"" +) +env = {**os.environ, "FILTER_BRANCH_SQUELCH_WARNING": "1"} +subprocess.run( + ["git", "filter-branch", "-f", "--tree-filter", tree_filter, "--", "--all"], + cwd=app, + check=True, + env=env, +) +refs = subprocess.run( + ["git", "for-each-ref", "--format=%(refname)", "refs/original"], + cwd=app, + check=True, + capture_output=True, + text=True, +).stdout.split() +for ref in refs: + subprocess.run(["git", "update-ref", "-d", ref], cwd=app, check=True) diff --git a/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/scrub-git-secret/task.toml b/bench/terminal_bench/tasks/scrub-git-secret/task.toml new file mode 100644 index 000000000..452f8a548 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "medium" +category = "security" +tags = ["git", "history-rewrite", "destructive"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh b/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py b/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py new file mode 100644 index 000000000..a98c50519 --- /dev/null +++ b/bench/terminal_bench/tasks/scrub-git-secret/tests/test_outputs.py @@ -0,0 +1,44 @@ +"""Verify the token is gone from every ref while history keeps its shape.""" + +import os +import subprocess +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +TOKEN = "ugb_live_7f3a9c2e1d4b8a6f0e5d" +EXPECTED = { + "main": [ + "Read token from the environment", + "Use settings in app", + "Add settings", + "Initial commit", + ], + "feature/report": ["Document deploy", "Use settings in app", "Add settings", "Initial commit"], +} + + +def git(*args: str) -> str: + return subprocess.run( + ["git", *args], cwd=APP, check=True, capture_output=True, text=True + ).stdout + + +def main() -> None: + for branch, messages in EXPECTED.items(): + assert git("log", "--format=%s", branch).splitlines() == messages, branch + + for commit in git("rev-list", "--all").split(): + found = subprocess.run(["git", "grep", "-F", TOKEN, commit], cwd=APP, capture_output=True) + assert found.returncode == 1, f"token still in {commit}" + assert TOKEN not in git("show", "-s", "--format=%B", commit) + + settings = git("show", "main~2:config/settings.py") + assert "REDACTED" in settings and "TIMEOUT = 30" in settings, settings + assert "REDACTED" in git("show", "feature/report:docs/deploy.md") + assert "os.environ" in git("show", "main:config/settings.py") + assert git("rev-parse", "--abbrev-ref", "HEAD").strip() == "main" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/ug_agent.py b/bench/terminal_bench/ug_agent.py new file mode 100644 index 000000000..f4fda0fd3 --- /dev/null +++ b/bench/terminal_bench/ug_agent.py @@ -0,0 +1,136 @@ +"""Harbor agents that run Claude Code and Codex inside a task container through ug. + +Use with `harbor run -a ug_agent:UgClaude` (or `UgCodex`) and this directory on +PYTHONPATH. The host must export UG_BENCH_WHEEL (a built ug wheel) plus the +workspace auth that bench_auth.py reads. +""" + +from __future__ import annotations + +import os +import shlex +from pathlib import Path, PurePosixPath + +import bench_auth +from harbor.agents.installed.base import BaseInstalledAgent +from harbor.agents.installed.node_install import nvm_node_install_snippet +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext + +INSTALL_DIR = PurePosixPath("/installed-agent") +UG = '"$HOME/.local/bin/ug"' +SHELL_PREFIX = 'export PATH="$HOME/.local/bin:$PATH"; if [ -s "$HOME/.nvm/nvm.sh" ]; then . "$HOME/.nvm/nvm.sh"; fi; ' + + +def _required_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise RuntimeError(f"{name} must be set on the host running harbor") + return value + + +class _UgAgent(BaseInstalledAgent): + AGENT = "" + + async def _install_agent_cli(self, environment: BaseEnvironment) -> None: + raise NotImplementedError + + def _launch_args(self) -> str: + raise NotImplementedError + + async def install(self, environment: BaseEnvironment) -> None: + wheel = Path(_required_env("UG_BENCH_WHEEL")) + await self.ensure_system_dependencies( + environment, ("curl", "bash", "git", "ca_certificates", "procps") + ) + await environment.upload_file(wheel, str(INSTALL_DIR / wheel.name)) + await self.exec_as_root( + environment, + command=( + "command -v databricks >/dev/null || curl -fsSL " + "https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh" + ), + ) + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; curl -LsSf https://astral.sh/uv/install.sh | sh && " + f'"$HOME/.local/bin/uv" tool install --python 3.12 {shlex.quote(str(INSTALL_DIR / wheel.name))}' + ), + ) + await self._install_agent_cli(environment) + + async def run( + self, instruction: str, environment: BaseEnvironment, context: AgentContext + ) -> None: + auth = bench_auth.agent_env({}) + workspace = auth["DATABRICKS_HOST"] + env = { + **auth, + # Lets Claude Code skip permissions as root inside the container. + "IS_SANDBOX": "1", + "UG_BENCH_INSTRUCTION": instruction, + } + await self.exec_as_agent( + environment, + command=( + f"{SHELL_PREFIX}{UG} configure --agents {self.AGENT} " + f"--workspace {shlex.quote(workspace)} --skip-validate --skip-upgrade " + "--disable-databricks-ai-tools &1 | tee {log.as_posix()}" + ), + env=env, + ) + + +class UgClaude(_UgAgent): + AGENT = "claude" + + @staticmethod + def name() -> str: + return "ug-claude" + + async def _install_agent_cli(self, environment: BaseEnvironment) -> None: + version = f" {shlex.quote(self._version)}" if self._version else "" + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; curl -fsSL " + f"https://downloads.claude.ai/claude-code-releases/bootstrap.sh | bash -s --{version} && " + '"$HOME/.local/bin/claude" --version' + ), + ) + + def _launch_args(self) -> str: + return "-p --verbose --output-format stream-json --dangerously-skip-permissions" + + +class UgCodex(_UgAgent): + AGENT = "codex" + + @staticmethod + def name() -> str: + return "ug-codex" + + async def _install_agent_cli(self, environment: BaseEnvironment) -> None: + version = f"@{self._version}" if self._version else "@latest" + await self.exec_as_agent( + environment, + command=( + f"set -euo pipefail; {nvm_node_install_snippet()} && " + f"npm install -g @openai/codex{shlex.quote(version)} && codex --version" + ), + env={"NVM_NODEJS_ORG_MIRROR": "https://nodejs.org/dist"}, + ) + + def _launch_args(self) -> str: + return "exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check --json -" From 9459197db16adf415de7175a5cb42fe88db81364 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:56:36 -0400 Subject: [PATCH 02/19] TEMP: run Terminal Bench on push to this branch Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml index 7d49ce11f..9f77b29cc 100644 --- a/.github/workflows/terminal-bench.yml +++ b/.github/workflows/terminal-bench.yml @@ -1,6 +1,9 @@ name: Terminal Bench on: + # TEMP: exercise the workflow from PR #1074. Remove before merge. + push: + branches: [max-rozen-oss-db/stack/terminal-bench] schedule: - cron: '0 7 * * *' workflow_dispatch: From a8d8068f59d3eeb1dbbce75c8492c7e5aa671b0b Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:35:19 -0400 Subject: [PATCH 03/19] Install unzip for the Databricks CLI installer in Harbor containers Co-authored-by: Isaac --- bench/terminal_bench/ug_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/terminal_bench/ug_agent.py b/bench/terminal_bench/ug_agent.py index f4fda0fd3..6659686e8 100644 --- a/bench/terminal_bench/ug_agent.py +++ b/bench/terminal_bench/ug_agent.py @@ -41,7 +41,7 @@ def _launch_args(self) -> str: async def install(self, environment: BaseEnvironment) -> None: wheel = Path(_required_env("UG_BENCH_WHEEL")) await self.ensure_system_dependencies( - environment, ("curl", "bash", "git", "ca_certificates", "procps") + environment, ("curl", "bash", "git", "ca_certificates", "procps", "unzip") ) await environment.upload_file(wheel, str(INSTALL_DIR / wheel.name)) await self.exec_as_root( From ff59bbca67e3c1e22f2582eca60d8f912599ff68 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:49:04 -0400 Subject: [PATCH 04/19] Launch Codex with unified_exec in Harbor, matching Harbor's own agent Co-authored-by: Isaac --- bench/terminal_bench/ug_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/terminal_bench/ug_agent.py b/bench/terminal_bench/ug_agent.py index 6659686e8..a9906ec9f 100644 --- a/bench/terminal_bench/ug_agent.py +++ b/bench/terminal_bench/ug_agent.py @@ -133,4 +133,4 @@ async def _install_agent_cli(self, environment: BaseEnvironment) -> None: ) def _launch_args(self) -> str: - return "exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check --json -" + return "exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check --json --enable unified_exec -" From 28ad85bd443cea45f6052d607e1b0c52d3e89eea Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:18:29 -0400 Subject: [PATCH 05/19] Add harness-feature tasks and multi-step resume to Terminal Bench Seven tasks check that agent features keep working through ug: project hooks, a project MCP server, CLAUDE.md/AGENTS.md conventions, a haiku subagent, image input, web fetch, and resuming a session across steps. Both runners filter tasks per agent and support multi-step tasks. Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 5 +- bench/terminal_bench/README.md | 29 +++- bench/terminal_bench/bench_tasks.py | 24 +++ bench/terminal_bench/run_native.py | 148 ++++++++++-------- .../tasks/image-code/environment/Dockerfile | 4 + .../tasks/image-code/environment/setup.py | 52 ++++++ .../tasks/image-code/instruction.md | 1 + .../tasks/image-code/solution/solve.py | 6 + .../tasks/image-code/solution/solve.sh | 2 + .../terminal_bench/tasks/image-code/task.toml | 18 +++ .../tasks/image-code/tests/test.sh | 3 + .../tasks/image-code/tests/test_outputs.py | 15 ++ .../tasks/mcp-server/environment/Dockerfile | 3 + .../environment/app/.claude/settings.json | 1 + .../mcp-server/environment/app/.mcp.json | 5 + .../mcp-server/environment/app/mcp_server.py | 51 ++++++ .../tasks/mcp-server/instruction.md | 5 + .../tasks/mcp-server/solution/solve.py | 35 +++++ .../tasks/mcp-server/solution/solve.sh | 2 + .../terminal_bench/tasks/mcp-server/task.toml | 18 +++ .../tasks/mcp-server/tests/test.sh | 3 + .../tasks/mcp-server/tests/test_outputs.py | 30 ++++ .../environment/Dockerfile | 3 + .../environment/app/AGENTS.md | 4 + .../environment/app/CLAUDE.md | 4 + .../environment/app/data.txt | 40 +++++ .../tasks/project-instructions/instruction.md | 1 + .../project-instructions/solution/solve.py | 16 ++ .../project-instructions/solution/solve.sh | 2 + .../tasks/project-instructions/task.toml | 18 +++ .../tasks/project-instructions/tests/test.sh | 3 + .../tests/test_outputs.py | 24 +++ .../resume-session/environment/Dockerfile | 4 + .../tasks/resume-session/environment/setup.py | 7 + .../resume-session/steps/learn/instruction.md | 3 + .../steps/learn/solution/solve.py | 9 ++ .../steps/learn/solution/solve.sh | 2 + .../resume-session/steps/learn/tests/test.sh | 3 + .../steps/learn/tests/test_outputs.py | 18 +++ .../steps/recall/instruction.md | 1 + .../steps/recall/solution/solve.py | 8 + .../steps/recall/solution/solve.sh | 2 + .../resume-session/steps/recall/tests/test.sh | 3 + .../steps/recall/tests/test_outputs.py | 15 ++ .../tasks/resume-session/task.toml | 26 +++ .../subagent-model/environment/Dockerfile | 5 + .../app/.claude/agents/checksum-auditor.md | 14 ++ .../tasks/subagent-model/environment/setup.py | 10 ++ .../tasks/subagent-model/instruction.md | 1 + .../tasks/subagent-model/solution/solve.py | 13 ++ .../tasks/subagent-model/solution/solve.sh | 2 + .../tasks/subagent-model/task.toml | 18 +++ .../tasks/subagent-model/tests/test.sh | 3 + .../subagent-model/tests/test_outputs.py | 43 +++++ .../tasks/user-hooks/environment/Dockerfile | 5 + .../environment/app/.claude/hooks/hook.py | 24 +++ .../environment/app/.claude/settings.json | 13 ++ .../tasks/user-hooks/environment/setup.py | 6 + .../tasks/user-hooks/instruction.md | 3 + .../tasks/user-hooks/solution/solve.py | 15 ++ .../tasks/user-hooks/solution/solve.sh | 2 + .../terminal_bench/tasks/user-hooks/task.toml | 18 +++ .../tasks/user-hooks/tests/test.sh | 3 + .../tasks/user-hooks/tests/test_outputs.py | 24 +++ .../tasks/web-fetch/environment/Dockerfile | 2 + .../tasks/web-fetch/instruction.md | 5 + .../tasks/web-fetch/solution/solve.py | 14 ++ .../tasks/web-fetch/solution/solve.sh | 2 + .../terminal_bench/tasks/web-fetch/task.toml | 18 +++ .../tasks/web-fetch/tests/test.sh | 3 + .../tasks/web-fetch/tests/test_outputs.py | 16 ++ bench/terminal_bench/ug_agent.py | 43 +++-- 72 files changed, 912 insertions(+), 89 deletions(-) create mode 100644 bench/terminal_bench/bench_tasks.py create mode 100644 bench/terminal_bench/tasks/image-code/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/image-code/environment/setup.py create mode 100644 bench/terminal_bench/tasks/image-code/instruction.md create mode 100644 bench/terminal_bench/tasks/image-code/solution/solve.py create mode 100755 bench/terminal_bench/tasks/image-code/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/image-code/task.toml create mode 100755 bench/terminal_bench/tasks/image-code/tests/test.sh create mode 100644 bench/terminal_bench/tasks/image-code/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/mcp-server/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json create mode 100644 bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json create mode 100644 bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py create mode 100644 bench/terminal_bench/tasks/mcp-server/instruction.md create mode 100644 bench/terminal_bench/tasks/mcp-server/solution/solve.py create mode 100755 bench/terminal_bench/tasks/mcp-server/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/mcp-server/task.toml create mode 100755 bench/terminal_bench/tasks/mcp-server/tests/test.sh create mode 100644 bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/project-instructions/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md create mode 100644 bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md create mode 100644 bench/terminal_bench/tasks/project-instructions/environment/app/data.txt create mode 100644 bench/terminal_bench/tasks/project-instructions/instruction.md create mode 100644 bench/terminal_bench/tasks/project-instructions/solution/solve.py create mode 100755 bench/terminal_bench/tasks/project-instructions/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/project-instructions/task.toml create mode 100755 bench/terminal_bench/tasks/project-instructions/tests/test.sh create mode 100644 bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/resume-session/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/resume-session/environment/setup.py create mode 100644 bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md create mode 100644 bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py create mode 100755 bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh create mode 100755 bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh create mode 100644 bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md create mode 100644 bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py create mode 100755 bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh create mode 100755 bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh create mode 100644 bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/resume-session/task.toml create mode 100644 bench/terminal_bench/tasks/subagent-model/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md create mode 100644 bench/terminal_bench/tasks/subagent-model/environment/setup.py create mode 100644 bench/terminal_bench/tasks/subagent-model/instruction.md create mode 100644 bench/terminal_bench/tasks/subagent-model/solution/solve.py create mode 100755 bench/terminal_bench/tasks/subagent-model/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/subagent-model/task.toml create mode 100755 bench/terminal_bench/tasks/subagent-model/tests/test.sh create mode 100644 bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/user-hooks/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py create mode 100644 bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json create mode 100644 bench/terminal_bench/tasks/user-hooks/environment/setup.py create mode 100644 bench/terminal_bench/tasks/user-hooks/instruction.md create mode 100644 bench/terminal_bench/tasks/user-hooks/solution/solve.py create mode 100755 bench/terminal_bench/tasks/user-hooks/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/user-hooks/task.toml create mode 100755 bench/terminal_bench/tasks/user-hooks/tests/test.sh create mode 100644 bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/web-fetch/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/web-fetch/instruction.md create mode 100644 bench/terminal_bench/tasks/web-fetch/solution/solve.py create mode 100755 bench/terminal_bench/tasks/web-fetch/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/web-fetch/task.toml create mode 100755 bench/terminal_bench/tasks/web-fetch/tests/test.sh create mode 100644 bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml index 9f77b29cc..3df4b6734 100644 --- a/.github/workflows/terminal-bench.yml +++ b/.github/workflows/terminal-bench.yml @@ -91,8 +91,11 @@ jobs: env: AGENT_VERSION: ${{ matrix.agent == 'claude' && env.CLAUDE_VERSION || env.CODEX_VERSION }} run: | + includes=() + while read -r task; do includes+=(-i "$task"); done \ + < <(python3 bench/terminal_bench/bench_tasks.py "${{ matrix.agent }}") uvx --python 3.12 --from "harbor==$HARBOR_VERSION" harbor run -y \ - -p bench/terminal_bench/tasks \ + -p bench/terminal_bench/tasks "${includes[@]}" --resume-trajectory \ -a "ug_agent:$AGENT_CLASS" --ak "version=$AGENT_VERSION" \ ${MODEL:+-m "$MODEL"} -n 3 -o "$RUNNER_TEMP/harbor-jobs/ug" - name: Summarize diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index dc217fcc6..1f64618f7 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -24,12 +24,29 @@ Each task follows the Harbor layout: `task.toml`, `instruction.md`, the task dir from `TASK_APP_DIR` (default `/app`) so the same files work in both lanes. Keep them standard-library Python so they run on Windows too. -| Task | Exercises | -|------|-----------| -| `fix-failing-tests` | Run tests, edit, rerun | -| `log-triage` | Independent per-service work that can fan out to subagents | -| `scrub-git-secret` | Destructive git history rewrite across branches | -| `background-service` | Start a background process, query it, shut it down | +Most tasks check one agent feature that should keep working when the agent +runs through ug. Each one leaves evidence only that feature can produce, such as +a hook log, an MCP call log, or a subagent's model in the transcript. Tasks that +only one agent supports list it in `[metadata] agents`. `bench_tasks.py ` +prints the tasks for an agent. + +| Task | Exercises | Agents | +|------|-----------|--------| +| `user-hooks` | Project `UserPromptSubmit` and `PostToolUse` hooks still run | Claude | +| `mcp-server` | A project stdio MCP server is loaded and called | Claude | +| `project-instructions` | `CLAUDE.md` / `AGENTS.md` conventions are followed | Both | +| `subagent-model` | A custom subagent pinned to `model: haiku` runs on the gateway's haiku model | Claude | +| `image-code` | Reading an image through the gateway | Both | +| `web-fetch` | Fetching a pinned URL | Both | +| `resume-session` | Two steps: the second resumes the first session and recalls a value | Both | +| `log-triage` | Independent per-service work that can fan out to subagents | Both | +| `fix-failing-tests` | Run tests, edit, rerun | Both | +| `scrub-git-secret` | Destructive git history rewrite across branches | Both | +| `background-service` | Start a background process, query it, shut it down | Both | + +Multi-step tasks put `instruction.md`, `tests/`, and `solution/` under +`steps//`. Harbor runs them with `--resume-trajectory`, and `run_native.py` +resumes with `--continue` or `exec resume --last`. ## Running locally diff --git a/bench/terminal_bench/bench_tasks.py b/bench/terminal_bench/bench_tasks.py new file mode 100644 index 000000000..8bb8cddb6 --- /dev/null +++ b/bench/terminal_bench/bench_tasks.py @@ -0,0 +1,24 @@ +"""List bench tasks for an agent. Tasks opt in with `[metadata] agents` in task.toml.""" + +import sys +import tomllib +from pathlib import Path + +TASKS_DIR = Path(__file__).resolve().parent / "tasks" + + +def task_agents(config: dict) -> list[str]: + return config.get("metadata", {}).get("agents", ["claude", "codex"]) + + +def names_for(agent: str) -> list[str]: + return [ + task.name + for task in sorted(TASKS_DIR.iterdir()) + if agent == "oracle" + or agent in task_agents(tomllib.loads((task / "task.toml").read_text())) + ] + + +if __name__ == "__main__": + print("\n".join(names_for(sys.argv[1]))) diff --git a/bench/terminal_bench/run_native.py b/bench/terminal_bench/run_native.py index 5c23950fc..4d81ae03b 100644 --- a/bench/terminal_bench/run_native.py +++ b/bench/terminal_bench/run_native.py @@ -19,39 +19,40 @@ from pathlib import Path import bench_auth +from bench_tasks import TASKS_DIR, names_for from ucode.os_compatibility.subprocess_cross_os import popen, run -TASKS_DIR = Path(__file__).resolve().parent / "tasks" - -def agent_command(agent: str, ug: str, model: str | None, task_dir: Path) -> list[str]: +def agent_command( + agent: str, ug: str, model: str | None, step_dir: Path, instruction: str, resume: bool +) -> tuple[list[str], str | None]: + """Return argv plus the stdin to send. Codex takes the prompt as an argument.""" model_args = ["--model", model] if model else [] if agent == "claude": - return [ - ug, - "claude", - *model_args, - "--", - "-p", - "--verbose", - "--output-format", - "stream-json", - "--dangerously-skip-permissions", - ] + argv = [ug, "claude", *model_args, "--", "-p", "--verbose", "--output-format"] + argv += ["stream-json", "--dangerously-skip-permissions"] + return argv + (["--continue"] if resume else []), instruction if agent == "codex": - return [ - ug, - "codex", - *model_args, - "--", - "exec", - "--dangerously-bypass-approvals-and-sandbox", - "--skip-git-repo-check", - "--json", - "-", - ] - return [sys.executable, str(task_dir / "solution" / "solve.py")] + argv = [ug, "codex", *model_args, "--", "exec", *(["resume", "--last"] if resume else [])] + argv += ["--dangerously-bypass-approvals-and-sandbox", "--skip-git-repo-check", "--json"] + return argv + ["--enable", "unified_exec", "--", instruction], None + return [sys.executable, str(step_dir / "solution" / "solve.py")], None + + +def task_steps(task_dir: Path, config: dict) -> list[tuple[str, Path, float]]: + """(name, dir holding instruction.md/tests/solution, agent timeout) for each step.""" + default = config.get("agent", {}).get("timeout_sec", 600) + if not config.get("steps"): + return [("agent", task_dir, default)] + return [ + ( + step["name"], + task_dir / "steps" / step["name"], + step.get("agent", {}).get("timeout_sec", default), + ) + for step in config["steps"] + ] def kill_tree(proc: subprocess.Popen) -> None: @@ -75,65 +76,80 @@ def prepare(task_dir: Path, workdir: Path) -> None: run([sys.executable, str(setup), str(workdir)], check=True) -def run_task(args: argparse.Namespace, task_dir: Path, out: Path) -> dict: - config = tomllib.loads((task_dir / "task.toml").read_text()) - agent_timeout = config.get("agent", {}).get("timeout_sec", 600) - verifier_timeout = config.get("verifier", {}).get("timeout_sec", 120) - workdir = out / "work" / task_dir.name - logs = out / "logs" / task_dir.name - logs.mkdir(parents=True, exist_ok=True) - prepare(task_dir, workdir) - - env = {**os.environ, "TASK_APP_DIR": str(workdir), "IS_SANDBOX": "1"} - if args.agent != "oracle": - env = bench_auth.agent_env(env) - # Tasks say "run it with Python", so put this interpreter first on PATH. - env["PATH"] = os.pathsep.join([str(Path(sys.executable).parent), env.get("PATH", "")]) - instruction = (task_dir / "instruction.md").read_text() - started = time.monotonic() - timed_out = False - with (logs / "agent.log").open("w", encoding="utf-8") as log: +def run_agent( + argv: list[str], stdin: str | None, workdir: Path, env: dict, log_path: Path, timeout: float +) -> tuple[subprocess.Popen, bool]: + with log_path.open("w", encoding="utf-8") as log: proc = popen( - agent_command(args.agent, args.ug, args.model, task_dir), + argv, cwd=workdir, env=env, - stdin=subprocess.PIPE, + stdin=subprocess.PIPE if stdin is not None else subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, text=True, start_new_session=os.name != "nt", ) try: - proc.communicate(instruction, timeout=agent_timeout) + proc.communicate(stdin, timeout=timeout) except subprocess.TimeoutExpired: - timed_out = True kill_tree(proc) - duration = time.monotonic() - started + return proc, True + return proc, False + - # Verify before cleanup so a server the agent left running still counts against it. +def verify(step_dir: Path, workdir: Path, env: dict, timeout: float) -> tuple[bool, str]: try: verifier = run( - [sys.executable, str(task_dir / "tests" / "test_outputs.py")], + [sys.executable, str(step_dir / "tests" / "test_outputs.py")], cwd=workdir, env=env, capture_output=True, text=True, - timeout=verifier_timeout, + timeout=timeout, ) - passed = verifier.returncode == 0 - verifier_log = verifier.stdout + verifier.stderr except subprocess.TimeoutExpired: - passed, verifier_log = False, f"verifier timed out after {verifier_timeout}s\n" - (logs / "verifier.log").write_text(verifier_log, encoding="utf-8") - if not timed_out and os.name != "nt": - kill_tree(proc) - return { - "task": task_dir.name, - "passed": passed, - "agent_exit": proc.returncode, - "timed_out": timed_out, - "duration_sec": round(duration, 1), - } + return False, f"verifier timed out after {timeout}s\n" + return verifier.returncode == 0, verifier.stdout + verifier.stderr + + +def run_task(args: argparse.Namespace, task_dir: Path, out: Path) -> dict: + config = tomllib.loads((task_dir / "task.toml").read_text()) + verifier_timeout = config.get("verifier", {}).get("timeout_sec", 120) + workdir = out / "work" / task_dir.name + logs = out / "logs" / task_dir.name + logs.mkdir(parents=True, exist_ok=True) + prepare(task_dir, workdir) + + env = {**os.environ, "TASK_APP_DIR": str(workdir), "IS_SANDBOX": "1"} + if args.agent == "oracle": + # Checks that read the agent transcript have nothing to read for the oracle. + env["TASK_ORACLE"] = "1" + # Tasks say "run it with Python", so put this interpreter first on PATH. + env["PATH"] = os.pathsep.join([str(Path(sys.executable).parent), env.get("PATH", "")]) + started = time.monotonic() + result = {"task": task_dir.name, "passed": True, "agent_exit": None, "timed_out": False} + for index, (name, step_dir, agent_timeout) in enumerate(task_steps(task_dir, config)): + if args.agent != "oracle": + env = bench_auth.agent_env(env) + log_path = logs / f"{name}.log" + env["TASK_AGENT_LOG"] = str(log_path) + instruction = (step_dir / "instruction.md").read_text() + argv, stdin = agent_command( + args.agent, args.ug, args.model, step_dir, instruction, resume=index > 0 + ) + proc, timed_out = run_agent(argv, stdin, workdir, env, log_path, agent_timeout) + # Verify before cleanup so a server the agent left running still counts against it. + passed, verifier_log = verify(step_dir, workdir, env, verifier_timeout) + (logs / f"{name}.verifier.log").write_text(verifier_log, encoding="utf-8") + if not timed_out and os.name != "nt": + kill_tree(proc) + result.update(agent_exit=proc.returncode, timed_out=timed_out) + if not passed: + result.update(passed=False, failed_step=name) + break + result["duration_sec"] = round(time.monotonic() - started, 1) + return result def write_summary(args: argparse.Namespace, results: list[dict], out: Path) -> None: @@ -164,7 +180,7 @@ def main() -> int: parser.add_argument("--model") args = parser.parse_args() - names = args.task or sorted(p.name for p in TASKS_DIR.iterdir() if p.is_dir()) + names = args.task or names_for(args.agent) out = args.output.resolve() if (out / "work").exists(): shutil.rmtree(out / "work") diff --git a/bench/terminal_bench/tasks/image-code/environment/Dockerfile b/bench/terminal_bench/tasks/image-code/environment/Dockerfile new file mode 100644 index 000000000..18ba6f9bd --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/image-code/environment/setup.py b/bench/terminal_bench/tasks/image-code/environment/setup.py new file mode 100644 index 000000000..65481d7bb --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/environment/setup.py @@ -0,0 +1,52 @@ +"""Draw a numeric code into code.png with a 5x7 bitmap font (standard library only).""" + +import struct +import sys +import zlib +from pathlib import Path + +CODE = "47193825" +FONT = { + "0": ["01110", "10001", "10011", "10101", "11001", "10001", "01110"], + "1": ["00100", "01100", "00100", "00100", "00100", "00100", "01110"], + "2": ["01110", "10001", "00001", "00010", "00100", "01000", "11111"], + "3": ["11110", "00001", "00001", "01110", "00001", "00001", "11110"], + "4": ["00010", "00110", "01010", "10010", "11111", "00010", "00010"], + "5": ["11111", "10000", "11110", "00001", "00001", "10001", "01110"], + "6": ["00110", "01000", "10000", "11110", "10001", "10001", "01110"], + "7": ["11111", "00001", "00010", "00100", "01000", "01000", "01000"], + "8": ["01110", "10001", "10001", "01110", "10001", "10001", "01110"], + "9": ["01110", "10001", "10001", "01111", "00001", "00010", "01100"], +} +SCALE, GAP, MARGIN = 12, 2, 3 + + +def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + +def main(app: Path) -> None: + cols = MARGIN * 2 + len(CODE) * (5 + GAP) - GAP + rows = MARGIN * 2 + 7 + grid = [[255] * cols for _ in range(rows)] + for i, digit in enumerate(CODE): + for y, row in enumerate(FONT[digit]): + for x, bit in enumerate(row): + if bit == "1": + grid[MARGIN + y][MARGIN + i * (5 + GAP) + x] = 0 + raw = b"".join( + b"\x00" + bytes(v for v in row for _ in range(SCALE)) for row in grid for _ in range(SCALE) + ) + header = struct.pack(">IIBBBBB", cols * SCALE, rows * SCALE, 8, 0, 0, 0, 0) + png = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(raw)) + + chunk(b"IEND", b"") + ) + app.mkdir(parents=True, exist_ok=True) + (app / "code.png").write_bytes(png) + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/bench/terminal_bench/tasks/image-code/instruction.md b/bench/terminal_bench/tasks/image-code/instruction.md new file mode 100644 index 000000000..ecc297105 --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/instruction.md @@ -0,0 +1 @@ +`code.png` shows an 8-digit code. Look at the image and write the code to `code.txt`. diff --git a/bench/terminal_bench/tasks/image-code/solution/solve.py b/bench/terminal_bench/tasks/image-code/solution/solve.py new file mode 100644 index 000000000..5381a4a5b --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/solution/solve.py @@ -0,0 +1,6 @@ +"""Reference solution for image-code.""" + +import os +from pathlib import Path + +Path(os.environ.get("TASK_APP_DIR", "/app"), "code.txt").write_text("47193825\n") diff --git a/bench/terminal_bench/tasks/image-code/solution/solve.sh b/bench/terminal_bench/tasks/image-code/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/image-code/task.toml b/bench/terminal_bench/tasks/image-code/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/image-code/tests/test.sh b/bench/terminal_bench/tasks/image-code/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/image-code/tests/test_outputs.py b/bench/terminal_bench/tasks/image-code/tests/test_outputs.py new file mode 100644 index 000000000..81b22af69 --- /dev/null +++ b/bench/terminal_bench/tasks/image-code/tests/test_outputs.py @@ -0,0 +1,15 @@ +"""Verify the code read from the image.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + assert (APP / "code.txt").read_text().strip() == "47193825" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile b/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json b/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json new file mode 100644 index 000000000..2fc9d2f15 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/environment/app/.claude/settings.json @@ -0,0 +1 @@ +{"enableAllProjectMcpServers": true} diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json b/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json new file mode 100644 index 000000000..e5413902e --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/environment/app/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "inventory": {"command": "python", "args": ["mcp_server.py"]} + } +} diff --git a/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py b/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py new file mode 100644 index 000000000..74704dc14 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/environment/app/mcp_server.py @@ -0,0 +1,51 @@ +"""Minimal stdio MCP server with one tool. Stock levels are random per server start.""" + +import json +import random +import sys +from pathlib import Path + +LOG = Path(__file__).resolve().parent / ".mcp_log" / "calls.jsonl" +STOCK: dict[str, int] = {} +TOOL = { + "name": "get_stock", + "description": "Return the current number of units in stock for a SKU.", + "inputSchema": { + "type": "object", + "properties": {"sku": {"type": "string"}}, + "required": ["sku"], + }, +} + + +def handle(method: str, params: dict) -> dict: + if method == "initialize": + return { + "protocolVersion": params.get("protocolVersion", "2025-06-18"), + "capabilities": {"tools": {}}, + "serverInfo": {"name": "inventory", "version": "1.0"}, + } + if method == "tools/list": + return {"tools": [TOOL]} + if method == "tools/call": + sku = params["arguments"]["sku"].strip().upper() + units = STOCK.setdefault(sku, random.randint(100, 999)) + LOG.parent.mkdir(exist_ok=True) + with LOG.open("a", encoding="utf-8") as f: + f.write(json.dumps({"sku": sku, "units": units}) + "\n") + return {"content": [{"type": "text", "text": f"{sku}: {units} units in stock"}]} + if method == "ping": + return {} + raise KeyError(method) + + +for line in sys.stdin: + message = json.loads(line) + if "id" not in message: + continue + try: + response = {"result": handle(message["method"], message.get("params") or {})} + except KeyError: + response = {"error": {"code": -32601, "message": f"unknown method {message['method']}"}} + sys.stdout.write(json.dumps({"jsonrpc": "2.0", "id": message["id"], **response}) + "\n") + sys.stdout.flush() diff --git a/bench/terminal_bench/tasks/mcp-server/instruction.md b/bench/terminal_bench/tasks/mcp-server/instruction.md new file mode 100644 index 000000000..9b5642bf9 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/instruction.md @@ -0,0 +1,5 @@ +This project has an `inventory` MCP server. Use its `get_stock` tool to look up the stock for SKUs `A-100`, `B-200`, and `C-300`. + +Write `stock.json` in the current directory mapping each SKU to its unit count as an integer, for example `{"A-100": 0, ...}`. + +Stock levels are only available from the MCP tool. diff --git a/bench/terminal_bench/tasks/mcp-server/solution/solve.py b/bench/terminal_bench/tasks/mcp-server/solution/solve.py new file mode 100644 index 000000000..dca647d48 --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/solution/solve.py @@ -0,0 +1,35 @@ +"""Reference run: speak MCP to the server over stdio.""" + +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +requests = [{"jsonrpc": "2.0", "id": 0, "method": "initialize", "params": {}}] +requests += [ + { + "jsonrpc": "2.0", + "id": i, + "method": "tools/call", + "params": {"name": "get_stock", "arguments": {"sku": sku}}, + } + for i, sku in enumerate(["A-100", "B-200", "C-300"], start=1) +] +out = subprocess.run( + [sys.executable, "mcp_server.py"], + cwd=app, + input="".join(json.dumps(r) + "\n" for r in requests), + capture_output=True, + text=True, + check=True, +).stdout +stock = {} +for line in out.splitlines()[1:]: + sku, units = re.match( + r"(\S+): (\d+)", json.loads(line)["result"]["content"][0]["text"] + ).groups() + stock[sku] = int(units) +(app / "stock.json").write_text(json.dumps(stock)) diff --git a/bench/terminal_bench/tasks/mcp-server/solution/solve.sh b/bench/terminal_bench/tasks/mcp-server/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/mcp-server/task.toml b/bench/terminal_bench/tasks/mcp-server/task.toml new file mode 100644 index 000000000..a699c38db --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/mcp-server/tests/test.sh b/bench/terminal_bench/tasks/mcp-server/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py b/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py new file mode 100644 index 000000000..cc3ff235f --- /dev/null +++ b/bench/terminal_bench/tasks/mcp-server/tests/test_outputs.py @@ -0,0 +1,30 @@ +"""Verify stock.json matches what the MCP server actually returned.""" + +import json +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +SKUS = ["A-100", "B-200", "C-300"] + + +def main() -> None: + log = APP / ".mcp_log" / "calls.jsonl" + assert log.exists(), "the MCP tool was never called" + served = {} + for line in log.read_text().splitlines(): + call = json.loads(line) + served[call["sku"]] = call["units"] + actual = json.loads((APP / "stock.json").read_text()) + for sku in SKUS: + assert sku in served, f"get_stock was never called for {sku}" + assert actual.get(sku) == served[sku], { + "sku": sku, + "served": served[sku], + "got": actual.get(sku), + } + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile b/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md b/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md new file mode 100644 index 000000000..22d1c0db1 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/environment/app/AGENTS.md @@ -0,0 +1,4 @@ +# Project conventions + +- Put every output file under `out/`. Never write output files to the project root. +- Every Python file you create must start with the line `# SPDX-License-Identifier: Apache-2.0`. diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md b/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md new file mode 100644 index 000000000..22d1c0db1 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/environment/app/CLAUDE.md @@ -0,0 +1,4 @@ +# Project conventions + +- Put every output file under `out/`. Never write output files to the project root. +- Every Python file you create must start with the line `# SPDX-License-Identifier: Apache-2.0`. diff --git a/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt b/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt new file mode 100644 index 000000000..f6b86b2b2 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/environment/app/data.txt @@ -0,0 +1,40 @@ +model cache gateway token tool token route shell gateway tool +gateway token cache cache token agent token tool +gateway shell token agent shell gateway shell shell cache gateway agent +tool model harness cache model +token shell harness tool model token shell shell agent route token tool token +gateway shell agent prompt tool cache route prompt shell prompt route harness agent model +token shell harness tool prompt route prompt harness +token token tool cache model route model prompt cache gateway token tool shell route +route shell prompt shell prompt token token harness prompt token +harness shell prompt harness cache +gateway prompt route model shell token prompt gateway agent harness +agent cache cache prompt token model prompt +tool harness model cache tool harness cache route cache agent model +model model agent agent gateway prompt +model harness harness gateway model cache tool route shell shell route model tool shell +prompt tool cache cache cache +token prompt cache gateway agent token agent prompt model token route +gateway token gateway shell model tool token route shell gateway token agent shell cache +harness route shell route prompt token token +prompt prompt prompt harness token model token route harness prompt model tool +agent tool route model tool +tool harness token harness tool +model route agent tool tool tool route agent shell agent +cache agent agent tool prompt route gateway gateway +prompt harness agent shell route prompt route route token +token agent prompt agent route agent prompt shell +gateway prompt route token token cache agent prompt model cache route token cache prompt +token model model model gateway model shell prompt model shell shell +route model tool tool model gateway gateway token tool model cache agent +gateway harness agent harness tool agent shell route +tool cache model gateway route prompt shell tool cache +model tool model tool tool gateway prompt model shell gateway model model model +shell token tool gateway route tool tool tool prompt token tool gateway +agent harness gateway token tool prompt tool gateway +prompt route shell tool shell tool +harness prompt tool tool prompt tool agent tool +tool agent prompt model cache token cache prompt route +agent cache token agent harness token +route model harness model prompt agent token +prompt model agent model cache tool cache route cache agent route diff --git a/bench/terminal_bench/tasks/project-instructions/instruction.md b/bench/terminal_bench/tasks/project-instructions/instruction.md new file mode 100644 index 000000000..27e157f0f --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/instruction.md @@ -0,0 +1 @@ +Write a Python script that counts the words in `data.txt`, run it, and save the count to a file named `count.txt`. diff --git a/bench/terminal_bench/tasks/project-instructions/solution/solve.py b/bench/terminal_bench/tasks/project-instructions/solution/solve.py new file mode 100644 index 000000000..1f9c1deab --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/solution/solve.py @@ -0,0 +1,16 @@ +"""Reference solution for project-instructions.""" + +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +(app / "out").mkdir(exist_ok=True) +script = app / "out" / "count_words.py" +script.write_text( + "# SPDX-License-Identifier: Apache-2.0\n" + "from pathlib import Path\n" + "Path('out/count.txt').write_text(str(len(Path('data.txt').read_text().split())))\n" +) +subprocess.run([sys.executable, str(script)], cwd=app, check=True) diff --git a/bench/terminal_bench/tasks/project-instructions/solution/solve.sh b/bench/terminal_bench/tasks/project-instructions/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/project-instructions/task.toml b/bench/terminal_bench/tasks/project-instructions/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/project-instructions/tests/test.sh b/bench/terminal_bench/tasks/project-instructions/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py b/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py new file mode 100644 index 000000000..3eb0f9c76 --- /dev/null +++ b/bench/terminal_bench/tasks/project-instructions/tests/test_outputs.py @@ -0,0 +1,24 @@ +"""Verify the agent followed the project's instruction file.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) +HEADER = "# SPDX-License-Identifier: Apache-2.0" + + +def main() -> None: + expected = len((APP / "data.txt").read_text().split()) + assert not (APP / "count.txt").exists(), "count.txt was written to the project root" + assert (APP / "out" / "count.txt").read_text().strip() == str(expected) + scripts = [p for p in APP.rglob("*.py") if ".claude" not in p.parts] + assert scripts, "no Python script was written" + for script in scripts: + assert script.read_text().splitlines()[0].strip() == HEADER, ( + f"{script.name} lacks the SPDX header" + ) + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/resume-session/environment/Dockerfile b/bench/terminal_bench/tasks/resume-session/environment/Dockerfile new file mode 100644 index 000000000..18ba6f9bd --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/resume-session/environment/setup.py b/bench/terminal_bench/tasks/resume-session/environment/setup.py new file mode 100644 index 000000000..0e4ea09b5 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/environment/setup.py @@ -0,0 +1,7 @@ +"""Write the vault file the first step reads.""" + +import sys +from pathlib import Path + +Path(sys.argv[1]).mkdir(parents=True, exist_ok=True) +Path(sys.argv[1], "vault.txt").write_text("passphrase: amber-falcon-7342\n") diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md b/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md new file mode 100644 index 000000000..1bf7f3a62 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/instruction.md @@ -0,0 +1,3 @@ +`vault.txt` holds a passphrase. Read it and remember it, then delete `vault.txt`. + +Don't write the passphrase to any file. You'll be asked for it later in this session. diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py new file mode 100644 index 000000000..fec3b5770 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.py @@ -0,0 +1,9 @@ +"""Reference step 1: the oracle has no memory, so keep the value outside the project.""" + +import os +import tempfile +from pathlib import Path + +vault = Path(os.environ.get("TASK_APP_DIR", "/app"), "vault.txt") +Path(tempfile.gettempdir(), "ug-bench-passphrase").write_text(vault.read_text().split(": ", 1)[1]) +vault.unlink() diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py new file mode 100644 index 000000000..0134d2485 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/learn/tests/test_outputs.py @@ -0,0 +1,18 @@ +"""Verify the vault is gone and the passphrase wasn't stashed in the project.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + assert not (APP / "vault.txt").exists(), "vault.txt still exists" + for path in APP.rglob("*"): + if path.is_file(): + assert b"amber-falcon-7342" not in path.read_bytes(), f"passphrase written to {path}" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md b/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md new file mode 100644 index 000000000..928e46bf9 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/instruction.md @@ -0,0 +1 @@ +Earlier in this session you read a passphrase from `vault.txt`. Write it to `answer.txt`. diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py new file mode 100644 index 000000000..b53573d04 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.py @@ -0,0 +1,8 @@ +"""Reference step 2.""" + +import os +import tempfile +from pathlib import Path + +value = Path(tempfile.gettempdir(), "ug-bench-passphrase").read_text() +Path(os.environ.get("TASK_APP_DIR", "/app"), "answer.txt").write_text(value) diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py new file mode 100644 index 000000000..f362c0f7e --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/steps/recall/tests/test_outputs.py @@ -0,0 +1,15 @@ +"""Verify the resumed session recalled the passphrase.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + assert "amber-falcon-7342" in (APP / "answer.txt").read_text() + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/resume-session/task.toml b/bench/terminal_bench/tasks/resume-session/task.toml new file mode 100644 index 000000000..84280a982 --- /dev/null +++ b/bench/terminal_bench/tasks/resume-session/task.toml @@ -0,0 +1,26 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 + +[[steps]] +name = "learn" +min_reward = 1.0 + +[[steps]] +name = "recall" +min_reward = 1.0 diff --git a/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile b/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile new file mode 100644 index 000000000..3a5c9c9b3 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md b/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md new file mode 100644 index 000000000..0088c6a87 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/environment/app/.claude/agents/checksum-auditor.md @@ -0,0 +1,14 @@ +--- +name: checksum-auditor +description: Audits a single file by recording its SHA-256 checksum. Use it for every checksum audit. +model: haiku +tools: Read, Bash, Write +--- + +You audit exactly one file. Compute its SHA-256 with Python's `hashlib`, then write +`audit/.txt` with two lines: + +``` +auditor: checksum-auditor +sha256: +``` diff --git a/bench/terminal_bench/tasks/subagent-model/environment/setup.py b/bench/terminal_bench/tasks/subagent-model/environment/setup.py new file mode 100644 index 000000000..1a053edb4 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/environment/setup.py @@ -0,0 +1,10 @@ +"""Write random data files to audit.""" + +import os +import sys +from pathlib import Path + +data = Path(sys.argv[1], "data") +data.mkdir(parents=True, exist_ok=True) +for name in ["alpha.bin", "bravo.bin", "charlie.bin"]: + (data / name).write_bytes(os.urandom(4096)) diff --git a/bench/terminal_bench/tasks/subagent-model/instruction.md b/bench/terminal_bench/tasks/subagent-model/instruction.md new file mode 100644 index 000000000..31562d60a --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/instruction.md @@ -0,0 +1 @@ +Audit every file in `data/` with the `checksum-auditor` subagent. Run one subagent per file, and don't compute the checksums yourself. diff --git a/bench/terminal_bench/tasks/subagent-model/solution/solve.py b/bench/terminal_bench/tasks/subagent-model/solution/solve.py new file mode 100644 index 000000000..5a0cf9623 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/solution/solve.py @@ -0,0 +1,13 @@ +"""Reference solution for subagent-model (the file side only).""" + +import hashlib +import os +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +(app / "audit").mkdir(exist_ok=True) +for path in (app / "data").iterdir(): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + (app / "audit" / f"{path.name}.txt").write_text( + f"auditor: checksum-auditor\nsha256: {digest}\n" + ) diff --git a/bench/terminal_bench/tasks/subagent-model/solution/solve.sh b/bench/terminal_bench/tasks/subagent-model/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/subagent-model/task.toml b/bench/terminal_bench/tasks/subagent-model/task.toml new file mode 100644 index 000000000..a699c38db --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/subagent-model/tests/test.sh b/bench/terminal_bench/tasks/subagent-model/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py b/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py new file mode 100644 index 000000000..798066f54 --- /dev/null +++ b/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py @@ -0,0 +1,43 @@ +"""Verify the audits and that a subagent ran on the haiku model.""" + +import glob +import hashlib +import json +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def agent_log() -> Path: + paths = [os.environ.get("TASK_AGENT_LOG", "")] + glob.glob("/logs/agent/ug-claude.txt") + return next(Path(p) for p in paths if p and Path(p).exists()) + + +def subagent_models(log: Path) -> set[str]: + models = set() + for line in log.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + event = json.loads(line) + except ValueError: + continue + if event.get("type") == "assistant" and event.get("parent_tool_use_id"): + models.add(event.get("message", {}).get("model", "")) + return models + + +def main() -> None: + for path in sorted((APP / "data").iterdir()): + audit = (APP / "audit" / f"{path.name}.txt").read_text().splitlines() + assert audit[0].strip() == "auditor: checksum-auditor", audit + assert audit[1].strip() == f"sha256: {hashlib.sha256(path.read_bytes()).hexdigest()}", audit + if os.environ.get("TASK_ORACLE"): + print("ok (oracle: skipped subagent check)") + return + models = subagent_models(agent_log()) + assert any("haiku" in m for m in models), f"no subagent ran on a haiku model: {models}" + print("ok", models) + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile b/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile new file mode 100644 index 000000000..3a5c9c9b3 --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/environment/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py new file mode 100644 index 000000000..6035f9883 --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/hooks/hook.py @@ -0,0 +1,24 @@ +"""Project hook: share the release codename and log every hook call.""" + +import json +import sys +from pathlib import Path + +root = Path(__file__).resolve().parents[2] +payload = json.load(sys.stdin) +log = root / ".hook_log" / "events.jsonl" +log.parent.mkdir(exist_ok=True) +with log.open("a", encoding="utf-8") as f: + f.write( + json.dumps( + { + "event": sys.argv[1], + "tool": payload.get("tool_name"), + "file": (payload.get("tool_input") or {}).get("file_path"), + } + ) + + "\n" + ) +if sys.argv[1] == "prompt": + codename = (root / ".claude" / "hooks" / "codename").read_text().strip() + print(f"The release codename for this session is {codename}.") diff --git a/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json new file mode 100644 index 000000000..d8c1cd98c --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/environment/app/.claude/settings.json @@ -0,0 +1,13 @@ +{ + "hooks": { + "UserPromptSubmit": [ + {"hooks": [{"type": "command", "command": "python \"$CLAUDE_PROJECT_DIR/.claude/hooks/hook.py\" prompt"}]} + ], + "PostToolUse": [ + { + "matcher": "Write|Edit", + "hooks": [{"type": "command", "command": "python \"$CLAUDE_PROJECT_DIR/.claude/hooks/hook.py\" tool"}] + } + ] + } +} diff --git a/bench/terminal_bench/tasks/user-hooks/environment/setup.py b/bench/terminal_bench/tasks/user-hooks/environment/setup.py new file mode 100644 index 000000000..9efc79370 --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/environment/setup.py @@ -0,0 +1,6 @@ +"""Store the codename the hook hands to the agent.""" + +import sys +from pathlib import Path + +Path(sys.argv[1], ".claude", "hooks", "codename").write_text("velvet-harbor-58\n") diff --git a/bench/terminal_bench/tasks/user-hooks/instruction.md b/bench/terminal_bench/tasks/user-hooks/instruction.md new file mode 100644 index 000000000..8b62f6606 --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/instruction.md @@ -0,0 +1,3 @@ +This project's hooks give you the release codename at the start of the session. + +Write a file `release.txt` in the current directory that contains only that codename. diff --git a/bench/terminal_bench/tasks/user-hooks/solution/solve.py b/bench/terminal_bench/tasks/user-hooks/solution/solve.py new file mode 100644 index 000000000..14ad8694f --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/solution/solve.py @@ -0,0 +1,15 @@ +"""Reference run: drive the hook the way Claude Code would.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +hook = [sys.executable, str(app / ".claude" / "hooks" / "hook.py")] +out = subprocess.run([*hook, "prompt"], input="{}", capture_output=True, text=True, check=True) +codename = out.stdout.strip().removesuffix(".").rsplit(" ", 1)[-1] +(app / "release.txt").write_text(codename + "\n") +payload = {"tool_name": "Write", "tool_input": {"file_path": str(app / "release.txt")}} +subprocess.run([*hook, "tool"], input=json.dumps(payload), text=True, check=True) diff --git a/bench/terminal_bench/tasks/user-hooks/solution/solve.sh b/bench/terminal_bench/tasks/user-hooks/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/user-hooks/task.toml b/bench/terminal_bench/tasks/user-hooks/task.toml new file mode 100644 index 000000000..a699c38db --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/user-hooks/tests/test.sh b/bench/terminal_bench/tasks/user-hooks/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py b/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py new file mode 100644 index 000000000..7968cb97c --- /dev/null +++ b/bench/terminal_bench/tasks/user-hooks/tests/test_outputs.py @@ -0,0 +1,24 @@ +"""Verify the codename came from the prompt hook and the write went through the tool hook.""" + +import json +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + assert (APP / "release.txt").read_text().strip() == "velvet-harbor-58" + log = APP / ".hook_log" / "events.jsonl" + assert log.exists(), "no project hook ran" + events = [json.loads(line) for line in log.read_text().splitlines()] + assert any(e["event"] == "prompt" for e in events), "UserPromptSubmit hook didn't run" + assert any( + e["event"] == "tool" and (e["file"] or "").replace("\\", "/").endswith("release.txt") + for e in events + ), f"PostToolUse hook didn't see release.txt being written: {events}" + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile b/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile new file mode 100644 index 000000000..1c0593216 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/environment/Dockerfile @@ -0,0 +1,2 @@ +FROM python:3.12-slim +WORKDIR /app diff --git a/bench/terminal_bench/tasks/web-fetch/instruction.md b/bench/terminal_bench/tasks/web-fetch/instruction.md new file mode 100644 index 000000000..7b6a09943 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/instruction.md @@ -0,0 +1,5 @@ +Fetch this file from the web: + +https://raw.githubusercontent.com/databricks/unity-gateway/ba0d79ed9d429e08af3e7cf110e379c004a3a503/pyproject.toml + +Write the exact value of the `description` field under `[project]` to `answer.txt`, without quotes. diff --git a/bench/terminal_bench/tasks/web-fetch/solution/solve.py b/bench/terminal_bench/tasks/web-fetch/solution/solve.py new file mode 100644 index 000000000..8c4faece9 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/solution/solve.py @@ -0,0 +1,14 @@ +"""Reference solution for web-fetch.""" + +import os +import tomllib +import urllib.request +from pathlib import Path + +url = ( + "https://raw.githubusercontent.com/databricks/unity-gateway/" + "ba0d79ed9d429e08af3e7cf110e379c004a3a503/pyproject.toml" +) +with urllib.request.urlopen(url, timeout=30) as response: + description = tomllib.loads(response.read().decode())["project"]["description"] +Path(os.environ.get("TASK_APP_DIR", "/app"), "answer.txt").write_text(description + "\n") diff --git a/bench/terminal_bench/tasks/web-fetch/solution/solve.sh b/bench/terminal_bench/tasks/web-fetch/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/web-fetch/task.toml b/bench/terminal_bench/tasks/web-fetch/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/web-fetch/tests/test.sh b/bench/terminal_bench/tasks/web-fetch/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py b/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py new file mode 100644 index 000000000..9a9648d82 --- /dev/null +++ b/bench/terminal_bench/tasks/web-fetch/tests/test_outputs.py @@ -0,0 +1,16 @@ +"""Verify the value fetched from the pinned file.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + answer = (APP / "answer.txt").read_text().strip().strip("\"'") + assert answer == "Bootstrap Codex against a Databricks workspace with minimal setup.", answer + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/ug_agent.py b/bench/terminal_bench/ug_agent.py index a9906ec9f..3fa8e000e 100644 --- a/bench/terminal_bench/ug_agent.py +++ b/bench/terminal_bench/ug_agent.py @@ -12,6 +12,7 @@ from pathlib import Path, PurePosixPath import bench_auth +from harbor.agents.capabilities import AgentCapabilities from harbor.agents.installed.base import BaseInstalledAgent from harbor.agents.installed.node_install import nvm_node_install_snippet from harbor.environments.base import BaseEnvironment @@ -31,11 +32,14 @@ def _required_env(name: str) -> str: class _UgAgent(BaseInstalledAgent): AGENT = "" + PROMPT_ON_STDIN = False + capabilities = AgentCapabilities(resume=True) async def _install_agent_cli(self, environment: BaseEnvironment) -> None: raise NotImplementedError - def _launch_args(self) -> str: + def _launch_args(self, resume: bool) -> str: + """Agent args after ug's `--`. Without PROMPT_ON_STDIN they pass $UG_BENCH_INSTRUCTION.""" raise NotImplementedError async def install(self, environment: BaseEnvironment) -> None: @@ -71,22 +75,24 @@ async def run( "IS_SANDBOX": "1", "UG_BENCH_INSTRUCTION": instruction, } - await self.exec_as_agent( - environment, - command=( - f"{SHELL_PREFIX}{UG} configure --agents {self.AGENT} " - f"--workspace {shlex.quote(workspace)} --skip-validate --skip-upgrade " - "--disable-databricks-ai-tools &1 | tee {log.as_posix()}" + f"{SHELL_PREFIX}{stdin}{UG} {self.AGENT} {model}-- {self._launch_args(self._resume)} " + f"2>&1 | tee -a {log.as_posix()}" ), env=env, ) @@ -94,6 +100,7 @@ async def run( class UgClaude(_UgAgent): AGENT = "claude" + PROMPT_ON_STDIN = True @staticmethod def name() -> str: @@ -110,8 +117,9 @@ async def _install_agent_cli(self, environment: BaseEnvironment) -> None: ), ) - def _launch_args(self) -> str: - return "-p --verbose --output-format stream-json --dangerously-skip-permissions" + def _launch_args(self, resume: bool) -> str: + args = "-p --verbose --output-format stream-json --dangerously-skip-permissions" + return f"{args} --continue" if resume else args class UgCodex(_UgAgent): @@ -132,5 +140,8 @@ async def _install_agent_cli(self, environment: BaseEnvironment) -> None: env={"NVM_NODEJS_ORG_MIRROR": "https://nodejs.org/dist"}, ) - def _launch_args(self) -> str: - return "exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check --json --enable unified_exec -" + def _launch_args(self, resume: bool) -> str: + return ( + f"exec {'resume --last ' if resume else ''}--dangerously-bypass-approvals-and-sandbox " + '--skip-git-repo-check --json --enable unified_exec -- "$UG_BENCH_INSTRUCTION" Date: Fri, 9 Oct 2026 13:35:40 -0400 Subject: [PATCH 06/19] Accept a policy-named fallback in the subagent-model check The CUJ workspace's managed config disallows haiku, so Claude Code runs the subagent on the fallback it names. Require that notice and the fallback model instead of failing. Co-authored-by: Isaac --- bench/terminal_bench/README.md | 2 +- .../subagent-model/tests/test_outputs.py | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index 1f64618f7..3242a460e 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -35,7 +35,7 @@ prints the tasks for an agent. | `user-hooks` | Project `UserPromptSubmit` and `PostToolUse` hooks still run | Claude | | `mcp-server` | A project stdio MCP server is loaded and called | Claude | | `project-instructions` | `CLAUDE.md` / `AGENTS.md` conventions are followed | Both | -| `subagent-model` | A custom subagent pinned to `model: haiku` runs on the gateway's haiku model | Claude | +| `subagent-model` | A custom subagent pinned to `model: haiku` runs on haiku, or on the fallback a workspace policy names | Claude | | `image-code` | Reading an image through the gateway | Both | | `web-fetch` | Fetching a pinned URL | Both | | `resume-session` | Two steps: the second resumes the first session and recalls a value | Both | diff --git a/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py b/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py index 798066f54..73cec0e47 100644 --- a/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py +++ b/bench/terminal_bench/tasks/subagent-model/tests/test_outputs.py @@ -4,6 +4,7 @@ import hashlib import json import os +import re from pathlib import Path APP = Path(os.environ.get("TASK_APP_DIR", "/app")) @@ -14,8 +15,9 @@ def agent_log() -> Path: return next(Path(p) for p in paths if p and Path(p).exists()) -def subagent_models(log: Path) -> set[str]: - models = set() +def read_transcript(log: Path) -> tuple[set[str], list[str]]: + """Models subagents ran on, and any notices that their requested model was restricted.""" + models, restricted = set(), [] for line in log.read_text(encoding="utf-8", errors="replace").splitlines(): try: event = json.loads(line) @@ -23,7 +25,9 @@ def subagent_models(log: Path) -> set[str]: continue if event.get("type") == "assistant" and event.get("parent_tool_use_id"): models.add(event.get("message", {}).get("model", "")) - return models + if str(event.get("key", "")).startswith("agent-model-restricted-checksum-auditor"): + restricted.append(event.get("text", "")) + return models, restricted def main() -> None: @@ -34,9 +38,16 @@ def main() -> None: if os.environ.get("TASK_ORACLE"): print("ok (oracle: skipped subagent check)") return - models = subagent_models(agent_log()) - assert any("haiku" in m for m in models), f"no subagent ran on a haiku model: {models}" - print("ok", models) + models, restricted = read_transcript(agent_log()) + assert models, "no subagent ran" + if any("haiku" in m for m in models): + print("ok", models) + return + # A workspace policy can disallow haiku; then the subagent must run on the named fallback. + assert restricted, f"subagent ignored model: haiku without a policy notice: {models}" + fallback = re.search(r"Using (\S+?) instead", restricted[0])[1] + assert all(fallback.endswith(m) for m in models), {"fallback": fallback, "models": models} + print("ok (haiku restricted by policy)", fallback, models) if __name__ == "__main__": From b2eaff957bff38b96d0bd4127e89e18552b993b7 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:18:45 -0400 Subject: [PATCH 07/19] Add skill, long-job, and PDF tasks; refocus the TB2 subset project-skill checks a project skill's procedure runs, background-shell needs a job past the default command timeout, and pdf-extract reads a compressed PDF. The TB2 default now favors realistic tasks on the same paths (images, documents, long edits, services) over model-skill tasks. Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 4 +- bench/terminal_bench/README.md | 9 ++-- .../background-shell/environment/Dockerfile | 3 ++ .../environment/app/slow_job.py | 14 +++++++ .../tasks/background-shell/instruction.md | 3 ++ .../tasks/background-shell/solution/solve.py | 10 +++++ .../tasks/background-shell/solution/solve.sh | 2 + .../tasks/background-shell/task.toml | 18 ++++++++ .../tasks/background-shell/tests/test.sh | 3 ++ .../background-shell/tests/test_outputs.py | 19 +++++++++ .../tasks/pdf-extract/environment/Dockerfile | 4 ++ .../tasks/pdf-extract/environment/setup.py | 42 +++++++++++++++++++ .../tasks/pdf-extract/instruction.md | 1 + .../tasks/pdf-extract/solution/solve.py | 6 +++ .../tasks/pdf-extract/solution/solve.sh | 2 + .../tasks/pdf-extract/task.toml | 18 ++++++++ .../tasks/pdf-extract/tests/test.sh | 3 ++ .../tasks/pdf-extract/tests/test_outputs.py | 16 +++++++ .../project-skill/environment/Dockerfile | 3 ++ .../app/.claude/skills/release-notes/SKILL.md | 10 +++++ .../app/.claude/skills/release-notes/build.py | 20 +++++++++ .../project-skill/environment/app/changes.txt | 5 +++ .../tasks/project-skill/instruction.md | 1 + .../tasks/project-skill/solution/solve.py | 17 ++++++++ .../tasks/project-skill/solution/solve.sh | 2 + .../tasks/project-skill/task.toml | 18 ++++++++ .../tasks/project-skill/tests/test.sh | 3 ++ .../tasks/project-skill/tests/test_outputs.py | 24 +++++++++++ 28 files changed, 275 insertions(+), 5 deletions(-) create mode 100644 bench/terminal_bench/tasks/background-shell/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py create mode 100644 bench/terminal_bench/tasks/background-shell/instruction.md create mode 100644 bench/terminal_bench/tasks/background-shell/solution/solve.py create mode 100755 bench/terminal_bench/tasks/background-shell/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/background-shell/task.toml create mode 100755 bench/terminal_bench/tasks/background-shell/tests/test.sh create mode 100644 bench/terminal_bench/tasks/background-shell/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/pdf-extract/environment/setup.py create mode 100644 bench/terminal_bench/tasks/pdf-extract/instruction.md create mode 100644 bench/terminal_bench/tasks/pdf-extract/solution/solve.py create mode 100755 bench/terminal_bench/tasks/pdf-extract/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/pdf-extract/task.toml create mode 100755 bench/terminal_bench/tasks/pdf-extract/tests/test.sh create mode 100644 bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py create mode 100644 bench/terminal_bench/tasks/project-skill/environment/Dockerfile create mode 100644 bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md create mode 100644 bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py create mode 100644 bench/terminal_bench/tasks/project-skill/environment/app/changes.txt create mode 100644 bench/terminal_bench/tasks/project-skill/instruction.md create mode 100644 bench/terminal_bench/tasks/project-skill/solution/solve.py create mode 100755 bench/terminal_bench/tasks/project-skill/solution/solve.sh create mode 100644 bench/terminal_bench/tasks/project-skill/task.toml create mode 100755 bench/terminal_bench/tasks/project-skill/tests/test.sh create mode 100644 bench/terminal_bench/tasks/project-skill/tests/test_outputs.py diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml index 3df4b6734..a74a43c4d 100644 --- a/.github/workflows/terminal-bench.yml +++ b/.github/workflows/terminal-bench.yml @@ -10,7 +10,7 @@ on: inputs: tb2_tasks: description: Comma-separated Terminal Bench 2 task names for the Linux Harbor lane - default: git-multibranch,financial-document-processor,hf-model-inference,nginx-request-logging,sanitize-git-repo,git-leak-recovery,multi-source-data-merger,kv-store-grpc + default: fix-git,financial-document-processor,code-from-image,chess-best-move,git-multibranch,kv-store-grpc,large-scale-text-editing required: true tb2_dataset: description: Harbor dataset for the TB2 lane @@ -38,7 +38,7 @@ concurrency: env: PYTHONUTF8: 1 - TB2_TASKS: ${{ inputs.tb2_tasks || 'git-multibranch,financial-document-processor,hf-model-inference,nginx-request-logging,sanitize-git-repo,git-leak-recovery,multi-source-data-merger,kv-store-grpc' }} + TB2_TASKS: ${{ inputs.tb2_tasks || 'fix-git,financial-document-processor,code-from-image,chess-best-move,git-multibranch,kv-store-grpc,large-scale-text-editing' }} TB2_DATASET: ${{ inputs.tb2_dataset || 'terminal-bench@2.0' }} MODEL: ${{ inputs.model }} CLAUDE_VERSION: ${{ inputs.claude_version || '2.1.290' }} diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index 3242a460e..134b470c4 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -12,9 +12,9 @@ There are two lanes: - Native (Linux and Windows). GitHub's Windows runners can't run Linux containers, so `run_native.py` runs the tasks in `tasks/` on the host. -The default TB2 subset was picked to make the agent spawn subagents, use a range -of tools, and touch system state (services, global installs, git history -rewrites). Change it with the workflow's `tb2_tasks` input. +The default TB2 subset is a small set of realistic tasks that go through the +same paths: images, documents, a long text edit, system services, and a +background server. Change it with the workflow's `tb2_tasks` input. ## Tasks @@ -39,6 +39,9 @@ prints the tasks for an agent. | `image-code` | Reading an image through the gateway | Both | | `web-fetch` | Fetching a pinned URL | Both | | `resume-session` | Two steps: the second resumes the first session and recalls a value | Both | +| `project-skill` | A project skill's procedure is followed, including its script | Claude | +| `background-shell` | A job longer than the default command timeout is run in the background and its output captured | Both | +| `pdf-extract` | Reading a value from a PDF with a compressed text stream | Both | | `log-triage` | Independent per-service work that can fan out to subagents | Both | | `fix-failing-tests` | Run tests, edit, rerun | Both | | `scrub-git-secret` | Destructive git history rewrite across branches | Both | diff --git a/bench/terminal_bench/tasks/background-shell/environment/Dockerfile b/bench/terminal_bench/tasks/background-shell/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py b/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py new file mode 100644 index 000000000..54fdafbfc --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/environment/app/slow_job.py @@ -0,0 +1,14 @@ +"""A slow job. It prints its result to stdout only once it finishes.""" + +import hashlib +import secrets +import time +from pathlib import Path + +result = secrets.token_hex(8) +state = Path(__file__).resolve().parent / ".job_state" +state.mkdir(exist_ok=True) +(state / "started").write_text(str(time.time())) +time.sleep(150) +(state / "result.sha256").write_text(hashlib.sha256(result.encode()).hexdigest()) +print(f"RESULT={result}", flush=True) diff --git a/bench/terminal_bench/tasks/background-shell/instruction.md b/bench/terminal_bench/tasks/background-shell/instruction.md new file mode 100644 index 000000000..5c60f0fbb --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/instruction.md @@ -0,0 +1,3 @@ +Run `python slow_job.py`. It takes about two and a half minutes and prints a line `RESULT=` only when it finishes. + +Write the value to `result.txt`. The result is only printed to the job's output. diff --git a/bench/terminal_bench/tasks/background-shell/solution/solve.py b/bench/terminal_bench/tasks/background-shell/solution/solve.py new file mode 100644 index 000000000..7894dd793 --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/solution/solve.py @@ -0,0 +1,10 @@ +"""Reference solution for background-shell.""" + +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +out = subprocess.run([sys.executable, "slow_job.py"], cwd=app, capture_output=True, text=True, check=True).stdout +(app / "result.txt").write_text(out.strip().removeprefix("RESULT=") + "\n") diff --git a/bench/terminal_bench/tasks/background-shell/solution/solve.sh b/bench/terminal_bench/tasks/background-shell/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/background-shell/task.toml b/bench/terminal_bench/tasks/background-shell/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/background-shell/tests/test.sh b/bench/terminal_bench/tasks/background-shell/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py b/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py new file mode 100644 index 000000000..845536a6a --- /dev/null +++ b/bench/terminal_bench/tasks/background-shell/tests/test_outputs.py @@ -0,0 +1,19 @@ +"""Verify the agent waited for the long job and captured its output.""" + +import hashlib +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + state = APP / ".job_state" + assert (state / "result.sha256").exists(), "slow_job.py never finished" + value = (APP / "result.txt").read_text().strip().removeprefix("RESULT=") + assert hashlib.sha256(value.encode()).hexdigest() == (state / "result.sha256").read_text() + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile b/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile new file mode 100644 index 000000000..18ba6f9bd --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/environment/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.12-slim +WORKDIR /app +COPY setup.py /tmp/setup.py +RUN python3 /tmp/setup.py /app && rm /tmp/setup.py diff --git a/bench/terminal_bench/tasks/pdf-extract/environment/setup.py b/bench/terminal_bench/tasks/pdf-extract/environment/setup.py new file mode 100644 index 000000000..f4a64e63e --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/environment/setup.py @@ -0,0 +1,42 @@ +"""Write invoice.pdf with a Flate-compressed text stream (standard library only).""" + +import sys +import zlib +from pathlib import Path + +LINES = [ + "Northwind Supplies GmbH", + "Invoice INV-2026-0419", + "Consulting hours: 38", + "Subtotal: 6,169.91 EUR", + "VAT (19%): 1,172.28 EUR", + "Total due: 7,342.19 EUR", +] + + +def main(app: Path) -> None: + text = "BT /F1 14 Tf 72 720 Td 20 TL " + " ".join(f"({line}) Tj T*" for line in LINES) + " ET" + stream = zlib.compress(text.encode()) + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Length %d /Filter /FlateDecode >>\nstream\n" % len(stream) + stream + b"\nendstream", + ] + pdf = bytearray(b"%PDF-1.4\n") + offsets = [] + for number, body in enumerate(objects, start=1): + offsets.append(len(pdf)) + pdf += b"%d 0 obj\n" % number + body + b"\nendobj\n" + xref = len(pdf) + pdf += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) + pdf += b"".join(b"%010d 00000 n \n" % offset for offset in offsets) + pdf += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (len(objects) + 1, xref) + app.mkdir(parents=True, exist_ok=True) + (app / "invoice.pdf").write_bytes(bytes(pdf)) + + +if __name__ == "__main__": + main(Path(sys.argv[1])) diff --git a/bench/terminal_bench/tasks/pdf-extract/instruction.md b/bench/terminal_bench/tasks/pdf-extract/instruction.md new file mode 100644 index 000000000..e931c3aea --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/instruction.md @@ -0,0 +1 @@ +Read `invoice.pdf` and write the invoice's total due to `total.txt` exactly as printed, for example `1,234.56 EUR`. diff --git a/bench/terminal_bench/tasks/pdf-extract/solution/solve.py b/bench/terminal_bench/tasks/pdf-extract/solution/solve.py new file mode 100644 index 000000000..b636f8c4b --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/solution/solve.py @@ -0,0 +1,6 @@ +"""Reference solution for pdf-extract.""" + +import os +from pathlib import Path + +Path(os.environ.get("TASK_APP_DIR", "/app"), "total.txt").write_text("7,342.19 EUR\n") diff --git a/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh b/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/pdf-extract/task.toml b/bench/terminal_bench/tasks/pdf-extract/task.toml new file mode 100644 index 000000000..9ed323ad6 --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude", "codex"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/pdf-extract/tests/test.sh b/bench/terminal_bench/tasks/pdf-extract/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py b/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py new file mode 100644 index 000000000..8306a656d --- /dev/null +++ b/bench/terminal_bench/tasks/pdf-extract/tests/test_outputs.py @@ -0,0 +1,16 @@ +"""Verify the total read from the PDF.""" + +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + total = (APP / "total.txt").read_text().strip() + assert total.removeprefix("Total due:").strip() == "7,342.19 EUR", total + print("ok") + + +if __name__ == "__main__": + main() diff --git a/bench/terminal_bench/tasks/project-skill/environment/Dockerfile b/bench/terminal_bench/tasks/project-skill/environment/Dockerfile new file mode 100644 index 000000000..ed07be056 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/environment/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.12-slim +WORKDIR /app +COPY app/ /app/ diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md new file mode 100644 index 000000000..f843dca9e --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/SKILL.md @@ -0,0 +1,10 @@ +--- +name: release-notes +description: Build release notes for this project. Use whenever someone asks for release notes or a changelog. +--- + +# Release notes + +1. Run `python .claude/skills/release-notes/build.py `. It reads `changes.txt` and prints a draft. +2. Save the draft to `dist/release-notes-.md`. +3. Add this as the last line of the file, exactly: `Signed-off-by: release-bot` diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py new file mode 100644 index 000000000..6188516e1 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/environment/app/.claude/skills/release-notes/build.py @@ -0,0 +1,20 @@ +"""Draft release notes from changes.txt, grouped by change type.""" + +import json +import sys +from pathlib import Path + +root = Path(__file__).resolve().parents[3] +version = sys.argv[1] +groups: dict[str, list[str]] = {} +for line in (root / "changes.txt").read_text().splitlines(): + kind, _, text = line.partition(": ") + groups.setdefault(kind.strip(), []).append(text.strip()) +print(f"# Release {version}\n") +for kind in sorted(groups): + print(f"## {kind.title()}\n") + print("\n".join(f"- {item}" for item in groups[kind]) + "\n") +log = root / ".skill_log" / "runs.jsonl" +log.parent.mkdir(exist_ok=True) +with log.open("a", encoding="utf-8") as f: + f.write(json.dumps({"version": version}) + "\n") diff --git a/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt b/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt new file mode 100644 index 000000000..ffd3e31e6 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/environment/app/changes.txt @@ -0,0 +1,5 @@ +fix: Retry gateway calls on HTTP 429 +feature: Add a Codex model catalog +fix: Keep custom headers on reconfigure +docs: Document managed settings +feature: Smart routing for subagents diff --git a/bench/terminal_bench/tasks/project-skill/instruction.md b/bench/terminal_bench/tasks/project-skill/instruction.md new file mode 100644 index 000000000..ca7a222cd --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/instruction.md @@ -0,0 +1 @@ +Write the release notes for version 2.3.0. diff --git a/bench/terminal_bench/tasks/project-skill/solution/solve.py b/bench/terminal_bench/tasks/project-skill/solution/solve.py new file mode 100644 index 000000000..fdedd7380 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/solution/solve.py @@ -0,0 +1,17 @@ +"""Reference solution: follow the skill.""" + +import os +import subprocess +import sys +from pathlib import Path + +app = Path(os.environ.get("TASK_APP_DIR", "/app")) +draft = subprocess.run( + [sys.executable, ".claude/skills/release-notes/build.py", "2.3.0"], + cwd=app, + capture_output=True, + text=True, + check=True, +).stdout +(app / "dist").mkdir(exist_ok=True) +(app / "dist" / "release-notes-2.3.0.md").write_text(draft.rstrip() + "\n\nSigned-off-by: release-bot\n") diff --git a/bench/terminal_bench/tasks/project-skill/solution/solve.sh b/bench/terminal_bench/tasks/project-skill/solution/solve.sh new file mode 100755 index 000000000..5dbbeddfd --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/solution/solve.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python3 /solution/solve.py diff --git a/bench/terminal_bench/tasks/project-skill/task.toml b/bench/terminal_bench/tasks/project-skill/task.toml new file mode 100644 index 000000000..a699c38db --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/task.toml @@ -0,0 +1,18 @@ +version = "1.0" + +[metadata] +author_name = "Unity Gateway" +difficulty = "easy" +category = "agent-harness" +agents = ["claude"] + +[verifier] +timeout_sec = 120.0 + +[agent] +timeout_sec = 600.0 + +[environment] +build_timeout_sec = 600.0 +cpus = 1 +memory_mb = 2048 diff --git a/bench/terminal_bench/tasks/project-skill/tests/test.sh b/bench/terminal_bench/tasks/project-skill/tests/test.sh new file mode 100755 index 000000000..48266bb0b --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +mkdir -p /logs/verifier +if python3 /tests/test_outputs.py; then echo 1 > /logs/verifier/reward.txt; else echo 0 > /logs/verifier/reward.txt; fi diff --git a/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py b/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py new file mode 100644 index 000000000..e54079506 --- /dev/null +++ b/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py @@ -0,0 +1,24 @@ +"""Verify the release-notes skill's procedure was followed.""" + +import json +import os +from pathlib import Path + +APP = Path(os.environ.get("TASK_APP_DIR", "/app")) + + +def main() -> None: + log = APP / ".skill_log" / "runs.jsonl" + assert log.exists(), "the skill's build script never ran" + assert any(json.loads(line)["version"].lstrip("v") == "2.3.0" for line in log.read_text().splitlines()) + notes = (APP / "dist" / "release-notes-2.3.0.md").read_text().strip().splitlines() + assert notes[0].strip() == "# Release 2.3.0", notes[0] + assert notes[-1].strip() == "Signed-off-by: release-bot", notes[-1] + body = "\n".join(notes) + for item in ["Retry gateway calls on HTTP 429", "Smart routing for subagents", "Document managed settings"]: + assert item in body, item + print("ok") + + +if __name__ == "__main__": + main() From 9c28190804eddcc7e62373728be4c58b499d234d Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:25:27 -0400 Subject: [PATCH 08/19] Remove the temporary push trigger from Terminal Bench Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml index a74a43c4d..6a66610bb 100644 --- a/.github/workflows/terminal-bench.yml +++ b/.github/workflows/terminal-bench.yml @@ -1,9 +1,6 @@ name: Terminal Bench on: - # TEMP: exercise the workflow from PR #1074. Remove before merge. - push: - branches: [max-rozen-oss-db/stack/terminal-bench] schedule: - cron: '0 7 * * *' workflow_dispatch: From 33fd8c93b3c02d004ac3081fda42e47cd50e426a Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:05:59 -0400 Subject: [PATCH 09/19] Rewrite the Terminal Bench README and add a task reference README covers running the workflow, reading results, running locally, credentials, and adding a task. TASKS.md documents the task format, the variables verifiers read, and what each task checks. Co-authored-by: Isaac --- bench/terminal_bench/README.md | 124 +++++++++++++++------------------ bench/terminal_bench/TASKS.md | 54 ++++++++++++++ 2 files changed, 109 insertions(+), 69 deletions(-) create mode 100644 bench/terminal_bench/TASKS.md diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index 134b470c4..9c4bb949a 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -1,85 +1,71 @@ -# Terminal Bench - -Runs Terminal Bench-style tasks through `ug claude` and `ug codex`. The -`Terminal Bench` workflow runs nightly and on demand. - -There are two lanes: - -- Harbor (Linux). Runs real [Terminal Bench 2](https://www.tbench.ai/) tasks - plus the tasks in `tasks/` inside Harbor's Docker containers. - `ug_agent.py` installs the ug wheel, the Databricks CLI, and the agent CLI in - each container, then launches the agent headlessly through ug. -- Native (Linux and Windows). GitHub's Windows runners can't run Linux - containers, so `run_native.py` runs the tasks in `tasks/` on the host. - -The default TB2 subset is a small set of realistic tasks that go through the -same paths: images, documents, a long text edit, system services, and a -background server. Change it with the workflow's `tb2_tasks` input. - -## Tasks - -Each task follows the Harbor layout: `task.toml`, `instruction.md`, -`environment/` (a Dockerfile, plus `app/` fixtures or a `setup.py`), -`tests/test_outputs.py`, and `solution/solve.py`. The verifier and solution read -the task dir from `TASK_APP_DIR` (default `/app`) so the same files work in -both lanes. Keep them standard-library Python so they run on Windows too. - -Most tasks check one agent feature that should keep working when the agent -runs through ug. Each one leaves evidence only that feature can produce, such as -a hook log, an MCP call log, or a subagent's model in the transcript. Tasks that -only one agent supports list it in `[metadata] agents`. `bench_tasks.py ` -prints the tasks for an agent. - -| Task | Exercises | Agents | -|------|-----------|--------| -| `user-hooks` | Project `UserPromptSubmit` and `PostToolUse` hooks still run | Claude | -| `mcp-server` | A project stdio MCP server is loaded and called | Claude | -| `project-instructions` | `CLAUDE.md` / `AGENTS.md` conventions are followed | Both | -| `subagent-model` | A custom subagent pinned to `model: haiku` runs on haiku, or on the fallback a workspace policy names | Claude | -| `image-code` | Reading an image through the gateway | Both | -| `web-fetch` | Fetching a pinned URL | Both | -| `resume-session` | Two steps: the second resumes the first session and recalls a value | Both | -| `project-skill` | A project skill's procedure is followed, including its script | Claude | -| `background-shell` | A job longer than the default command timeout is run in the background and its output captured | Both | -| `pdf-extract` | Reading a value from a PDF with a compressed text stream | Both | -| `log-triage` | Independent per-service work that can fan out to subagents | Both | -| `fix-failing-tests` | Run tests, edit, rerun | Both | -| `scrub-git-secret` | Destructive git history rewrite across branches | Both | -| `background-service` | Start a background process, query it, shut it down | Both | - -Multi-step tasks put `instruction.md`, `tests/`, and `solution/` under -`steps//`. Harbor runs them with `--resume-trajectory`, and `run_native.py` -resumes with `--continue` or `exec resume --last`. - -## Running locally - -Check tasks with the reference solutions (no gateway needed): +# Terminal Bench for ug + +This directory runs Terminal Bench tasks through `ug claude` and `ug codex`. It checks that an agent keeps working when ug configures and launches it. The `Terminal Bench` workflow in `.github/workflows/terminal-bench.yml` runs every night at 07:00 UTC and on manual dispatch. + +The workflow has two lanes. The Harbor lane runs on Linux. It runs a subset of [Terminal Bench 2](https://www.tbench.ai/) and the tasks in `tasks/` inside Harbor's Docker containers, and `ug_agent.py` installs ug in each container. The native lane runs on Linux and Windows. GitHub's Windows runners can't run Linux containers, so `run_native.py` runs the tasks in `tasks/` directly on the runner. + +[TASKS.md](TASKS.md) describes the task format and lists every task. + +## Run the workflow with other tasks or a model + +To change the TB2 tasks or pin a model, dispatch the workflow with inputs: ```sh -uv run python bench/terminal_bench/run_native.py --agent oracle --output /tmp/tb +gh workflow run terminal-bench.yml -f tb2_tasks=fix-git,code-from-image -f model= ``` -## Auth +The workflow also takes `tb2_dataset`, `claude_version`, and `codex_version`. GitHub can dispatch the workflow only from a file that is already on `main`. -`bench_auth.py` picks the credentials. With `UG_BENCH_CLIENT_ID`, -`UG_BENCH_CLIENT_SECRET`, and `UG_BENCH_SP_WORKSPACE` set, it mints a fresh -OAuth M2M token on the host before each task. Only that token reaches the agent, -never the SP secret. Otherwise it uses `DATABRICKS_BEARER` against -`UCODE_TEST_WORKSPACE`. CI uses the CUJ service principal and `UG_CUJ1_WORKSPACE`. +## Read the results -Uploaded artifacts have tokens redacted by `scrub_secrets.py`. +Each job writes a table of task results to its run summary. -Run the tasks through ug with either set of credentials exported. This rewrites -your `ug configure` state for the agent: +A Harbor job fails only when a trial hits a harness error, such as a failed agent install or an API error. A task with reward 0 doesn't fail the job, because TB2 tasks are meant to be hard. A native job fails when any task fails, so a native job is often red while the Harbor jobs in the same run pass. + +Each job uploads its files as an artifact named `terminal-bench-harbor-` or `terminal-bench-native--`. `scrub_secrets.py` replaces bearer tokens in those files before upload. + +To tell a ug problem from an agent mistake, read the verifier output first and the agent log second: + +- In a Harbor artifact, read `//verifier/test-stdout.txt`, then `//agent/ug-.txt`. +- In a native artifact, read `logs//.verifier.log`, then `logs//.log`. A single-step task names its step `agent`. + +## Run tasks locally + +To check the tasks without a gateway, run the reference solutions. Every task passes. + +```sh +uv run python bench/terminal_bench/run_native.py --agent oracle --output /tmp/tb +``` + +Add `--task ` to run one task. + +To run the tasks through ug, export credentials as described in [Credentials](#credentials) and pick an agent. The runner calls `ug configure` for that agent, so it replaces your own configuration for it. ```sh uv run python bench/terminal_bench/run_native.py --agent claude --output /tmp/tb ``` -Harbor needs Docker, `UG_BENCH_WHEEL`, and the same credentials: +To run in Harbor, you need Docker. Build a ug wheel, point `UG_BENCH_WHEEL` at it, and start a job: ```sh -uv build --wheel --out-dir /tmp/dist && export UG_BENCH_WHEEL=$(ls /tmp/dist/*.whl) +uv build --wheel --out-dir /tmp/dist +export UG_BENCH_WHEEL=$(ls /tmp/dist/*.whl) PYTHONPATH=bench/terminal_bench uvx --python 3.12 --from harbor==0.23.0 harbor run -y \ - -d terminal-bench@2.0 -i fix-git -a ug_agent:UgClaude -o /tmp/harbor-jobs + -d terminal-bench@2.0 -i fix-git -a ug_agent:UgClaude -o /tmp/harbor-jobs ``` + +For Codex, use `ug_agent:UgCodex`. To run ug's own tasks, replace `-d terminal-bench@2.0` with `-p bench/terminal_bench/tasks --resume-trajectory`. + +## Credentials + +`bench_auth.py` picks the credentials for each task. If `UG_BENCH_CLIENT_ID`, `UG_BENCH_CLIENT_SECRET`, and `UG_BENCH_SP_WORKSPACE` are set, it mints a new OAuth M2M token on the host before each task. The agent gets that token and never the service principal secret, because the agent has a full shell. If those variables aren't set, it uses `DATABRICKS_BEARER` against `UCODE_TEST_WORKSPACE`. + +CI uses the `UG_CUJ_SP_CLIENT_ID` service principal against `UG_CUJ1_WORKSPACE`. That workspace publishes a managed coding-agent config, so its model policy applies to every CI run. The policy doesn't allow haiku, for example. + +## Add a task + +1. Copy the task in `tasks/` that is closest to yours. [TASKS.md](TASKS.md) describes each file. +2. Write `tests/test_outputs.py` so it checks evidence that only the feature can produce, such as a hook log or an MCP call log, as well as the answer. An agent can read every file in the task, so the answer alone is easy to fake. +3. If only one agent supports the feature, set `agents` under `[metadata]` in `task.toml`. +4. Run `run_native.py --agent oracle --task ` and confirm the task passes. +5. Confirm the verifier fails when the agent does nothing. Copy `environment/app/` into an empty directory, or run `environment/setup.py `. Then run `tests/test_outputs.py` with `TASK_APP_DIR` set to that directory. diff --git a/bench/terminal_bench/TASKS.md b/bench/terminal_bench/TASKS.md new file mode 100644 index 000000000..c16ef5e75 --- /dev/null +++ b/bench/terminal_bench/TASKS.md @@ -0,0 +1,54 @@ +# Task reference + +Each directory in `tasks/` is one task in Harbor's task format. Harbor and `run_native.py` read the same files. `bench_tasks.py ` prints the tasks that one agent runs. + +## Files in a task + +| Path | Contents | +|------|----------| +| `task.toml` | Timeouts, resources, and metadata. `agents` under `[metadata]` lists the agents that run the task, and defaults to Claude and Codex. Each `[[steps]]` entry adds a step. | +| `instruction.md` | The prompt the agent gets. | +| `environment/Dockerfile` | The Harbor image. It copies `environment/app/` to `/app` and runs `environment/setup.py /app` if the task has them. | +| `environment/app/` | Fixture files. The runner copies them into the task directory. | +| `environment/setup.py` | Generates fixtures. It takes the task directory as its only argument. | +| `tests/test_outputs.py` | The verifier. Exit code 0 is a pass. | +| `tests/test.sh` | Runs the verifier in Harbor and writes the reward to `/logs/verifier/reward.txt`. | +| `solution/solve.py` | The reference solution that `--agent oracle` runs. | +| `solution/solve.sh` | Runs the reference solution in Harbor. | + +A multi-step task has `instruction.md`, `tests/`, and `solution/` under `steps//` for each step. Each step after the first resumes the previous session. Harbor resumes with `--resume-trajectory`. `run_native.py` passes `--continue` to Claude and `exec resume --last` to Codex. A failed step stops the task. + +Verifiers and solutions use only the Python standard library. They run on Windows and in `python:3.12-slim`. + +## Environment variables + +Verifiers and solutions read these variables. + +| Variable | Value | +|----------|-------| +| `TASK_APP_DIR` | The task directory. Defaults to `/app`, which is the task directory in Harbor. | +| `TASK_AGENT_LOG` | The agent's log for the current step. Only `run_native.py` sets it. In Harbor, read `/logs/agent/ug-.txt`. | +| `TASK_ORACLE` | `1` when the oracle runs. A check that reads the agent log skips itself. | + +## ug's tasks + +| Task | Agents | Passes when | +|------|--------|-------------| +| `user-hooks` | Claude | `release.txt` holds the codename from the project's `UserPromptSubmit` hook, and the `PostToolUse` hook logged the write. | +| `mcp-server` | Claude | `stock.json` matches the random values that the project's stdio MCP server logged for each lookup. | +| `project-instructions` | Claude, Codex | The output is under `out/`, and each new Python file starts with the license header that `CLAUDE.md` and `AGENTS.md` require. | +| `project-skill` | Claude | The release notes follow the project skill, and the skill's script logged a run. | +| `subagent-model` | Claude | Each file has a correct audit, and the transcript shows a subagent on a haiku model. If a workspace policy disallows haiku, the transcript must show the policy notice and the subagent on the fallback model it names. | +| `image-code` | Claude, Codex | `code.txt` holds the 8-digit code drawn in `code.png`. | +| `pdf-extract` | Claude, Codex | `total.txt` holds the total from `invoice.pdf`, whose text stream is compressed. | +| `web-fetch` | Claude, Codex | `answer.txt` holds a value from a file in this repo, pinned to a fixed commit. | +| `background-shell` | Claude, Codex | `result.txt` holds the value that a 150-second job prints when it finishes. Claude's default command timeout is 2 minutes. | +| `resume-session` | Claude, Codex | Step 1 deletes `vault.txt` without saving its passphrase to a file. Step 2 resumes the session and writes the passphrase to `answer.txt`. | +| `log-triage` | Claude, Codex | `report.json` has the correct counts for five independent service directories. | +| `fix-failing-tests` | Claude, Codex | The unit tests and extra hidden checks pass, and `tests/` is unchanged. | +| `scrub-git-secret` | Claude, Codex | Every copy of the token in history is replaced with `REDACTED`, and both branches keep their commit messages in order. | +| `background-service` | Claude, Codex | `token.txt` matches the token the server served, and nothing listens on the server's port afterward. | + +## Default TB2 subset + +The Harbor lane runs these Terminal Bench 2 tasks unless the `tb2_tasks` input overrides them: `fix-git`, `financial-document-processor`, `code-from-image`, `chess-best-move`, `git-multibranch`, `kv-store-grpc`, and `large-scale-text-editing`. From 84ce04ba1c23d4fa3814d351c4b266fc434d2825 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:08:15 -0400 Subject: [PATCH 10/19] Add lane and task-run diagrams to the Terminal Bench README Co-authored-by: Isaac --- bench/terminal_bench/README.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index 9c4bb949a..ce700525f 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -4,7 +4,37 @@ This directory runs Terminal Bench tasks through `ug claude` and `ug codex`. It The workflow has two lanes. The Harbor lane runs on Linux. It runs a subset of [Terminal Bench 2](https://www.tbench.ai/) and the tasks in `tasks/` inside Harbor's Docker containers, and `ug_agent.py` installs ug in each container. The native lane runs on Linux and Windows. GitHub's Windows runners can't run Linux containers, so `run_native.py` runs the tasks in `tasks/` directly on the runner. -[TASKS.md](TASKS.md) describes the task format and lists every task. +```mermaid +flowchart LR + workflow["Terminal Bench workflow"] --> harbor["Harbor lane
Linux, Docker"] + workflow --> native["Native lane
Linux and Windows"] + harbor --> tb2["TB2 subset"] + harbor --> own["tasks/"] + native --> own +``` + +Each lane runs every task for Claude and for Codex. [TASKS.md](TASKS.md) describes the task format and lists every task. + +## How a task runs + +Both lanes run a task the same way. The host mints the gateway token, so the agent never sees the service principal secret. The verifier runs after the agent exits and decides the result. + +```mermaid +sequenceDiagram + participant host as Runner host + participant env as Task directory + participant agent as ug and agent CLI + participant gateway as AI Gateway + host->>env: Copy environment/app/ and run environment/setup.py + host->>host: bench_auth.py mints a token + host->>agent: ug configure, then ug claude or ug codex with instruction.md + agent->>gateway: Model calls with the token + agent->>env: Read files, run commands, write output + host->>env: Run tests/test_outputs.py + env-->>host: Exit code 0 is a pass +``` + +The Harbor lane runs `ug configure` in each container. The native lane runs it once at the start of the job. A multi-step task mints a new token, launches the agent, and runs the verifier again for each step, resuming the agent's session each time. ## Run the workflow with other tasks or a model From f9ed36949f100e13938ff947397e1b23e5641ee4 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:14:35 -0400 Subject: [PATCH 11/19] Add a figlet banner and explain the two lanes in the Terminal Bench README Co-authored-by: Isaac --- bench/terminal_bench/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index ce700525f..f864b46b7 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -1,8 +1,17 @@ # Terminal Bench for ug +```text + _ _ _ _ _ +| |_ ___ _ _ _ __ (_)_ _ __ _| | | |__ ___ _ _ __| |_ +| _/ -_) '_| ' \| | ' \/ _` | | | '_ \/ -_) ' \/ _| ' \ + \__\___|_| |_|_|_|_|_||_\__,_|_| |_.__/\___|_||_\__|_||_| +``` + This directory runs Terminal Bench tasks through `ug claude` and `ug codex`. It checks that an agent keeps working when ug configures and launches it. The `Terminal Bench` workflow in `.github/workflows/terminal-bench.yml` runs every night at 07:00 UTC and on manual dispatch. -The workflow has two lanes. The Harbor lane runs on Linux. It runs a subset of [Terminal Bench 2](https://www.tbench.ai/) and the tasks in `tasks/` inside Harbor's Docker containers, and `ug_agent.py` installs ug in each container. The native lane runs on Linux and Windows. GitHub's Windows runners can't run Linux containers, so `run_native.py` runs the tasks in `tasks/` directly on the runner. +The workflow has two lanes because Harbor needs Linux containers. Harbor runs each task in its own Docker container, and [Terminal Bench 2](https://www.tbench.ai/) tasks depend on the images they ship with. GitHub's Windows runners can't run Linux containers. + +The Harbor lane runs on Linux. It runs a TB2 subset and the tasks in `tasks/`, and `ug_agent.py` installs ug in each container. The native lane is how ug gets tested on Windows. `run_native.py` runs the tasks in `tasks/` directly on Linux and Windows runners. Those tasks use only the Python standard library so they run on both. ```mermaid flowchart LR From 876cff7297ef3a3039468db732b0d6391fd30fb8 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:18:32 -0400 Subject: [PATCH 12/19] Pass native jobs at a 75% task pass rate and explain the two lanes One agent miss out of 14 tasks no longer fails a native job. The README gets its own section on why Harbor and native lanes both exist. Co-authored-by: Isaac --- bench/terminal_bench/README.md | 18 +++++++++++++----- bench/terminal_bench/run_native.py | 12 ++++++++++-- .../tasks/background-shell/solution/solve.py | 4 +++- .../tasks/pdf-extract/environment/setup.py | 5 ++++- .../tasks/project-skill/solution/solve.py | 4 +++- .../tasks/project-skill/tests/test_outputs.py | 10 ++++++++-- 6 files changed, 41 insertions(+), 12 deletions(-) diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index f864b46b7..bfc694c97 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -9,9 +9,7 @@ This directory runs Terminal Bench tasks through `ug claude` and `ug codex`. It checks that an agent keeps working when ug configures and launches it. The `Terminal Bench` workflow in `.github/workflows/terminal-bench.yml` runs every night at 07:00 UTC and on manual dispatch. -The workflow has two lanes because Harbor needs Linux containers. Harbor runs each task in its own Docker container, and [Terminal Bench 2](https://www.tbench.ai/) tasks depend on the images they ship with. GitHub's Windows runners can't run Linux containers. - -The Harbor lane runs on Linux. It runs a TB2 subset and the tasks in `tasks/`, and `ug_agent.py` installs ug in each container. The native lane is how ug gets tested on Windows. `run_native.py` runs the tasks in `tasks/` directly on Linux and Windows runners. Those tasks use only the Python standard library so they run on both. +The workflow runs tasks in two lanes. The Harbor lane runs a [Terminal Bench 2](https://www.tbench.ai/) subset and the tasks in `tasks/` in Docker containers on Linux. The native lane runs the tasks in `tasks/` directly on Linux and Windows runners. ```mermaid flowchart LR @@ -22,7 +20,17 @@ flowchart LR native --> own ``` -Each lane runs every task for Claude and for Codex. [TASKS.md](TASKS.md) describes the task format and lists every task. +Each lane runs its tasks for Claude and for Codex. [TASKS.md](TASKS.md) describes the task format and lists every task. + +## Why there are two lanes + +Each lane tests something the other can't. + +The Harbor lane exists for real TB2 tasks. Each TB2 task ships a Linux Docker image with the services, packages, and root access it needs. Harbor starts a fresh container per trial, so a task can't change the runner or another trial. `ug_agent.py` installs ug, the Databricks CLI, and the agent CLI in each container. + +The native lane exists for Windows. GitHub's Windows runners can't run Linux containers, and TB2 has no Windows images, so Harbor can't test ug on Windows. `run_native.py` runs the tasks in `tasks/` on the runner itself instead. Those tasks use only the Python standard library so they run on both operating systems. + +The native Linux job overlaps with Harbor. It adds one thing: ug running outside a container, on a machine with a normal user home. ## How a task runs @@ -59,7 +67,7 @@ The workflow also takes `tb2_dataset`, `claude_version`, and `codex_version`. Gi Each job writes a table of task results to its run summary. -A Harbor job fails only when a trial hits a harness error, such as a failed agent install or an API error. A task with reward 0 doesn't fail the job, because TB2 tasks are meant to be hard. A native job fails when any task fails, so a native job is often red while the Harbor jobs in the same run pass. +A Harbor job fails only when a trial hits a harness error, such as a failed agent install or an API error. A task with reward 0 doesn't fail the job, because TB2 tasks are meant to be hard. A native job fails when fewer than 75% of its tasks pass. Agents miss a different task or two from run to run, so one miss doesn't fail the job. Change the bar with `run_native.py --min-pass-rate`. Each job uploads its files as an artifact named `terminal-bench-harbor-` or `terminal-bench-native--`. `scrub_secrets.py` replaces bearer tokens in those files before upload. diff --git a/bench/terminal_bench/run_native.py b/bench/terminal_bench/run_native.py index 4d81ae03b..8443b019f 100644 --- a/bench/terminal_bench/run_native.py +++ b/bench/terminal_bench/run_native.py @@ -156,7 +156,8 @@ def write_summary(args: argparse.Namespace, results: list[dict], out: Path) -> N (out / "results.json").write_text(json.dumps(results, indent=2)) passed = sum(r["passed"] for r in results) lines = [ - f"### Native tasks · {args.agent} · {sys.platform}: {passed}/{len(results)} passed", + f"### Native tasks · {args.agent} · {sys.platform}: {passed}/{len(results)} passed" + f" (needs {args.min_pass_rate:.0%})", "", "| Task | Result | Agent exit | Time (s) |", "|------|--------|-----------|----------|", @@ -178,6 +179,12 @@ def main() -> int: parser.add_argument("--output", type=Path, required=True) parser.add_argument("--ug", default=shutil.which("ug"), help="Absolute path to ug.") parser.add_argument("--model") + parser.add_argument( + "--min-pass-rate", + type=float, + default=0.75, + help="Fraction of tasks that must pass for exit code 0. Agents miss some tasks run to run.", + ) args = parser.parse_args() names = args.task or names_for(args.agent) @@ -210,7 +217,8 @@ def main() -> int: results = [run_task(args, TASKS_DIR / name, out) for name in names] write_summary(args, results, out) - return 0 if all(r["passed"] for r in results) else 1 + passed = sum(r["passed"] for r in results) + return 0 if results and passed / len(results) >= args.min_pass_rate else 1 if __name__ == "__main__": diff --git a/bench/terminal_bench/tasks/background-shell/solution/solve.py b/bench/terminal_bench/tasks/background-shell/solution/solve.py index 7894dd793..d744f624c 100644 --- a/bench/terminal_bench/tasks/background-shell/solution/solve.py +++ b/bench/terminal_bench/tasks/background-shell/solution/solve.py @@ -6,5 +6,7 @@ from pathlib import Path app = Path(os.environ.get("TASK_APP_DIR", "/app")) -out = subprocess.run([sys.executable, "slow_job.py"], cwd=app, capture_output=True, text=True, check=True).stdout +out = subprocess.run( + [sys.executable, "slow_job.py"], cwd=app, capture_output=True, text=True, check=True +).stdout (app / "result.txt").write_text(out.strip().removeprefix("RESULT=") + "\n") diff --git a/bench/terminal_bench/tasks/pdf-extract/environment/setup.py b/bench/terminal_bench/tasks/pdf-extract/environment/setup.py index f4a64e63e..86476b4ef 100644 --- a/bench/terminal_bench/tasks/pdf-extract/environment/setup.py +++ b/bench/terminal_bench/tasks/pdf-extract/environment/setup.py @@ -33,7 +33,10 @@ def main(app: Path) -> None: xref = len(pdf) pdf += b"xref\n0 %d\n0000000000 65535 f \n" % (len(objects) + 1) pdf += b"".join(b"%010d 00000 n \n" % offset for offset in offsets) - pdf += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (len(objects) + 1, xref) + pdf += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % ( + len(objects) + 1, + xref, + ) app.mkdir(parents=True, exist_ok=True) (app / "invoice.pdf").write_bytes(bytes(pdf)) diff --git a/bench/terminal_bench/tasks/project-skill/solution/solve.py b/bench/terminal_bench/tasks/project-skill/solution/solve.py index fdedd7380..278a7430c 100644 --- a/bench/terminal_bench/tasks/project-skill/solution/solve.py +++ b/bench/terminal_bench/tasks/project-skill/solution/solve.py @@ -14,4 +14,6 @@ check=True, ).stdout (app / "dist").mkdir(exist_ok=True) -(app / "dist" / "release-notes-2.3.0.md").write_text(draft.rstrip() + "\n\nSigned-off-by: release-bot\n") +(app / "dist" / "release-notes-2.3.0.md").write_text( + draft.rstrip() + "\n\nSigned-off-by: release-bot\n" +) diff --git a/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py b/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py index e54079506..8c409b092 100644 --- a/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py +++ b/bench/terminal_bench/tasks/project-skill/tests/test_outputs.py @@ -10,12 +10,18 @@ def main() -> None: log = APP / ".skill_log" / "runs.jsonl" assert log.exists(), "the skill's build script never ran" - assert any(json.loads(line)["version"].lstrip("v") == "2.3.0" for line in log.read_text().splitlines()) + assert any( + json.loads(line)["version"].lstrip("v") == "2.3.0" for line in log.read_text().splitlines() + ) notes = (APP / "dist" / "release-notes-2.3.0.md").read_text().strip().splitlines() assert notes[0].strip() == "# Release 2.3.0", notes[0] assert notes[-1].strip() == "Signed-off-by: release-bot", notes[-1] body = "\n".join(notes) - for item in ["Retry gateway calls on HTTP 429", "Smart routing for subagents", "Document managed settings"]: + for item in [ + "Retry gateway calls on HTTP 429", + "Smart routing for subagents", + "Document managed settings", + ]: assert item in body, item print("ok") From 362113757320a0db0dcb29edc4d7daac82f58a63 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:22:45 -0400 Subject: [PATCH 13/19] Document one-shot task runs and what the Terminal Bench doesn't test Co-authored-by: Isaac --- bench/terminal_bench/README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index bfc694c97..c6b156e7f 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -34,7 +34,9 @@ The native Linux job overlaps with Harbor. It adds one thing: ug running outside ## How a task runs -Both lanes run a task the same way. The host mints the gateway token, so the agent never sees the service principal secret. The verifier runs after the agent exits and decides the result. +Both lanes run a task the same way. The agent gets the task's `instruction.md` once, at launch, and works on its own until it exits or hits the task's `timeout_sec`. Nothing sends it more instructions while it runs. That matches Terminal Bench and Harbor's built-in `claude-code` and `codex` agents, which also launch `claude --print` and `codex exec` with the instruction and wait. + +Claude reads the instruction on stdin in print mode (`-p`). Codex gets it as an argument to `codex exec`. The host mints the gateway token, so the agent never sees the service principal secret. The verifier runs after the agent exits and decides the result. ```mermaid sequenceDiagram @@ -51,7 +53,20 @@ sequenceDiagram env-->>host: Exit code 0 is a pass ``` -The Harbor lane runs `ug configure` in each container. The native lane runs it once at the start of the job. A multi-step task mints a new token, launches the agent, and runs the verifier again for each step, resuming the agent's session each time. +The Harbor lane runs `ug configure` in each container. The native lane runs it once at the start of the job. + +A multi-step task launches the agent again for each step, with that step's instruction and a new token. The new launch resumes the previous session with `--continue` for Claude or `exec resume --last` for Codex, and the verifier runs after each step. All TB2 tasks have one step. `resume-session` is the only multi-step task here. + +## What the bench doesn't test + +The bench treats ug as a black box. It installs ug from this checkout, runs `ug configure`, and launches the agent with `ug claude` or `ug codex`. Verifiers check only what the agent left behind. Some of ug runs differently from a user's setup: + +- Auth goes through `DATABRICKS_BEARER` with a token the host mints. ug's `databricks auth login` and OAuth flows don't run. +- Databricks AI Tools, the skills and plugins that `--enable-databricks-ai-tools` installs, aren't installed. That's ug's default. +- The CUJ1 workspace's managed config lists no MCP servers, so ug registers none. +- The harness installs a pinned Claude Code or Codex before ug starts, so ug's agent install doesn't run for that agent. It can still run for the other one. In a Claude job, ug tries to install Codex because the CUJ1 managed config enables both. +- Claude runs with `--dangerously-skip-permissions` and Codex with `--dangerously-bypass-approvals-and-sandbox`. Permission prompts and the Codex sandbox don't run. +- Smart routing is off. ## Run the workflow with other tasks or a model From 177bb1cc9523c2052f00b2d2e93fe40e0435c9de Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:26:07 -0400 Subject: [PATCH 14/19] Run every Terminal Bench launch with smart routing on Co-authored-by: Isaac --- bench/terminal_bench/README.md | 3 ++- bench/terminal_bench/run_native.py | 1 + bench/terminal_bench/ug_agent.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index c6b156e7f..f280d089b 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -53,6 +53,8 @@ sequenceDiagram env-->>host: Exit code 0 is a pass ``` +Every launch runs with smart routing on (`ENABLE_SMART_ROUTING_V2=1`). On Linux, ug routes Claude's first prompt and its subagents. On Windows, ug routes only Claude's subagents, because the first-prompt router needs a Unix PTY. Codex sends ug's router header on every model call. + The Harbor lane runs `ug configure` in each container. The native lane runs it once at the start of the job. A multi-step task launches the agent again for each step, with that step's instruction and a new token. The new launch resumes the previous session with `--continue` for Claude or `exec resume --last` for Codex, and the verifier runs after each step. All TB2 tasks have one step. `resume-session` is the only multi-step task here. @@ -66,7 +68,6 @@ The bench treats ug as a black box. It installs ug from this checkout, runs `ug - The CUJ1 workspace's managed config lists no MCP servers, so ug registers none. - The harness installs a pinned Claude Code or Codex before ug starts, so ug's agent install doesn't run for that agent. It can still run for the other one. In a Claude job, ug tries to install Codex because the CUJ1 managed config enables both. - Claude runs with `--dangerously-skip-permissions` and Codex with `--dangerously-bypass-approvals-and-sandbox`. Permission prompts and the Codex sandbox don't run. -- Smart routing is off. ## Run the workflow with other tasks or a model diff --git a/bench/terminal_bench/run_native.py b/bench/terminal_bench/run_native.py index 8443b019f..4d3966300 100644 --- a/bench/terminal_bench/run_native.py +++ b/bench/terminal_bench/run_native.py @@ -122,6 +122,7 @@ def run_task(args: argparse.Namespace, task_dir: Path, out: Path) -> dict: prepare(task_dir, workdir) env = {**os.environ, "TASK_APP_DIR": str(workdir), "IS_SANDBOX": "1"} + env["ENABLE_SMART_ROUTING_V2"] = "1" if args.agent == "oracle": # Checks that read the agent transcript have nothing to read for the oracle. env["TASK_ORACLE"] = "1" diff --git a/bench/terminal_bench/ug_agent.py b/bench/terminal_bench/ug_agent.py index 3fa8e000e..ebc4daad5 100644 --- a/bench/terminal_bench/ug_agent.py +++ b/bench/terminal_bench/ug_agent.py @@ -73,6 +73,7 @@ async def run( **auth, # Lets Claude Code skip permissions as root inside the container. "IS_SANDBOX": "1", + "ENABLE_SMART_ROUTING_V2": "1", "UG_BENCH_INSTRUCTION": instruction, } if not self._resume: From dce5c375dff7672aa52b8155904f36bcb97f2c4f Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:02:27 -0400 Subject: [PATCH 15/19] TEMP: run Terminal Bench from the PR again, and keep ug routing logs Re-adds the push trigger to verify smart routing before merge. Both runners copy ~/.ucode/*.log into task artifacts as evidence that routing ran. Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 3 +++ bench/terminal_bench/run_native.py | 4 ++++ bench/terminal_bench/ug_agent.py | 23 +++++++++++++++-------- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml index 6a66610bb..7f3f992ac 100644 --- a/.github/workflows/terminal-bench.yml +++ b/.github/workflows/terminal-bench.yml @@ -1,6 +1,9 @@ name: Terminal Bench on: + # TEMP: run from PR #1074 while smart routing is verified. Remove before merge. + push: + branches: [max-rozen-oss-db/stack/terminal-bench] schedule: - cron: '0 7 * * *' workflow_dispatch: diff --git a/bench/terminal_bench/run_native.py b/bench/terminal_bench/run_native.py index 4d3966300..3ca1a9da5 100644 --- a/bench/terminal_bench/run_native.py +++ b/bench/terminal_bench/run_native.py @@ -143,6 +143,10 @@ def run_task(args: argparse.Namespace, task_dir: Path, out: Path) -> dict: # Verify before cleanup so a server the agent left running still counts against it. passed, verifier_log = verify(step_dir, workdir, env, verifier_timeout) (logs / f"{name}.verifier.log").write_text(verifier_log, encoding="utf-8") + # ug's routing wrappers log under ~/.ucode; keep them as evidence that routing ran. + for ucode_log in (Path.home() / ".ucode").glob("*.log"): + (logs / f"{name}.ucode").mkdir(exist_ok=True) + shutil.copy2(ucode_log, logs / f"{name}.ucode" / ucode_log.name) if not timed_out and os.name != "nt": kill_tree(proc) result.update(agent_exit=proc.returncode, timed_out=timed_out) diff --git a/bench/terminal_bench/ug_agent.py b/bench/terminal_bench/ug_agent.py index ebc4daad5..da146b9db 100644 --- a/bench/terminal_bench/ug_agent.py +++ b/bench/terminal_bench/ug_agent.py @@ -89,14 +89,21 @@ async def run( model = f"--model {shlex.quote(self.model_name)} " if self.model_name else "" log = self.environment_logs_dir / f"ug-{self.AGENT}.txt" stdin = 'printf "%s" "$UG_BENCH_INSTRUCTION" | ' if self.PROMPT_ON_STDIN else "" - await self.exec_as_agent( - environment, - command=( - f"{SHELL_PREFIX}{stdin}{UG} {self.AGENT} {model}-- {self._launch_args(self._resume)} " - f"2>&1 | tee -a {log.as_posix()}" - ), - env=env, - ) + try: + await self.exec_as_agent( + environment, + command=( + f"{SHELL_PREFIX}{stdin}{UG} {self.AGENT} {model}-- {self._launch_args(self._resume)} " + f"2>&1 | tee -a {log.as_posix()}" + ), + env=env, + ) + finally: + # ug's routing wrappers log under ~/.ucode; keep them as evidence that routing ran. + ucode_logs = (self.environment_logs_dir / "ucode").as_posix() + await environment.exec( + command=f'mkdir -p {ucode_logs} && cp "$HOME"/.ucode/*.log {ucode_logs}/ 2>/dev/null || true' + ) class UgClaude(_UgAgent): From dde374df8619ef3cb4d6cbe81bd98ba393a614ca Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:11:54 -0400 Subject: [PATCH 16/19] TEMP: also run Terminal Bench on the headless-routing stacked branch Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml index 7f3f992ac..6db12cb07 100644 --- a/.github/workflows/terminal-bench.yml +++ b/.github/workflows/terminal-bench.yml @@ -3,7 +3,7 @@ name: Terminal Bench on: # TEMP: run from PR #1074 while smart routing is verified. Remove before merge. push: - branches: [max-rozen-oss-db/stack/terminal-bench] + branches: [max-rozen-oss-db/stack/terminal-bench, max-rozen-oss-db/stack/claude-headless-routing] schedule: - cron: '0 7 * * *' workflow_dispatch: From 7f19dc963db9ad5599e99aa6da92de0f236508e1 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:19:25 -0400 Subject: [PATCH 17/19] Don't persist checkout credentials in the Terminal Bench workflow zizmor flags artipacked: both jobs upload artifacts after checkout. Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml index 6db12cb07..48976afb4 100644 --- a/.github/workflows/terminal-bench.yml +++ b/.github/workflows/terminal-bench.yml @@ -70,6 +70,8 @@ jobs: AGENT_CLASS: ${{ matrix.agent == 'claude' && 'UgClaude' || 'UgCodex' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Build the ug wheel installed into each task container run: | @@ -144,6 +146,8 @@ jobs: AGENT_PACKAGE: ${{ matrix.agent == 'claude' && '@anthropic-ai/claude-code' || '@openai/codex' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22.19.0 From ca09220b2521a15b73b5655f01062e4ccad9030f Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:24:01 -0400 Subject: [PATCH 18/19] Describe headless smart routing in the Terminal Bench README Co-authored-by: Isaac --- bench/terminal_bench/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/terminal_bench/README.md b/bench/terminal_bench/README.md index f280d089b..6cc4dce76 100644 --- a/bench/terminal_bench/README.md +++ b/bench/terminal_bench/README.md @@ -53,7 +53,7 @@ sequenceDiagram env-->>host: Exit code 0 is a pass ``` -Every launch runs with smart routing on (`ENABLE_SMART_ROUTING_V2=1`). On Linux, ug routes Claude's first prompt and its subagents. On Windows, ug routes only Claude's subagents, because the first-prompt router needs a Unix PTY. Codex sends ug's router header on every model call. +Every launch runs with smart routing on (`ENABLE_SMART_ROUTING_V2=1`). ug routes Claude's prompt before launch, because print mode (`-p`) has no terminal UI to switch models in, and it routes Claude's subagents. Codex sends ug's router header on every model call. The Harbor lane runs `ug configure` in each container. The native lane runs it once at the start of the job. From 2ad978e991d5571f0ef36333f783f9cb11aafaf6 Mon Sep 17 00:00:00 2001 From: max-rozen-oss-db <291811446+max-rozen-oss-db@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:50:25 -0400 Subject: [PATCH 19/19] Remove the temporary push trigger from Terminal Bench Co-authored-by: Isaac --- .github/workflows/terminal-bench.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/terminal-bench.yml b/.github/workflows/terminal-bench.yml index 48976afb4..9cc05d23e 100644 --- a/.github/workflows/terminal-bench.yml +++ b/.github/workflows/terminal-bench.yml @@ -1,9 +1,6 @@ name: Terminal Bench on: - # TEMP: run from PR #1074 while smart routing is verified. Remove before merge. - push: - branches: [max-rozen-oss-db/stack/terminal-bench, max-rozen-oss-db/stack/claude-headless-routing] schedule: - cron: '0 7 * * *' workflow_dispatch: