diff --git a/src/benchmarks/agent-cli/generation-proxy.test.ts b/src/benchmarks/agent-cli/generation-proxy.test.ts new file mode 100644 index 0000000..b7d1e36 --- /dev/null +++ b/src/benchmarks/agent-cli/generation-proxy.test.ts @@ -0,0 +1,265 @@ +import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import type { Subprocess } from "bun"; + +import { + buildGenerationProxyPrelude, + GENERATION_ID_LINE_PREFIX, + GENERATION_PROXY_BASE_URL, + GENERATION_PROXY_BASE_URL_ENV, + GENERATION_PROXY_SCRIPT, + GENERATION_PROXY_SCRIPT_PATH, + parseProxyGenerationIds, +} from "./generation-proxy"; + +const FIRST_ID = "gen-1788472540-mosiIhRdbQJ7cMvPxJTN"; +const SECOND_ID = "gen-1788472541-8GGFjohh9vkZ0U4fMJyo"; +const THIRD_ID = "gen-1788472726-5s5Zxbng20ez3mXkOmb1"; + +describe("parseProxyGenerationIds", () => { + it("collects prefixed ids in order and drops duplicates and other lines", () => { + const stdout = [ + '{"type":"turn.started"}', + `${GENERATION_ID_LINE_PREFIX}${FIRST_ID}`, + ` ${GENERATION_ID_LINE_PREFIX}${SECOND_ID} `, + `${GENERATION_ID_LINE_PREFIX}${FIRST_ID}`, + GENERATION_ID_LINE_PREFIX.trim(), + `not ${GENERATION_ID_LINE_PREFIX}${THIRD_ID}`, + ].join("\n"); + expect(parseProxyGenerationIds(stdout)).toEqual([FIRST_ID, SECOND_ID]); + }); +}); + +describe("buildGenerationProxyPrelude", () => { + it("writes the proxy, waits for its port and exports the base url", () => { + const prelude = buildGenerationProxyPrelude("/logs/agent/codex.txt").join( + "\n" + ); + expect(prelude).toContain(": > /logs/agent/codex.txt"); + expect(prelude).toContain( + `cat > ${GENERATION_PROXY_SCRIPT_PATH} <<'OR_GENERATION_PROXY_EOF'` + ); + expect(prelude).toContain(GENERATION_PROXY_SCRIPT.trimEnd()); + expect(prelude).toContain("GEN_PROXY_UPSTREAM=https://openrouter.ai"); + expect(prelude).toContain("GEN_PROXY_LOG_PATH=/logs/agent/codex.txt"); + expect(prelude).toContain('trap \'kill "$OR_GENERATION_PROXY_PID"'); + expect(prelude).toContain("generation proxy failed to start"); + expect(prelude).toContain( + `export ${GENERATION_PROXY_BASE_URL_ENV}="http://127.0.0.1:$(cat ${GENERATION_PROXY_SCRIPT_PATH}.$$.port)/api/v1"` + ); + expect(GENERATION_PROXY_BASE_URL).toBe(`$${GENERATION_PROXY_BASE_URL_ENV}`); + }); +}); + +interface UpstreamRequest { + readonly method: string; + readonly path: string; + readonly headers: Record; + readonly body: string; +} + +describe("generation proxy script", () => { + const dir = mkdtempSync(join(tmpdir(), "generation-proxy-test-")); + const scriptPath = join(dir, "proxy.cjs"); + const portFile = join(dir, "proxy.port"); + const logPath = join(dir, "proxy.log"); + const upstreamRequests: UpstreamRequest[] = []; + let upstream: ReturnType; + let proxy: Subprocess<"ignore", "pipe", "pipe">; + let proxyBase = ""; + + const UPSTREAM_ROUTES: Readonly Response>> = { + "/api/v1/chat/completions": () => + Response.json( + { id: FIRST_ID, object: "chat.completion", choices: [] }, + { headers: { "x-upstream": "yes" } } + ), + "/api/v1/responses": () => + new Response( + [ + `data: {"type":"response.created","response":{"id":"${SECOND_ID}","object":"response"}}`, + "", + `data: {"type":"response.output_item.done","item":{"id":"msg_1","type":"message"}}`, + "", + `data: {"type":"response.completed","response":{"id":"${SECOND_ID}","object":"response"}}`, + "", + "data: [DONE]", + "", + ].join("\n"), + { headers: { "content-type": "text/event-stream" } } + ), + "/api/v1/repeat": () => + Response.json({ id: FIRST_ID, other: { id: THIRD_ID } }), + "/api/v1/models": () => + Response.json({ + data: [{ id: "openai/gpt-5-mini" }, { id: "gen-eric/model" }], + }), + "/api/v1/malformed": () => + new Response('{"id": "gen-', { + headers: { "content-type": "application/json" }, + }), + "/api/v1/failure": () => + Response.json( + { error: { message: "rate limited", code: 429 } }, + { status: 429 } + ), + }; + + beforeAll(async () => { + upstream = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + idleTimeout: 0, + fetch: async (request) => { + const url = new URL(request.url); + upstreamRequests.push({ + method: request.method, + path: url.pathname + url.search, + headers: Object.fromEntries(request.headers.entries()), + body: await request.text(), + }); + const route = UPSTREAM_ROUTES[url.pathname]; + return route === undefined + ? new Response("not found", { status: 404 }) + : route(); + }, + }); + writeFileSync(scriptPath, GENERATION_PROXY_SCRIPT); + writeFileSync(logPath, ""); + proxy = Bun.spawn(["node", scriptPath], { + env: { + ...process.env, + GEN_PROXY_PORT_FILE: portFile, + GEN_PROXY_UPSTREAM: `http://127.0.0.1:${upstream.port}`, + GEN_PROXY_LINE_PREFIX: GENERATION_ID_LINE_PREFIX, + GEN_PROXY_LOG_PATH: logPath, + }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + for (let attempt = 0; attempt < 100 && !existsSync(portFile); attempt++) { + await Bun.sleep(50); + } + proxyBase = `http://127.0.0.1:${readFileSync(portFile, "utf8")}/api/v1`; + }); + + afterAll(() => { + proxy.kill(); + upstream.stop(true); + }); + + function loggedIds(): string[] { + return parseProxyGenerationIds(readFileSync(logPath, "utf8")); + } + + it("forwards method, path, headers and body and relays the json response", async () => { + const response = await fetch(`${proxyBase}/chat/completions?x=1`, { + method: "POST", + headers: { + authorization: "Bearer test-key", + "content-type": "application/json", + "x-session-id": "sess-1", + connection: "keep-alive", + }, + body: JSON.stringify({ model: "openai/gpt-5-mini", messages: [] }), + }); + expect(response.status).toBe(200); + expect(response.headers.get("x-upstream")).toBe("yes"); + expect(await response.json()).toEqual({ + id: FIRST_ID, + object: "chat.completion", + choices: [], + }); + const forwarded = upstreamRequests.at(-1); + expect(forwarded?.method).toBe("POST"); + expect(forwarded?.path).toBe("/api/v1/chat/completions?x=1"); + expect(forwarded?.headers["authorization"]).toBe("Bearer test-key"); + expect(forwarded?.headers["x-session-id"]).toBe("sess-1"); + expect(forwarded?.headers["host"]).toBe(`127.0.0.1:${upstream.port}`); + expect(forwarded?.body).toBe( + JSON.stringify({ model: "openai/gpt-5-mini", messages: [] }) + ); + expect(loggedIds()).toEqual([FIRST_ID]); + }); + + it("relays sse streams and records each response id once", async () => { + const response = await fetch(`${proxyBase}/responses`, { + method: "POST", + body: "{}", + }); + expect(response.headers.get("content-type")).toBe("text/event-stream"); + const text = await response.text(); + expect(text).toContain(`"id":"${SECOND_ID}"`); + expect(text).toContain("data: [DONE]"); + expect(loggedIds()).toEqual([FIRST_ID, SECOND_ID]); + }); + + it("records every generation id in a single response and suppresses repeats", async () => { + await fetch(`${proxyBase}/repeat`, { method: "POST", body: "{}" }); + await fetch(`${proxyBase}/repeat`, { method: "POST", body: "{}" }); + expect(loggedIds()).toEqual([FIRST_ID, SECOND_ID, THIRD_ID]); + }); + + it("ignores model catalog ids and malformed bodies", async () => { + await fetch(`${proxyBase}/models`); + const malformed = await fetch(`${proxyBase}/malformed`, { + method: "POST", + body: "{}", + }); + expect(await malformed.text()).toBe('{"id": "gen-'); + expect(loggedIds()).toEqual([FIRST_ID, SECOND_ID, THIRD_ID]); + }); + + it("relays upstream error statuses unchanged", async () => { + const response = await fetch(`${proxyBase}/failure`, { method: "POST" }); + expect(response.status).toBe(429); + expect(await response.json()).toEqual({ + error: { message: "rate limited", code: 429 }, + }); + }); + + it("answers 502 when the upstream is unreachable", async () => { + const deadPortFile = join(dir, "dead.port"); + const dead = Bun.spawn(["node", scriptPath], { + env: { + ...process.env, + GEN_PROXY_PORT_FILE: deadPortFile, + GEN_PROXY_UPSTREAM: "http://127.0.0.1:9", + GEN_PROXY_LINE_PREFIX: GENERATION_ID_LINE_PREFIX, + GEN_PROXY_LOG_PATH: join(dir, "dead.log"), + }, + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + }); + for ( + let attempt = 0; + attempt < 100 && !existsSync(deadPortFile); + attempt++ + ) { + await Bun.sleep(50); + } + const response = await fetch( + `http://127.0.0.1:${readFileSync(deadPortFile, "utf8")}/api/v1/chat/completions`, + { method: "POST", body: "{}" } + ); + dead.kill(); + expect(response.status).toBe(502); + const body: unknown = await response.json(); + expect(JSON.stringify(body)).toContain("could not reach upstream"); + }); + + it("echoes recorded ids on stdout with the parseable prefix", async () => { + proxy.kill(); + const stdout = await new Response(proxy.stdout).text(); + expect(parseProxyGenerationIds(stdout)).toEqual([ + FIRST_ID, + SECOND_ID, + THIRD_ID, + ]); + }); +}); diff --git a/src/benchmarks/agent-cli/generation-proxy.ts b/src/benchmarks/agent-cli/generation-proxy.ts new file mode 100644 index 0000000..1b87f17 --- /dev/null +++ b/src/benchmarks/agent-cli/generation-proxy.ts @@ -0,0 +1,132 @@ +export const GENERATION_PROXY_SCRIPT_PATH = + "/tmp/openrouter-generation-proxy.cjs" as const; + +const GENERATION_PROXY_PORT_FILE = `${GENERATION_PROXY_SCRIPT_PATH}.$$.port`; + +export const GENERATION_PROXY_UPSTREAM = "https://openrouter.ai" as const; + +export const GENERATION_PROXY_BASE_URL_ENV = + "OR_GENERATION_PROXY_BASE_URL" as const; + +export const GENERATION_PROXY_BASE_URL = + `$${GENERATION_PROXY_BASE_URL_ENV}` as const; + +export const GENERATION_ID_LINE_PREFIX = "OR_GENERATION_ID " as const; + +const GENERATION_ID_PATTERN = /"id"\s*:\s*"(gen-\d+-[A-Za-z0-9_-]+)"/g; + +const PROXY_READY_ATTEMPTS = 50; + +export const GENERATION_PROXY_SCRIPT = String.raw`"use strict"; +const http = require("node:http"); +const fs = require("node:fs"); +const portFile = process.env.GEN_PROXY_PORT_FILE; +const upstream = new URL(process.env.GEN_PROXY_UPSTREAM); +const prefix = process.env.GEN_PROXY_LINE_PREFIX; +const logPath = process.env.GEN_PROXY_LOG_PATH; +const idPattern = ${GENERATION_ID_PATTERN.toString()}; +const hopHeaders = new Set(["host", "connection", "content-length", "transfer-encoding", "content-encoding"]); +const seen = new Set(); +function record(id) { + if (seen.has(id)) return; + seen.add(id); + process.stdout.write(prefix + id + "\n"); + fs.appendFileSync(logPath, prefix + id + "\n"); +} +function scan(text) { + for (const match of text.matchAll(idPattern)) record(match[1]); +} +async function readBody(req) { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + return Buffer.concat(chunks); +} +async function handle(req, res) { + const body = await readBody(req); + const headers = {}; + for (const [name, value] of Object.entries(req.headers)) { + if (!hopHeaders.has(name) && typeof value === "string") headers[name] = value; + } + headers.host = upstream.host; + let upstreamRes; + try { + upstreamRes = await fetch(new URL(req.url, upstream), { + method: req.method, + headers, + body: body.length > 0 ? body : undefined, + redirect: "manual", + }); + } catch (error) { + res.writeHead(502, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: { message: "generation proxy could not reach upstream: " + String(error) } })); + return; + } + const responseHeaders = {}; + upstreamRes.headers.forEach((value, name) => { + if (!hopHeaders.has(name)) responseHeaders[name] = value; + }); + res.writeHead(upstreamRes.status, responseHeaders); + if (upstreamRes.body === null) { + res.end(); + return; + } + const decoder = new TextDecoder(); + const inspect = req.method !== "GET"; + let pending = ""; + for await (const chunk of upstreamRes.body) { + res.write(chunk); + if (!inspect) continue; + pending += decoder.decode(chunk, { stream: true }); + const cut = pending.lastIndexOf("\n"); + if (cut >= 0) { + scan(pending.slice(0, cut + 1)); + pending = pending.slice(cut + 1); + } + } + scan(pending + decoder.decode()); + res.end(); +} +const server = http.createServer((req, res) => { + handle(req, res).catch((error) => { + process.stderr.write("generation proxy request failed: " + String(error) + "\n"); + if (!res.headersSent) res.writeHead(502); + res.end(); + }); +}); +server.keepAliveTimeout = 0; +server.listen(0, "127.0.0.1", () => { + fs.writeFileSync(portFile + ".tmp", String(server.address().port)); + fs.renameSync(portFile + ".tmp", portFile); +}); +`; + +export function buildGenerationProxyPrelude(logPath: string): string[] { + return [ + `: > ${logPath}`, + `cat > ${GENERATION_PROXY_SCRIPT_PATH} <<'OR_GENERATION_PROXY_EOF'`, + GENERATION_PROXY_SCRIPT.trimEnd(), + "OR_GENERATION_PROXY_EOF", + `rm -f ${GENERATION_PROXY_PORT_FILE}`, + `GEN_PROXY_PORT_FILE=${GENERATION_PROXY_PORT_FILE} GEN_PROXY_UPSTREAM=${GENERATION_PROXY_UPSTREAM} GEN_PROXY_LINE_PREFIX=${JSON.stringify(GENERATION_ID_LINE_PREFIX)} GEN_PROXY_LOG_PATH=${logPath} node ${GENERATION_PROXY_SCRIPT_PATH} 2>>/tmp/openrouter-generation-proxy.err &`, + "OR_GENERATION_PROXY_PID=$!", + `trap 'kill "$OR_GENERATION_PROXY_PID" 2>/dev/null || true; rm -f ${GENERATION_PROXY_PORT_FILE}' EXIT`, + `for _ in $(seq ${PROXY_READY_ATTEMPTS}); do [ -s ${GENERATION_PROXY_PORT_FILE} ] && break; sleep 0.2; done`, + `[ -s ${GENERATION_PROXY_PORT_FILE} ] || { echo "generation proxy failed to start" >&2; exit 3; }`, + `export ${GENERATION_PROXY_BASE_URL_ENV}="http://127.0.0.1:$(cat ${GENERATION_PROXY_PORT_FILE})/api/v1"`, + ]; +} + +export function parseProxyGenerationIds(stdout: string): string[] { + const ids: string[] = []; + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (!trimmed.startsWith(GENERATION_ID_LINE_PREFIX)) { + continue; + } + const id = trimmed.slice(GENERATION_ID_LINE_PREFIX.length).trim(); + if (id.length > 0 && !ids.includes(id)) { + ids.push(id); + } + } + return ids; +} diff --git a/src/benchmarks/agent-cli/harness.test.ts b/src/benchmarks/agent-cli/harness.test.ts new file mode 100644 index 0000000..abafc03 --- /dev/null +++ b/src/benchmarks/agent-cli/harness.test.ts @@ -0,0 +1,267 @@ +import { describe, expect, it } from "bun:test"; + +import { GENERATION_ID_LINE_PREFIX } from "./generation-proxy"; +import type { OriRunScriptOptions } from "./harness"; +import { ORI_HARNESSES } from "./harness"; + +const FIRST_ID = "gen-1788473388-R386NZLdIly5wVTk0l5f"; +const SECOND_ID = "gen-1788473571-DnfmpihRmJf33b9GDEF1"; + +const RUN_OPTIONS: OriRunScriptOptions = { + instructionPath: "/instruction.md", + logPath: "/logs/agent/agent.txt", + reasoningEffort: "high", + hasSystemPrompt: false, + hasAppendSystemPrompt: true, + hasAllowedTools: false, + hasDisallowedTools: true, + isolateAgentConfig: true, +}; + +const CODEX_STREAM = [ + "Reading additional input from stdin...", + JSON.stringify({ + type: "thread.started", + thread_id: "01a06952-2c38-7a13-9c4b-6c0f69fee5e0", + }), + JSON.stringify({ type: "turn.started" }), + JSON.stringify({ + type: "item.completed", + item: { + id: "item_0", + type: "error", + message: "Model metadata for `openai/gpt-5-mini` not found.", + }, + }), + `${GENERATION_ID_LINE_PREFIX}${FIRST_ID}`, + JSON.stringify({ + type: "item.completed", + item: { + id: "item_1", + type: "command_execution", + command: "ls", + status: "completed", + }, + }), + `${GENERATION_ID_LINE_PREFIX}${SECOND_ID}`, + `${GENERATION_ID_LINE_PREFIX}${FIRST_ID}`, + JSON.stringify({ + type: "item.completed", + item: { id: "item_2", type: "agent_message", text: "The sky is blue." }, + }), + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 7620, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 136, + reasoning_output_tokens: 64, + }, + }), +].join("\n"); + +const OPENCODE_STREAM = [ + JSON.stringify({ + type: "step_start", + sessionID: "ses_1", + part: { id: "prt_0", type: "step-start" }, + }), + `${GENERATION_ID_LINE_PREFIX}${FIRST_ID}`, + JSON.stringify({ + type: "tool_use", + sessionID: "ses_1", + part: { + id: "prt_1", + type: "tool", + tool: "read", + state: { status: "completed" }, + }, + }), + JSON.stringify({ + type: "text", + sessionID: "ses_1", + part: { id: "prt_2", type: "text", text: "LE CIEL EST BLEU." }, + }), + JSON.stringify({ + type: "step_finish", + sessionID: "ses_1", + part: { + id: "prt_3", + type: "step-finish", + reason: "stop", + tokens: { + total: 4886, + input: 4642, + output: 116, + reasoning: 128, + cache: { write: 10, read: 20 }, + }, + cost: 0.0016485, + }, + }), +].join("\n"); + +describe("codex harness", () => { + it("parses usage, tool calls, the final message and proxy generation ids", () => { + const run = ORI_HARNESSES.codex.parseRun(CODEX_STREAM); + expect(run.generationIds).toEqual([FIRST_ID, SECOND_ID]); + expect(run.usage).toEqual({ + inputTokens: 7620, + outputTokens: 136, + totalTokens: 7756, + reasoningTokens: 64, + totalCost: 0, + }); + expect(run.finalText).toBe("The sky is blue."); + expect(run.assistantMessages).toEqual([ + { role: "assistant", content: "The sky is blue." }, + ]); + expect(run.turns).toBe(1); + expect(run.toolCalls).toBe(1); + expect(run.isError).toBe(false); + expect(run.responseItems).toHaveLength(6); + }); + + it("flags failed turns with the upstream message", () => { + const run = ORI_HARNESSES.codex.parseRun( + [ + JSON.stringify({ type: "turn.started" }), + JSON.stringify({ + type: "turn.failed", + error: { message: "unexpected status 401 Unauthorized" }, + }), + ].join("\n") + ); + expect(run.isError).toBe(true); + expect(run.apiErrorStatus).toBe("unexpected status 401 Unauthorized"); + expect(run.usage).toBeUndefined(); + expect(run.generationIds).toEqual([]); + }); + + it("routes codex through the proxy with exec-level provider overrides", () => { + const script = ORI_HARNESSES.codex.buildRunScript({ + ...RUN_OPTIONS, + hasSystemPrompt: true, + hasDisallowedTools: false, + }); + expect(script).toContain('ori codex --model "$TB_MODEL"'); + expect(script).toContain("--reasoning-effort high --"); + expect(script).toContain("exec --json --ephemeral --skip-git-repo-check"); + expect(script).toContain('-m "$TB_MODEL"'); + expect(script).toContain("-c model_reasoning_effort=high"); + expect(script).toContain("-c model_provider=openrouter"); + expect(script).toContain( + "-c \"model_providers.openrouter.base_url='$OR_GENERATION_PROXY_BASE_URL'\"" + ); + expect(script).toContain( + "-c \"model_providers.openrouter.wire_api='responses'\"" + ); + expect(script).toContain( + "-c \"model_providers.openrouter.env_http_headers.X-Session-Id='ORI_OPENROUTER_SESSION_ID'\"" + ); + expect(script).toContain( + "-c \"model_providers.openrouter.auth.args=['-c','echo \\$OPENROUTER_API_KEY']\"" + ); + expect(script).toContain( + "-c \"model_instructions_file='/tmp/agent-system-prompt.md'\"" + ); + expect(script).toContain("developer_instructions="); + expect(script).toContain("--ignore-user-config"); + expect(script).toContain("--ignore-rules"); + expect(script).toContain("-c project_doc_max_bytes=0"); + expect(script).toContain('"$(cat /instruction.md)"'); + expect(script).toContain("tee -a /logs/agent/agent.txt"); + expect(script.indexOf("export OR_GENERATION_PROXY_BASE_URL")).toBeLessThan( + script.indexOf("ori codex") + ); + }); + + it("rejects tool allow and deny lists", () => { + const script = ORI_HARNESSES.codex.buildRunScript(RUN_OPTIONS); + expect(script).toContain( + "Codex does not support allowedTools or disallowedTools" + ); + expect(script).toContain("exit 2"); + }); +}); + +describe("opencode and kilo harnesses", () => { + it("parses usage, cost, tool calls, the final message and proxy generation ids", () => { + for (const harness of [ORI_HARNESSES.opencode, ORI_HARNESSES.kilo]) { + const run = harness.parseRun(OPENCODE_STREAM); + expect(run.generationIds).toEqual([FIRST_ID]); + expect(run.usage).toEqual({ + inputTokens: 4672, + outputTokens: 244, + totalTokens: 4886, + reasoningTokens: 128, + totalCost: 0.0016485, + }); + expect(run.finalText).toBe("LE CIEL EST BLEU."); + expect(run.turns).toBe(1); + expect(run.toolCalls).toBe(1); + expect(run.isError).toBe(false); + } + }); + + it("flags error events with the nested message", () => { + const run = ORI_HARNESSES.opencode.parseRun( + JSON.stringify({ + type: "error", + error: { name: "ProviderAuthError", data: { message: "bad key" } }, + }) + ); + expect(run.isError).toBe(true); + expect(run.apiErrorStatus).toBe("bad key"); + }); + + it("builds the proxy config after the proxy is up and disables denied tools", () => { + for (const binary of ["opencode", "kilo"] as const) { + const script = ORI_HARNESSES[binary].buildRunScript(RUN_OPTIONS); + expect(script).toContain( + `export ${binary.toUpperCase()}_CONFIG_CONTENT=` + ); + expect(script).toContain( + "baseURL: process.env.OR_GENERATION_PROXY_BASE_URL" + ); + expect(script).toContain( + '"X-Session-Id": "{env:ORI_OPENROUTER_SESSION_ID}"' + ); + expect(script).toContain('instructions: ["/tmp/agent-append-prompt.md"]'); + expect(script).toContain("process.env.TB_DISALLOWED_TOOLS"); + expect(script).toContain(`ori ${binary} --model "$TB_MODEL"`); + expect(script).toContain("--reasoning-effort high --"); + expect(script).toContain("run --format json --auto"); + expect(script).toContain("--pure"); + const prefix = binary.toUpperCase(); + expect(script).toContain( + `export ${prefix}_DISABLE_PROJECT_CONFIG=1 ${prefix}_DISABLE_CLAUDE_CODE=1 ${prefix}_DISABLE_EXTERNAL_SKILLS=1` + ); + expect( + script.indexOf("export OR_GENERATION_PROXY_BASE_URL") + ).toBeLessThan(script.indexOf("_CONFIG_CONTENT=")); + } + }); + + it("omits optional configuration and rejects unsupported options", () => { + const plain = ORI_HARNESSES.kilo.buildRunScript({ + ...RUN_OPTIONS, + hasAppendSystemPrompt: false, + hasDisallowedTools: false, + isolateAgentConfig: false, + }); + expect(plain).not.toContain("instructions:"); + expect(plain).not.toContain("TB_DISALLOWED_TOOLS"); + expect(plain).not.toContain("--pure"); + expect(plain).not.toContain("_DISABLE_PROJECT_CONFIG"); + const rejected = ORI_HARNESSES.opencode.buildRunScript({ + ...RUN_OPTIONS, + hasAllowedTools: true, + }); + expect(rejected).toContain( + "opencode does not support systemPrompt or allowedTools" + ); + expect(rejected).toContain("exit 2"); + }); +}); diff --git a/src/benchmarks/agent-cli/harness.ts b/src/benchmarks/agent-cli/harness.ts index 173450b..7e9cdbb 100644 --- a/src/benchmarks/agent-cli/harness.ts +++ b/src/benchmarks/agent-cli/harness.ts @@ -6,6 +6,12 @@ import type { import { MessageRole } from "../../harness/core"; import { Either } from "../../internal/either"; import { definedValues, isRecord } from "../../internal/guards"; +import { + buildGenerationProxyPrelude, + GENERATION_PROXY_BASE_URL, + GENERATION_PROXY_BASE_URL_ENV, + parseProxyGenerationIds, +} from "./generation-proxy"; import type { OriAgent, OriChannel, OriReasoningEffort } from "./schema"; import { assertValidAgentPackage, DEFAULT_CLAUDE_PACKAGE } from "./schema"; @@ -27,6 +33,12 @@ export const DEFAULT_PRIME_AGENT_PACKAGE = export const DEFAULT_OMP_PACKAGE = "@oh-my-pi/pi-coding-agent@18.1.2" as const; +export const DEFAULT_CODEX_PACKAGE = "@openai/codex@0.153.1" as const; + +export const DEFAULT_OPENCODE_PACKAGE = "opencode-ai@1.18.27" as const; + +export const DEFAULT_KILO_PACKAGE = "@kilocode/cli@7.5.9" as const; + export const OMP_BUN_VERSION = "bun-v1.3.14" as const; export const BUN_RELEASE_URL = @@ -518,11 +530,153 @@ const OMP_HARNESS: OriHarnessDef = { parseRun: parseJsonAgentStream, }; +const CODEX_SYSTEM_PROMPT_PATH = "/tmp/agent-system-prompt.md" as const; +const OPENCODE_APPEND_PROMPT_PATH = "/tmp/agent-append-prompt.md" as const; + +const CODEX_HARNESS: OriHarnessDef = { + id: "codex", + defaultPackage: DEFAULT_CODEX_PACKAGE, + binaryName: "codex", + remoteLogPath: "/logs/agent/codex.txt", + imageBuildSteps: (options) => + buildImageSteps({ ...options, binaryName: "codex" }), + buildBootstrapScript: (options) => + buildBootstrapScript({ ...options, binaryName: "codex" }), + buildRunScript: (options) => + [ + "set -euo pipefail", + "export HOME=/root", + "export RUST_LOG=error", + "mkdir -p /logs/agent", + ...(options.hasAllowedTools || options.hasDisallowedTools + ? [ + 'echo "Codex does not support allowedTools or disallowedTools" >&2', + "exit 2", + ] + : []), + ...(options.hasSystemPrompt + ? [`printf '%s' "$TB_SYSTEM_PROMPT" > ${CODEX_SYSTEM_PROMPT_PATH}`] + : []), + ...buildGenerationProxyPrelude(options.logPath), + 'ori codex --model "$TB_MODEL" \\', + ` --reasoning-effort ${options.reasoningEffort} -- \\`, + " exec --json --ephemeral --skip-git-repo-check \\", + " --dangerously-bypass-approvals-and-sandbox \\", + ' -m "$TB_MODEL" \\', + ` -c model_reasoning_effort=${options.reasoningEffort} \\`, + " -c model_provider=openrouter \\", + " -c \"model_providers.openrouter.name='OpenRouter'\" \\", + ` -c "model_providers.openrouter.base_url='${GENERATION_PROXY_BASE_URL}'" \\`, + " -c \"model_providers.openrouter.wire_api='responses'\" \\", + " -c \"model_providers.openrouter.env_http_headers.X-Session-Id='ORI_OPENROUTER_SESSION_ID'\" \\", + " -c \"model_providers.openrouter.auth.command='sh'\" \\", + " -c \"model_providers.openrouter.auth.args=['-c','echo \\$OPENROUTER_API_KEY']\" \\", + ...(options.hasSystemPrompt + ? [` -c "model_instructions_file='${CODEX_SYSTEM_PROMPT_PATH}'" \\`] + : []), + ...(options.hasAppendSystemPrompt + ? [ + ` -c "developer_instructions=$(node -e 'process.stdout.write(JSON.stringify(process.env.TB_APPEND_SYSTEM_PROMPT))')" \\`, + ] + : []), + ...(options.isolateAgentConfig + ? [ + " --ignore-user-config \\", + " --ignore-rules \\", + " -c project_doc_max_bytes=0 \\", + ] + : []), + ` "$(cat ${options.instructionPath})" \\`, + ` 2>&1 [tool, false])),', + ] + : []), + "};", + "process.stdout.write(JSON.stringify(config));", + ].join(" "); + return [ + "set -euo pipefail", + "export HOME=/root", + "mkdir -p /logs/agent", + ...(options.hasSystemPrompt || options.hasAllowedTools + ? [ + `echo "${binary} does not support systemPrompt or allowedTools" >&2`, + "exit 2", + ] + : []), + ...(options.hasAppendSystemPrompt + ? [ + `printf '%s' "$TB_APPEND_SYSTEM_PROMPT" > ${OPENCODE_APPEND_PROMPT_PATH}`, + ] + : []), + ...(options.isolateAgentConfig + ? [ + `export ${envPrefix}_DISABLE_PROJECT_CONFIG=1 ${envPrefix}_DISABLE_CLAUDE_CODE=1 ${envPrefix}_DISABLE_EXTERNAL_SKILLS=1`, + ] + : []), + ...buildGenerationProxyPrelude(options.logPath), + `export ${configEnv}="$(node -e '${configScript}')"`, + `ori ${binary} --model "$TB_MODEL" \\`, + ` --reasoning-effort ${options.reasoningEffort} -- \\`, + " run --format json --auto \\", + ...(options.isolateAgentConfig ? [" --pure \\"] : []), + ` "$(cat ${options.instructionPath})" \\`, + ` 2>&1 + buildImageSteps({ ...options, binaryName: "opencode" }), + buildBootstrapScript: (options) => + buildBootstrapScript({ ...options, binaryName: "opencode" }), + buildRunScript: (options) => buildOpencodeRunScript("opencode", options), + parseRun: parseOpencodeStream, +}; + +const KILO_HARNESS: OriHarnessDef = { + id: "kilo", + defaultPackage: DEFAULT_KILO_PACKAGE, + binaryName: "kilo", + remoteLogPath: "/logs/agent/kilo.txt", + imageBuildSteps: (options) => + buildImageSteps({ ...options, binaryName: "kilo" }), + buildBootstrapScript: (options) => + buildBootstrapScript({ ...options, binaryName: "kilo" }), + buildRunScript: (options) => buildOpencodeRunScript("kilo", options), + parseRun: parseOpencodeStream, +}; + export const ORI_HARNESSES: Readonly> = { claude: CLAUDE_HARNESS, pi: ORI_PI_HARNESS, "prime-agent": PRIME_AGENT_HARNESS, omp: OMP_HARNESS, + codex: CODEX_HARNESS, + opencode: OPENCODE_HARNESS, + kilo: KILO_HARNESS, }; export function getOriHarness(agent: OriAgent): OriHarnessDef { @@ -668,3 +822,199 @@ function parseJsonAgentStream(stdout: string): OriAgentRun { toolCalls, }; } + +function parseJsonLines(stdout: string): Record[] { + const events: Record[] = []; + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) { + continue; + } + const parsed = Either.try(() => JSON.parse(trimmed)); + if (Either.isRight(parsed) && isRecord(parsed.right)) { + events.push(parsed.right); + } + } + return events; +} + +const CODEX_TOOL_ITEM_TYPES = new Set([ + "command_execution", + "file_change", + "mcp_tool_call", + "web_search", +]); + +function parseCodexStream(stdout: string): OriAgentRun { + let inputTokens = 0; + let outputTokens = 0; + let reasoningTokens = 0; + let turns = 0; + let toolCalls = 0; + let isError = false; + let apiErrorStatus: string | undefined; + let finalText: string | undefined; + const assistantMessages: ModelMessage[] = []; + const responseItems: ResponseItem[] = []; + for (const event of parseJsonLines(stdout)) { + responseItems.push(event); + const eventType = event["type"]; + if (eventType === "error" || eventType === "turn.failed") { + isError = true; + const errorRecord = isRecord(event["error"]) ? event["error"] : event; + apiErrorStatus = + optionalStringField(errorRecord, "message") ?? apiErrorStatus; + continue; + } + if (eventType === "turn.completed") { + turns++; + const { usage } = event; + if (isRecord(usage)) { + inputTokens += optionalNumberField(usage, "input_tokens") ?? 0; + outputTokens += optionalNumberField(usage, "output_tokens") ?? 0; + reasoningTokens += + optionalNumberField(usage, "reasoning_output_tokens") ?? 0; + } + continue; + } + if (eventType !== "item.completed") { + continue; + } + const { item } = event; + if (!isRecord(item)) { + continue; + } + const itemType = item["type"]; + if (typeof itemType === "string" && CODEX_TOOL_ITEM_TYPES.has(itemType)) { + toolCalls++; + continue; + } + if (itemType !== "agent_message") { + continue; + } + const text = optionalStringField(item, "text"); + if (text !== undefined && text.length > 0) { + finalText = text; + assistantMessages.push({ role: MessageRole.Assistant, content: text }); + } + } + const totalTokens = inputTokens + outputTokens; + return { + usage: + totalTokens !== 0 + ? { + inputTokens, + outputTokens, + totalTokens, + reasoningTokens, + totalCost: 0, + } + : undefined, + generationIds: parseProxyGenerationIds(stdout), + generationTimeMs: undefined, + finalText, + assistantMessages, + responseItems, + isError, + apiErrorStatus, + turns: turns > 0 ? turns : undefined, + toolCalls, + }; +} + +function parseOpencodeStream(stdout: string): OriAgentRun { + let inputTokens = 0; + let outputTokens = 0; + let cacheRead = 0; + let cacheWrite = 0; + let totalTokens = 0; + let reasoningTokens = 0; + let totalCost = 0; + let turns = 0; + let toolCalls = 0; + let isError = false; + let apiErrorStatus: string | undefined; + let finalText: string | undefined; + const assistantMessages: ModelMessage[] = []; + const responseItems: ResponseItem[] = []; + for (const event of parseJsonLines(stdout)) { + responseItems.push(event); + const eventType = event["type"]; + if (eventType === "error") { + isError = true; + const error = event["error"]; + if (isRecord(error)) { + const data = isRecord(error["data"]) ? error["data"] : error; + apiErrorStatus = + optionalStringField(data, "message") ?? + optionalStringField(error, "name") ?? + apiErrorStatus; + } + continue; + } + if (eventType === "tool_use") { + toolCalls++; + continue; + } + const { part } = event; + if (!isRecord(part)) { + continue; + } + if (eventType === "text") { + const text = optionalStringField(part, "text"); + if (text !== undefined && text.length > 0) { + finalText = text; + assistantMessages.push({ role: MessageRole.Assistant, content: text }); + } + continue; + } + if (eventType !== "step_finish") { + continue; + } + turns++; + totalCost += optionalNumberField(part, "cost") ?? 0; + const { tokens } = part; + if (!isRecord(tokens)) { + continue; + } + const stepInput = optionalNumberField(tokens, "input") ?? 0; + const stepOutput = optionalNumberField(tokens, "output") ?? 0; + const stepReasoning = optionalNumberField(tokens, "reasoning") ?? 0; + const { cache } = tokens; + const stepCacheRead = isRecord(cache) + ? (optionalNumberField(cache, "read") ?? 0) + : 0; + const stepCacheWrite = isRecord(cache) + ? (optionalNumberField(cache, "write") ?? 0) + : 0; + inputTokens += stepInput; + outputTokens += stepOutput + stepReasoning; + cacheRead += stepCacheRead; + cacheWrite += stepCacheWrite; + reasoningTokens += stepReasoning; + totalTokens += + optionalNumberField(tokens, "total") ?? + stepInput + stepCacheRead + stepCacheWrite + stepOutput + stepReasoning; + } + return { + usage: + totalTokens !== 0 + ? { + inputTokens: inputTokens + cacheRead + cacheWrite, + outputTokens, + totalTokens, + reasoningTokens, + totalCost, + } + : undefined, + generationIds: parseProxyGenerationIds(stdout), + generationTimeMs: undefined, + finalText, + assistantMessages, + responseItems, + isError, + apiErrorStatus, + turns: turns > 0 ? turns : undefined, + toolCalls, + }; +} diff --git a/src/benchmarks/agent-cli/schema.ts b/src/benchmarks/agent-cli/schema.ts index 57fc926..b6aefdb 100644 --- a/src/benchmarks/agent-cli/schema.ts +++ b/src/benchmarks/agent-cli/schema.ts @@ -1,6 +1,14 @@ import type { ValueOf } from "../../internal/guards"; -export const ORI_AGENTS = ["pi", "claude", "prime-agent", "omp"] as const; +export const ORI_AGENTS = [ + "pi", + "claude", + "prime-agent", + "omp", + "codex", + "opencode", + "kilo", +] as const; export type OriAgent = ValueOf; diff --git a/src/benchmarks/registry.test.ts b/src/benchmarks/registry.test.ts index 39082a0..ab5059a 100644 --- a/src/benchmarks/registry.test.ts +++ b/src/benchmarks/registry.test.ts @@ -300,7 +300,7 @@ describe("benchmark registry", () => { const result = parseSchema(BenchmarkRunConfigSchema, { benchmarkId: "terminal_bench", model: "openai/gpt-5.4", - agent: "codex", + agent: "grok", reasoningEffort: "high", agentReasoningEffort: "high", }); @@ -365,7 +365,7 @@ describe("benchmark registry", () => { const result = parseSchema(BenchmarkRunConfigSchema, { benchmarkId: "deep_swe", model: "anthropic/claude-opus-5", - agent: "codex", + agent: "grok", reasoningEffort: "high", agentReasoningEffort: "high", }); diff --git a/src/benchmarks/terminal-bench/ori-solver.test.ts b/src/benchmarks/terminal-bench/ori-solver.test.ts index 969f237..cf68e77 100644 --- a/src/benchmarks/terminal-bench/ori-solver.test.ts +++ b/src/benchmarks/terminal-bench/ori-solver.test.ts @@ -41,12 +41,16 @@ import { getCollectedGenerationIds, resetGenerationIds, } from "../../runtime/generation-ids"; +import { GENERATION_ID_LINE_PREFIX } from "../agent-cli/generation-proxy"; import { BUN_RELEASE_SHA256, BUN_RELEASE_URL, DEFAULT_AGENT_RUNTIME_SHA256, DEFAULT_AGENT_RUNTIME_URL, + DEFAULT_CODEX_PACKAGE, + DEFAULT_KILO_PACKAGE, DEFAULT_OMP_PACKAGE, + DEFAULT_OPENCODE_PACKAGE, DEFAULT_PI_AGENT_PACKAGE, DEFAULT_PRIME_AGENT_PACKAGE, getOriHarness, @@ -1812,3 +1816,107 @@ describe("terminal-bench omp via ori", () => { expect(steps.join("\n")).toContain('bun install -g "file:///opt/omp.tgz"'); }); }); + +describe("terminal-bench proxy-attributed harnesses via ori", () => { + const PROXY_GENERATION_ID = "gen-1788473388-R386NZLdIly5wVTk0l5f"; + const CODEX_STREAM = [ + JSON.stringify({ type: "thread.started", thread_id: "thread-1" }), + JSON.stringify({ type: "turn.started" }), + `${GENERATION_ID_LINE_PREFIX}${PROXY_GENERATION_ID}`, + JSON.stringify({ + type: "item.completed", + item: { id: "item_0", type: "agent_message", text: "OK" }, + }), + JSON.stringify({ + type: "turn.completed", + usage: { + input_tokens: 100, + cached_input_tokens: 0, + cache_write_input_tokens: 0, + output_tokens: 5, + reasoning_output_tokens: 2, + }, + }), + ].join("\n"); + + it("registers codex, opencode and kilo as ori agents", () => { + for (const agent of ["codex", "opencode", "kilo"] as const) { + expect(ORI_AGENTS).toContain(agent); + expect(getOriHarness(agent).binaryName).toBe(agent); + expect(getOriHarness(agent).remoteLogPath).toBe( + `/logs/agent/${agent}.txt` + ); + } + expect(getOriHarness("codex").defaultPackage).toBe(DEFAULT_CODEX_PACKAGE); + expect(getOriHarness("opencode").defaultPackage).toBe( + DEFAULT_OPENCODE_PACKAGE + ); + expect(getOriHarness("kilo").defaultPackage).toBe(DEFAULT_KILO_PACKAGE); + }); + + it("records the proxy-captured generation ids from the codex run", async () => { + const layer = makeTerminalBenchFakeSandboxLayer({ + reward: 1, + testOutput: "1 passed", + agentEventStream: CODEX_STREAM, + agentExitCode: 0, + }); + const execCalls: ExecCalls = []; + const solverLayer = layerEffect(Solver)( + gen(function* () { + const sessionFactory = yield* SandboxSession; + return Solver.of( + oriSolver(sessionFactory, SOLVER_OPTS, getOriHarness("codex")) + ); + }) + ); + const finalState = await runPromise( + gen(function* () { + const solver = yield* Solver; + return yield* solver(sampleState()); + }).pipe( + provide( + layerMergeAll( + solverLayer.pipe( + layerProvide( + makeTerminalBenchFakeSandboxLayer({ + reward: 1, + testOutput: "1 passed", + agentEventStream: CODEX_STREAM, + agentExitCode: 0, + execCalls, + }) + ) + ), + noopProgressLayer, + noopCheckpointLayer + ) + ) + ) + ); + expect(finalState.output?.completion).toBe("OK"); + expect(finalState.output?.usage).toEqual({ + inputTokens: 100, + outputTokens: 5, + totalTokens: 105, + reasoningTokens: 2, + totalCost: 0, + }); + expect(finalState.sample.metadata?.["generationIds"]).toEqual([ + PROXY_GENERATION_ID, + ]); + expect(finalState.sample.metadata?.["agent"]).toBe("codex"); + expect( + await runAndCollectGenerationIds(layer, getOriHarness("codex")) + ).toEqual([PROXY_GENERATION_ID]); + const agentCall = execCalls.find((call) => + call.argv[2]?.includes("ori codex") + ); + expect(agentCall?.argv[2]).toContain( + "node /tmp/openrouter-generation-proxy.cjs" + ); + expect(agentCall?.argv[2]).toContain( + "GEN_PROXY_UPSTREAM=https://openrouter.ai" + ); + }); +}); diff --git a/test/helpers/terminal-bench-sandbox.ts b/test/helpers/terminal-bench-sandbox.ts index c30bc34..4b2a934 100644 --- a/test/helpers/terminal-bench-sandbox.ts +++ b/test/helpers/terminal-bench-sandbox.ts @@ -35,6 +35,9 @@ const AGENT_COMMAND_MARKERS = [ "ori claude", "ori prime-agent", "ori omp", + "ori codex", + "ori opencode", + "ori kilo", ] as const; const ORI_INSTALL_MARKER = "ORI_INSTALL_DIR=" as const;