Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion tools/code-exec-harness/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ Scenarios are JSON files. Common fields:
turns resume the first turn's session id
- `responses_api`: start a local fake `/v1/responses` server and point the run
at it with a dummy API key, allowing request-body assertions without spending
live model tokens
live model tokens. When `responses_api` is set, inherited Code auth is
suppressed even if `inherit_auth` is requested.
- `expect`: simple assertions over the final answer, commands, fake `gh` calls,
and exit code

Expand Down
30 changes: 20 additions & 10 deletions tools/code-exec-harness/harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from typing import Any, cast
from urllib.parse import urlparse


Expand Down Expand Up @@ -348,6 +348,14 @@ def inherit_auth(paths: RunPaths) -> dict[str, str]:
return env_overrides


def auth_inheritance_requested(scenario: dict[str, Any], args: argparse.Namespace) -> bool:
return bool(args.inherit_auth or scenario.get("inherit_auth", False))


def fake_responses_enabled(scenario: dict[str, Any]) -> bool:
return isinstance(scenario.get("responses_api"), dict)


FAKE_GH = r'''#!/usr/bin/env python3
import json
import os
Expand Down Expand Up @@ -820,8 +828,11 @@ def run_scenario(path: Path, args: argparse.Namespace) -> int:
materialize_workspace(scenario, paths)
materialize_skills(scenario, paths, scenario_dir, extra_roots)
write_config(scenario, paths)
use_fake_responses = fake_responses_enabled(scenario)
fake_responses = cast(dict[str, Any], scenario["responses_api"]) if use_fake_responses else None
inherit_requested = auth_inheritance_requested(scenario, args)
inherited_env = {}
if args.inherit_auth or scenario.get("inherit_auth", False):
if inherit_requested and not use_fake_responses:
inherited_env = inherit_auth(paths)
gh_paths = write_fake_gh(scenario, paths)

Expand All @@ -845,21 +856,16 @@ def run_scenario(path: Path, args: argparse.Namespace) -> int:
env["CODE_EXEC_HARNESS_GH_LOG"] = str(gh_paths["log"])
env["CODE_EXEC_HARNESS_GH_STATE"] = str(gh_paths["state"])

fake_responses = scenario.get("responses_api")
server_context = (
FakeResponsesServer(fake_responses, paths.artifacts)
if isinstance(fake_responses, dict)
else None
)
server_context = FakeResponsesServer(fake_responses, paths.artifacts) if fake_responses is not None else None

def run_with_env(
fake_server: FakeResponsesServer | None,
) -> tuple[int, list[dict[str, Any]], list[list[str]]]:
run_env = env.copy()
if fake_server is not None:
run_env["OPENAI_BASE_URL"] = fake_server.base_url
run_env.setdefault("OPENAI_API_KEY", "harness-test-key")
run_env.setdefault("OPENAI_WIRE_API", "responses")
run_env["OPENAI_API_KEY"] = "harness-test-key"
run_env["OPENAI_WIRE_API"] = "responses"

turn_prompts = scenario.get("turns")
if isinstance(turn_prompts, list) and turn_prompts:
Expand Down Expand Up @@ -889,6 +895,10 @@ def run_with_env(
put_json(paths.artifacts / "manifest.json", {
"scenario": str(path),
"code_home": str(paths.code_home),
"fake_responses": use_fake_responses,
"inherit_auth_applied": bool(inherited_env),
"inherit_auth_requested": inherit_requested,
"inherit_auth_suppressed": bool(inherit_requested and use_fake_responses),
"workspace": str(paths.workspace),
})
if args.dry_run:
Expand Down