diff --git a/tests/README.md b/tests/README.md index c4450b54d..c6719fe61 100644 --- a/tests/README.md +++ b/tests/README.md @@ -151,7 +151,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | Test | User action | Expected evidence | | --- | --- | --- | -| `test_cuj_configuration`, `test_cuj_codex_inference`, `test_cuj_claude_inference` in `test_cuj2_mps_mcp.py` | Read the permanently preconfigured two-agent MPS/MCP config; run separate configure, Codex TUI, and Claude TUI cases | Configuration verifies both native MPS APIs, generated settings, and sandbox inclusion with web_search excluded; each TUI case records its exact MPS target header and model, paired HTTP 200 response, and final marker. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. No workspace config CRUD is performed | +| `test_cuj_configuration`, `test_cuj_codex_inference`, `test_cuj_claude_inference` in `test_cuj2_mps_mcp.py` | Read the permanently preconfigured two-agent MPS/MCP config; run separate configure, Codex TUI, and Claude TUI cases | Configuration verifies the exact MPS/defaults, sandbox inclusion, and web_search exclusion. The Codex TUI case checks ug's real app-server `model/list`; the Claude TUI case checks Claude's native gateway cache and `/model` picker. Each TUI case also records its exact MPS target header and model, paired HTTP 200 response, and final marker. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. No workspace config CRUD is performed | | `test_ug_configure_claude_databricks` | Configure Databricks Hosted; execute the generated auth helper; launch plain `ug claude`, read a file, and open `/model` | Generated helper invokes `ug` with clean token stdout; assistant returns an unpredictable file value; native discovery caches `system.ai` models and the picker shows a discovered model without an opt-in flag; normal exit; reopen with working keyboard input | | `test_ug_configure_claude_anthropic_mps` | Select Anthropic MPS in the real configure picker; launch Claude | Saved provider in status; completed TUI file task; normal exit | | `test_ug_configure_codex_databricks` | Configure Databricks Hosted; execute the generated auth helper; open Codex TUI and read a file | Generated helper invokes `ug` with clean token stdout; completed assistant answer contains the file value; normal exit and reopen | @@ -231,9 +231,10 @@ one configuration case and separate Codex and Claude TUI task cases. The workspa preconfigured for CUJ2 and is read-only; every case validates it, and the fixture compares it again during teardown. The TUI cases forward agent requests through the shared per-test loopback recorder to the real workspace and assert the provider-service header, model, -paired HTTP 200 response, and final answer. The configuration case verifies sandbox MCP -selection and generated client listings; live sandbox execution remains deferred because -MAS cannot downscope the CI service principal. Failed TUI runs +paired HTTP 200 response, and final answer. The Codex case also checks ug's real app-server +`model/list`; the Claude case checks Claude's native gateway cache and `/model` picker. The +configuration case verifies sandbox MCP selection and generated client listings; live sandbox +execution remains deferred because MAS cannot downscope the CI service principal. Failed TUI runs retain terminal output/actions/screen and best-effort agent transcript snapshots under the runner artifact directory before the temporary homes are deleted. The suite requires `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET`, with `ug`, `claude`, `codex`, and diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index 6961d9093..ee2ed1032 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -52,8 +52,10 @@ reads and validates the exact two-agent MPS/MCP shape. No case creates, updates, or restores CodingAgentConfig. The fixture's final read-only equality check is the only workspace mutation guard. The test uses the shared `UserSession`, `Workspace`, and `TuiRequestRecorder`, `AgentTerminal`, and integration transcript readers directly. The -configuration case verifies the selected sandbox service and generated MCP listings; live -`run_code` execution remains deferred because MAS cannot downscope the CI service principal. +configuration case verifies the selected sandbox service and generated MCP listings. The Codex +and Claude TUI cases additionally verify Codex's real `model/list` and Claude's native cache plus +`/model` picker, respectively; live `run_code` execution remains deferred because MAS cannot +downscope the CI service principal. Use a clean disposable POSIX runner without existing machine-wide agent settings. Install the intended versions of `ug`, `claude`, `codex`, and `databricks` on PATH. diff --git a/tests/e2e_cuj/test_cuj2_mps_mcp.py b/tests/e2e_cuj/test_cuj2_mps_mcp.py index 0af6311df..e6faa92ce 100644 --- a/tests/e2e_cuj/test_cuj2_mps_mcp.py +++ b/tests/e2e_cuj/test_cuj2_mps_mcp.py @@ -15,6 +15,7 @@ assistant_answers, is_child_session, ) +from tests.integration.utils.model_discovery import claude_model_in_picker from tests.integration.utils.terminal import AgentTerminal from .base import BaseCujTest @@ -133,6 +134,7 @@ def _assert_configured_files(session, workspace_url: str) -> None: assert codex_path.is_file(), codex_path codex_config = tomllib.loads(codex_path.read_text(encoding="utf-8")) assert codex_config["model_provider"] == "Databricks", codex_config + assert codex_config["model"] == MODEL_PROVIDER_SERVICE_FIXTURES[CODEX][1], codex_config codex_provider = codex_config["model_providers"]["Databricks"] assert codex_provider["wire_api"] == "responses", codex_provider assert codex_provider["base_url"] == workspace_url.rstrip("/") + "/ai-gateway/codex/v1" @@ -153,6 +155,21 @@ def _assert_generated_mcp_listings(session) -> None: assert not _mcp_name_listed(output, WEB_SEARCH_MCP_SERVICE_NAME), output +def _assert_codex_provider_discovery(session, expected_model: str) -> None: + model_ids = session.codex_model_ids( + ["app-server", "--listen", "stdio://"], name="cuj2-codex-models" + ) + assert model_ids == [expected_model], model_ids + + +def _assert_claude_provider_discovery(session, screen: str, expected_model: str) -> None: + models = session.claude_gateway_models("cuj2-claude-models") + model_ids = [model["id"] for model in models] + assert model_ids == [expected_model], models + display_name = models[0].get("display_name") + assert claude_model_in_picker(screen, expected_model, display_name), screen + + def _parent_transcripts(session, agent: str) -> dict[str, list[dict]]: return { path: records @@ -204,8 +221,8 @@ def test_cuj_configuration(self, cuj): """Scenario: configure ug from the preconfigured two-agent MPS/MCP workspace. Expected: the read-only CodingAgentConfig selects the two exact MPS resources and sandbox - MCP; both native MPS APIs advertise their selected model, generated agent settings use - the exact provider/model values, and agent MCP listings exclude web_search. + MCP; generated agent settings use the exact provider/model values, and agent MCP listings + exclude web_search. The agent-specific TUI cases assert provider discovery surfaces. """ session, workspace, _ = cuj published = workspace.config() @@ -230,7 +247,8 @@ class TestCuj2CodexInference(_Cuj2Base): def test_cuj_codex_inference(self, cuj): """Scenario: configure through the recorder and complete a real Codex inference. - Expected: Codex sends a Responses request for gpt-5-nano with the + Expected: ug's real Codex app-server `model/list` exposes only gpt-5-nano through the + configured provider; Codex then sends a Responses request for gpt-5-nano with the ug_e2e.providers.openai target header and receives its paired HTTP 200 response; its parent transcript contains the final marker. """ @@ -250,6 +268,7 @@ def test_cuj_codex_inference(self, cuj): ) _assert_configured_files(session, workspace.url) _assert_generated_mcp_listings(session) + _assert_codex_provider_discovery(session, MODEL_PROVIDER_SERVICE_FIXTURES[CODEX][1]) recorder.prepare_launch() marker = f"CUJ2-CODEX-{uuid.uuid4().hex}" @@ -286,9 +305,10 @@ class TestCuj2ClaudeInference(_Cuj2Base): def test_cuj_claude_inference(self, cuj): """Scenario: configure through the recorder and complete a real Claude inference. - Expected: Claude sends a Messages request for claude-haiku-4-5-20251001 with the - ug_e2e.providers.anthropic target header and receives its paired HTTP 200 response; its - parent transcript contains the final marker. + Expected: Claude's native gateway discovery cache and real `/model` picker expose only + claude-haiku-4-5-20251001 through the configured provider; Claude then sends a Messages + request for that model with the ug_e2e.providers.anthropic target header and receives its + paired HTTP 200 response; its parent transcript contains the final marker. """ session, workspace, recorder = cuj published = workspace.config() @@ -310,8 +330,15 @@ def test_cuj_claude_inference(self, cuj): recorder.prepare_launch() marker = f"CUJ2-CLAUDE-{uuid.uuid4().hex}" checkpoint = recorder.checkpoint() + expected_model = MODEL_PROVIDER_SERVICE_FIXTURES[CLAUDE][1] with AgentTerminal(session, CLAUDE, [str(session.binary), CLAUDE], "cuj2-claude") as tui: tui.boot() + screen = tui.open_model_picker( + model_visible=lambda text: ( + session.claude_gateway_cache_ready([expected_model]) + and claude_model_in_picker(text, expected_model, None) + ) + ) tui.submit(_agent_prompt(marker)) def completed(screen): @@ -321,6 +348,8 @@ def completed(screen): tui.wait_for(completed, "completed Claude task", timeout=240) tui.exit_normally() + _assert_claude_provider_discovery(session, screen, expected_model) + request = recorder.expect_request( method="POST", path=INFERENCE_PATHS[CLAUDE],