diff --git a/src/ucode/agents/codex.py b/src/ucode/agents/codex.py index 8b0bcd3b0..e63463113 100644 --- a/src/ucode/agents/codex.py +++ b/src/ucode/agents/codex.py @@ -613,6 +613,11 @@ def _reconcile_managed_config(state: dict, compose: Callable[[dict], dict]) -> N MANAGED_MCP_CONFIG_KEY = "mcp_servers" +_CODEX_MCP_AUTH_ENV_VARS = [ + "DATABRICKS_BEARER", + "DATABRICKS_BEARER_COMMAND", + "DATABRICKS_CONFIG_FILE", +] def managed_mcp_uses_managed_file() -> bool: @@ -627,7 +632,11 @@ def managed_mcp_uses_managed_file() -> bool: def managed_mcp_entry(argv: list[str]) -> dict: """A ``[mcp_servers.]`` stdio entry from the ``ug mcp-proxy`` argv (same as user scope).""" - return {"command": argv[0], "args": list(argv[1:])} + return { + "command": argv[0], + "args": list(argv[1:]), + "env_vars": list(_CODEX_MCP_AUTH_ENV_VARS), + } def managed_mcp_http_entry(url: str, client_id: str) -> dict: diff --git a/src/ucode/databricks.py b/src/ucode/databricks.py index 970f51ecc..b67ba8f7e 100644 --- a/src/ucode/databricks.py +++ b/src/ucode/databricks.py @@ -2384,7 +2384,7 @@ def list_model_provider_services( # reach the next. return [dict(service) for service in cached], None - hostname = workspace_hostname(workspace) + origin = normalize_workspace_url(workspace) services: list[dict] = [] page_token: str | None = None seen_tokens: set[str] = set() @@ -2395,9 +2395,7 @@ def list_model_provider_services( params["parent"] = f"schemas/{parent}" if page_token: params["page_token"] = page_token - url = ( - f"https://{hostname}/api/2.1/unity-catalog/model-provider-services?{urlencode(params)}" - ) + url = f"{origin}/api/2.1/unity-catalog/model-provider-services?{urlencode(params)}" payload, reason = _http_get_json(url, token, timeout=30) if payload is None: # Surface the failure only if we have nothing yet; a mid-pagination blip still @@ -2470,8 +2468,8 @@ def get_model_provider_service( server-side filter) makes a service that plainly exists look absent. Addressing it directly removes that whole class of false negative. """ - hostname = workspace_hostname(workspace) - url = f"https://{hostname}/api/2.1/unity-catalog/model-provider-services/{service_name}" + origin = normalize_workspace_url(workspace) + url = f"{origin}/api/2.1/unity-catalog/model-provider-services/{service_name}" payload, reason = _http_get_json(url, token, timeout=30) if payload is None: return None, reason @@ -2956,14 +2954,14 @@ def _get_anthropic_models_json( parent_schema: str | None = None, provider: str | None = None, ) -> tuple[dict | list | None, str | None]: - hostname = workspace_hostname(workspace) + origin = normalize_workspace_url(workspace) headers = None if provider is not None: headers = {MODEL_PROVIDER_SERVICE_HEADER: provider} elif parent_schema is not None: headers = {MODEL_SERVICE_PARENT_SCHEMA_HEADER: parent_schema} return _http_get_json( - f"https://{hostname}{ANTHROPIC_MODELS_PATH}?limit=1000", + f"{origin}{ANTHROPIC_MODELS_PATH}?limit=1000", token, max_retries=_ANTHROPIC_MODEL_DISCOVERY_SETUP_MAX_RETRIES, **({"headers": headers} if headers is not None else {}), diff --git a/tests/README.md b/tests/README.md index f9e9a8130..c125b45bc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -131,10 +131,13 @@ These are **implemented assertions**, not a claim that every version passes. Consult the run's JUnit report and artifacts for results. Each function states its **Scenario** and **Expected** outcome and shows its configure and launch commands. Fixtures supply fresh environments and credentials, never configured ug. -All tests live directly in `integration/`; shared mechanics live in `utils/`. +Integration tests live directly in `integration/`; shared mechanics live in `utils/`. +Dedicated-workspace CUJs live in `e2e_cuj/` and use the shared `cuj` fixture plus +integration utilities; only CUJ-specific evidence correlation stays in a test file. | Test | User action | Expected evidence | | --- | --- | --- | +| `test_cuj_configuration`, `test_cuj_codex_inference`, `test_cuj_claude_inference` in `test_cuj2_mps_mcp.py` | Read the permanently preconfigured two-agent MPS/MCP config; run separate configure, Codex TUI, and Claude TUI cases | Configuration verifies both native MPS APIs, generated settings, and sandbox inclusion with web_search excluded; each TUI case records its exact MPS target header and model, paired HTTP 200 response, and final marker. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. No workspace config CRUD is performed | | `test_ug_configure_claude_databricks` | Configure Databricks Hosted; execute the generated auth helper; launch plain `ug claude`, read a file, and open `/model` | Generated helper invokes `ug` with clean token stdout; assistant returns an unpredictable file value; native discovery caches `system.ai` models and the picker shows a discovered model without an opt-in flag; normal exit; reopen with working keyboard input | | `test_ug_configure_claude_anthropic_mps` | Select Anthropic MPS in the real configure picker; launch Claude | Saved provider in status; completed TUI file task; normal exit | | `test_ug_configure_codex_databricks` | Configure Databricks Hosted; execute the generated auth helper; open Codex TUI and read a file | Generated helper invokes `ug` with clean token stdout; completed assistant answer contains the file value; normal exit and reopen | @@ -209,6 +212,23 @@ deduplication, and raw catalog ID/display-name rows for scoped pickers. Managed discovery expectations come from separate read-only, provider-scoped model-list requests; they do not rely solely on ug's generated catalog. +The dedicated-workspace CUJ suite adds three independently runnable CUJ2 MPS+MCP cases: +one configuration case and separate Codex and Claude TUI task cases. The workspace is +preconfigured for CUJ2 and is read-only; every case validates it, and the fixture compares +it again during teardown. The TUI cases forward agent requests through the shared per-test +loopback recorder to the real workspace and assert the provider-service header, model, +paired HTTP 200 response, and final answer. The configuration case verifies sandbox MCP +selection and generated client listings; live sandbox execution remains deferred because +MAS cannot downscope the CI service principal. Failed TUI runs +retain terminal output/actions/screen and best-effort agent transcript snapshots under the +runner artifact directory before the temporary homes are deleted. The suite requires +`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET`, with `ug`, `claude`, `codex`, and +`databricks` on PATH; run it with +`uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj`. +Pytest prints the per-test artifact directory; the fixture forwards only its short-lived +bearer to the isolated session. It reads the published config as an input only and does not +use `UCODE_MANAGED_CONFIG_STUB`. + ug no longer runs a post-configure agent probe; the deprecated `--skip-validate` flag is accepted as a no-op where older journeys still pass it. Tests retain `--skip-upgrade` as a deprecated no-op too; UG only upgrades agents below its @@ -288,7 +308,8 @@ These are unit/component checks; they do not establish live sudo password-prompt | Scenario | Status / requirement | | --- | --- | -| Live MCP and skills functionality | Deferred; installation tests cover the local web-search MCP handshake and tool listing, not upstream proxying or a real search request. Custom OAuth search dispatch/refresh has component coverage; live parent/child search and permission decisions remain unverified | +| Broad live MCP functionality | CUJ2 covers sandbox MCP configuration and generated client listings. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. Installation tests cover the local web-search MCP handshake and tool listing, not upstream proxying or a real search request. Other MCP services, live parent/child search, and permission decisions remain deferred | +| Dedicated CUJ Skill coverage | Deferred because of a backend storage-path issue; CUJ2 has no Skill configuration, download, invocation, or assertion path | | Broad configure flags, multiple workspaces, and PAT flows | Deferred while focusing on basic CUJs | | Workspace-switch MCP cleanup | The `workspace_switch` CUJ covers real registration, cleanup, repeat configure, and a completed Claude task. Unit/component tests cover duplicate attempts and injected removal failures; the CUJ does not force an agent timeout. It runs in the required managed CI lane for full/live runs. | | Relayed/subscription MPS discovery | Not covered by the scoped discovery journeys | diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index 448158862..6961d9093 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -3,6 +3,8 @@ Each CUJ owns a separate workspace. Subclass `BaseCujTest` from `base.py` and set `WORKSPACE_URL`. Setup provides `self.workspace`, a Databricks SDK client using `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` with OAuth M2M authentication. +The workspaces share one metastore, so metastore-scoped fixture names belong in +`helpers/constants.py`; keep each workspace URL in its test class. All workspace configurations are READ ONLY. Never create, publish, update, delete, or restore workspace configuration, including during setup or teardown. Validate the preconfigured workspace and fail on a mismatch; never repair it. Concurrent @@ -44,10 +46,14 @@ In `helpers/evidence.py`, `ClaudeCujHelper` and `CodexCujHelper` implement Model evidence combines observed gateway requests/responses with native completed-turn records; neither routing banners nor native records alone prove an applied decision. -`test_cuj_smart_routing.py` runs four fresh interactive sessions: routed and explicit -model, for both Claude and Codex. It uses the published models/defaults and asserts -configuration is unchanged, including during teardown after failures. Routing-disabled -coverage requires a separately preconfigured workspace and is not part of this CUJ. +`test_cuj2_mps_mcp.py` runs three independently collected cases against a permanently +preconfigured workspace: configuration, Codex inference, and Claude inference. Each case +reads and validates the exact two-agent MPS/MCP shape. No case creates, updates, deletes, +or restores CodingAgentConfig. The fixture's final read-only equality check is the only +workspace mutation guard. The test uses the shared `UserSession`, `Workspace`, and +`TuiRequestRecorder`, `AgentTerminal`, and integration transcript readers directly. The +configuration case verifies the selected sandbox service and generated MCP listings; live +`run_code` execution remains deferred because MAS cannot downscope the CI service principal. Use a clean disposable POSIX runner without existing machine-wide agent settings. Install the intended versions of `ug`, `claude`, `codex`, and `databricks` on PATH. diff --git a/tests/e2e_cuj/conftest.py b/tests/e2e_cuj/conftest.py index 2306b8c12..b11fde12a 100644 --- a/tests/e2e_cuj/conftest.py +++ b/tests/e2e_cuj/conftest.py @@ -8,6 +8,8 @@ import pytest from databricks.sdk.errors import DatabricksError +from tests.integration.utils.terminal import TerminalProcess + from .helpers.constants import CLAUDE, CODEX, MANAGED_PATHS from .helpers.session import UserSession from .helpers.tui_request_recorder import TuiRequestRecorder @@ -45,7 +47,24 @@ def cuj(request, setup_workspace, tmp_path_factory): try: yield session, workspace, recorder finally: - workspace.assert_unchanged(published) + try: + workspace.assert_unchanged(published) + finally: + state_dir = session.home / ".ucode" + if any( + (state_dir / name).is_file() + for name in ("state.json", "managed-backups/manifest.json") + ): + with TerminalProcess( + session, + "ug", + [str(session.binary), "revert"], + "cleanup-revert", + ) as terminal: + terminal.finish() + assert not any(path.exists() for path in MANAGED_PATHS), ( + "CUJ teardown left machine-wide agent settings" + ) except DatabricksError as error: # Server messages may echo credentials; retain only the SDK error type. raise RuntimeError(f"Workspace API failed: {type(error).__name__}") from None diff --git a/tests/e2e_cuj/helpers/constants.py b/tests/e2e_cuj/helpers/constants.py index fbd144e36..3a934d550 100644 --- a/tests/e2e_cuj/helpers/constants.py +++ b/tests/e2e_cuj/helpers/constants.py @@ -1,4 +1,4 @@ -"""CLI agent names and public API wire values shared by CUJs.""" +"""Agent, metastore fixture, and transcript values shared by CUJs.""" from enum import StrEnum from pathlib import Path @@ -20,3 +20,17 @@ class CodingAgent(StrEnum): CLAUDE_CODE = "CODING_AGENT_CLAUDE_CODE" CODEX = "CODING_AGENT_CODEX" + + +CODING_AGENT_BY_CLI_NAME = { + CLAUDE: CodingAgent.CLAUDE_CODE, + CODEX: CodingAgent.CODEX, +} + +MODEL_PROVIDER_SERVICE_FIXTURES = { + CLAUDE: ("ug_e2e.providers.anthropic", "claude-haiku-4-5-20251001"), + CODEX: ("ug_e2e.providers.openai", "gpt-5-nano"), +} + +SANDBOX_MCP_SERVICE_NAME = "system.ai.sandbox" +WEB_SEARCH_MCP_SERVICE_NAME = "system.ai.web_search" diff --git a/tests/e2e_cuj/test_cuj2_mps_mcp.py b/tests/e2e_cuj/test_cuj2_mps_mcp.py new file mode 100644 index 000000000..0af6311df --- /dev/null +++ b/tests/e2e_cuj/test_cuj2_mps_mcp.py @@ -0,0 +1,337 @@ +"""CUJ2: configure two real agents from per-agent MPS resources with one MCP.""" + +from __future__ import annotations + +import json +import shutil +import tomllib +import uuid + +from databricks.sdk.service.catalog import ListModelProviderServicesRequestView + +from tests.integration.utils.evidence import ( + agent_sessions, + assert_no_terminal_api_error, + assistant_answers, + is_child_session, +) +from tests.integration.utils.terminal import AgentTerminal + +from .base import BaseCujTest +from .helpers.constants import ( + CLAUDE, + CODEX, + CODING_AGENT_BY_CLI_NAME, + INFERENCE_PATHS, + MODEL_PROVIDER_SERVICE_FIXTURES, + SANDBOX_MCP_SERVICE_NAME, + WEB_SEARCH_MCP_SERVICE_NAME, +) +from .helpers.workspace import Workspace + +WORKSPACE_URL = "https://dbc-0dcf95cf-e357.cloud.databricks.com" + + +def expected_agent_config(agent: str) -> dict: + mps_name, model = MODEL_PROVIDER_SERVICE_FIXTURES[agent] + return { + "agent": CODING_AGENT_BY_CLI_NAME[agent], + "config": { + "models": {"model_provider_service": mps_name}, + "default_models": {"default_model": model}, + "smart_routing": {"enabled": False}, + "tracing": {"enabled": False}, + }, + } + + +def _assert_cuj2_config(config: dict) -> None: + actual = Workspace.policy(config) + actual.pop("name", None) + expected = { + "spec_version": 1, + "default_agent": CODING_AGENT_BY_CLI_NAME[CODEX], + "enabled_agents": [expected_agent_config(CLAUDE), expected_agent_config(CODEX)], + "mcp_servers": {"names": [SANDBOX_MCP_SERVICE_NAME]}, + } + assert actual == expected, config + + +def _assert_mps_catalog(workspace) -> None: + listed = list( + workspace.client.ai_gateway.list_model_provider_services( + parent="schemas/ug_e2e.providers", + view=ListModelProviderServicesRequestView.FULL, + ) + ) + listed_by_name = { + item.name.removeprefix("model-provider-services/"): item for item in listed if item.name + } + for agent, (mps_name, model) in MODEL_PROVIDER_SERVICE_FIXTURES.items(): + service = workspace.client.ai_gateway.get_model_provider_service( + f"model-provider-services/{mps_name}" + ) + assert service.name and service.name.endswith(mps_name), service + config = service.config + assert config is not None, service + targets = config.targets or [] + assert [target.model for target in targets] == [model], service + assert config.allow_all_targets is not True, service + expected_type = ( + "EXTERNAL_MODEL_PROVIDER_TYPE_ANTHROPIC" + if agent == CLAUDE + else "EXTERNAL_MODEL_PROVIDER_TYPE_OPENAI" + ) + provider_type = getattr(config.provider_type, "value", config.provider_type) + assert provider_type == expected_type, config + expected_api = "anthropic/v1/messages" if agent == CLAUDE else "openai/v1/responses" + native_apis = [ + getattr(api, "value", api) + for target in targets + for api in (target.native_api_types or []) + ] + assert native_apis == [expected_api], targets + + matching = listed_by_name.get(mps_name) + assert matching is not None, [item.name for item in listed] + listed_targets = matching.config.targets if matching.config else None + assert [target.model for target in listed_targets or []] == [model], matching + + +def _assert_mcp_services(workspace) -> None: + for name in (SANDBOX_MCP_SERVICE_NAME, WEB_SEARCH_MCP_SERVICE_NAME): + service = workspace.client.ai_gateway.get_mcp_service(f"mcp-services/{name}") + assert service.name and service.name.endswith(name), service + + +def _assert_configured_files(session, workspace_url: str) -> None: + managed_cache = json.loads( + (session.home / ".ucode" / "managed-config.json").read_text(encoding="utf-8") + ) + managed_config = managed_cache.get("config") + assert isinstance(managed_config, dict), managed_cache + _assert_cuj2_config(managed_config) + + claude_settings = session.home / ".claude" / "ucode-settings.json" + assert claude_settings.is_file(), claude_settings + settings = json.loads(claude_settings.read_text(encoding="utf-8")) + claude_env = settings.get("env") + assert isinstance(claude_env, dict), settings + claude_mps, claude_model = MODEL_PROVIDER_SERVICE_FIXTURES[CLAUDE] + custom_headers = claude_env.get("ANTHROPIC_CUSTOM_HEADERS") + assert isinstance(custom_headers, str), claude_env + claude_headers = {} + for line in custom_headers.splitlines(): + name, separator, value = line.partition(":") + assert separator, line + claude_headers[name.strip().casefold()] = value.strip() + assert claude_headers["databricks-model-provider-service"] == claude_mps, claude_headers + assert claude_env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] == claude_model, claude_env + assert claude_env["ANTHROPIC_BASE_URL"] == workspace_url.rstrip("/") + "/ai-gateway/anthropic" + + codex_path = session.home / ".codex" / "ucode.config.toml" + assert codex_path.is_file(), codex_path + codex_config = tomllib.loads(codex_path.read_text(encoding="utf-8")) + assert codex_config["model_provider"] == "Databricks", codex_config + codex_provider = codex_config["model_providers"]["Databricks"] + assert codex_provider["wire_api"] == "responses", codex_provider + assert codex_provider["base_url"] == workspace_url.rstrip("/") + "/ai-gateway/codex/v1" + + +def _mcp_name_listed(output: str, name: str) -> bool: + dashed = name.replace(".", "-") + return any(name in line or dashed in line for line in output.splitlines()) + + +def _assert_generated_mcp_listings(session) -> None: + for agent in (CLAUDE, CODEX): + binary = shutil.which(agent, path=session.env["PATH"]) + assert binary, f"Required agent is not on the isolated PATH: {agent}" + listing = session.run("mcp", "list", binary=binary, timeout=120) + output = f"{listing.stdout}\n{listing.stderr}" + assert _mcp_name_listed(output, SANDBOX_MCP_SERVICE_NAME), output + assert not _mcp_name_listed(output, WEB_SEARCH_MCP_SERVICE_NAME), output + + +def _parent_transcripts(session, agent: str) -> dict[str, list[dict]]: + return { + path: records + for path, records in agent_sessions(session, agent).items() + if not is_child_session(agent, path, records) + } + + +def _task_complete(session, agent: str, marker: str) -> bool: + transcripts = _parent_transcripts(session, agent) + return any( + marker in answer + for records in transcripts.values() + for answer in assistant_answers(agent, records) + ) + + +def _assert_task_evidence(session, agent: str, marker: str) -> None: + transcripts = _parent_transcripts(session, agent) + answers = [ + answer for records in transcripts.values() for answer in assistant_answers(agent, records) + ] + assert any(marker in answer for answer in answers), answers + session.record( + f"cuj2-{agent}-evidence", + {"marker": marker, "answers": answers}, + ) + + +def _agent_prompt(marker: str) -> str: + return f"Reply with the exact marker `{marker}` and no other text. Do not use tools." + + +def _assert_inference_request(recorder, request, *, provider: str, model: str, marker: str) -> None: + assert request.headers["databricks-model-provider-service"] == provider + assert request.payload["model"] == model + assert marker.encode() in request.body, request.payload + response = recorder.response_for(request, timeout=240) + assert response.status_code == 200, response.status_code + assert response.body, "Inference response was empty" + + +class _Cuj2Base(BaseCujTest): + WORKSPACE_URL = WORKSPACE_URL + + +class TestCuj2Configuration(_Cuj2Base): + def test_cuj_configuration(self, cuj): + """Scenario: configure ug from the preconfigured two-agent MPS/MCP workspace. + + Expected: the read-only CodingAgentConfig selects the two exact MPS resources and sandbox + MCP; both native MPS APIs advertise their selected model, generated agent settings use + the exact provider/model values, and agent MCP listings exclude web_search. + """ + session, workspace, _ = cuj + published = workspace.config() + _assert_cuj2_config(published) + _assert_mps_catalog(workspace) + _assert_mcp_services(workspace) + + configured = session.run( + "configure", + "--workspace", + workspace.url, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=300, + ) + assert "Select coding agents to configure:" not in configured.stdout, configured.stdout + _assert_configured_files(session, workspace.url) + _assert_generated_mcp_listings(session) + + +class TestCuj2CodexInference(_Cuj2Base): + def test_cuj_codex_inference(self, cuj): + """Scenario: configure through the recorder and complete a real Codex inference. + + Expected: Codex sends a Responses request for gpt-5-nano with the + ug_e2e.providers.openai target header and receives its paired HTTP 200 response; its + parent transcript contains the final marker. + """ + session, workspace, recorder = cuj + published = workspace.config() + _assert_cuj2_config(published) + _assert_mps_catalog(workspace) + _assert_mcp_services(workspace) + + recorder.configure_session( + session, + [ + "configure", + "--skip-upgrade", + "--disable-databricks-ai-tools", + ], + ) + _assert_configured_files(session, workspace.url) + _assert_generated_mcp_listings(session) + + recorder.prepare_launch() + marker = f"CUJ2-CODEX-{uuid.uuid4().hex}" + checkpoint = recorder.checkpoint() + with AgentTerminal(session, CODEX, [str(session.binary), CODEX], "cuj2-codex") as tui: + tui.boot() + tui.submit(_agent_prompt(marker)) + + def completed(screen): + assert_no_terminal_api_error(screen) + assert "Do you want to proceed?" not in screen, screen + return _task_complete(session, CODEX, marker) + + tui.wait_for(completed, "completed Codex task", timeout=240) + tui.exit_normally() + + request = recorder.expect_request( + method="POST", + path=INFERENCE_PATHS[CODEX], + after=checkpoint, + timeout=240, + ) + _assert_inference_request( + recorder, + request, + provider=MODEL_PROVIDER_SERVICE_FIXTURES[CODEX][0], + model=MODEL_PROVIDER_SERVICE_FIXTURES[CODEX][1], + marker=marker, + ) + _assert_task_evidence(session, CODEX, marker) + + +class TestCuj2ClaudeInference(_Cuj2Base): + def test_cuj_claude_inference(self, cuj): + """Scenario: configure through the recorder and complete a real Claude inference. + + Expected: Claude sends a Messages request for claude-haiku-4-5-20251001 with the + ug_e2e.providers.anthropic target header and receives its paired HTTP 200 response; its + parent transcript contains the final marker. + """ + session, workspace, recorder = cuj + published = workspace.config() + _assert_cuj2_config(published) + _assert_mps_catalog(workspace) + _assert_mcp_services(workspace) + + recorder.configure_session( + session, + [ + "configure", + "--skip-upgrade", + "--disable-databricks-ai-tools", + ], + ) + _assert_configured_files(session, workspace.url) + _assert_generated_mcp_listings(session) + + recorder.prepare_launch() + marker = f"CUJ2-CLAUDE-{uuid.uuid4().hex}" + checkpoint = recorder.checkpoint() + with AgentTerminal(session, CLAUDE, [str(session.binary), CLAUDE], "cuj2-claude") as tui: + tui.boot() + tui.submit(_agent_prompt(marker)) + + def completed(screen): + assert_no_terminal_api_error(screen) + return _task_complete(session, CLAUDE, marker) + + tui.wait_for(completed, "completed Claude task", timeout=240) + tui.exit_normally() + + request = recorder.expect_request( + method="POST", + path=INFERENCE_PATHS[CLAUDE], + after=checkpoint, + timeout=240, + ) + _assert_inference_request( + recorder, + request, + provider=MODEL_PROVIDER_SERVICE_FIXTURES[CLAUDE][0], + model=MODEL_PROVIDER_SERVICE_FIXTURES[CLAUDE][1], + marker=marker, + ) + _assert_task_evidence(session, CLAUDE, marker) diff --git a/tests/integration/README.md b/tests/integration/README.md index 5f631a5eb..3fb8c5457 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -6,8 +6,8 @@ It reuses the session, terminal, file-task, and transcript helpers in `utils/`. Its Claude/Codex evidence helpers keep scenario-specific assertions separate from shared mechanics. Workspace config/catalog reads use its base class's Databricks SDK client. Configuration is read-only and checked for changes at teardown; concurrent readers need no reservation. -The smart-routing CUJ's four sessions cover routing and explicit-model bypass, not -routing-disabled behavior. +CUJ2 adds three separately collected cases for exact MPS/MCP configuration, Codex inference, +and Claude inference. This suite runs the **installed product** through subprocesses, against the same `UCODE_TEST_WORKSPACE` used by the existing e2e tests. It does not import `ucode`, diff --git a/tests/integration/test_ug_smart_routing_hooks.py b/tests/integration/test_ug_smart_routing_hooks.py index 67f55076c..58ddcffe3 100644 --- a/tests/integration/test_ug_smart_routing_hooks.py +++ b/tests/integration/test_ug_smart_routing_hooks.py @@ -131,7 +131,10 @@ def _toggle_with_skill(tui, session, agent: str, enabled: bool) -> None: lambda screen: ( "Smart Router" in screen and confirmation in screen - and assistant_answer_contains(session, agent, confirmation) + and ( + assistant_answer_contains(session, agent, confirmation) + or assistant_answer_contains(session, agent, f"**{state}** for this session") + ) ), f"the installed Smart Router skill to turn routing {state}", timeout=120, diff --git a/tests/test_agent_codex.py b/tests/test_agent_codex.py index bde491007..c7f1e866a 100644 --- a/tests/test_agent_codex.py +++ b/tests/test_agent_codex.py @@ -1926,6 +1926,11 @@ def test_writes_mcp_servers_preserving_model_keys(self, monkeypatch): assert doc["model_provider"] == "Databricks" assert doc["mcp_servers"]["system-ai-github"]["command"] == "/opt/ug" assert doc["mcp_servers"]["system-ai-github"]["args"][:2] == ["mcp-proxy", "--url"] + assert doc["mcp_servers"]["system-ai-github"]["env_vars"] == [ + "DATABRICKS_BEARER", + "DATABRICKS_BEARER_COMMAND", + "DATABRICKS_CONFIG_FILE", + ] assert captured["owned_paths"] == [["mcp_servers"]] assert captured["tool"] == "codex" @@ -2055,6 +2060,11 @@ def test_adds_and_preserves_other_tables(self, tmp_path, monkeypatch): assert dict(doc["mcp_servers"]["system-ai-github"]) == { "command": "ug", "args": ["mcp-proxy", "https://ws/svc"], + "env_vars": [ + "DATABRICKS_BEARER", + "DATABRICKS_BEARER_COMMAND", + "DATABRICKS_CONFIG_FILE", + ], } def test_removes_named_entries_only(self, tmp_path, monkeypatch): diff --git a/tests/test_databricks.py b/tests/test_databricks.py index 40a47fcc0..ca619be8b 100644 --- a/tests/test_databricks.py +++ b/tests/test_databricks.py @@ -318,6 +318,21 @@ def test_multiple_locations_preserve_order(self): class TestDiscoverClaudeModels: + def test_preserves_explicit_origin(self, monkeypatch): + captured = {} + + def fake_get(url, token, **kwargs): + captured["request"] = (url, token, kwargs) + return {"data": []}, None + + monkeypatch.setattr(db_mod, "_http_get_json", fake_get) + + db_mod.list_anthropic_models("http://127.0.0.1:43123", "token") + + assert captured["request"][0] == ( + "http://127.0.0.1:43123/ai-gateway/anthropic/v1/models?limit=1000" + ) + def test_lists_all_anthropic_model_ids_without_legacy_validation(self, monkeypatch): captured = {} payload = { @@ -1035,6 +1050,20 @@ def fake_get(url, token, **kwargs): assert "page_size=" in seen["url"] + def test_preserves_loopback_workspace_origin(self, monkeypatch): + seen: dict = {} + + def fake_get(url, token, **kwargs): + seen["url"] = url + return self._page(["main.s.one"]), None + + monkeypatch.setattr(db_mod, "_http_get_json", fake_get) + db_mod.list_model_provider_services("http://127.0.0.1:54321", "tok", use_cache=False) + + assert seen["url"].startswith( + "http://127.0.0.1:54321/api/2.1/unity-catalog/model-provider-services?" + ) + class TestGetModelProviderService: def test_addresses_the_service_directly(self, monkeypatch): @@ -1061,6 +1090,26 @@ def fake_get(url, token, **kwargs): assert service["allow_all_targets"] is True assert seen["url"].endswith("/model-provider-services/main.tien_le.openai_all") + def test_preserves_loopback_workspace_origin(self, monkeypatch): + seen: dict = {} + + def fake_get(url, token, **kwargs): + seen["url"] = url + return { + "name": "model-provider-services/main.tien_le.openai_all", + "config": {"provider_type": "EXTERNAL_MODEL_PROVIDER_TYPE_OPENAI"}, + }, None + + monkeypatch.setattr(db_mod, "_http_get_json", fake_get) + db_mod.get_model_provider_service( + "main.tien_le.openai_all", "http://127.0.0.1:54321", "tok" + ) + + assert seen["url"] == ( + "http://127.0.0.1:54321/api/2.1/unity-catalog/" + "model-provider-services/main.tien_le.openai_all" + ) + def test_missing_service_returns_the_reason(self, monkeypatch): monkeypatch.setattr( db_mod, "_http_get_json", lambda url, token, **kw: (None, "HTTP 404 Not Found")