From 5586380722fff864737db471abd178ff0fcee3d3 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Mon, 5 Oct 2026 19:15:20 +0000 Subject: [PATCH 1/7] AIGTWY-4878 Test MCP registration and read-only tool use in agent TUIs --- tests/e2e_cuj/MCP.md | 31 ++++++ tests/e2e_cuj/pytest.ini | 1 + tests/e2e_cuj/test_mcp_registration.py | 127 +++++++++++++++++++++++++ tests/integration/utils/mcp.py | 67 +++++++++++++ 4 files changed, 226 insertions(+) create mode 100644 tests/e2e_cuj/MCP.md create mode 100644 tests/e2e_cuj/test_mcp_registration.py create mode 100644 tests/integration/utils/mcp.py diff --git a/tests/e2e_cuj/MCP.md b/tests/e2e_cuj/MCP.md new file mode 100644 index 000000000..3e0fe3c0f --- /dev/null +++ b/tests/e2e_cuj/MCP.md @@ -0,0 +1,31 @@ +# MCP registration CUJ + +The dedicated workspace must publish `mcp_servers.unity_catalog_location = ug_e2e.tools` +for both agents. Fixtures are provisioned separately; this suite does not modify them. + +| Service | Read-only tool | +| --- | --- | +| `ug_e2e.tools.fixture_reader` | `read_fixture(run_id)` | +| `ug_e2e.tools.fixture_metadata` | `describe_fixture(run_id)` | + +Each journey configures installed UG, opens the real agent MCP inventory, and requires +connected servers with their tools loaded. Claude opens each server's details and tools +view; Codex uses `/mcp verbose`. Both then call the two tools with a fresh run ID and return +their receipts. Expected receipts are computed locally and withheld from the prompt. +Terminal actions, rendered screens, and native transcripts are recorded by the harness. + +CI runs one lane per agent using the existing `UG_CUJ3_WORKSPACE` and CUJ SP secrets. +The lanes run after catalog discovery, even if it fails, and share its workspace +concurrency lock. Each lane uploads terminal and native-transcript evidence. + +Run on a clean POSIX host with the CUJ SP credentials and dedicated workspace: + +```bash +python3.12 scripts/run_integration.py --suite e2e-cuj \ + --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ + --workspace "$CATALOG_DISCOVERY_WORKSPACE" \ + -- -m 'managed and mcp_registration and workspace_isolated' +``` + +Only live passes establish coverage. Out-of-scope exclusion and UG status/list checks +remain separate follow-ups. Skill storage and provisioning are not part of this suite. diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini index 008cc4a9e..ae59c16cd 100644 --- a/tests/e2e_cuj/pytest.ini +++ b/tests/e2e_cuj/pytest.ini @@ -5,6 +5,7 @@ addopts = --strict-markers --tb=short markers = managed: requires an admin CodingAgentConfig published on the selected workspace catalog_discovery: managed schema-scoped model discovery and inference + mcp_registration: managed MCP registration, connection, and tool use workspace_isolated: reads preconfigured CUJ workspace state with isolated local sessions tui: real interactive terminal and completed task claude: only runs when Claude Code is explicitly selected diff --git a/tests/e2e_cuj/test_mcp_registration.py b/tests/e2e_cuj/test_mcp_registration.py new file mode 100644 index 000000000..fc036e05f --- /dev/null +++ b/tests/e2e_cuj/test_mcp_registration.py @@ -0,0 +1,127 @@ +"""Managed MCP registration, connection, and tool use in both real agent TUIs.""" + +import hashlib +import json +import os +import uuid + +import pytest +from utils.evidence import ( + agent_sessions, + assert_no_terminal_api_error, + assistant_answers, + is_child_session, +) +from utils.mcp import open_claude_mcp_inventory, open_codex_mcp_inventory +from utils.terminal import AgentTerminal + +from .base import BaseCujTest + +pytestmark = [pytest.mark.managed, pytest.mark.mcp_registration, pytest.mark.workspace_isolated] + +SERVICES = { + "ug_e2e.tools.fixture_metadata": "describe_fixture", + "ug_e2e.tools.fixture_reader": "read_fixture", +} + + +class McpFixtureTask: + def __init__(self): + run_id = uuid.uuid4().hex + self.expected = { + tool: hashlib.sha256( + b"unity-gateway-cuj-fixture\0" + + tool.encode("ascii") + + b"\0" + + run_id.encode("ascii") + ).hexdigest() + for tool in SERVICES.values() + } + calls = "; ".join( + f"call {tool} on {service.replace('.', '-')} with run_id={run_id}" + for service, tool in SERVICES.items() + ) + self.prompt = ( + f"Use these read-only MCP tools: {calls}. " + "Do not use shell commands, files, or subagents. Reply with only a JSON object " + "mapping each tool name to the text it returned, without code fences." + ) + + def completed(self, session, agent: str) -> bool: + for path, records in agent_sessions(session, agent).items(): + if is_child_session(agent, path, records): + continue + for answer in assistant_answers(agent, records): + try: + observed = json.loads(answer) + except json.JSONDecodeError: + continue + if observed == self.expected: + return True + return False + + +def _wait_for_mcp_task(tui, task): + def completed(screen): + assert_no_terminal_api_error(screen) + return task.completed(tui.session, tui.agent) + + tui.wait_for( + completed, + "a completed assistant answer with both verified MCP receipts", + timeout=240, + ) + + +class TestMcpRegistration(BaseCujTest): + WORKSPACE_URL = os.environ.get("UCODE_TEST_WORKSPACE", "") + + @pytest.mark.claude + @pytest.mark.tui + def test_mcp_registration_claude_servers_connect_and_work(self, live_session, workspace): + """Scenario: configure UG, inspect Claude's MCP menu, then call both fixture tools. + + Expected: both scoped servers are registered, connected, and expose their tools; + a parent assistant answer contains the correct receipts for a fresh run ID. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = McpFixtureTask() + allowed_tools = ",".join( + f"mcp__{service.replace('.', '-')}__{tool}" for service, tool in SERVICES.items() + ) + with AgentTerminal( + session, + "claude", + [str(session.binary), "claude", "--", "--allowedTools", allowed_tools], + "mcp-registration-claude", + ) as tui: + tui.boot() + open_claude_mcp_inventory(tui, SERVICES) + tui.submit(task.prompt) + _wait_for_mcp_task(tui, task) + tui.exit_normally() + assert task.completed(session, "claude") + session.assert_not_routed() + + @pytest.mark.codex + @pytest.mark.tui + def test_mcp_registration_codex_servers_connect_and_work(self, live_session, workspace): + """Scenario: configure UG, inspect Codex's MCP inventory, then call both fixture tools. + + Expected: both scoped servers are registered, connected, and expose their tools; + a completed parent turn returns the correct receipts for a fresh run ID. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = McpFixtureTask() + with AgentTerminal( + session, "codex", [str(session.binary), "codex"], "mcp-registration-codex" + ) as tui: + tui.boot() + open_codex_mcp_inventory(tui, SERVICES) + tui.submit(task.prompt) + _wait_for_mcp_task(tui, task) + tui.exit_normally() + assert task.completed(session, "codex") + session.assert_not_routed() diff --git a/tests/integration/utils/mcp.py b/tests/integration/utils/mcp.py new file mode 100644 index 000000000..89297a1e5 --- /dev/null +++ b/tests/integration/utils/mcp.py @@ -0,0 +1,67 @@ +"""Inspect MCP inventories through the real agent terminals.""" + +import re + + +def open_claude_mcp_inventory(tui, services: dict[str, str]) -> None: + tui.submit("/mcp") + tui.wait_for( + lambda screen: "Manage MCP servers" in screen, + "Claude's MCP menu", + timeout=120, + ) + for service, tool in services.items(): + name = service.replace(".", "-") + tui.choose("Manage MCP servers", name) + tui.wait_for( + lambda screen, name=name: ( + name in screen + and re.search(r"(?m)^\s*Status:\s*(?:[✓✔]\s*)?connected\s*$", screen) + and "View tools" in screen + and "tools fetch failed" not in screen + ), + f"{name} connected with tools loaded", + timeout=120, + ) + tui.actions.append({"reason": f"mcp-connected-{name}", "screen": tui.visible}) + tui.choose("MCP Server", "View tools") + tui.wait_for( + lambda screen, tool=tool: re.search(rf"\b{re.escape(tool)}\b", screen), + f"{tool} in {name}'s tools menu", + ) + tui.actions.append({"reason": f"mcp-tools-{name}", "screen": tui.visible}) + tui.send("\x1b", "return from the MCP tools view") + tui.wait_for( + lambda screen: "MCP Server" in screen or "Manage MCP servers" in screen, + "the MCP server detail or inventory", + ) + if "Manage MCP servers" not in tui.visible: + tui.send("\x1b", "return to the MCP inventory") + tui.wait_for( + lambda screen: "Manage MCP servers" in screen, + "Claude's MCP inventory", + ) + tui.send("\x1b", "close Claude's MCP menu") + tui.wait_for( + lambda screen: ( + "Manage MCP servers" not in screen and re.search(r"(?m)^\s*[❯›>]\s*(?!\d+[.)])", screen) + ), + "Claude's prompt after closing the MCP menu", + ) + + +def open_codex_mcp_inventory(tui, services: dict[str, str]) -> None: + tui.submit("/mcp verbose") + for service, tool in services.items(): + name = re.escape(service.replace(".", "-")) + tui.wait_for( + lambda screen, name=name, tool=tool: re.search( + rf"(?m)^\s*• {name}: connected \(1 tool\)\s*\n" + rf"\s*• Auth: [^\n]*\n" + rf"\s*• Tools: {re.escape(tool)}\s*$", + screen, + ), + f"{service} connected with {tool} loaded in Codex's MCP inventory", + timeout=120, + ) + tui.actions.append({"reason": "codex-mcp-inventory", "screen": tui.visible}) From a6d1c378da2f0a501185ee67e51c99ff91224d4f Mon Sep 17 00:00:00 2001 From: Tien Le Date: Mon, 5 Oct 2026 19:58:11 +0000 Subject: [PATCH 2/7] AIGTWY-4878 Fix MCP discovery CI and forward Codex proxy auth --- tests/e2e_cuj/MCP.md | 8 +++-- tests/e2e_cuj/test_mcp_registration.py | 10 ++++-- tests/integration/utils/mcp.py | 45 +++++++++++++++++++++++--- tests/test_agent_codex.py | 10 ++++++ 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/tests/e2e_cuj/MCP.md b/tests/e2e_cuj/MCP.md index 3e0fe3c0f..f2adafdf7 100644 --- a/tests/e2e_cuj/MCP.md +++ b/tests/e2e_cuj/MCP.md @@ -14,6 +14,10 @@ view; Codex uses `/mcp verbose`. Both then call the two tools with a fresh run I their receipts. Expected receipts are computed locally and withheld from the prompt. Terminal actions, rendered screens, and native transcripts are recorded by the harness. +Both inventories must omit `ug_e2e.other_tools`, including the accessible +`fixture_decoy` service and its `decoy_status` tool. Claude traverses the server menu +until it wraps; Codex includes rendered scrollback from `/mcp verbose`. + CI runs one lane per agent using the existing `UG_CUJ3_WORKSPACE` and CUJ SP secrets. The lanes run after catalog discovery, even if it fails, and share its workspace concurrency lock. Each lane uploads terminal and native-transcript evidence. @@ -27,5 +31,5 @@ python3.12 scripts/run_integration.py --suite e2e-cuj \ -- -m 'managed and mcp_registration and workspace_isolated' ``` -Only live passes establish coverage. Out-of-scope exclusion and UG status/list checks -remain separate follow-ups. Skill storage and provisioning are not part of this suite. +Only live passes establish coverage. UG status/list checks remain a separate follow-up. +Skill storage and provisioning are not part of this suite. diff --git a/tests/e2e_cuj/test_mcp_registration.py b/tests/e2e_cuj/test_mcp_registration.py index fc036e05f..0d4d1eadf 100644 --- a/tests/e2e_cuj/test_mcp_registration.py +++ b/tests/e2e_cuj/test_mcp_registration.py @@ -82,6 +82,7 @@ def test_mcp_registration_claude_servers_connect_and_work(self, live_session, wo """Scenario: configure UG, inspect Claude's MCP menu, then call both fixture tools. Expected: both scoped servers are registered, connected, and expose their tools; + no ug_e2e.other_tools server or decoy tool appears in the complete inventory; a parent assistant answer contains the correct receipts for a fresh run ID. """ session = live_session @@ -97,7 +98,9 @@ def test_mcp_registration_claude_servers_connect_and_work(self, live_session, wo "mcp-registration-claude", ) as tui: tui.boot() - open_claude_mcp_inventory(tui, SERVICES) + inventory = open_claude_mcp_inventory(tui, SERVICES) + assert "ug_e2e-other_tools" not in inventory, inventory + assert "decoy_status" not in inventory, inventory tui.submit(task.prompt) _wait_for_mcp_task(tui, task) tui.exit_normally() @@ -110,6 +113,7 @@ def test_mcp_registration_codex_servers_connect_and_work(self, live_session, wor """Scenario: configure UG, inspect Codex's MCP inventory, then call both fixture tools. Expected: both scoped servers are registered, connected, and expose their tools; + no ug_e2e.other_tools server or decoy tool appears in the complete inventory; a completed parent turn returns the correct receipts for a fresh run ID. """ session = live_session @@ -119,7 +123,9 @@ def test_mcp_registration_codex_servers_connect_and_work(self, live_session, wor session, "codex", [str(session.binary), "codex"], "mcp-registration-codex" ) as tui: tui.boot() - open_codex_mcp_inventory(tui, SERVICES) + inventory = open_codex_mcp_inventory(tui, SERVICES) + assert "ug_e2e-other_tools" not in inventory, inventory + assert "decoy_status" not in inventory, inventory tui.submit(task.prompt) _wait_for_mcp_task(tui, task) tui.exit_normally() diff --git a/tests/integration/utils/mcp.py b/tests/integration/utils/mcp.py index 89297a1e5..85e550db0 100644 --- a/tests/integration/utils/mcp.py +++ b/tests/integration/utils/mcp.py @@ -2,20 +2,43 @@ import re +import pyte -def open_claude_mcp_inventory(tui, services: dict[str, str]) -> None: + +def open_claude_mcp_inventory(tui, services: dict[str, str]) -> str: tui.submit("/mcp") tui.wait_for( - lambda screen: "Manage MCP servers" in screen, + lambda screen: "Manage MCP servers" in screen and tui.selected_line(), "Claude's MCP menu", timeout=120, ) + tui.wait_for( + lambda screen: "connecting" not in screen.casefold(), + "Claude's MCP startup checks to finish before opening server details", + timeout=120, + ) + first = tui.selected_line() + screens = [] + for _ in range(100): + screens.append(tui.visible) + current = tui.selected_line() + tui.send("\x1b[B", "inspect the next MCP inventory row") + tui.wait_for( + lambda screen, current=current: tui.selected_line() != current, + "the next MCP inventory row", + ) + if tui.selected_line() == first: + break + else: + raise AssertionError("Claude's MCP inventory did not wrap within 100 rows") + inventory = "\n".join(screens) + tui.actions.append({"reason": "claude-mcp-inventory", "screen": inventory}) for service, tool in services.items(): name = service.replace(".", "-") tui.choose("Manage MCP servers", name) tui.wait_for( lambda screen, name=name: ( - name in screen + name in screen.casefold() and re.search(r"(?m)^\s*Status:\s*(?:[✓✔]\s*)?connected\s*$", screen) and "View tools" in screen and "tools fetch failed" not in screen @@ -48,9 +71,11 @@ def open_claude_mcp_inventory(tui, services: dict[str, str]) -> None: ), "Claude's prompt after closing the MCP menu", ) + return inventory -def open_codex_mcp_inventory(tui, services: dict[str, str]) -> None: +def open_codex_mcp_inventory(tui, services: dict[str, str]) -> str: + output_start = len(tui.output) tui.submit("/mcp verbose") for service, tool in services.items(): name = re.escape(service.replace(".", "-")) @@ -64,4 +89,14 @@ def open_codex_mcp_inventory(tui, services: dict[str, str]) -> None: f"{service} connected with {tool} loaded in Codex's MCP inventory", timeout=120, ) - tui.actions.append({"reason": "codex-mcp-inventory", "screen": tui.visible}) + screen = pyte.HistoryScreen(140, 60, history=1000) + pyte.Stream(screen).feed("".join(tui.output[output_start:])) + assert len(screen.history.top) < screen.history.size, "Codex's MCP inventory was truncated" + history = [ + "".join(line[column].data for column in range(screen.columns)) + for line in screen.history.top + ] + inventory = "\n".join([*history, *screen.display]) + assert "MCP Tools" in inventory, inventory + tui.actions.append({"reason": "codex-mcp-inventory", "screen": inventory}) + return inventory diff --git a/tests/test_agent_codex.py b/tests/test_agent_codex.py index 935e9712e..e0c6424a5 100644 --- a/tests/test_agent_codex.py +++ b/tests/test_agent_codex.py @@ -2100,6 +2100,16 @@ def test_token_provider_uses_custom_oauth_when_configured(self, monkeypatch): class TestWriteUserMcpServers: """Batched user-scope `[mcp_servers]` writes for the workspace-managed reconcile path.""" + def test_passes_external_auth_by_name_without_persisting_credentials(self, monkeypatch): + monkeypatch.setenv("DATABRICKS_BEARER", "test-bearer-not-for-persistence") + monkeypatch.setenv("DATABRICKS_BEARER_COMMAND", "test-token-command") + + assert codex.managed_mcp_entry(["ug", "mcp-proxy", "https://ws/svc"]) == { + "command": "ug", + "args": ["mcp-proxy", "https://ws/svc"], + "env_vars": ["DATABRICKS_BEARER", "DATABRICKS_BEARER_COMMAND"], + } + def test_adds_and_preserves_other_tables(self, tmp_path, monkeypatch): path = tmp_path / "config.toml" path.write_text('model = "gpt-5"\n\n[mcp_servers.mine]\ncommand = "x"\nargs = []\n') From 63b76f5ff64f4e004ad4103da5b0f472af5fa137 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Tue, 6 Oct 2026 13:43:24 +0000 Subject: [PATCH 3/7] Use shared dedicated CUJ framework for MCP registration --- tests/e2e_cuj/MCP.md | 18 ++++++------ tests/e2e_cuj/test_mcp_registration.py | 39 +++++++++++++------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/tests/e2e_cuj/MCP.md b/tests/e2e_cuj/MCP.md index f2adafdf7..25afe0ba6 100644 --- a/tests/e2e_cuj/MCP.md +++ b/tests/e2e_cuj/MCP.md @@ -18,18 +18,18 @@ Both inventories must omit `ug_e2e.other_tools`, including the accessible `fixture_decoy` service and its `decoy_status` tool. Claude traverses the server menu until it wraps; Codex includes rendered scrollback from `/mcp verbose`. -CI runs one lane per agent using the existing `UG_CUJ3_WORKSPACE` and CUJ SP secrets. -The lanes run after catalog discovery, even if it fails, and share its workspace -concurrency lock. Each lane uploads terminal and native-transcript evidence. - -Run on a clean POSIX host with the CUJ SP credentials and dedicated workspace: +Run on a clean disposable POSIX host with `ug`, `claude`, `codex`, and `databricks` on `PATH`. +Set `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET`, install `pexpect==4.9.0` and +`pyte==0.8.2`, and use a host without machine-wide agent settings. The test uses +the dedicated workspace URL declared on `TestMcpRegistration`; no workspace URL environment +variable is required. ```bash -python3.12 scripts/run_integration.py --suite e2e-cuj \ - --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ - --workspace "$CATALOG_DISCOVERY_WORKSPACE" \ - -- -m 'managed and mcp_registration and workspace_isolated' +uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest \ + --confcutdir=tests/e2e_cuj tests/e2e_cuj \ + -m 'managed and mcp_registration and workspace_isolated' -v ``` +CI discovers these tests in the shared `dedicated-cuj` job alongside catalog discovery. Only live passes establish coverage. UG status/list checks remain a separate follow-up. Skill storage and provisioning are not part of this suite. diff --git a/tests/e2e_cuj/test_mcp_registration.py b/tests/e2e_cuj/test_mcp_registration.py index 0d4d1eadf..ffb041f73 100644 --- a/tests/e2e_cuj/test_mcp_registration.py +++ b/tests/e2e_cuj/test_mcp_registration.py @@ -2,20 +2,21 @@ import hashlib import json -import os import uuid import pytest -from utils.evidence import ( + +from tests.integration.utils.evidence import ( agent_sessions, assert_no_terminal_api_error, assistant_answers, is_child_session, ) -from utils.mcp import open_claude_mcp_inventory, open_codex_mcp_inventory -from utils.terminal import AgentTerminal +from tests.integration.utils.mcp import open_claude_mcp_inventory, open_codex_mcp_inventory from .base import BaseCujTest +from .helpers.constants import CLAUDE, CODEX +from .helpers.terminal import Terminal pytestmark = [pytest.mark.managed, pytest.mark.mcp_registration, pytest.mark.workspace_isolated] @@ -64,6 +65,9 @@ def completed(self, session, agent: str) -> bool: def _wait_for_mcp_task(tui, task): def completed(screen): assert_no_terminal_api_error(screen) + assert "Do you want to proceed?" not in screen, ( + "Unexpected permission request; inspect the actual command:\n" + screen + ) return task.completed(tui.session, tui.agent) tui.wait_for( @@ -74,28 +78,27 @@ def completed(screen): class TestMcpRegistration(BaseCujTest): - WORKSPACE_URL = os.environ.get("UCODE_TEST_WORKSPACE", "") + WORKSPACE_URL = "https://dbc-bbdd5508-648e.cloud.databricks.com" @pytest.mark.claude @pytest.mark.tui - def test_mcp_registration_claude_servers_connect_and_work(self, live_session, workspace): + def test_mcp_registration_claude_servers_connect_and_work(self, cuj): """Scenario: configure UG, inspect Claude's MCP menu, then call both fixture tools. Expected: both scoped servers are registered, connected, and expose their tools; no ug_e2e.other_tools server or decoy tool appears in the complete inventory; a parent assistant answer contains the correct receipts for a fresh run ID. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = McpFixtureTask() allowed_tools = ",".join( f"mcp__{service.replace('.', '-')}__{tool}" for service, tool in SERVICES.items() ) - with AgentTerminal( + with Terminal( session, - "claude", - [str(session.binary), "claude", "--", "--allowedTools", allowed_tools], "mcp-registration-claude", + [CLAUDE, "--", "--allowedTools", allowed_tools], ) as tui: tui.boot() inventory = open_claude_mcp_inventory(tui, SERVICES) @@ -104,24 +107,22 @@ def test_mcp_registration_claude_servers_connect_and_work(self, live_session, wo tui.submit(task.prompt) _wait_for_mcp_task(tui, task) tui.exit_normally() - assert task.completed(session, "claude") + assert task.completed(session, CLAUDE) session.assert_not_routed() @pytest.mark.codex @pytest.mark.tui - def test_mcp_registration_codex_servers_connect_and_work(self, live_session, workspace): + def test_mcp_registration_codex_servers_connect_and_work(self, cuj): """Scenario: configure UG, inspect Codex's MCP inventory, then call both fixture tools. Expected: both scoped servers are registered, connected, and expose their tools; no ug_e2e.other_tools server or decoy tool appears in the complete inventory; a completed parent turn returns the correct receipts for a fresh run ID. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = McpFixtureTask() - with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "mcp-registration-codex" - ) as tui: + with Terminal(session, "mcp-registration-codex", [CODEX]) as tui: tui.boot() inventory = open_codex_mcp_inventory(tui, SERVICES) assert "ug_e2e-other_tools" not in inventory, inventory @@ -129,5 +130,5 @@ def test_mcp_registration_codex_servers_connect_and_work(self, live_session, wor tui.submit(task.prompt) _wait_for_mcp_task(tui, task) tui.exit_normally() - assert task.completed(session, "codex") + assert task.completed(session, CODEX) session.assert_not_routed() From 91ba621a645fcf127784682af0d9065060d2ce70 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Tue, 6 Oct 2026 14:13:16 +0000 Subject: [PATCH 4/7] Move Codex MCP auth fix into separate prerequisite PR --- tests/test_agent_codex.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/test_agent_codex.py b/tests/test_agent_codex.py index e0c6424a5..935e9712e 100644 --- a/tests/test_agent_codex.py +++ b/tests/test_agent_codex.py @@ -2100,16 +2100,6 @@ def test_token_provider_uses_custom_oauth_when_configured(self, monkeypatch): class TestWriteUserMcpServers: """Batched user-scope `[mcp_servers]` writes for the workspace-managed reconcile path.""" - def test_passes_external_auth_by_name_without_persisting_credentials(self, monkeypatch): - monkeypatch.setenv("DATABRICKS_BEARER", "test-bearer-not-for-persistence") - monkeypatch.setenv("DATABRICKS_BEARER_COMMAND", "test-token-command") - - assert codex.managed_mcp_entry(["ug", "mcp-proxy", "https://ws/svc"]) == { - "command": "ug", - "args": ["mcp-proxy", "https://ws/svc"], - "env_vars": ["DATABRICKS_BEARER", "DATABRICKS_BEARER_COMMAND"], - } - def test_adds_and_preserves_other_tables(self, tmp_path, monkeypatch): path = tmp_path / "config.toml" path.write_text('model = "gpt-5"\n\n[mcp_servers.mine]\ncommand = "x"\nargs = []\n') From 8cc7a286150278fa716e5bfcff4975054e9e5fee Mon Sep 17 00:00:00 2001 From: Tien Le Date: Wed, 7 Oct 2026 04:00:37 +0000 Subject: [PATCH 5/7] Revert managed MCP test settings after each agent journey --- tests/e2e_cuj/test_mcp_registration.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/e2e_cuj/test_mcp_registration.py b/tests/e2e_cuj/test_mcp_registration.py index ffb041f73..0398ec803 100644 --- a/tests/e2e_cuj/test_mcp_registration.py +++ b/tests/e2e_cuj/test_mcp_registration.py @@ -13,9 +13,10 @@ is_child_session, ) from tests.integration.utils.mcp import open_claude_mcp_inventory, open_codex_mcp_inventory +from tests.integration.utils.terminal import TerminalProcess from .base import BaseCujTest -from .helpers.constants import CLAUDE, CODEX +from .helpers.constants import CLAUDE, CODEX, MANAGED_PATHS from .helpers.terminal import Terminal pytestmark = [pytest.mark.managed, pytest.mark.mcp_registration, pytest.mark.workspace_isolated] @@ -77,6 +78,26 @@ def completed(screen): ) +@pytest.fixture(autouse=True) +def _revert_mcp_test_state(cuj): + yield + session, _, _ = cuj + state_dir = session.home / ".ucode" + if any( + (state_dir / name).is_file() for name in ("state.json", "managed-backups/manifest.json") + ): + with TerminalProcess( + session, + "ug", + [str(session.binary), "revert"], + "mcp-registration-cleanup-revert", + ) as terminal: + terminal.finish() + assert not any(path.exists() for path in MANAGED_PATHS), ( + "MCP registration CUJ teardown left machine-wide agent settings" + ) + + class TestMcpRegistration(BaseCujTest): WORKSPACE_URL = "https://dbc-bbdd5508-648e.cloud.databricks.com" From 0e93af3f79647935e3b61880772e8fa2d1fa7f8d Mon Sep 17 00:00:00 2001 From: Tien Le Date: Wed, 7 Oct 2026 04:32:54 +0000 Subject: [PATCH 6/7] Test managed MCP listing for both agents --- tests/e2e_cuj/MCP.md | 11 ++++++--- tests/e2e_cuj/test_mcp_registration.py | 31 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/tests/e2e_cuj/MCP.md b/tests/e2e_cuj/MCP.md index 25afe0ba6..04e50ac77 100644 --- a/tests/e2e_cuj/MCP.md +++ b/tests/e2e_cuj/MCP.md @@ -1,4 +1,4 @@ -# MCP registration CUJ +# MCP registration and listing CUJ The dedicated workspace must publish `mcp_servers.unity_catalog_location = ug_e2e.tools` for both agents. Fixtures are provisioned separately; this suite does not modify them. @@ -8,7 +8,7 @@ for both agents. Fixtures are provisioned separately; this suite does not modify | `ug_e2e.tools.fixture_reader` | `read_fixture(run_id)` | | `ug_e2e.tools.fixture_metadata` | `describe_fixture(run_id)` | -Each journey configures installed UG, opens the real agent MCP inventory, and requires +The two TUI journeys configure installed UG, open the real agent MCP inventory, and require connected servers with their tools loaded. Claude opens each server's details and tools view; Codex uses `/mcp verbose`. Both then call the two tools with a fresh run ID and return their receipts. Expected receipts are computed locally and withheld from the prompt. @@ -18,6 +18,11 @@ Both inventories must omit `ug_e2e.other_tools`, including the accessible `fixture_decoy` service and its `decoy_status` tool. Claude traverses the server menu until it wraps; Codex includes rendered scrollback from `/mcp verbose`. +The command journey checks that `ug mcp list` shows both managed fixture rows for `claude, codex` +with aggregate `connected` status and no `ug_e2e.other_tools` servers. This combines Claude's +health check with Codex's `enabled` registration; Codex connectivity is verified by its TUI/tool +journey, not the aggregate label. Local-server counts are not asserted. + Run on a clean disposable POSIX host with `ug`, `claude`, `codex`, and `databricks` on `PATH`. Set `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET`, install `pexpect==4.9.0` and `pyte==0.8.2`, and use a host without machine-wide agent settings. The test uses @@ -31,5 +36,5 @@ uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest \ ``` CI discovers these tests in the shared `dedicated-cuj` job alongside catalog discovery. -Only live passes establish coverage. UG status/list checks remain a separate follow-up. +Only live passes establish coverage. `ug status` is not part of this CUJ. Skill storage and provisioning are not part of this suite. diff --git a/tests/e2e_cuj/test_mcp_registration.py b/tests/e2e_cuj/test_mcp_registration.py index 0398ec803..f48dda5aa 100644 --- a/tests/e2e_cuj/test_mcp_registration.py +++ b/tests/e2e_cuj/test_mcp_registration.py @@ -101,6 +101,37 @@ def _revert_mcp_test_state(cuj): class TestMcpRegistration(BaseCujTest): WORKSPACE_URL = "https://dbc-bbdd5508-648e.cloud.databricks.com" + def test_mcp_list_reports_managed_fixture_servers(self, cuj): + """Scenario: configure both agents and inspect unscoped `ug mcp list` output. + + Expected: both managed fixture rows list Claude/Codex and aggregate `connected` status; + no `ug_e2e.other_tools` server appears. Codex connectivity is verified by its TUI journey. + """ + session, workspace, _recorder = cuj + session.configure( + [ + "configure", + "--agents", + f"{CLAUDE},{CODEX}", + "--workspace", + workspace.url, + "--skip-upgrade", + ] + ) + listing = session.run("mcp", "list", timeout=120) + output = f"{listing.stdout}\n{listing.stderr}" + for service in SERVICES: + server_name = service.replace(".", "-") + rows = [ + " ".join(line.split()) + for line in output.splitlines() + if line.split()[:1] == [server_name] + ] + expected_row = f"{server_name} {service} (managed) {CLAUDE}, {CODEX} connected" + assert rows == [expected_row], output + assert "ug_e2e.other_tools" not in output, output + assert "ug_e2e-other_tools" not in output, output + @pytest.mark.claude @pytest.mark.tui def test_mcp_registration_claude_servers_connect_and_work(self, cuj): From 1df7a6e07c0f781c62e3faef8d523e154e8655bb Mon Sep 17 00:00:00 2001 From: Tien Le Date: Wed, 7 Oct 2026 05:12:38 +0000 Subject: [PATCH 7/7] Approve expected Codex MCP fixture calls through the TUI --- tests/e2e_cuj/test_mcp_registration.py | 57 +++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/tests/e2e_cuj/test_mcp_registration.py b/tests/e2e_cuj/test_mcp_registration.py index f48dda5aa..7188004d5 100644 --- a/tests/e2e_cuj/test_mcp_registration.py +++ b/tests/e2e_cuj/test_mcp_registration.py @@ -2,6 +2,7 @@ import hashlib import json +import re import uuid import pytest @@ -29,18 +30,18 @@ class McpFixtureTask: def __init__(self): - run_id = uuid.uuid4().hex + self.run_id = uuid.uuid4().hex self.expected = { tool: hashlib.sha256( b"unity-gateway-cuj-fixture\0" + tool.encode("ascii") + b"\0" - + run_id.encode("ascii") + + self.run_id.encode("ascii") ).hexdigest() for tool in SERVICES.values() } calls = "; ".join( - f"call {tool} on {service.replace('.', '-')} with run_id={run_id}" + f"call {tool} on {service.replace('.', '-')} with run_id={self.run_id}" for service, tool in SERVICES.items() ) self.prompt = ( @@ -64,11 +65,55 @@ def completed(self, session, agent: str) -> bool: def _wait_for_mcp_task(tui, task): + approved_calls = set() + expected_tools = {(service.replace(".", "-"), tool) for service, tool in SERVICES.items()} + def completed(screen): assert_no_terminal_api_error(screen) - assert "Do you want to proceed?" not in screen, ( - "Unexpected permission request; inspect the actual command:\n" + screen - ) + assert not any( + prompt in screen for prompt in ("Do you want to proceed?", "Do you want to allow") + ), "Unexpected permission request; inspect the actual command:\n" + screen + + if ( + "Allow for this session" in screen + and "Always allow" in screen + and "Cancel" in screen + and re.search(r"(?m)^[ \t]*[›❯>][ \t]*[1-4]\.", screen) + ): + questions = list(re.finditer(r"(?m)^[ \t]*(Allow\b[^\n?]+\?)[ \t]*$", screen)) + assert questions, "Unexpected approval menu; inspect the actual command:\n" + screen + question = questions[-1].group(1) + approval = re.fullmatch( + r'Allow the (?P\S+) MCP server to run tool "(?P[^"]+)"\?', + question, + ) + assert tui.agent == CODEX and approval, ( + "Unexpected Codex permission request; inspect the actual command:\n" + screen + ) + tail = screen[questions[-1].end() :] + run_match = re.search(r"(?m)^[ \t]*run_id:[ \t]*(\S+)[ \t]*$", tail) + selected = re.search( + r"(?m)^[ \t]*[›❯>][ \t]*([1-4])\.[ \t]*(Allow(?: for this session)?|Always allow|Cancel)\b", + tail, + ) + assert run_match and selected, ( + "Unexpected Codex approval menu; inspect the actual command:\n" + screen + ) + server = approval.group("server") + tool = approval.group("tool") + run_id = run_match.group(1) + assert (server, tool) in expected_tools and run_id == task.run_id, ( + "Unexpected MCP permission request; inspect the actual command:\n" + screen + ) + assert selected.group(1) == "1" and selected.group(2) == "Allow", ( + "Codex MCP approval must remain a per-call Allow; inspect the actual menu:\n" + + screen + ) + call = (server, tool, run_id) + if call not in approved_calls: + tui.send("\r", f"allow one MCP call: {server}.{tool}") + approved_calls.add(call) + return False return task.completed(tui.session, tui.agent) tui.wait_for(