From 780466559604064e0902b6ba96ad24a2d96e1829 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Thu, 1 Oct 2026 19:34:52 +0000 Subject: [PATCH 1/4] Add real managed CUJ3 provider override coverage --- tests/integration/pytest.ini | 1 + .../test_ug_claude_managed_model_discovery.py | 36 +++++++++++--- .../test_ug_codex_managed_model_discovery.py | 34 +++++++++++--- tests/integration/utils/managed.py | 47 +++++++++++++++++++ 4 files changed, 104 insertions(+), 14 deletions(-) diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 75fdc9621..ea25cc88d 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -10,6 +10,7 @@ markers = smoke: Databricks Hosted, custom OAuth CLI TUI, and headless prompt for each agent tui: real interactive terminal boot, keyboard input, exit and reopen workspace_isolated: requires a dedicated workspace instead of the shared integration workspace + cuj3: managed Claude and Codex provider-override guards against the dedicated CUJ3 workspace cuj7: configured Claude and Codex model discovery against the dedicated CUJ7 workspace claude: only runs when Claude Code is explicitly selected codex: only runs when Codex is explicitly selected diff --git a/tests/integration/test_ug_claude_managed_model_discovery.py b/tests/integration/test_ug_claude_managed_model_discovery.py index f674d225e..46190d543 100644 --- a/tests/integration/test_ug_claude_managed_model_discovery.py +++ b/tests/integration/test_ug_claude_managed_model_discovery.py @@ -1,8 +1,8 @@ """Claude managed-config CUJs for repository scenarios 1, 3, and 5. -The admin CodingAgentConfig is fetched once from the managed workspace, its Claude model source is -set to the dedicated test MPS, and the result is reused through ``UCODE_MANAGED_CONFIG_STUB`` in -each isolated session. Normalization, config writers, the gateway, and Claude Code remain real. +Cases 1 and 5 use the managed-config fixture seam to cover model-discovery shapes that the shared +workspace does not publish. Case 3 runs against the selected workspace's published config, which +must be the dedicated CUJ3 managed workspace. """ import json @@ -12,6 +12,7 @@ from utils.constants import MANAGED_CLAUDE_PROVIDER_SERVICE from utils.managed import ( fetch_managed_config_stub, + fetch_published_managed_config, is_managed_config_control_plane_cache, use_managed_config_stub, ) @@ -22,7 +23,7 @@ ) from utils.terminal import AgentTerminal -pytestmark = [pytest.mark.managed_fixture, pytest.mark.claude] +pytestmark = pytest.mark.claude @pytest.fixture(scope="module") @@ -46,11 +47,16 @@ def _managed_claude_provider_catalog(workspace): ) -@pytest.fixture(autouse=True) +@pytest.fixture def _managed_claude_config(live_session, _managed_claude_config_stub): use_managed_config_stub(live_session, _managed_claude_config_stub) +@pytest.fixture(scope="module") +def _published_managed_config(workspace): + return fetch_published_managed_config(workspace, os.environ["DATABRICKS_BEARER"]) + + def _claude_state_and_agent_files(session): paths = [session.home / ".claude.json"] for directory in (session.home / ".ucode", session.home / ".claude"): @@ -124,6 +130,8 @@ def _assert_managed_provider_in_picker( ), screen +@pytest.mark.managed_fixture +@pytest.mark.usefixtures("_managed_claude_config") @pytest.mark.tui def test_case_01_managed_claude_uses_admin_discovery_after_configure( live_session, workspace, _managed_claude_provider_catalog @@ -160,6 +168,8 @@ def test_case_01_managed_claude_uses_admin_discovery_after_configure( _assert_managed_provider_in_picker(session, workspace, screen, _managed_claude_provider_catalog) +@pytest.mark.managed_fixture +@pytest.mark.usefixtures("_managed_claude_config") @pytest.mark.tui def test_case_01_fresh_managed_claude_uses_admin_discovery( live_session, workspace, _managed_claude_provider_catalog @@ -186,7 +196,12 @@ def test_case_01_fresh_managed_claude_uses_admin_discovery( _assert_managed_provider_in_picker(session, workspace, screen, _managed_claude_provider_catalog) -def test_case_03_managed_claude_rejects_provider_override(live_session, workspace, claude_provider): +@pytest.mark.managed +@pytest.mark.cuj3 +@pytest.mark.workspace_isolated +def test_case_03_managed_claude_rejects_provider_override( + live_session, workspace, claude_provider, _published_managed_config +): """Scenario: configure managed Claude, then pass --provider. Expected: ug rejects the override without changing agent-owned state/files. @@ -215,8 +230,11 @@ def test_case_03_managed_claude_rejects_provider_override(live_session, workspac _assert_rejected_before_claude_started(session, result, before) +@pytest.mark.managed +@pytest.mark.cuj3 +@pytest.mark.workspace_isolated def test_case_03_fresh_managed_claude_rejects_provider_override( - live_session, workspace, claude_provider + live_session, workspace, claude_provider, _published_managed_config ): """Scenario: pass --provider while launching managed Claude from fresh state. @@ -239,6 +257,8 @@ def test_case_03_fresh_managed_claude_rejects_provider_override( _assert_rejected_before_claude_started(session, result) +@pytest.mark.managed_fixture +@pytest.mark.usefixtures("_managed_claude_config") def test_case_05_managed_claude_rejects_model_location_override( live_session, workspace, parent_schema ): @@ -270,6 +290,8 @@ def test_case_05_managed_claude_rejects_model_location_override( _assert_rejected_before_claude_started(session, result, before) +@pytest.mark.managed_fixture +@pytest.mark.usefixtures("_managed_claude_config") def test_case_05_fresh_managed_claude_rejects_model_location_override( live_session, workspace, parent_schema ): diff --git a/tests/integration/test_ug_codex_managed_model_discovery.py b/tests/integration/test_ug_codex_managed_model_discovery.py index 980f69085..31259c541 100644 --- a/tests/integration/test_ug_codex_managed_model_discovery.py +++ b/tests/integration/test_ug_codex_managed_model_discovery.py @@ -1,8 +1,8 @@ """Codex managed-config CUJs for repository scenarios 2, 4, and 6. -The admin CodingAgentConfig is fetched once from the managed workspace, its Codex model source is -set to the dedicated test MPS, and the result is reused through ``UCODE_MANAGED_CONFIG_STUB`` in -each isolated session. Normalization, config writers, the gateway, and Codex remain real. +Cases 2 and 6 use the managed-config fixture seam to cover model-discovery shapes that the shared +workspace does not publish. Case 4 runs against the selected workspace's published config, which +must be the dedicated CUJ3 managed workspace. """ import json @@ -13,6 +13,7 @@ from utils.constants import MANAGED_CODEX_PROVIDER_SERVICE from utils.managed import ( fetch_managed_config_stub, + fetch_published_managed_config, is_managed_config_control_plane_cache, use_managed_config_stub, ) @@ -23,7 +24,7 @@ ) from utils.terminal import TerminalProcess -pytestmark = [pytest.mark.managed_fixture, pytest.mark.codex] +pytestmark = pytest.mark.codex @pytest.fixture(scope="module") @@ -47,11 +48,16 @@ def _managed_codex_provider_catalog(workspace): ) -@pytest.fixture(autouse=True) +@pytest.fixture def _managed_codex_config(live_session, _managed_codex_config_stub): use_managed_config_stub(live_session, _managed_codex_config_stub) +@pytest.fixture(scope="module") +def _published_managed_config(workspace): + return fetch_published_managed_config(workspace, os.environ["DATABRICKS_BEARER"]) + + def _codex_state_and_agent_files(session): paths = [] for directory in (session.home / ".ucode", session.home / ".codex"): @@ -128,6 +134,8 @@ def _assert_managed_provider_catalog(session, models, expected: CodexProviderCat assert models == list(catalog_ids), (models, catalog) +@pytest.mark.managed_fixture +@pytest.mark.usefixtures("_managed_codex_config") def test_case_02_managed_codex_uses_admin_discovery_after_configure( live_session, workspace, _managed_codex_provider_catalog ): @@ -173,6 +181,8 @@ def test_case_02_managed_codex_uses_admin_discovery_after_configure( assert not (session.home / ".ucode" / "codex-model-catalog.json").exists() +@pytest.mark.managed_fixture +@pytest.mark.usefixtures("_managed_codex_config") def test_case_02_managed_codex_uses_admin_discovery_from_fresh_state( live_session, workspace, _managed_codex_provider_catalog ): @@ -193,8 +203,11 @@ def test_case_02_managed_codex_uses_admin_discovery_from_fresh_state( assert app_models == models, (app_models, models) +@pytest.mark.managed +@pytest.mark.cuj3 +@pytest.mark.workspace_isolated def test_case_04_managed_codex_rejects_provider_override_after_configure( - live_session, workspace, codex_provider + live_session, workspace, codex_provider, _published_managed_config ): """Scenario: configure managed Codex, then pass a --provider override. @@ -225,8 +238,11 @@ def test_case_04_managed_codex_rejects_provider_override_after_configure( _assert_rejected_before_codex_started(session, result, before) +@pytest.mark.managed +@pytest.mark.cuj3 +@pytest.mark.workspace_isolated def test_case_04_managed_codex_rejects_provider_override_from_fresh_state( - live_session, workspace, codex_provider + live_session, workspace, codex_provider, _published_managed_config ): """Scenario: pass --workspace and a --provider override from fresh state. @@ -249,6 +265,8 @@ def test_case_04_managed_codex_rejects_provider_override_from_fresh_state( _assert_rejected_before_codex_started(session, result) +@pytest.mark.managed_fixture +@pytest.mark.usefixtures("_managed_codex_config") def test_case_06_managed_codex_rejects_model_location_override_after_configure( live_session, workspace, parent_schema ): @@ -281,6 +299,8 @@ def test_case_06_managed_codex_rejects_model_location_override_after_configure( _assert_rejected_before_codex_started(session, result, before) +@pytest.mark.managed_fixture +@pytest.mark.usefixtures("_managed_codex_config") def test_case_06_managed_codex_rejects_model_location_override_from_fresh_state( live_session, workspace, parent_schema ): diff --git a/tests/integration/utils/managed.py b/tests/integration/utils/managed.py index 3599d38f9..68cd9a1d3 100644 --- a/tests/integration/utils/managed.py +++ b/tests/integration/utils/managed.py @@ -13,6 +13,53 @@ MANAGED_CONFIGS_PATH = "/api/ai-gateway/v2/coding-agent-configs" +def fetch_published_managed_config(workspace: str, token: str) -> dict: + """Fetch and validate the selected workspace's published CodingAgentConfig. + + This is a read-only prerequisite check for journeys that must exercise the real managed + workspace. It deliberately does not alter, replay, or inject the response into a test + session; the installed CLI performs its own control-plane read. + """ + request = urllib.request.Request( + workspace.rstrip("/") + MANAGED_CONFIGS_PATH, + headers={"Authorization": f"Bearer {token}", "Accept": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.load(response) + + if isinstance(payload, dict): + configs = payload.get("coding_agent_configs") + elif isinstance(payload, list): + configs = payload + else: + configs = None + assert isinstance(configs, list) and configs, ( + "CUJ3 requires a published CodingAgentConfig in the selected workspace; " + "the workspace returned no config" + ) + assert len(configs) == 1, ( + "CUJ3 requires exactly one published CodingAgentConfig in the selected workspace; " + f"found {len(configs)}" + ) + config = configs[0] + assert isinstance(config, dict), "published CodingAgentConfig was not an object" + enabled_agents = config.get("enabled_agents") + assert isinstance(enabled_agents, list), ( + "published CodingAgentConfig had no enabled_agents list" + ) + agents = [ + entry.get("agent") + for entry in enabled_agents + if isinstance(entry, dict) and isinstance(entry.get("agent"), str) + ] + expected = {"CODING_AGENT_CLAUDE_CODE", "CODING_AGENT_CODEX"} + assert all(agents.count(agent) == 1 for agent in expected), ( + "CUJ3 requires one published CodingAgentConfig enabling both Claude and Codex; " + f"found {agents}" + ) + return config + + def assert_no_managed_config(payload: object) -> None: """Validate the real List response before claiming unmanaged-workspace coverage.""" configs = payload.get("coding_agent_configs", []) if isinstance(payload, dict) else payload From 67005edb7e36d55d4f714b81ff9b78ea439a5e70 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Thu, 1 Oct 2026 19:36:07 +0000 Subject: [PATCH 2/4] Add CUJ3 managed workspace CI lane --- .github/workflows/integration.yml | 42 +++++++++++++++---- tests/README.md | 12 +++--- tests/integration/README.md | 42 ++++++++++++++----- tests/test_integration_contract.py | 65 +++++++++++++++++++++++++++++- 4 files changed, 136 insertions(+), 25 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d89bac7ad..5dacd6452 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -212,11 +212,17 @@ jobs: - name: Run the selected live cases against the e2e workspace shell: bash run: | - case "$AGENT" in - claude) args=(--claude-version "$CLAUDE_VERSION") ;; - codex) args=(--codex-version "$CODEX_VERSION") ;; - opencode) args=(--opencode-version "$OPENCODE_VERSION") ;; - esac + if [[ "${INSTALL_BOTH_AGENTS:-}" == "true" ]]; then + # Managed config is published for both agents, so configure must be able to + # install both CLIs even though this matrix leg asserts only one agent. + args=(--claude-version "$CLAUDE_VERSION" --codex-version "$CODEX_VERSION") + else + case "$AGENT" in + claude) args=(--claude-version "$CLAUDE_VERSION") ;; + codex) args=(--codex-version "$CODEX_VERSION") ;; + opencode) args=(--opencode-version "$OPENCODE_VERSION") ;; + esac + fi if [[ -n "$DEPENDENCY" ]]; then args+=(--dependency "$DEPENDENCY") fi @@ -365,6 +371,26 @@ jobs: ARTIFACT_NAME: integration-cuj7-${{ matrix.agent }} steps: *live-steps + cuj3: + name: "CUJ3: managed journey · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }}" + if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && inputs.suite != 'tui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + agent: [claude, codex] + env: + UCODE_TEST_WORKSPACE: ${{ secrets.UG_CUJ3_WORKSPACE }} + DATABRICKS_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + DATABRICKS_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + DEPENDENCY: ${{ inputs.dependency }} + AGENT: ${{ matrix.agent }} + INSTALL_BOTH_AGENTS: "true" + TEST_MARKER: managed and cuj3 and workspace_isolated and ${{ matrix.agent }} + ARTIFACT_NAME: integration-cuj3-${{ matrix.agent }} + steps: *live-steps + # Non-blocking: continue-on-error and absent from the cujs `needs`, so a flaky OpenCode # leg cannot block merges. Add it to cujs once the lane is stable. opencode: @@ -432,7 +458,7 @@ jobs: uv run --no-project --python 3.12 python scripts/run_integration.py \ --python 3.12 --ug-version "$UG_VERSION" --entry-point "$ENTRY_POINT" \ --default-index "$PACKAGE_INDEX" --output "$RUNNER_TEMP/ug-integration" \ - "${args[@]}" -- -m "(managed or managed_fixture or workspace_switch) and $AGENT" + "${args[@]}" -- -m "(managed or managed_fixture or workspace_switch) and not workspace_isolated and $AGENT" - name: Upload managed test evidence if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -451,7 +477,7 @@ jobs: name: All integration tests if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} # installation-windows remains advisory until its initial platform issues are fixed. - needs: [installation, workspace, smoke, full, cuj7, managed] + needs: [installation, workspace, smoke, full, cuj7, cuj3, managed] runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -473,6 +499,8 @@ jobs: if suite != "smoke": required.append("full") required.append("cuj7") + if suite in {"full", "live"}: + required.append("cuj3") if suite in {"full", "live"}: required.append("managed") failed = [job for job in required if results[job]["result"] != "success"] diff --git a/tests/README.md b/tests/README.md index 4bdd2a8e5..cd652a76c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -115,6 +115,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | +| CUJ3 managed Claude/Codex journeys | Configure in the dedicated managed workspace with the published config enabled for both agents, from configured and fresh state | Each real-config journey uses the shared service-principal credentials, configures the selected agent, and rejects a personal provider override before the agent starts; the CI matrix runs two Claude and two Codex cases | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -131,9 +132,9 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_and_ucode_web_search_helpers_preserve_mcp_stdio` | Initialize and list tools through both web-search helper commands | Exactly the MCP JSON-RPC responses; no text/ANSI contamination; existing server/tool identities preserved; no model request | With Claude and Codex selected there are **68 live cases** (12 marked TUI cases), -**6 managed-workspace cases** (marker `managed`, run against workspaces that +**10 managed-workspace cases** (marker `managed`, run against workspaces that publish a CodingAgentConfig), **1 two-workspace case** (marker `workspace_switch`), -**25 managed-fixture cases** (marker `managed_fixture`, with only +**21 managed-fixture cases** (marker `managed_fixture`, with only the CodingAgentConfig input injected), and **7 installation checks**. The 14 retained numbered scenarios comprise **24 explicit journeys**: 12 managed configured/fresh executions and 12 unmanaged executions. Thirteen additional managed-fixture cases cover focused model, MCP, skills, @@ -180,8 +181,9 @@ Databricks Hosted configure/TUI, custom OAuth CLI TUI, and headless argument journeys for both agents, in two parallel jobs. After smoke finishes, the full suite runs 66 live cases across the shared Claude and Codex jobs. CUJ7 runs two matrix legs in the required dedicated-workspace job, one per agent, bringing live -coverage to 68 cases. Each job installs its agent and runs its selected cases -serially. +coverage to 68 cases. The dedicated CUJ3 matrix adds four managed-workspace cases, +two per agent. Each job installs its selected agent and runs its selected cases +serially; CUJ3 installs both agent CLIs because its published config enables both. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites share workspace capacity; overlapping their requests can still encounter rate limits. @@ -196,7 +198,7 @@ journey is deferred while its npm proxy access is blocked. Native TUI and managed-settings coverage remain deferred; a green installation check alone does not establish a successful live task. The `All integration tests` check requires every selected integration job to pass, including -CUJ7 and both managed-config lanes for full/live runs; full coverage does not depend on a label or +CUJ7, CUJ3, and both managed-config lanes for full/live runs; full coverage does not depend on a label or a manual request. The existing e2e workflow runs seven parallel shards: gateway checks plus one for diff --git a/tests/integration/README.md b/tests/integration/README.md index 4fc0da6e2..362a89e4b 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ This suite runs the **installed product** through subprocesses. Most live cases use the same `UCODE_TEST_WORKSPACE` as existing e2e tests; CUJ7 uses a dedicated -unmanaged workspace in CI. It does not import `ucode`, +unmanaged workspace and CUJ3 uses a dedicated managed workspace in CI. It does not import `ucode`, patch application functions, substitute agent executables, run a fake gateway, or construct ug state files. The normal test suite checks these boundaries. @@ -313,14 +313,15 @@ fresh Claude model-location task also pins Haiku 4.5 to keep inference inexpensi fresh `--provider` journeys check setup and launch with dummy MPS credentials, without inference. There are **68 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent -re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against +checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **10 managed-workspace cases** (the existing per-agent, idempotent +re-configure, cache-TTL, and two Claude defaults cases plus four CUJ3 cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** uses two real workspaces and checks skills MCP cleanup and a completed Claude task. -A further **25 `managed_fixture` -cases** use `UCODE_MANAGED_CONFIG_STUB`. Twelve explicit configured/fresh Claude and Codex +A further **21 `managed_fixture` +cases** use `UCODE_MANAGED_CONFIG_STUB`. Eight configured/fresh Claude and Codex discovery and source-override journeys fetch the published config once per agent, replace that -agent's static source with its dedicated MPS, and reuse the result. Thirteen other collected cases +agent's static source with its dedicated MPS, and reuse the result. The four CUJ3 configured/fresh +source-override journeys use the selected workspace's published config directly. Thirteen other collected cases cover focused model, MCP, skills, and lifecycle shapes, including per-agent model reconciliation and managed skill cleanup. The two Claude default-model cases read published MPS and Unity Catalog sources directly from `eng-ml-inference-batch-inference-us-west-2` and @@ -479,6 +480,16 @@ published CodingAgentConfig and must expose discoverable `system.ai` models for both agents. The dedicated job is required for full, live, and TUI CI suites. +CUJ3 adds four real-config managed journeys, two per agent, in its own required CI job. The +workspace URL comes from the `UG_CUJ3_WORKSPACE` repository secret and must be exactly +`https://dbc-bbdd5508-648e.cloud.databricks.com` (without a query string; a trailing slash is +not needed). The shared CUJ service-principal credentials come from `UG_CUJ_SP_CLIENT_ID` and +`UG_CUJ_SP_CLIENT_SECRET`, and the runner mints a short-lived workspace bearer from them. The +tests carry `managed`, `cuj3`, and `workspace_isolated` markers, so the shared Claude and Codex +jobs exclude them. CUJ3 runs for full/live suites only; it is not selected by smoke, TUI, or +installation runs. Because the published managed config enables both agents, each CUJ3 matrix leg +installs both pinned agent CLIs before selecting its one agent's tests. + The workspace check requires the secret to match `https://eng-ml-inference-team-us-east-1.cloud.databricks.com` (a trailing slash is accepted). It never changes the secret or switches workspaces. There is no @@ -489,8 +500,9 @@ each test; only explicit-model scenarios choose and record a discovered Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, -two agent jobs). Full runs 66 live cases in the shared Claude and Codex lanes and -CUJ7 in two dedicated-workspace matrix legs: +two agent jobs). Full runs 66 live cases in the shared Claude and Codex lanes, CUJ7 in two +dedicated-workspace matrix legs, and CUJ3's four managed cases in two dedicated-workspace +matrix legs: | Agent lane | Marker | Cases | | --- | --- | --- | @@ -498,23 +510,26 @@ CUJ7 in two dedicated-workspace matrix legs: | Codex | `live and codex and not workspace_isolated` | 36 | | CUJ7: unmanaged journey · Claude | `live and claude and cuj7 and workspace_isolated` | 1 | | CUJ7: unmanaged journey · Codex | `live and codex and cuj7 and workspace_isolated` | 1 | +| CUJ3: managed journey · Claude | `managed and cuj3 and workspace_isolated and claude` | 2 | +| CUJ3: managed journey · Codex | `managed and cuj3 and workspace_isolated and codex` | 2 | A non-blocking **OpenCode** job (`live and opencode`, one case) runs alongside them with `continue-on-error` and is not part of the required `cujs` gate until it is stable. -Each lane installs only its agent CLI, once, and runs all its configure, headless, +Each shared agent lane installs only its agent CLI, once, and runs all its configure, headless, commands, lifecycle, and applicable app-server journeys. Cases remain serial inside each fresh VM because configure/revert can touch machine-level settings; separate runners isolate those writes as well as the PTYs. Claude and Codex run in parallel, with at most one full-suite job per agent in a workflow run. This -avoids six serial job startups without overlapping same-agent shards. The two +avoids six serial job startups without overlapping same-agent shards. CUJ3 installs +both agent CLIs because its published config enables both. The two lanes still share workspace capacity with the concurrently running agent e2e shards and other PRs; this limit does not guarantee freedom from rate limits. No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, shared workspace validation, -smoke, both full lanes, both CUJ7 matrix legs, and both **Managed config** lanes to pass for full/live runs. Each tracing +smoke, both full lanes, both CUJ7 matrix legs, both CUJ3 matrix legs, and both **Managed config** lanes to pass for full/live runs. Each tracing journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke, TUI, and installation subsets do not select managed tests and do not require them. @@ -540,6 +555,11 @@ publish a Claude `system.ai` parent-schema source with Unity Catalog family defa fetches the config; the tests assert the generated private and OS-managed Claude settings after launch. Their exact required defaults and source headers are reflected in the tests. +The four CUJ3 real-config cases use the dedicated `UG_CUJ3_WORKSPACE` host described above and +carry `managed`, `cuj3`, and `workspace_isolated` markers. They run twice for Claude and twice for +Codex in the dedicated full/live matrix; the shared managed jobs explicitly exclude +`workspace_isolated`. + `test_unmanaged_claude_preserves_preexisting_family_defaults` is a live lifecycle journey against one real workspace. A read-only check first proves that the workspace publishes no CodingAgentConfig. The test then seeds `/etc/claude-code/managed-settings.json`, runs public diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 863da8c40..83e7ebe7c 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -45,9 +45,10 @@ def test_managed_integration_ci_is_blocking(): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" managed, gate = workflow.read_text().split("\n managed:\n", 1)[1].split("\n cujs:\n", 1) assert "continue-on-error:" not in managed + assert "not workspace_isolated" in managed needs = re.search(r"(?m)^ needs: \[([^\]]+)\]$", gate) assert needs is not None - assert "managed" in {job.strip() for job in needs.group(1).split(",")} + assert {"cuj3", "managed"} <= {job.strip() for job in needs.group(1).split(",")} assert ( "if: ${{ always() && (github.event_name != 'pull_request' || " "github.event.pull_request.head.repo.full_name == github.repository) }}" @@ -62,6 +63,19 @@ def test_windows_integration_ci_uses_shared_claude_version(): assert contents.count('"--claude-version", $env:CLAUDE_VERSION,') == 2 +def test_cuj3_integration_ci_uses_the_dedicated_managed_workspace(): + workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" + cuj3 = workflow.read_text().split("\n cuj3:\n", 1)[1].split("\n opencode:\n", 1)[0] + + assert "agent: [claude, codex]" in cuj3 + assert "secrets.UG_CUJ3_WORKSPACE" in cuj3 + assert "secrets.UG_CUJ_SP_CLIENT_ID" in cuj3 + assert "secrets.UG_CUJ_SP_CLIENT_SECRET" in cuj3 + assert 'INSTALL_BOTH_AGENTS: "true"' in cuj3 + assert "TEST_MARKER: managed and cuj3 and workspace_isolated and ${{ matrix.agent }}" in cuj3 + assert "inputs.suite != 'tui'" in cuj3 + + @pytest.mark.parametrize("suite", ["full", "live", "smoke", "tui", "installation"]) @pytest.mark.parametrize("managed_result", ["success", "failure", "cancelled", "skipped"]) def test_integration_ci_gate_requires_selected_managed_jobs(suite, managed_result): @@ -77,6 +91,7 @@ def test_integration_ci_gate_requires_selected_managed_jobs(suite, managed_resul "smoke", "full", "cuj7", + "cuj3", "managed", ) } @@ -102,6 +117,47 @@ def test_integration_ci_gate_requires_selected_managed_jobs(suite, managed_resul assert "All selected integration jobs passed:" in result.stdout +@pytest.mark.parametrize("suite", ["full", "live", "smoke", "tui", "installation"]) +@pytest.mark.parametrize("cuj3_result", ["success", "failure", "cancelled", "skipped"]) +def test_integration_ci_gate_requires_selected_cuj3_jobs(suite, cuj3_result): + workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" + gate = workflow.read_text().split("\n cujs:\n", 1)[1] + script = re.search(r" python3 - <<'PY'\n(.*?) PY", gate, re.DOTALL) + assert script is not None + results = { + job: {"result": "success"} + for job in ( + "installation", + "workspace", + "smoke", + "full", + "cuj7", + "cuj3", + "managed", + ) + } + results["cuj3"]["result"] = cuj3_result + for job in { + "installation": ("workspace", "smoke", "full", "cuj7", "cuj3"), + "smoke": ("full", "cuj7", "cuj3"), + "tui": ("smoke", "cuj3"), + }.get(suite, ()): + results[job]["result"] = "skipped" + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(script.group(1))], + env={"RESULTS": json.dumps(results), "SUITE": suite}, + capture_output=True, + text=True, + timeout=10, + ) + if suite in {"full", "live"} and cuj3_result != "success": + assert result.returncode != 0 + assert "Integration jobs did not pass: cuj3" in result.stderr + else: + assert result.returncode == 0, result.stderr + assert "All selected integration jobs passed:" in result.stdout + + @pytest.mark.parametrize("suite", ["full", "live", "tui", "smoke", "installation"]) def test_integration_ci_gate_requires_cuj7_for_live_suites(suite): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" @@ -116,6 +172,7 @@ def test_integration_ci_gate_requires_cuj7_for_live_suites(suite): "smoke", "full", "cuj7", + "cuj3", "managed", ) } @@ -220,8 +277,12 @@ def test_model_discovery_cases_match_current_launch_contract(): case = int(match.group(1)) seen.append(case) marks = module_marks | _markers(node.decorator_list) - expected = {"managed_fixture"} if case <= 6 else {"live"} + expected = ( + {"managed"} if case in {3, 4} else ({"managed_fixture"} if case <= 6 else {"live"}) + ) assert marks & {"managed_fixture", "managed", "live"} == expected, node.name + if case in {3, 4}: + assert {"managed", "cuj3", "workspace_isolated"} <= marks, node.name assert marks & {"claude", "codex"} == ({"claude"} if case % 2 else {"codex"}), node.name assert not any(arg.arg == "configured" for arg in node.args.args), node.name for value in ast.walk(node): From 9d04a71296d4a1e371c29152bdb2385d1ed9be3c Mon Sep 17 00:00:00 2001 From: Tien Le Date: Thu, 1 Oct 2026 19:37:14 +0000 Subject: [PATCH 3/4] Document CUJ3 workspace through its secret --- tests/integration/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index 362a89e4b..b6150e0e1 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -481,9 +481,8 @@ both agents. The dedicated job is required for full, live, and TUI CI suites. CUJ3 adds four real-config managed journeys, two per agent, in its own required CI job. The -workspace URL comes from the `UG_CUJ3_WORKSPACE` repository secret and must be exactly -`https://dbc-bbdd5508-648e.cloud.databricks.com` (without a query string; a trailing slash is -not needed). The shared CUJ service-principal credentials come from `UG_CUJ_SP_CLIENT_ID` and +workspace URL comes from the `UG_CUJ3_WORKSPACE` repository secret as a base HTTPS URL +without a query string. The shared CUJ service-principal credentials come from `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET`, and the runner mints a short-lived workspace bearer from them. The tests carry `managed`, `cuj3`, and `workspace_isolated` markers, so the shared Claude and Codex jobs exclude them. CUJ3 runs for full/live suites only; it is not selected by smoke, TUI, or From 2b1adbe3984ee7380373a97ecf48f9221d526610 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Thu, 1 Oct 2026 21:01:55 +0000 Subject: [PATCH 4/4] Exercise CUJ3 schema pointers across Claude and Codex --- tests/README.md | 10 +- tests/integration/README.md | 30 +- .../test_ug_claude_managed_model_discovery.py | 38 +- .../test_ug_codex_managed_model_discovery.py | 38 +- .../test_ug_cuj3_model_discovery.py | 498 ++++++++++++++++++ tests/integration/utils/cuj3.py | 365 +++++++++++++ tests/test_integration_contract.py | 5 +- 7 files changed, 906 insertions(+), 78 deletions(-) create mode 100644 tests/integration/test_ug_cuj3_model_discovery.py create mode 100644 tests/integration/utils/cuj3.py diff --git a/tests/README.md b/tests/README.md index cd652a76c..5d55d6c88 100644 --- a/tests/README.md +++ b/tests/README.md @@ -115,11 +115,11 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| CUJ3 managed Claude/Codex journeys | Configure in the dedicated managed workspace with the published config enabled for both agents, from configured and fresh state | Each real-config journey uses the shared service-principal credentials, configures the selected agent, and rejects a personal provider override before the agent starts; the CI matrix runs two Claude and two Codex cases | +| CUJ3 managed Claude/Codex journeys | Configure in the dedicated managed workspace with the published schema pointers for models, MCPs, and skills | Each agent discovers the scoped inventory, excludes accessible decoys, invokes each compatible in-scope model, calls both MCP tools, and uses both downloaded skills with native execution evidence | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | -| `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | +| Managed fixture Claude override cases (`test_managed_fixture_*`, `test_case_05_*`) | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | -| `test_case_04_*`, `test_case_06_*` | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state | +| Managed fixture Codex override cases (`test_managed_fixture_*`, `test_case_06_*`) | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state | | `test_ug_configure_managed_codex_catalog_fallback` | Configure from an injected managed response containing a GPT model absent from Codex's bundled catalog | Actionable metadata warning; conservative catalog entry for the unknown model; real Codex prompt on the valid default model | | `test_managed_fixture_codex_http_headers_in_managed_file` | Interactive PTY configure with injected managed `http_headers` for Codex | The specified header (`x-databricks-workspace`) lands in `model_providers.Databricks.http_headers` in `/etc/codex/managed_config.toml` with the exact admin value | | `test_managed_claude_mps_defaults_accompany_discovery`, `test_managed_claude_parent_schema_defaults_accompany_discovery` | Configure from the published admin config and launch Claude with MPS on `eng-ml-inference-batch-inference-us-west-2` and Unity Catalog discovery on `eng-ml-inference-ap-northeast-2`, respectively | Both generated settings files retain every admin-authored default alongside the source header; MPS pickers show labeled default rows first and the independently fetched catalog below, including models also used as defaults, while retaining catalog labels; only UC Opus/Sonnet family ids gain `[1m]` | @@ -181,8 +181,8 @@ Databricks Hosted configure/TUI, custom OAuth CLI TUI, and headless argument journeys for both agents, in two parallel jobs. After smoke finishes, the full suite runs 66 live cases across the shared Claude and Codex jobs. CUJ7 runs two matrix legs in the required dedicated-workspace job, one per agent, bringing live -coverage to 68 cases. The dedicated CUJ3 matrix adds four managed-workspace cases, -two per agent. Each job installs its selected agent and runs its selected cases +coverage to 68 cases. The dedicated CUJ3 matrix adds schema-pointer journeys +for both agents. Each job installs its selected agent and runs its selected cases serially; CUJ3 installs both agent CLIs because its published config enables both. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites diff --git a/tests/integration/README.md b/tests/integration/README.md index b6150e0e1..5b20bbf8c 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -313,15 +313,15 @@ fresh Claude model-location task also pins Haiku 4.5 to keep inference inexpensi fresh `--provider` journeys check setup and launch with dummy MPS credentials, without inference. There are **68 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **10 managed-workspace cases** (the existing per-agent, idempotent -re-configure, cache-TTL, and two Claude defaults cases plus four CUJ3 cases; marker `managed`) run against +checks** with Claude and Codex; selecting OpenCode adds one live headless case. Separate managed-workspace cases (the existing per-agent, idempotent +re-configure, cache-TTL, and two Claude defaults cases plus CUJ3; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** uses two real workspaces and checks skills MCP cleanup and a completed Claude task. A further **21 `managed_fixture` cases** use `UCODE_MANAGED_CONFIG_STUB`. Eight configured/fresh Claude and Codex discovery and source-override journeys fetch the published config once per agent, replace that -agent's static source with its dedicated MPS, and reuse the result. The four CUJ3 configured/fresh -source-override journeys use the selected workspace's published config directly. Thirteen other collected cases +agent's static source with its dedicated MPS, and reuse the result. CUJ3 exercises the selected +workspace's published schema-pointer config directly. Thirteen other collected cases cover focused model, MCP, skills, and lifecycle shapes, including per-agent model reconciliation and managed skill cleanup. The two Claude default-model cases read published MPS and Unity Catalog sources directly from `eng-ml-inference-batch-inference-us-west-2` and @@ -480,7 +480,7 @@ published CodingAgentConfig and must expose discoverable `system.ai` models for both agents. The dedicated job is required for full, live, and TUI CI suites. -CUJ3 adds four real-config managed journeys, two per agent, in its own required CI job. The +CUJ3 runs the published schema-pointer config for both agents in its own required CI job. The workspace URL comes from the `UG_CUJ3_WORKSPACE` repository secret as a base HTTPS URL without a query string. The shared CUJ service-principal credentials come from `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET`, and the runner mints a short-lived workspace bearer from them. The @@ -489,6 +489,13 @@ jobs exclude them. CUJ3 runs for full/live suites only; it is not selected by sm installation runs. Because the published managed config enables both agents, each CUJ3 matrix leg installs both pinned agent CLIs before selecting its one agent's tests. +The CUJ3 fixture inventory and exact managed config are in `fixtures/cuj/README.md`. Model +services in `ug_e2e.models` include the configured default and an additional compatible model +per agent. The test must invoke both with completed inference and native model-identity evidence. +It also checks accessible decoys in `ug_e2e.other_models`, `ug_e2e.other_tools`, and +`ug_e2e.other_skills` while rejecting them from the managed agent views. MCP and skill checks +require real calls and output values, not just listings. + The workspace check requires the secret to match `https://eng-ml-inference-team-us-east-1.cloud.databricks.com` (a trailing slash is accepted). It never changes the secret or switches workspaces. There is no @@ -500,7 +507,7 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs 66 live cases in the shared Claude and Codex lanes, CUJ7 in two -dedicated-workspace matrix legs, and CUJ3's four managed cases in two dedicated-workspace +dedicated-workspace matrix legs, and CUJ3's schema-pointer cases in two dedicated-workspace matrix legs: | Agent lane | Marker | Cases | @@ -509,8 +516,8 @@ matrix legs: | Codex | `live and codex and not workspace_isolated` | 36 | | CUJ7: unmanaged journey · Claude | `live and claude and cuj7 and workspace_isolated` | 1 | | CUJ7: unmanaged journey · Codex | `live and codex and cuj7 and workspace_isolated` | 1 | -| CUJ3: managed journey · Claude | `managed and cuj3 and workspace_isolated and claude` | 2 | -| CUJ3: managed journey · Codex | `managed and cuj3 and workspace_isolated and codex` | 2 | +| CUJ3: managed journey · Claude | `managed and cuj3 and workspace_isolated and claude` | 1 | +| CUJ3: managed journey · Codex | `managed and cuj3 and workspace_isolated and codex` | 1 | A non-blocking **OpenCode** job (`live and opencode`, one case) runs alongside them with `continue-on-error` and is not part of the required `cujs` gate until it is stable. @@ -554,10 +561,9 @@ publish a Claude `system.ai` parent-schema source with Unity Catalog family defa fetches the config; the tests assert the generated private and OS-managed Claude settings after launch. Their exact required defaults and source headers are reflected in the tests. -The four CUJ3 real-config cases use the dedicated `UG_CUJ3_WORKSPACE` host described above and -carry `managed`, `cuj3`, and `workspace_isolated` markers. They run twice for Claude and twice for -Codex in the dedicated full/live matrix; the shared managed jobs explicitly exclude -`workspace_isolated`. +The CUJ3 schema-pointer cases use the dedicated `UG_CUJ3_WORKSPACE` host described above and +carry `managed`, `cuj3`, and `workspace_isolated` markers. The dedicated full/live matrix runs +both agents; the shared managed jobs explicitly exclude `workspace_isolated`. `test_unmanaged_claude_preserves_preexisting_family_defaults` is a live lifecycle journey against one real workspace. A read-only check first proves that the workspace publishes no diff --git a/tests/integration/test_ug_claude_managed_model_discovery.py b/tests/integration/test_ug_claude_managed_model_discovery.py index 46190d543..e759823e2 100644 --- a/tests/integration/test_ug_claude_managed_model_discovery.py +++ b/tests/integration/test_ug_claude_managed_model_discovery.py @@ -1,8 +1,8 @@ """Claude managed-config CUJs for repository scenarios 1, 3, and 5. -Cases 1 and 5 use the managed-config fixture seam to cover model-discovery shapes that the shared -workspace does not publish. Case 3 runs against the selected workspace's published config, which -must be the dedicated CUJ3 managed workspace. +The admin CodingAgentConfig is fetched once from the managed workspace, its Claude model source is +set to the dedicated test MPS, and the result is reused through ``UCODE_MANAGED_CONFIG_STUB`` in +each isolated session. Normalization, config writers, the gateway, and Claude Code remain real. """ import json @@ -12,7 +12,6 @@ from utils.constants import MANAGED_CLAUDE_PROVIDER_SERVICE from utils.managed import ( fetch_managed_config_stub, - fetch_published_managed_config, is_managed_config_control_plane_cache, use_managed_config_stub, ) @@ -23,7 +22,7 @@ ) from utils.terminal import AgentTerminal -pytestmark = pytest.mark.claude +pytestmark = [pytest.mark.managed_fixture, pytest.mark.claude] @pytest.fixture(scope="module") @@ -47,16 +46,11 @@ def _managed_claude_provider_catalog(workspace): ) -@pytest.fixture +@pytest.fixture(autouse=True) def _managed_claude_config(live_session, _managed_claude_config_stub): use_managed_config_stub(live_session, _managed_claude_config_stub) -@pytest.fixture(scope="module") -def _published_managed_config(workspace): - return fetch_published_managed_config(workspace, os.environ["DATABRICKS_BEARER"]) - - def _claude_state_and_agent_files(session): paths = [session.home / ".claude.json"] for directory in (session.home / ".ucode", session.home / ".claude"): @@ -130,8 +124,6 @@ def _assert_managed_provider_in_picker( ), screen -@pytest.mark.managed_fixture -@pytest.mark.usefixtures("_managed_claude_config") @pytest.mark.tui def test_case_01_managed_claude_uses_admin_discovery_after_configure( live_session, workspace, _managed_claude_provider_catalog @@ -168,8 +160,6 @@ def test_case_01_managed_claude_uses_admin_discovery_after_configure( _assert_managed_provider_in_picker(session, workspace, screen, _managed_claude_provider_catalog) -@pytest.mark.managed_fixture -@pytest.mark.usefixtures("_managed_claude_config") @pytest.mark.tui def test_case_01_fresh_managed_claude_uses_admin_discovery( live_session, workspace, _managed_claude_provider_catalog @@ -196,12 +186,7 @@ def test_case_01_fresh_managed_claude_uses_admin_discovery( _assert_managed_provider_in_picker(session, workspace, screen, _managed_claude_provider_catalog) -@pytest.mark.managed -@pytest.mark.cuj3 -@pytest.mark.workspace_isolated -def test_case_03_managed_claude_rejects_provider_override( - live_session, workspace, claude_provider, _published_managed_config -): +def test_managed_fixture_claude_rejects_provider_override(live_session, workspace, claude_provider): """Scenario: configure managed Claude, then pass --provider. Expected: ug rejects the override without changing agent-owned state/files. @@ -230,11 +215,8 @@ def test_case_03_managed_claude_rejects_provider_override( _assert_rejected_before_claude_started(session, result, before) -@pytest.mark.managed -@pytest.mark.cuj3 -@pytest.mark.workspace_isolated -def test_case_03_fresh_managed_claude_rejects_provider_override( - live_session, workspace, claude_provider, _published_managed_config +def test_managed_fixture_fresh_claude_rejects_provider_override( + live_session, workspace, claude_provider ): """Scenario: pass --provider while launching managed Claude from fresh state. @@ -257,8 +239,6 @@ def test_case_03_fresh_managed_claude_rejects_provider_override( _assert_rejected_before_claude_started(session, result) -@pytest.mark.managed_fixture -@pytest.mark.usefixtures("_managed_claude_config") def test_case_05_managed_claude_rejects_model_location_override( live_session, workspace, parent_schema ): @@ -290,8 +270,6 @@ def test_case_05_managed_claude_rejects_model_location_override( _assert_rejected_before_claude_started(session, result, before) -@pytest.mark.managed_fixture -@pytest.mark.usefixtures("_managed_claude_config") def test_case_05_fresh_managed_claude_rejects_model_location_override( live_session, workspace, parent_schema ): diff --git a/tests/integration/test_ug_codex_managed_model_discovery.py b/tests/integration/test_ug_codex_managed_model_discovery.py index 31259c541..f7484759b 100644 --- a/tests/integration/test_ug_codex_managed_model_discovery.py +++ b/tests/integration/test_ug_codex_managed_model_discovery.py @@ -1,8 +1,8 @@ """Codex managed-config CUJs for repository scenarios 2, 4, and 6. -Cases 2 and 6 use the managed-config fixture seam to cover model-discovery shapes that the shared -workspace does not publish. Case 4 runs against the selected workspace's published config, which -must be the dedicated CUJ3 managed workspace. +The admin CodingAgentConfig is fetched once from the managed workspace, its Codex model source is +set to the dedicated test MPS, and the result is reused through ``UCODE_MANAGED_CONFIG_STUB`` in +each isolated session. Normalization, config writers, the gateway, and Codex remain real. """ import json @@ -13,7 +13,6 @@ from utils.constants import MANAGED_CODEX_PROVIDER_SERVICE from utils.managed import ( fetch_managed_config_stub, - fetch_published_managed_config, is_managed_config_control_plane_cache, use_managed_config_stub, ) @@ -24,7 +23,7 @@ ) from utils.terminal import TerminalProcess -pytestmark = pytest.mark.codex +pytestmark = [pytest.mark.managed_fixture, pytest.mark.codex] @pytest.fixture(scope="module") @@ -48,16 +47,11 @@ def _managed_codex_provider_catalog(workspace): ) -@pytest.fixture +@pytest.fixture(autouse=True) def _managed_codex_config(live_session, _managed_codex_config_stub): use_managed_config_stub(live_session, _managed_codex_config_stub) -@pytest.fixture(scope="module") -def _published_managed_config(workspace): - return fetch_published_managed_config(workspace, os.environ["DATABRICKS_BEARER"]) - - def _codex_state_and_agent_files(session): paths = [] for directory in (session.home / ".ucode", session.home / ".codex"): @@ -134,8 +128,6 @@ def _assert_managed_provider_catalog(session, models, expected: CodexProviderCat assert models == list(catalog_ids), (models, catalog) -@pytest.mark.managed_fixture -@pytest.mark.usefixtures("_managed_codex_config") def test_case_02_managed_codex_uses_admin_discovery_after_configure( live_session, workspace, _managed_codex_provider_catalog ): @@ -181,8 +173,6 @@ def test_case_02_managed_codex_uses_admin_discovery_after_configure( assert not (session.home / ".ucode" / "codex-model-catalog.json").exists() -@pytest.mark.managed_fixture -@pytest.mark.usefixtures("_managed_codex_config") def test_case_02_managed_codex_uses_admin_discovery_from_fresh_state( live_session, workspace, _managed_codex_provider_catalog ): @@ -203,11 +193,8 @@ def test_case_02_managed_codex_uses_admin_discovery_from_fresh_state( assert app_models == models, (app_models, models) -@pytest.mark.managed -@pytest.mark.cuj3 -@pytest.mark.workspace_isolated -def test_case_04_managed_codex_rejects_provider_override_after_configure( - live_session, workspace, codex_provider, _published_managed_config +def test_managed_fixture_codex_rejects_provider_override_after_configure( + live_session, workspace, codex_provider ): """Scenario: configure managed Codex, then pass a --provider override. @@ -238,11 +225,8 @@ def test_case_04_managed_codex_rejects_provider_override_after_configure( _assert_rejected_before_codex_started(session, result, before) -@pytest.mark.managed -@pytest.mark.cuj3 -@pytest.mark.workspace_isolated -def test_case_04_managed_codex_rejects_provider_override_from_fresh_state( - live_session, workspace, codex_provider, _published_managed_config +def test_managed_fixture_codex_rejects_provider_override_from_fresh_state( + live_session, workspace, codex_provider ): """Scenario: pass --workspace and a --provider override from fresh state. @@ -265,8 +249,6 @@ def test_case_04_managed_codex_rejects_provider_override_from_fresh_state( _assert_rejected_before_codex_started(session, result) -@pytest.mark.managed_fixture -@pytest.mark.usefixtures("_managed_codex_config") def test_case_06_managed_codex_rejects_model_location_override_after_configure( live_session, workspace, parent_schema ): @@ -299,8 +281,6 @@ def test_case_06_managed_codex_rejects_model_location_override_after_configure( _assert_rejected_before_codex_started(session, result, before) -@pytest.mark.managed_fixture -@pytest.mark.usefixtures("_managed_codex_config") def test_case_06_managed_codex_rejects_model_location_override_from_fresh_state( live_session, workspace, parent_schema ): diff --git a/tests/integration/test_ug_cuj3_model_discovery.py b/tests/integration/test_ug_cuj3_model_discovery.py new file mode 100644 index 000000000..8cb03f1f3 --- /dev/null +++ b/tests/integration/test_ug_cuj3_model_discovery.py @@ -0,0 +1,498 @@ +"""CUJ3 journeys for managed Unity Catalog model, MCP, and skill locations.""" + +from __future__ import annotations + +import json +import os +import re + +import pytest +from utils.cuj3 import ( + CLAUDE_DECOY, + CLAUDE_MODELS, + CODEX_DECOY, + CODEX_MODELS, + MCP_DECOY, + MCP_SCHEMA, + MCP_SERVERS, + MODEL_SCHEMA, + OTHER_MCP_SCHEMA, + OTHER_MODEL_SCHEMA, + OTHER_SKILL_SCHEMA, + SKILL_DECOY, + SKILL_SCHEMA, + SKILLS, + assert_cuj3_config, + assert_native_model_identity, + assert_persisted_config, + assert_skill_transcript, + fetch_claude_parent_catalog, + fetch_codex_parent_catalog, + fetch_mcp_names, + fetch_skill_names, + fixture_value, + native_tool_evidence, +) +from utils.evidence import FileTask, agent_sessions, assert_no_terminal_api_error, assistant_answers +from utils.managed import fetch_published_managed_config +from utils.terminal import AgentTerminal + +pytestmark = [ + pytest.mark.managed, + pytest.mark.cuj3, + pytest.mark.workspace_isolated, +] +MCP_PERMISSION_TARGETS = ( + (MCP_SERVERS["fixture_reader"], "read_fixture"), + (MCP_SERVERS["fixture_metadata"], "describe_fixture"), +) + + +def _assistant_answers(session, agent: str) -> list[str]: + return [ + answer + for records in agent_sessions(session, agent).values() + for answer in assistant_answers(agent, records) + ] + + +def _wait_for_native_answer( + tui: AgentTerminal, + session, + agent: str, + expected_parts: tuple[str, ...], + permission_targets: tuple[tuple[str, str], ...], +) -> None: + """Wait for an assistant result, approving only the fixture MCP calls if asked.""" + permission_in_progress = False + + def completed(screen: str) -> bool: + nonlocal permission_in_progress + assert_no_terminal_api_error(screen) + lowered = screen.casefold() + if "Do you want to proceed?" in screen: + matching = [ + tool + for server, tool in permission_targets + if server.casefold() in lowered and tool.casefold() in lowered + ] + selected_yes = re.search(r"(?m)^\s*[›❯>]\s*1[.)]\s*(?:Yes|Allow)\b", screen) + assert matching and selected_yes, "Unrecognized fixture tool permission:\n" + screen + if not permission_in_progress: + tui.send("\r", f"allow expected fixture MCP tool {matching[0]}") + permission_in_progress = True + return False + permission_in_progress = False + answers = _assistant_answers(session, agent) + return any(all(part in answer for part in expected_parts) for answer in answers) + + tui.wait_for(completed, "the native assistant result", timeout=240) + + +def _assert_downloaded_skills(session) -> None: + for root_name in (".claude/skills", ".agents/skills"): + root = session.home / root_name + assert root.is_dir(), root + bundles = {path.name for path in root.iterdir() if path.is_dir()} + assert bundles == SKILLS, (root, bundles) + for bundle in SKILLS: + assert (root / bundle / "SKILL.md").is_file(), root / bundle / "SKILL.md" + assert SKILL_DECOY not in bundles + + +def _assert_mcp_list(output: str) -> None: + for name in MCP_SERVERS.values(): + row = next((line for line in output.splitlines() if name in line), "") + assert row and "managed" in row, output + assert "connected" in row or "enabled" in row, row + assert MCP_DECOY not in output, output + + +@pytest.mark.claude +def test_case_03_managed_schema_pointers_claude(live_session, workspace): + """Scenario: configure Claude from CUJ3's published Unity Catalog locations. + + Expected: Claude sees exactly the scoped model, MCP, and skill inventory, completes tasks on + both managed models with native model identity, invokes both fixture tools, and executes both + downloaded skills while accessible decoys remain absent. + """ + session = live_session + bearer = os.environ["DATABRICKS_BEARER"] + published = fetch_published_managed_config(workspace, bearer) + assert_cuj3_config(published) + + parent_catalog = fetch_claude_parent_catalog(workspace, bearer, MODEL_SCHEMA) + decoy_catalog = fetch_claude_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) + assert set(parent_catalog.model_ids) == CLAUDE_MODELS, parent_catalog + assert set(decoy_catalog.model_ids) == {CLAUDE_DECOY}, decoy_catalog + + skills = fetch_skill_names(workspace, bearer, SKILL_SCHEMA) + decoy_skills = fetch_skill_names(workspace, bearer, OTHER_SKILL_SCHEMA) + assert skills == SKILLS, skills + assert decoy_skills == {SKILL_DECOY}, decoy_skills + scoped_mcps = fetch_mcp_names(workspace, bearer, MCP_SCHEMA) + decoy_mcps = fetch_mcp_names(workspace, bearer, OTHER_MCP_SCHEMA) + assert scoped_mcps == {f"{MCP_SCHEMA}.{leaf}" for leaf in MCP_SERVERS}, scoped_mcps + assert decoy_mcps == {f"{OTHER_MCP_SCHEMA}.fixture_decoy"}, decoy_mcps + session.record( + "cuj3-parent-inventory.json", + { + "claude": { + "scoped": parent_catalog.model_ids, + "decoy": decoy_catalog.model_ids, + "pages": parent_catalog.payloads, + }, + "skills": {"scoped": sorted(skills), "decoy": sorted(decoy_skills)}, + "mcps": {"scoped": sorted(scoped_mcps), "decoy": sorted(decoy_mcps)}, + }, + ) + + configured = session.run( + "configure", + "--workspace", + workspace, + "--skip-upgrade", + timeout=300, + ) + assert "Select coding agents to configure:" not in configured.stdout, configured.stdout + assert_persisted_config(session, workspace, published) + _assert_downloaded_skills(session) + + with AgentTerminal( + session, "claude", [str(session.binary), "claude"], "cuj3-claude-model-picker" + ) as tui: + tui.boot() + picker_screen = tui.open_model_picker( + model_visible=lambda text: any(model in text for model in CLAUDE_MODELS) + ) + tui.exit_normally() + assert all(model in picker_screen for model in CLAUDE_MODELS), picker_screen + assert CLAUDE_DECOY not in picker_screen, picker_screen + + gateway_ids = set(session.claude_gateway_model_ids()) + assert gateway_ids == CLAUDE_MODELS, gateway_ids + assert CLAUDE_DECOY not in gateway_ids + settings_path = session.home / ".claude" / "ucode-settings.json" + settings = json.loads(settings_path.read_text()) + options = (settings.get("modelPicker") or {}).get("options") + assert isinstance(options, list), settings + picker_ids = {option.get("model") for option in options if isinstance(option, dict)} + assert picker_ids == CLAUDE_MODELS, settings + assert CLAUDE_DECOY not in picker_ids + + default_task = FileTask(session) + default_result = session.run( + "claude", + "--", + "-p", + default_task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + timeout=240, + ) + default_task.assert_headless_answer("claude", default_result) + assert_native_model_identity( + session, + "claude", + default_task.value, + f"{MODEL_SCHEMA}.claude_sonnet", + ) + + extra_task = FileTask(session) + extra_result = session.run( + "claude", + "--", + "-p", + extra_task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + "--model", + f"{MODEL_SCHEMA}.claude_extra", + timeout=240, + ) + extra_task.assert_headless_answer("claude", extra_result) + assert_native_model_identity( + session, + "claude", + extra_task.value, + f"{MODEL_SCHEMA}.claude_extra", + ) + + run_id = "cuj3-claude-" + os.urandom(8).hex() + reader_value = fixture_value("read_fixture", run_id) + metadata_value = fixture_value("describe_fixture", run_id) + summary_id = "cuj3-summary-" + os.urandom(8).hex() + summary_value = fixture_value("read_fixture", summary_id) + audit_id = "cuj3-audit-" + os.urandom(8).hex() + audit_reader = fixture_value("read_fixture", audit_id) + audit_metadata = fixture_value("describe_fixture", audit_id) + mcp_prompt = ( + f"On server {MCP_SERVERS['fixture_reader']}, call read_fixture with run_id {run_id}. " + f"Then on server {MCP_SERVERS['fixture_metadata']}, call describe_fixture with the same " + "run_id. Return the two opaque tool results with labels reader= and metadata=. Do not " + "calculate or infer either value." + ) + with AgentTerminal( + session, "claude", [str(session.binary), "claude"], "cuj3-claude-tools-skills" + ) as tui: + tui.boot() + tui.submit(mcp_prompt) + _wait_for_native_answer( + tui, + session, + "claude", + (f"reader={reader_value}", f"metadata={metadata_value}"), + MCP_PERMISSION_TARGETS, + ) + tui.submit(f"/fixture-summary {summary_id}") + _wait_for_native_answer( + tui, + session, + "claude", + (f"fixture-summary: {summary_value}",), + MCP_PERMISSION_TARGETS, + ) + tui.submit(f"/fixture-audit {audit_id}") + _wait_for_native_answer( + tui, + session, + "claude", + (f"fixture-audit: {audit_reader} {audit_metadata}",), + MCP_PERMISSION_TARGETS, + ) + tui.exit_normally() + + native_tool_evidence( + session, + "claude", + run_id, + MCP_SERVERS["fixture_reader"], + "read_fixture", + reader_value, + ) + native_tool_evidence( + session, + "claude", + run_id, + MCP_SERVERS["fixture_metadata"], + "describe_fixture", + metadata_value, + ) + native_tool_evidence( + session, "claude", summary_id, MCP_SERVERS["fixture_reader"], "read_fixture", summary_value + ) + native_tool_evidence( + session, "claude", audit_id, MCP_SERVERS["fixture_reader"], "read_fixture", audit_reader + ) + native_tool_evidence( + session, + "claude", + audit_id, + MCP_SERVERS["fixture_metadata"], + "describe_fixture", + audit_metadata, + ) + assert_skill_transcript(session, "claude", "fixture-summary", "read_fixture", summary_id) + assert_skill_transcript(session, "claude", "fixture-audit", "read_fixture", audit_id) + assert any( + "fixture-audit:" in answer and audit_metadata in answer + for answer in _assistant_answers(session, "claude") + ) + + mcp_list = session.run("mcp", "list", timeout=180) + _assert_mcp_list(mcp_list.stdout) + status = session.run("status", timeout=180) + assert len(re.findall(r"MCP servers:\s+2\b", status.stdout)) >= 2, status.stdout + assert len(re.findall(r"Skills:\s+2\b", status.stdout)) >= 2, status.stdout + skills_list = session.run("skills", "list", timeout=180) + assert all(skill in skills_list.stdout for skill in SKILLS), skills_list.stdout + assert SKILL_DECOY not in skills_list.stdout, skills_list.stdout + session.assert_not_routed() + + +@pytest.mark.codex +def test_case_04_managed_schema_pointers_codex(live_session, workspace): + """Scenario: configure Codex from CUJ3's published Unity Catalog locations. + + Expected: Codex sees exactly the scoped model, MCP, and skill inventory, completes tasks on + both managed models with native model identity, invokes both fixture tools, and executes both + downloaded skills while accessible decoys remain absent. + """ + session = live_session + bearer = os.environ["DATABRICKS_BEARER"] + published = fetch_published_managed_config(workspace, bearer) + assert_cuj3_config(published) + + parent_catalog = fetch_codex_parent_catalog(workspace, bearer, MODEL_SCHEMA) + decoy_catalog = fetch_codex_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) + assert set(parent_catalog.model_ids) == CODEX_MODELS, parent_catalog + assert set(decoy_catalog.model_ids) == {CODEX_DECOY}, decoy_catalog + + skills = fetch_skill_names(workspace, bearer, SKILL_SCHEMA) + decoy_skills = fetch_skill_names(workspace, bearer, OTHER_SKILL_SCHEMA) + assert skills == SKILLS, skills + assert decoy_skills == {SKILL_DECOY}, decoy_skills + scoped_mcps = fetch_mcp_names(workspace, bearer, MCP_SCHEMA) + decoy_mcps = fetch_mcp_names(workspace, bearer, OTHER_MCP_SCHEMA) + assert scoped_mcps == {f"{MCP_SCHEMA}.{leaf}" for leaf in MCP_SERVERS}, scoped_mcps + assert decoy_mcps == {f"{OTHER_MCP_SCHEMA}.fixture_decoy"}, decoy_mcps + session.record( + "cuj3-parent-inventory.json", + { + "codex": { + "scoped": parent_catalog.model_ids, + "decoy": decoy_catalog.model_ids, + "payloads": parent_catalog.payloads, + }, + "skills": {"scoped": sorted(skills), "decoy": sorted(decoy_skills)}, + "mcps": {"scoped": sorted(scoped_mcps), "decoy": sorted(decoy_mcps)}, + }, + ) + + configured = session.run( + "configure", + "--workspace", + workspace, + "--skip-upgrade", + timeout=300, + ) + assert "Select coding agents to configure:" not in configured.stdout, configured.stdout + assert_persisted_config(session, workspace, published) + _assert_downloaded_skills(session) + + codex_ids = set(session.codex_model_ids(["app-server", "--listen", "stdio://"])) + assert codex_ids == CODEX_MODELS, codex_ids + assert CODEX_DECOY not in codex_ids + + default_task = FileTask(session) + default_result = session.run( + "codex", + "--", + "exec", + "--skip-git-repo-check", + "--json", + default_task.prompt, + timeout=240, + ) + default_task.assert_headless_answer("codex", default_result) + assert_native_model_identity( + session, + "codex", + default_task.value, + f"{MODEL_SCHEMA}.codex_primary", + ) + + extra_task = FileTask(session) + extra_result = session.run( + "codex", + "--", + "exec", + "--skip-git-repo-check", + "--json", + "--model", + f"{MODEL_SCHEMA}.codex_extra", + extra_task.prompt, + timeout=240, + ) + extra_task.assert_headless_answer("codex", extra_result) + assert_native_model_identity( + session, + "codex", + extra_task.value, + f"{MODEL_SCHEMA}.codex_extra", + ) + + run_id = "cuj3-codex-" + os.urandom(8).hex() + reader_value = fixture_value("read_fixture", run_id) + metadata_value = fixture_value("describe_fixture", run_id) + summary_id = "cuj3-summary-" + os.urandom(8).hex() + summary_value = fixture_value("read_fixture", summary_id) + audit_id = "cuj3-audit-" + os.urandom(8).hex() + audit_reader = fixture_value("read_fixture", audit_id) + audit_metadata = fixture_value("describe_fixture", audit_id) + mcp_prompt = ( + f"On server {MCP_SERVERS['fixture_reader']}, call read_fixture with run_id {run_id}. " + f"Then on server {MCP_SERVERS['fixture_metadata']}, call describe_fixture with the same " + "run_id. Return the two opaque tool results with labels reader= and metadata=. Do not " + "calculate or infer either value." + ) + with AgentTerminal( + session, "codex", [str(session.binary), "codex"], "cuj3-codex-tools-skills" + ) as tui: + tui.boot() + tui.submit(mcp_prompt) + _wait_for_native_answer( + tui, + session, + "codex", + (f"reader={reader_value}", f"metadata={metadata_value}"), + MCP_PERMISSION_TARGETS, + ) + tui.submit(f"$fixture-summary {summary_id}") + _wait_for_native_answer( + tui, + session, + "codex", + (f"fixture-summary: {summary_value}",), + MCP_PERMISSION_TARGETS, + ) + tui.submit(f"$fixture-audit {audit_id}") + _wait_for_native_answer( + tui, + session, + "codex", + (f"fixture-audit: {audit_reader} {audit_metadata}",), + MCP_PERMISSION_TARGETS, + ) + tui.exit_normally() + + native_tool_evidence( + session, + "codex", + run_id, + MCP_SERVERS["fixture_reader"], + "read_fixture", + reader_value, + ) + native_tool_evidence( + session, + "codex", + run_id, + MCP_SERVERS["fixture_metadata"], + "describe_fixture", + metadata_value, + ) + native_tool_evidence( + session, "codex", summary_id, MCP_SERVERS["fixture_reader"], "read_fixture", summary_value + ) + native_tool_evidence( + session, "codex", audit_id, MCP_SERVERS["fixture_reader"], "read_fixture", audit_reader + ) + native_tool_evidence( + session, + "codex", + audit_id, + MCP_SERVERS["fixture_metadata"], + "describe_fixture", + audit_metadata, + ) + assert_skill_transcript(session, "codex", "fixture-summary", "read_fixture", summary_id) + assert_skill_transcript(session, "codex", "fixture-audit", "read_fixture", audit_id) + assert any( + "fixture-audit:" in answer and audit_metadata in answer + for answer in _assistant_answers(session, "codex") + ) + + mcp_list = session.run("mcp", "list", timeout=180) + _assert_mcp_list(mcp_list.stdout) + status = session.run("status", timeout=180) + assert len(re.findall(r"MCP servers:\s+2\b", status.stdout)) >= 2, status.stdout + assert len(re.findall(r"Skills:\s+2\b", status.stdout)) >= 2, status.stdout + skills_list = session.run("skills", "list", timeout=180) + assert all(skill in skills_list.stdout for skill in SKILLS), skills_list.stdout + assert SKILL_DECOY not in skills_list.stdout, skills_list.stdout + session.assert_not_routed() diff --git a/tests/integration/utils/cuj3.py b/tests/integration/utils/cuj3.py new file mode 100644 index 000000000..480e758b3 --- /dev/null +++ b/tests/integration/utils/cuj3.py @@ -0,0 +1,365 @@ +"""Black-box evidence helpers for the real CUJ3 managed workspace.""" + +from __future__ import annotations + +import hashlib +import json +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + +from .evidence import agent_sessions, assistant_answers +from .provider_catalog import parse_codex_provider_catalog + +MODEL_SCHEMA = "ug_e2e.models" +OTHER_MODEL_SCHEMA = "ug_e2e.other_models" +MCP_SCHEMA = "ug_e2e.tools" +OTHER_MCP_SCHEMA = "ug_e2e.other_tools" +SKILL_SCHEMA = "ug_e2e.skills" +OTHER_SKILL_SCHEMA = "ug_e2e.other_skills" +MODEL_HEADER = "Databricks-Model-Service-Parent-Schema" + +CLAUDE_MODELS = frozenset({f"{MODEL_SCHEMA}.claude_sonnet", f"{MODEL_SCHEMA}.claude_extra"}) +CODEX_MODELS = frozenset({f"{MODEL_SCHEMA}.codex_primary", f"{MODEL_SCHEMA}.codex_extra"}) +CLAUDE_DECOY = f"{OTHER_MODEL_SCHEMA}.claude_decoy" +CODEX_DECOY = f"{OTHER_MODEL_SCHEMA}.codex_decoy" +MCP_SERVERS = { + "fixture_reader": f"{MCP_SCHEMA}.fixture_reader".replace(".", "-"), + "fixture_metadata": f"{MCP_SCHEMA}.fixture_metadata".replace(".", "-"), +} +MCP_DECOY = f"{OTHER_MCP_SCHEMA}.fixture_decoy".replace(".", "-") +SKILLS = frozenset({"fixture-summary", "fixture-audit"}) +SKILL_DECOY = "fixture-decoy" + + +@dataclass(frozen=True) +class ParentCatalog: + """The model ids returned by one parent-schema API.""" + + model_ids: tuple[str, ...] + payloads: tuple[dict, ...] + + +def fixture_value(domain: str, run_id: str) -> str: + """Derive the opaque value returned by the published fixture app.""" + payload = b"unity-gateway-cuj-fixture\0" + domain.encode("ascii") + b"\0" + run_id.encode() + return hashlib.sha256(payload).hexdigest() + + +def _get_json(url: str, token: str, headers: dict[str, str]) -> object: + request = urllib.request.Request( + url, + headers={"Authorization": f"Bearer {token}", "Accept": "application/json", **headers}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + raise AssertionError(f"GET {url} returned HTTP {error.code}: {error.reason}") from error + except (urllib.error.URLError, OSError) as error: + raise AssertionError(f"GET {url} failed: {error}") from error + + +def _base_url(workspace: str) -> str: + return workspace.rstrip("/") + + +def fetch_claude_parent_catalog(workspace: str, token: str, parent_schema: str) -> ParentCatalog: + """Fetch the complete Anthropic catalog through the real parent-schema API.""" + model_ids: list[str] = [] + pages: list[dict] = [] + after_id: str | None = None + seen_cursors: set[str] = set() + for _ in range(20): + query = {"limit": "1000"} + if after_id: + query["after_id"] = after_id + url = ( + f"{_base_url(workspace)}/ai-gateway/anthropic/v1/models?{urllib.parse.urlencode(query)}" + ) + payload = _get_json(url, token, {MODEL_HEADER: parent_schema}) + assert isinstance(payload, dict), payload + pages.append(payload) + raw_models = payload.get("data") + assert isinstance(raw_models, list), payload + for model in raw_models: + assert isinstance(model, dict), model + model_id = model.get("id") + assert isinstance(model_id, str) and model_id, model + assert model_id not in model_ids, f"duplicate Claude model id: {model_id}" + model_ids.append(model_id) + has_more = payload.get("has_more", False) + assert isinstance(has_more, bool), payload + if not has_more: + break + after_id = payload.get("last_id") + assert isinstance(after_id, str) and after_id and after_id not in seen_cursors, payload + seen_cursors.add(after_id) + else: + raise AssertionError("Claude parent catalog exceeded 20 pages") + assert model_ids, pages + return ParentCatalog(tuple(model_ids), tuple(pages)) + + +def fetch_codex_parent_catalog(workspace: str, token: str, parent_schema: str) -> ParentCatalog: + """Fetch the list-visible Codex catalog through the real parent-schema API.""" + url = f"{_base_url(workspace)}/ai-gateway/codex/v1/models" + payload = _get_json(url, token, {MODEL_HEADER: parent_schema}) + assert isinstance(payload, dict), payload + model_ids = parse_codex_provider_catalog(payload) + return ParentCatalog(tuple(model_ids), (payload,)) + + +def fetch_skill_names(workspace: str, token: str, location: str) -> frozenset[str]: + """Read finalized skill bundle names from one Unity Catalog schema.""" + names: set[str] = set() + page_token: str | None = None + seen_tokens: set[str] = set() + for _ in range(20): + query_values = {"parent": f"schemas/{location}"} + if page_token: + query_values["page_token"] = page_token + query = urllib.parse.urlencode(query_values) + payload = _get_json( + f"{_base_url(workspace)}/api/2.1/unity-catalog/skills?{query}", token, {} + ) + assert isinstance(payload, dict), payload + raw_skills = payload.get("skills") + assert isinstance(raw_skills, list), payload + for skill in raw_skills: + assert isinstance(skill, dict), skill + name = skill.get("bundle_name") or skill.get("name") or skill.get("id") + assert isinstance(name, str) and name, skill + names.add(name) + next_token = payload.get("next_page_token") + if not next_token: + return frozenset(names) + assert isinstance(next_token, str) and next_token not in seen_tokens, payload + seen_tokens.add(next_token) + page_token = next_token + raise AssertionError("Unity Catalog skill inventory exceeded 20 pages") + + +def fetch_mcp_names(workspace: str, token: str, location: str) -> frozenset[str]: + """Read the complete visible MCP service inventory for one UC schema.""" + names: set[str] = set() + page_token: str | None = None + seen_tokens: set[str] = set() + for _ in range(20): + query_values = {"parent": f"schemas/{location}", "view": "BASIC"} + if page_token: + query_values["page_token"] = page_token + query = urllib.parse.urlencode(query_values) + payload = _get_json( + f"{_base_url(workspace)}/api/2.1/unity-catalog/mcp-services?{query}", token, {} + ) + assert isinstance(payload, dict), payload + raw_services = payload.get("mcp_services") + assert isinstance(raw_services, list), payload + for service in raw_services: + assert isinstance(service, dict), service + name = service.get("name") + assert isinstance(name, str), service + fqn = name.removeprefix("mcp-services/") + assert fqn.startswith(f"{location}."), service + names.add(fqn) + next_token = payload.get("next_page_token") + if not next_token: + return frozenset(names) + assert isinstance(next_token, str) and next_token not in seen_tokens, payload + seen_tokens.add(next_token) + page_token = next_token + raise AssertionError("Unity Catalog MCP inventory exceeded 20 pages") + + +def assert_cuj3_config(config: dict) -> None: + """Assert the published policy's exact schema pointers and model defaults.""" + assert config.get("spec_version") == 1, config + assert config.get("default_agent") == "CODING_AGENT_CLAUDE_CODE", config + assert config.get("mcp_servers") == {"unity_catalog_location": MCP_SCHEMA}, config + assert config.get("skills") == {"unity_catalog_location": SKILL_SCHEMA}, config + entries = config.get("enabled_agents") + assert isinstance(entries, list), config + by_agent = {entry.get("agent"): entry for entry in entries if isinstance(entry, dict)} + assert set(by_agent) == {"CODING_AGENT_CLAUDE_CODE", "CODING_AGENT_CODEX"}, by_agent + claude = by_agent["CODING_AGENT_CLAUDE_CODE"].get("config") + assert isinstance(claude, dict), claude + assert claude.get("models") == {"unity_catalog_location": MODEL_SCHEMA}, claude + assert claude.get("default_models") == { + "default_model": f"{MODEL_SCHEMA}.claude_sonnet", + "default_sonnet_model": f"{MODEL_SCHEMA}.claude_sonnet", + }, claude + assert claude.get("smart_routing") == {"enabled": False}, claude + assert claude.get("tracing") == {"enabled": False}, claude + codex = by_agent["CODING_AGENT_CODEX"].get("config") + assert isinstance(codex, dict), codex + assert codex.get("models") == {"unity_catalog_location": MODEL_SCHEMA}, codex + assert codex.get("default_models") == {"default_model": f"{MODEL_SCHEMA}.codex_primary"}, codex + assert codex.get("smart_routing") == {"enabled": False}, codex + assert codex.get("tracing") == {"enabled": False}, codex + + +def assert_persisted_config(session, workspace: str, published: dict) -> dict: + """Require ug's raw persisted config to match the real control-plane response.""" + path = session.home / ".ucode" / "managed-config.json" + assert path.is_file(), path + persisted = json.loads(path.read_text()) + assert persisted.get("workspace") == workspace, persisted + assert persisted.get("config") == published, (persisted, published) + assert_cuj3_config(persisted["config"]) + session.record("cuj3-managed-config.json", persisted) + return persisted["config"] + + +def _walk(value: object): + if isinstance(value, dict): + yield value + for child in value.values(): + yield from _walk(child) + elif isinstance(value, list): + for child in value: + yield from _walk(child) + + +def _text(value: object) -> str: + if isinstance(value, str): + return value + if isinstance(value, list): + return " ".join(_text(item) for item in value) + if isinstance(value, dict): + return " ".join(_text(item) for item in value.values()) + return str(value) + + +def native_model_ids(session, agent: str, answer_value: str) -> set[str]: + """Extract model identity from the native agent transcript for one completed answer.""" + found: set[str] = set() + sessions = agent_sessions(session, agent) + for records in sessions.values(): + if agent == "claude": + for record in records: + if record.get("type") != "assistant": + continue + message = record.get("message") or {} + answer = _text(message.get("content")) + model = message.get("model") + if answer_value in answer and isinstance(model, str) and model: + found.add(model.removesuffix("[1m]")) + else: + completed = { + (record.get("payload") or {}).get("turn_id") + for record in records + if record.get("type") == "event_msg" + and (record.get("payload") or {}).get("type") == "task_complete" + and answer_value in ((record.get("payload") or {}).get("last_agent_message") or "") + } + for record in records: + if record.get("type") != "turn_context": + continue + payload = record.get("payload") or {} + if payload.get("turn_id") in completed and isinstance(payload.get("model"), str): + found.add(payload["model"]) + return found + + +def assert_native_model_identity(session, agent: str, answer_value: str, expected: str) -> None: + observed = native_model_ids(session, agent, answer_value) + assert expected in observed, {"expected": expected, "observed": sorted(observed)} + session.record( + f"native-model-{agent}-{answer_value[:12]}.json", + {"expected": expected, "observed": sorted(observed)}, + ) + + +def native_tool_evidence( + session, agent: str, run_id: str, server_fragment: str, tool_name: str, expected_value: str +) -> None: + """Require a native tool call with the requested run id and its real result.""" + calls: list[dict] = [] + results: list[str] = [] + for records in agent_sessions(session, agent).values(): + for node in _walk(records): + kind = node.get("type") + name = node.get("name") + if kind in {"tool_use", "function_call", "mcp_tool_call"} and isinstance(name, str): + arguments = node.get("input", node.get("arguments", {})) + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except json.JSONDecodeError: + arguments = {} + if ( + server_fragment in name + and tool_name in name + and isinstance(arguments, dict) + and arguments.get("run_id") == run_id + ): + calls.append(node) + if kind in {"tool_result", "function_call_output", "mcp_tool_result"}: + results.append(_text(node.get("content", node.get("output", "")))) + assert calls, {"agent": agent, "server": server_fragment, "tool": tool_name, "run_id": run_id} + assert any(expected_value in result for result in results), { + "agent": agent, + "server": server_fragment, + "tool": tool_name, + "expected_result": expected_value, + "results": results, + } + session.record( + f"native-tool-{agent}-{tool_name}-{run_id}.json", + {"calls": calls, "expected_result": expected_value, "results": results}, + ) + + +def assert_skill_transcript(session, agent: str, skill: str, domain: str, run_id: str) -> None: + """Require a skill result and the native skill-loading event that preceded it.""" + expected = fixture_value(domain, run_id) + sessions = agent_sessions(session, agent) + records = [record for rows in sessions.values() for record in rows] + answers = assistant_answers(agent, records) + assert any(f"{skill}:" in answer and expected in answer for answer in answers), answers + loads: list[dict] = [] + load_results: list[dict] = [] + load_ids: set[str] = set() + for node in _walk(records): + kind = str(node.get("type") or "").lower() + name = str(node.get("name") or node.get("tool") or "").lower() + serialized = json.dumps(node, sort_keys=True) + skill_path = f"{skill}/SKILL.md" + is_skill_load = ( + (name in {"skill", "load_skill", "loadskill"} and skill in serialized) + or ("read" in name and skill_path in serialized) + or (kind in {"function_call", "tool_use"} and skill_path in serialized) + or (kind == "skill" and skill in serialized) + ) + if is_skill_load: + loads.append(node) + for key in ("id", "call_id", "tool_use_id"): + value = node.get(key) + if isinstance(value, str): + load_ids.add(value) + if kind in {"tool_result", "function_call_output", "mcp_tool_result"}: + result_ids = { + node[key] + for key in ("id", "call_id", "tool_use_id") + if isinstance(node.get(key), str) + } + if result_ids & load_ids or skill in serialized or skill_path in serialized: + load_results.append(node) + assert loads, f"native transcript did not load skill {skill}" + assert load_results, { + "skill": skill, + "loads": loads, + "answers": answers, + } + session.record( + f"skill-{agent}-{skill}.json", + { + "skill": skill, + "run_id": run_id, + "expected_result": expected, + "loads": loads, + "load_results": load_results, + }, + ) diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 83e7ebe7c..8b7fbca94 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -294,9 +294,10 @@ def test_model_discovery_cases_match_current_launch_contract(): # design document. Configured/fresh variants share their scenario number. expected_cases = set(range(1, 15)) assert set(seen) == expected_cases - assert len(seen) == 24 + # CUJ3 covers both agents in one complete schema-pointer journey each. + assert len(seen) == 22 for case in expected_cases: - assert seen.count(case) == (1 if 7 <= case <= 10 else 2), case + assert seen.count(case) == (1 if case in {3, 4} or 7 <= case <= 10 else 2), case @pytest.mark.parametrize("payload", [{}, {"coding_agent_configs": []}, []])