AIGTWY-4878: exercise managed schema-pointer CUJ3 for both agents

.github/workflows/integration.yml

@@ -212,11 +212,17 @@ jobs:
       - name: Run the selected live cases against the e2e workspace
         shell: bash
         run: |
-          case "$AGENT" in
-            claude) args=(--claude-version "$CLAUDE_VERSION") ;;
-            codex) args=(--codex-version "$CODEX_VERSION") ;;
-            opencode) args=(--opencode-version "$OPENCODE_VERSION") ;;
-          esac
+          if [[ "${INSTALL_BOTH_AGENTS:-}" == "true" ]]; then
+            # Managed config is published for both agents, so configure must be able to
+            # install both CLIs even though this matrix leg asserts only one agent.
+            args=(--claude-version "$CLAUDE_VERSION" --codex-version "$CODEX_VERSION")
+          else
+            case "$AGENT" in
+              claude) args=(--claude-version "$CLAUDE_VERSION") ;;
+              codex) args=(--codex-version "$CODEX_VERSION") ;;
+              opencode) args=(--opencode-version "$OPENCODE_VERSION") ;;
+            esac
+          fi
           if [[ -n "$DEPENDENCY" ]]; then
             args+=(--dependency "$DEPENDENCY")
           fi
@@ -365,6 +371,26 @@ jobs:
       ARTIFACT_NAME: integration-cuj7-${{ matrix.agent }}
     steps: *live-steps
 
+  cuj3:
+    name: "CUJ3: managed journey · ${{ matrix.agent == 'claude' && 'Claude' || 'Codex' }}"
+    if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && inputs.suite != 'tui' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        agent: [claude, codex]
+    env:
+      UCODE_TEST_WORKSPACE: ${{ secrets.UG_CUJ3_WORKSPACE }}
+      DATABRICKS_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }}
+      DATABRICKS_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }}
+      DEPENDENCY: ${{ inputs.dependency }}
+      AGENT: ${{ matrix.agent }}
+      INSTALL_BOTH_AGENTS: "true"
+      TEST_MARKER: managed and cuj3 and workspace_isolated and ${{ matrix.agent }}
+      ARTIFACT_NAME: integration-cuj3-${{ matrix.agent }}
+    steps: *live-steps
+
   # Non-blocking: continue-on-error and absent from the cujs `needs`, so a flaky OpenCode
   # leg cannot block merges. Add it to cujs once the lane is stable.
   opencode:
@@ -432,7 +458,7 @@ jobs:
           uv run --no-project --python 3.12 python scripts/run_integration.py \
             --python 3.12 --ug-version "$UG_VERSION" --entry-point "$ENTRY_POINT" \
             --default-index "$PACKAGE_INDEX" --output "$RUNNER_TEMP/ug-integration" \
-            "${args[@]}" -- -m "(managed or managed_fixture or workspace_switch) and $AGENT"
+            "${args[@]}" -- -m "(managed or managed_fixture or workspace_switch) and not workspace_isolated and $AGENT"
       - name: Upload managed test evidence
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
@@ -451,7 +477,7 @@ jobs:
     name: All integration tests
     if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
     # installation-windows remains advisory until its initial platform issues are fixed.
-    needs: [installation, workspace, smoke, full, cuj7, managed]
+    needs: [installation, workspace, smoke, full, cuj7, cuj3, managed]
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -473,6 +499,8 @@ jobs:
               if suite != "smoke":
                   required.append("full")
                   required.append("cuj7")
+              if suite in {"full", "live"}:
+                  required.append("cuj3")
           if suite in {"full", "live"}:
               required.append("managed")
           failed = [job for job in required if results[job]["result"] != "success"]

tests/README.md

@@ -115,10 +115,11 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`.
 | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace |
 | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup |
 | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list |
+| CUJ3 managed Claude/Codex journeys | Configure in the dedicated managed workspace with the published schema pointers for models, MCPs, and skills | Each agent discovers the scoped inventory, excludes accessible decoys, invokes each compatible in-scope model, calls both MCP tools, and uses both downloaded skills with native execution evidence |
 | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row |
-| `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state |
+| Managed fixture Claude override cases (`test_managed_fixture_*`, `test_case_05_*`) | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state |
 | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting |
-| `test_case_04_*`, `test_case_06_*` | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state |
+| Managed fixture Codex override cases (`test_managed_fixture_*`, `test_case_06_*`) | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state |
 | `test_ug_configure_managed_codex_catalog_fallback` | Configure from an injected managed response containing a GPT model absent from Codex's bundled catalog | Actionable metadata warning; conservative catalog entry for the unknown model; real Codex prompt on the valid default model |
 | `test_managed_fixture_codex_http_headers_in_managed_file` | Interactive PTY configure with injected managed `http_headers` for Codex | The specified header (`x-databricks-workspace`) lands in `model_providers.Databricks.http_headers` in `/etc/codex/managed_config.toml` with the exact admin value |
 | `test_managed_claude_mps_defaults_accompany_discovery`, `test_managed_claude_parent_schema_defaults_accompany_discovery` | Configure from the published admin config and launch Claude with MPS on `eng-ml-inference-batch-inference-us-west-2` and Unity Catalog discovery on `eng-ml-inference-ap-northeast-2`, respectively | Both generated settings files retain every admin-authored default alongside the source header; MPS pickers show labeled default rows first and the independently fetched catalog below, including models also used as defaults, while retaining catalog labels; only UC Opus/Sonnet family ids gain `[1m]` |
@@ -131,9 +132,9 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`.
 | `test_ug_and_ucode_web_search_helpers_preserve_mcp_stdio` | Initialize and list tools through both web-search helper commands | Exactly the MCP JSON-RPC responses; no text/ANSI contamination; existing server/tool identities preserved; no model request |
 
 With Claude and Codex selected there are **68 live cases** (12 marked TUI cases),
-**6 managed-workspace cases** (marker `managed`, run against workspaces that
+**10 managed-workspace cases** (marker `managed`, run against workspaces that
 publish a CodingAgentConfig), **1 two-workspace case** (marker `workspace_switch`),
-**25 managed-fixture cases** (marker `managed_fixture`, with only
+**21 managed-fixture cases** (marker `managed_fixture`, with only
 the CodingAgentConfig input injected), and **7 installation checks**. The 14 retained numbered scenarios
 comprise **24 explicit journeys**: 12 managed configured/fresh executions and 12 unmanaged
 executions. Thirteen additional managed-fixture cases cover focused model, MCP, skills,
@@ -180,8 +181,9 @@ Databricks Hosted configure/TUI, custom OAuth CLI TUI, and headless argument
 journeys for both agents, in two parallel jobs. After smoke finishes, the full
 suite runs 66 live cases across the shared Claude and Codex jobs. CUJ7 runs two
 matrix legs in the required dedicated-workspace job, one per agent, bringing live
-coverage to 68 cases. Each job installs its agent and runs its selected cases
-serially.
+coverage to 68 cases. The dedicated CUJ3 matrix adds schema-pointer journeys
+for both agents. Each job installs its selected agent and runs its selected cases
+serially; CUJ3 installs both agent CLIs because its published config enables both.
 CI starts integration alongside unit tests and the existing e2e shards. Integration
 does not wait for agent e2e or get skipped when an agent shard fails. These suites
 share workspace capacity; overlapping their requests can still encounter rate limits.
@@ -196,7 +198,7 @@ journey is deferred while its npm proxy access is blocked. Native TUI
 and managed-settings coverage remain deferred; a green installation check alone
 does not establish a successful live task.
 The `All integration tests` check requires every selected integration job to pass, including
-CUJ7 and both managed-config lanes for full/live runs; full coverage does not depend on a label or
+CUJ7, CUJ3, and both managed-config lanes for full/live runs; full coverage does not depend on a label or
 a manual request.
 
 The existing e2e workflow runs seven parallel shards: gateway checks plus one for

tests/integration/README.md

@@ -2,7 +2,7 @@
 
 This suite runs the **installed product** through subprocesses. Most live cases use
 the same `UCODE_TEST_WORKSPACE` as existing e2e tests; CUJ7 uses a dedicated
-unmanaged workspace in CI. It does not import `ucode`,
+unmanaged workspace and CUJ3 uses a dedicated managed workspace in CI. It does not import `ucode`,
 patch application functions, substitute agent executables, run a fake gateway,
 or construct ug state files. The normal test suite checks these boundaries.
 
@@ -313,14 +313,15 @@ fresh Claude model-location task also pins Haiku 4.5 to keep inference inexpensi
 fresh `--provider` journeys check setup and launch with dummy MPS credentials, without inference.
 
 There are **68 live cases** (including 12 marked TUI journeys) and **7 installation
-checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent
-re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against
+checks** with Claude and Codex; selecting OpenCode adds one live headless case. Separate managed-workspace cases (the existing per-agent, idempotent
+re-configure, cache-TTL, and two Claude defaults cases plus CUJ3; marker `managed`) run against
 workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case**
 uses two real workspaces and checks skills MCP cleanup and a completed Claude task.
-A further **25 `managed_fixture`
-cases** use `UCODE_MANAGED_CONFIG_STUB`. Twelve explicit configured/fresh Claude and Codex
+A further **21 `managed_fixture`
+cases** use `UCODE_MANAGED_CONFIG_STUB`. Eight configured/fresh Claude and Codex
 discovery and source-override journeys fetch the published config once per agent, replace that
-agent's static source with its dedicated MPS, and reuse the result. Thirteen other collected cases
+agent's static source with its dedicated MPS, and reuse the result. CUJ3 exercises the selected
+workspace's published schema-pointer config directly. Thirteen other collected cases
 cover focused model, MCP, skills, and lifecycle shapes, including per-agent model reconciliation
 and managed skill cleanup. The two Claude default-model cases read published MPS and Unity
 Catalog sources directly from `eng-ml-inference-batch-inference-us-west-2` and
@@ -479,6 +480,22 @@ published CodingAgentConfig and must expose discoverable `system.ai` models for
 both agents. The dedicated job is required for
 full, live, and TUI CI suites.
 
+CUJ3 runs the published schema-pointer config for both agents in its own required CI job. The
+workspace URL comes from the `UG_CUJ3_WORKSPACE` repository secret as a base HTTPS URL
+without a query string. The shared CUJ service-principal credentials come from `UG_CUJ_SP_CLIENT_ID` and
+`UG_CUJ_SP_CLIENT_SECRET`, and the runner mints a short-lived workspace bearer from them. The
+tests carry `managed`, `cuj3`, and `workspace_isolated` markers, so the shared Claude and Codex
+jobs exclude them. CUJ3 runs for full/live suites only; it is not selected by smoke, TUI, or
+installation runs. Because the published managed config enables both agents, each CUJ3 matrix leg
+installs both pinned agent CLIs before selecting its one agent's tests.
+
+The CUJ3 fixture inventory and exact managed config are in `fixtures/cuj/README.md`. Model
+services in `ug_e2e.models` include the configured default and an additional compatible model
+per agent. The test must invoke both with completed inference and native model-identity evidence.
+It also checks accessible decoys in `ug_e2e.other_models`, `ug_e2e.other_tools`, and
+`ug_e2e.other_skills` while rejecting them from the managed agent views. MCP and skill checks
+require real calls and output values, not just listings.
+
 The workspace check requires the secret to match
 `https://eng-ml-inference-team-us-east-1.cloud.databricks.com` (a trailing slash
 is accepted). It never changes the secret or switches workspaces. There is no
@@ -489,32 +506,36 @@ each test; only explicit-model scenarios choose and record a discovered
 Every same-repository PR and push to `main` runs **Smoke journeys**, followed by
 **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI,
 headless argument, and custom OAuth CLI TUI journeys for each agent (six cases,
-two agent jobs). Full runs 66 live cases in the shared Claude and Codex lanes and
-CUJ7 in two dedicated-workspace matrix legs:
+two agent jobs). Full runs 66 live cases in the shared Claude and Codex lanes, CUJ7 in two
+dedicated-workspace matrix legs, and CUJ3's schema-pointer cases in two dedicated-workspace
+matrix legs:
 
 | Agent lane | Marker | Cases |
 | --- | --- | --- |
 | Claude | `live and claude and not workspace_isolated` | 30 |
 | Codex | `live and codex and not workspace_isolated` | 36 |
 | CUJ7: unmanaged journey · Claude | `live and claude and cuj7 and workspace_isolated` | 1 |
 | CUJ7: unmanaged journey · Codex | `live and codex and cuj7 and workspace_isolated` | 1 |
+| CUJ3: managed journey · Claude | `managed and cuj3 and workspace_isolated and claude` | 1 |
+| CUJ3: managed journey · Codex | `managed and cuj3 and workspace_isolated and codex` | 1 |
 
 A non-blocking **OpenCode** job (`live and opencode`, one case) runs alongside them with
 `continue-on-error` and is not part of the required `cujs` gate until it is stable.
 
-Each lane installs only its agent CLI, once, and runs all its configure, headless,
+Each shared agent lane installs only its agent CLI, once, and runs all its configure, headless,
 commands, lifecycle, and applicable app-server journeys. Cases remain serial
 inside each fresh VM because configure/revert can touch machine-level settings;
 separate runners isolate those writes as well as the PTYs. Claude and Codex run
 in parallel, with at most one full-suite job per agent in a workflow run. This
-avoids six serial job startups without overlapping same-agent shards. The two
+avoids six serial job startups without overlapping same-agent shards. CUJ3 installs
+both agent CLIs because its published config enables both. The two
 lanes still share workspace capacity with the concurrently running agent e2e
 shards and other PRs; this limit does not guarantee freedom from rate limits.
 No test retries or assertion changes
 compensate for capacity failures. Both matrices use `fail-fast: false` and upload
 uniquely named evidence even when the other agent fails.
 The **All integration tests** check requires installation, shared workspace validation,
-smoke, both full lanes, both CUJ7 matrix legs, and both **Managed config** lanes to pass for full/live runs. Each tracing
+smoke, both full lanes, both CUJ7 matrix legs, both CUJ3 matrix legs, and both **Managed config** lanes to pass for full/live runs. Each tracing
 journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`:
 a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke, TUI,
 and installation subsets do not select managed tests and do not require them.
@@ -540,6 +561,10 @@ publish a Claude `system.ai` parent-schema source with Unity Catalog family defa
 fetches the config; the tests assert the generated private and OS-managed Claude settings after
 launch. Their exact required defaults and source headers are reflected in the tests.
 
+The CUJ3 schema-pointer cases use the dedicated `UG_CUJ3_WORKSPACE` host described above and
+carry `managed`, `cuj3`, and `workspace_isolated` markers. The dedicated full/live matrix runs
+both agents; the shared managed jobs explicitly exclude `workspace_isolated`.
+
 `test_unmanaged_claude_preserves_preexisting_family_defaults` is a live lifecycle journey against
 one real workspace. A read-only check first proves that the workspace publishes no
 CodingAgentConfig. The test then seeds `/etc/claude-code/managed-settings.json`, runs public

tests/integration/pytest.ini

@@ -10,6 +10,7 @@ markers =
     smoke: Databricks Hosted, custom OAuth CLI TUI, and headless prompt for each agent
     tui: real interactive terminal boot, keyboard input, exit and reopen
     workspace_isolated: requires a dedicated workspace instead of the shared integration workspace
+    cuj3: managed Claude and Codex provider-override guards against the dedicated CUJ3 workspace
     cuj7: configured Claude and Codex model discovery against the dedicated CUJ7 workspace
     claude: only runs when Claude Code is explicitly selected
     codex: only runs when Codex is explicitly selected

tests/integration/test_ug_claude_managed_model_discovery.py

@@ -186,7 +186,7 @@ def test_case_01_fresh_managed_claude_uses_admin_discovery(
     _assert_managed_provider_in_picker(session, workspace, screen, _managed_claude_provider_catalog)
 
 
-def test_case_03_managed_claude_rejects_provider_override(live_session, workspace, claude_provider):
+def test_managed_fixture_claude_rejects_provider_override(live_session, workspace, claude_provider):
     """Scenario: configure managed Claude, then pass --provider.
 
     Expected: ug rejects the override without changing agent-owned state/files.
@@ -215,7 +215,7 @@ def test_case_03_managed_claude_rejects_provider_override(live_session, workspac
     _assert_rejected_before_claude_started(session, result, before)
 
 
-def test_case_03_fresh_managed_claude_rejects_provider_override(
+def test_managed_fixture_fresh_claude_rejects_provider_override(
     live_session, workspace, claude_provider
 ):
     """Scenario: pass --provider while launching managed Claude from fresh state.