From 3d048381072b958aaa61044cb3998db3de411f70 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Fri, 2 Oct 2026 17:29:09 +0000 Subject: [PATCH 01/19] [AIGTWY-4880] Add CUJ5 budget smart-default integration coverage --- .github/workflows/integration.yml | 29 +- scripts/run_integration.py | 39 ++- src/ucode/agents/codex.py | 5 +- tests/README.md | 2 + tests/integration/README.md | 62 +++- tests/integration/pytest.ini | 1 + tests/integration/test_ug_budget_defaults.py | 188 +++++++++++++ tests/integration/utils/budget_defaults.py | 281 +++++++++++++++++++ tests/integration/utils/evidence.py | 96 +++++++ tests/integration/utils/harness.py | 7 +- tests/test_integration_contract.py | 2 +- 11 files changed, 694 insertions(+), 18 deletions(-) create mode 100644 tests/integration/test_ug_budget_defaults.py create mode 100644 tests/integration/utils/budget_defaults.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7d4365166..fcccb6b73 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -215,6 +215,7 @@ jobs: case "$AGENT" in claude) args=(--claude-version "$CLAUDE_VERSION") ;; codex) args=(--codex-version "$CODEX_VERSION") ;; + both) args=(--claude-version "$CLAUDE_VERSION" --codex-version "$CODEX_VERSION") ;; opencode) args=(--opencode-version "$OPENCODE_VERSION") ;; esac if [[ -n "$DEPENDENCY" ]]; then @@ -346,6 +347,31 @@ jobs: ARTIFACT_NAME: integration-full-${{ matrix.agent }} steps: *live-steps + cuj5: + name: "CUJ5: budget smart defaults · Claude and Codex" + if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} + runs-on: ubuntu-22.04 + timeout-minutes: 70 + concurrency: + group: cuj5-budget-defaults + cancel-in-progress: false + env: + UCODE_TEST_WORKSPACE: ${{ secrets.UG_CUJ5_WORKSPACE }} + DATABRICKS_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + DATABRICKS_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + UG_BUDGET_WORKSPACE_ID: "7474645051570450" + UG_BUDGET_ACCOUNT_HOST: https://accounts.cloud.databricks.com + UG_BUDGET_ACCOUNT_ID: 2860b24d-05af-4fb5-83b4-6539492f8180 + UG_BUDGET_ID: 25c4ce5c-fcd6-4d00-8266-69030cc1a236 + UG_BUDGET_SONNET_MODEL: system.ai.claude-sonnet-4-6 + UG_BUDGET_SOL_MODEL: system.ai.gpt-5-6-sol + UG_BUDGET_LUNA_MODEL: system.ai.gpt-5-6-luna + DEPENDENCY: ${{ inputs.dependency }} + AGENT: both + TEST_MARKER: cuj5 + ARTIFACT_NAME: integration-cuj5 + steps: *live-steps + # Non-blocking: continue-on-error and absent from the cujs `needs`, so a flaky OpenCode # leg cannot block merges. Add it to cujs once the lane is stable. opencode: @@ -432,7 +458,7 @@ jobs: name: All integration tests if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} # installation-windows remains advisory until its initial platform issues are fixed. - needs: [installation, workspace, smoke, full, managed] + needs: [installation, workspace, smoke, full, cuj5, managed] runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -453,6 +479,7 @@ jobs: required.append("smoke") if suite != "smoke": required.append("full") + required.append("cuj5") if suite in {"full", "live"}: required.append("managed") failed = [job for job in required if results[job]["result"] != "success"] diff --git a/scripts/run_integration.py b/scripts/run_integration.py index bc81650d3..cc7070b93 100644 --- a/scripts/run_integration.py +++ b/scripts/run_integration.py @@ -220,6 +220,27 @@ def mint_m2m_token(workspace: str, client_id: str, client_secret: str) -> str: return token +def mint_account_m2m_token( + account_host: str, client_id: str, client_secret: str, *, account_id: str +) -> str: + """Mint a short-lived account token for the account-scoped budget API.""" + basic = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode() + body = urllib.parse.urlencode( + {"grant_type": "client_credentials", "scope": "all-apis"} + ).encode() + request = urllib.request.Request( + account_host.rstrip("/") + + f"/oidc/accounts/{urllib.parse.quote(account_id, safe='')}/v1/token", + data=body, + headers={ + "Authorization": f"Basic {basic}", + "Content-Type": "application/x-www-form-urlencoded", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 (https account URL) + return json.load(response)["access_token"] + + @contextlib.contextmanager def managed_process(command, *, interrupt=False, **kwargs): """Bound child lifetimes, including descendants that outlive their parent.""" @@ -521,6 +542,9 @@ def terminate(signum, frame): second_bearer = os.environ.get("DATABRICKS_SECOND_BEARER", "").strip() oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN", "").strip() target_bearers: dict[str, str] = {} + budget_environment = { + key: value for key, value in os.environ.items() if key.startswith("UG_BUDGET_") + } client_secrets = ( os.environ.get("DATABRICKS_CLIENT_SECRET", ""), os.environ.get("UG_MPS_DEFAULTS_CLIENT_SECRET", ""), @@ -535,6 +559,7 @@ def redact(value: str) -> str: second_bearer, oauth_token, *target_bearers.values(), + budget_environment.get("UG_BUDGET_ACCOUNT_TOKEN", ""), *client_secrets, *installer_secrets, ), @@ -825,13 +850,19 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: if not bearer: raise RuntimeError("Selected profile returned no access token.") - if not bearer and not args.profile and not args.installation_only: + if not args.installation_only: client_id = os.environ.get("DATABRICKS_CLIENT_ID", "").strip() client_secret = os.environ.get("DATABRICKS_CLIENT_SECRET", "").strip() - if client_id and client_secret: + if not bearer and not args.profile and client_id and client_secret: bearer = mint_m2m_token(args.workspace, client_id, client_secret) - if not args.installation_only: + if budget_environment: + budget_environment["UG_BUDGET_ACCOUNT_TOKEN"] = mint_account_m2m_token( + budget_environment["UG_BUDGET_ACCOUNT_HOST"], + client_id, + client_secret, + account_id=budget_environment["UG_BUDGET_ACCOUNT_ID"], + ) for bearer_env, target_workspace, client_id, secret_env in MANAGED_DEFAULTS_TARGETS: secret = os.environ.get(secret_env, "").strip() if args.workspace.rstrip("/") == target_workspace: @@ -920,7 +951,7 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: f"--junitxml={output / 'junit.xml'}", *extra, ], - env=runtime_env, + env={**runtime_env, **budget_environment}, cwd=output, stdin=subprocess.DEVNULL, interrupt=True, diff --git a/src/ucode/agents/codex.py b/src/ucode/agents/codex.py index 75606811d..9fcd62527 100644 --- a/src/ucode/agents/codex.py +++ b/src/ucode/agents/codex.py @@ -405,9 +405,12 @@ def write_tool_config( workspace = state["workspace"] # Leave model selection to Codex. The gateway still receives the configured # provider and authentication settings, while Codex uses its own default. - # A managed default is the sole exception. + # A managed launch model is the sole exception: it may be the configured + # default or a budget recommendation that supersedes that default. managed_model = state.get("codex_default_model") chosen_model = managed_model if isinstance(managed_model, str) else None + if chosen_model is not None and isinstance(model, str): + chosen_model = model databricks_profile = state.get("profile") static_models = state.get("codex_static_models") static_models = static_models if isinstance(static_models, list) and static_models else None diff --git a/tests/README.md b/tests/README.md index 2245d68b4..0e0599e35 100644 --- a/tests/README.md +++ b/tests/README.md @@ -147,6 +147,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | +| `test_ug_budget_defaults.py` (CUJ5) | Five bare launches across the 50%/80% tiers and an explicit Claude launch at 80% | Real native processes and completed file tasks; model evidence from native session logs; backend spend/threshold agreement with `ug usage` and the launch panel; recommendation display; budget restoration. Requires existing positive spend. Select with `-m cuj5`; prerequisites and coverage limits are in the integration README. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -172,6 +173,7 @@ executions. Thirteen additional managed-fixture cases cover focused model, MCP, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. +CUJ5 adds six live cases: five bare launches and one explicit Claude override at 80%. Generated-file cleanup and strict app-server stdout assertions remain enforced. Unmanaged discovery Cases 7–14 configure, list models, or open the picker without submitting inference prompts; separate task journeys still perform inference. diff --git a/tests/integration/README.md b/tests/integration/README.md index 55f5ac9aa..adde8007b 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -13,6 +13,42 @@ The `smart_defaults` wire schema, legacy `spend_tiers` cache reads, and recommen request gating are covered by unit/component tests listed in `../README.md`. This suite does not yet assert live `recommendModel` request counts for configs with and without tiers. +## CUJ5: Budget-driven smart defaults (AIGTWY-4880) + +`test_ug_budget_defaults.py` contains six live journeys: + +| Launch | Budget usage | Expected agent / model | +| --- | --- | --- | +| Bare `ug` | 40% | Claude / `system.ai.claude-sonnet-4-6` | +| Bare `ug` | Exactly 50% and 62.5% | Codex / `system.ai.gpt-5-6-sol` | +| Bare `ug` | Exactly 80% and approximately 88.9% | Codex / `system.ai.gpt-5-6-luna` | +| Explicit `ug claude` | Exactly 80% | Claude / Sonnet; display the Codex Luna recommendation | + +Each case verifies the real budget and recommendation, runs `ug usage`, checks the native +process and displayed spend/threshold/recommendation, and completes a unique file task on +the expected model. Model evidence comes from native session logs, not HTTP capture. +The fixture changes the dedicated budget's per-user BLOCK_USAGE threshold relative to existing +positive spend and restores it on exit. Setup uses no mocks or generated spend. Detected +instability at an exact boundary fails as "unverified"; usage is sampled, not locked. + +The CUJ5 job in `.github/workflows/integration.yml` owns all workspace/account/budget/model +values. It uses `UG_CUJ5_WORKSPACE` and shared `UG_CUJ_SP_CLIENT_ID` / +`UG_CUJ_SP_CLIENT_SECRET` secrets. The principal needs workspace/model access and account +budget read/update permission. CI runs both agents serially and serializes CUJ5 jobs because +they share one budget. The job is required for `full`, `live`, and `tui` suites. + +For a local run, export the job's `UCODE_TEST_WORKSPACE`, `DATABRICKS_CLIENT_ID`, +`DATABRICKS_CLIENT_SECRET`, and `UG_BUDGET_*` values, then run on a clean POSIX runner: + +```bash +python3.12 scripts/run_integration.py \ + --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ + -- -m cuj5 +``` + +Do not overlap local and CI budget runs. A killed process or forced cancellation can prevent +restoration; verify the budget against its recorded original threshold before rerunning. + Shared subprocess command resolution is covered by `../test_subprocess_cross_os.py` and enforced by Ruff. These component checks do not establish native Windows coverage for every agent; the Windows journeys below validate their explicitly selected paths. @@ -179,6 +215,7 @@ test_ug_codex_managed_model_discovery.py # fetched/reused Codex MPS policy case test_ug_claude_model_discovery.py # unmanaged scenarios 7, 9, 11, 13 test_ug_codex_model_discovery.py # unmanaged scenarios 8, 10, 12, 14 test_ug_configure_managed.py # managed workspace: static model list/catalog pointer, no agent selector +test_ug_budget_defaults.py # live budget tiers, agent/model choice, explicit-agent override, usage test_ug_configure_managed_models.py # injected model sources, smart-routing banner, Codex fallback metadata test_ug_configure_managed_mcp.py # injected managed MCP list test_ug_configure_managed_skills.py # injected managed skills: download, coexist, reconcile away @@ -308,7 +345,9 @@ they only configure, list models, and open/close the picker. Other live CUJs per real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent +checks** with Claude and Codex; six CUJ5 budget cases are selected by `cuj5` +and run in their dedicated job. +Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** uses two real workspaces and checks skills MCP cleanup and a completed Claude task. @@ -332,7 +371,7 @@ constants in the runner; CI only needs `UG_MPS_DEFAULTS_CLIENT_SECRET` for west- mints short-lived tokens and passes bearers to pytest; each test selects its target bearer for `ug configure` and Claude. The client secrets do not enter the pytest process. The 14 retained numbered scenarios comprise 24 explicit journeys: 12 managed and 12 unmanaged -executions; the complete integration suite collects 101 executions. See the named coverage and gaps matrix in +executions; the complete integration suite collects 108 executions. See the named coverage and gaps matrix in [../README.md](../README.md). ```bash @@ -476,7 +515,8 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs all 62 live cases, including those smoke cases, in two -disjoint agent lanes: +disjoint agent lanes. The dedicated CUJ5 job runs its six budget cases with +both agent binaries on one runner: | Agent lane | Marker | Cases | | --- | --- | --- | @@ -498,10 +538,11 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, -both full lanes, and both **Managed config** lanes to pass for full/live runs. Each tracing -journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: -a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke, TUI, -and installation subsets do not select managed tests and do not require them. +both full lanes, both managed lanes, and CUJ5 to pass for full/live runs. TUI runs also require +CUJ5; managed jobs are intentionally not selected for TUI. Each tracing journey is +included in its agent's Full lane. The managed lanes do not use `continue-on-error`: +a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke +and installation subsets do not select CUJ5 or managed tests. The advisory Windows installation and headless lanes are not yet included in that aggregate check. The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. @@ -620,8 +661,9 @@ comment removes the label and reruns the gate; manually adding the label does no For a manual run, use **Actions → Integration → Run workflow**, select the branch, and choose `full` (default), `smoke`, `tui`, or `installation`. `live` remains an alias for `full`. Manual subsets are explicit: `smoke` runs just the six smoke -cases; `tui` adds `and tui` to each agent lane's marker and runs all 12 live TUI cases. Installation -checks always run. Set the ug/agent versions. From the CLI: +cases; `tui` adds `and tui` to each agent lane's marker and runs all 12 live TUI cases, +and also runs the dedicated CUJ5 job. Installation checks always run. Set the ug/agent +versions. From the CLI: ```bash gh workflow run integration.yml -R databricks/unity-gateway --ref YOUR_BRANCH \ @@ -649,7 +691,7 @@ gh run download RUN_ID -R databricks/unity-gateway \ ``` Use `integration-full-AGENT` for a full lane, `integration-smoke-AGENT` for -smoke, `integration-installation` for Linux package failures, or +smoke, or `integration-cuj5` for the budget journeys. Use `integration-installation` for Linux package failures, or `integration-installation-windows` for native Windows package failures, or `integration-headless-windows-claude` for the Windows gateway journey. Older runs used `integration-full-AGENT-GROUP`, `integration-cujs`, or numbered `integration-live-*` diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index cc0d9b0ca..854c55e94 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -7,6 +7,7 @@ markers = managed: requires an admin CodingAgentConfig published on the selected workspace managed_fixture: real ug/TUI against a real workspace, but the managed CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB workspace_switch: requires two real workspace hosts and their respective credentials + cuj5: budget smart-defaults journeys using the selected workspace and shared CUJ service principal smoke: Databricks Hosted, custom OAuth CLI TUI, and headless prompt for each agent tui: real interactive terminal boot, keyboard input, exit and reopen claude: only runs when Claude Code is explicitly selected diff --git a/tests/integration/test_ug_budget_defaults.py b/tests/integration/test_ug_budget_defaults.py new file mode 100644 index 000000000..d55c0a5ea --- /dev/null +++ b/tests/integration/test_ug_budget_defaults.py @@ -0,0 +1,188 @@ +"""Live customer journeys for workspace smart budget defaults.""" + +import re +from decimal import ROUND_HALF_UP, Decimal + +import pytest +from utils.budget_defaults import BudgetDefaults, BudgetTarget +from utils.evidence import ( + FileTask, + assert_native_process, + assert_task_model, +) +from utils.terminal import AgentTerminal + +pytestmark = [pytest.mark.tui, pytest.mark.cuj5] + + +@pytest.fixture +def budget_defaults(live_session, workspace): + """Supply the target; each journey opens and restores its own budget context.""" + return BudgetDefaults(live_session, workspace, BudgetTarget.from_environment()) + + +def _assert_spend(session, text, recommendation, *, usage=False): + """Compare displayed dollars and percentage with the independent backend response.""" + text = session.redact(text) + money = r"\$([\d,]+\.\d{2})" + pattern = ( + rf"Budget spend:\s*{money}\s+of\s+{money}\s+\((\d+)%\)" + if usage + else rf"{money}\s*/\s*{money}\s+(\d+)%\s+used" + ) + matches = re.findall(pattern, text) + assert matches, f"Budget spend was not displayed:\n{text}" + actual_spend, actual_threshold, actual_percent = matches[-1] + spend = Decimal(str(recommendation["current_spend"])) + threshold = Decimal(str(recommendation["effective_threshold"])) + if usage: + # The usage command formats Decimal dollars half-up and percent with .0%. + expected_money = [ + value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) for value in (spend, threshold) + ] + expected_percent = int(format(float(spend / threshold), ".0%").removesuffix("%")) + else: + # The launch panel formats float dollars and rounds its percentage half-up. + expected_money = [Decimal(format(float(value), ".2f")) for value in (spend, threshold)] + expected_percent = int(float(spend) / float(threshold) * 100 + 0.5) + assert [ + Decimal(actual_spend.replace(",", "")), + Decimal(actual_threshold.replace(",", "")), + ] == expected_money, text + assert int(actual_percent) == expected_percent, text + + +def _assert_launch(session, tui, budget, before, label): + assert_native_process(tui, tui.agent, label, session) + budget.assert_stable(before, label) + transcript = "".join(tui.output) + "\n" + tui.visible + _assert_spend(session, transcript, before) + text = session.redact(transcript) + agent = before.get("recommended_agent") + model = before.get("recommended_model") + if agent: + name = {"CODING_AGENT_CODEX": "Codex", "CODING_AGENT_CLAUDE_CODE": "Claude Code"}[agent] + recommendation = f"Recommended agent is {name}" + if model: + recommendation += f" with model {model}" + assert recommendation in text, text + elif model: + assert f"Recommended model is {model}" in text, text + else: + assert "Recommended agent is" not in text, text + assert "Recommended model is" not in text, text + + +def test_bare_ug_launches_claude_below_50_percent(live_session, budget_defaults): + """Scenario: configure the workspace and launch bare ``ug`` at 40% budget usage. + + Expected: Claude completes a file task on Sonnet; its real process and the + usage/launch displays agree with the stable backend budget response. + """ + session = live_session + label = "bare-40-percent" + with budget_defaults as budget: + session.run( + "configure", + "--workspace", + budget.workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + before = budget.prepare(Decimal("2.5"), label) + assert before.get("recommended_agent") in (None, "CODING_AGENT_CLAUDE_CODE"), before + assert before.get("recommended_model") in (None, budget.sonnet_model), before + usage = session.run("usage", timeout=120) + _assert_spend(session, usage.stdout + usage.stderr, before, usage=True) + + task = FileTask(session) + with AgentTerminal(session, "claude", [str(session.binary)], label) as tui: + tui.boot(timeout=240) + _assert_launch(session, tui, budget, before, label) + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + assert_task_model(session, "claude", task, budget.sonnet_model, label) + tui.exit_normally() + session.assert_not_routed() + + +@pytest.mark.parametrize( + ("multiplier", "model_attribute"), + [ + pytest.param("2", "sol_model", id="exact-50"), + pytest.param("1.6", "sol_model", id="between-50-and-80"), + pytest.param("1.25", "luna_model", id="exact-80"), + pytest.param("1.125", "luna_model", id="between-80-and-100"), + ], +) +def test_bare_ug_launches_codex_at_budget_tier( + live_session, budget_defaults, multiplier, model_attribute +): + """Scenario: launch bare ``ug`` at 50%, 62.5%, 80%, or approximately 88.9% usage. + + Expected: the highest applicable tier selects Codex Sol or Luna, superseding + Codex's Sol default at 80%+. Check the backend response before launch, the native + process, usage display, and completed task's model. Unstable exact boundaries fail. + """ + session = live_session + label = f"bare-codex-{multiplier}" + with budget_defaults as budget: + session.run( + "configure", + "--workspace", + budget.workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + model = getattr(budget, model_attribute) + before = budget.prepare(Decimal(multiplier), label) + assert before.get("recommended_agent") == "CODING_AGENT_CODEX", before + assert before.get("recommended_model") == model, before + usage = session.run("usage", timeout=120) + _assert_spend(session, usage.stdout + usage.stderr, before, usage=True) + + task = FileTask(session) + with AgentTerminal(session, "codex", [str(session.binary)], label) as tui: + tui.boot(timeout=240) + _assert_launch(session, tui, budget, before, label) + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + assert_task_model(session, "codex", task, model, label) + tui.exit_normally() + session.assert_not_routed() + + +def test_ug_claude_keeps_its_default_when_budget_recommends_codex(live_session, budget_defaults): + """Scenario: run explicit ``ug claude`` at 80%, where the budget recommends Codex Luna. + + Expected: the real Claude TUI displays that recommendation but completes a file + task on its own Sonnet default. The Codex-only Luna model never reaches Claude. + """ + session = live_session + label = "explicit-claude-luna-tier" + with budget_defaults as budget: + session.run( + "configure", + "--workspace", + budget.workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + before = budget.prepare(Decimal("1.25"), label) + assert before.get("recommended_agent") == "CODING_AGENT_CODEX", before + assert before.get("recommended_model") == budget.luna_model, before + usage = session.run("usage", timeout=120) + _assert_spend(session, usage.stdout + usage.stderr, before, usage=True) + + task = FileTask(session) + with AgentTerminal(session, "claude", [str(session.binary), "claude"], label) as tui: + tui.boot(timeout=240) + _assert_launch(session, tui, budget, before, label) + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + assert_task_model(session, "claude", task, budget.sonnet_model, label) + tui.exit_normally() + session.assert_not_routed() diff --git a/tests/integration/utils/budget_defaults.py b/tests/integration/utils/budget_defaults.py new file mode 100644 index 000000000..01be2d883 --- /dev/null +++ b/tests/integration/utils/budget_defaults.py @@ -0,0 +1,281 @@ +"""Small live fixture for the CUJ5 budget smart-default journeys.""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Mapping +from dataclasses import dataclass, field +from decimal import Decimal, localcontext +from typing import Any + +_CONFIG_PATH = "/api/ai-gateway/v2/coding-agent-configs" +_RECOMMEND_PATH = _CONFIG_PATH + ":recommendModel" +_BUDGET_PATH = "/api/2.1/accounts/{account_id}/budgets/{budget_id}" +_PER_USER = "ALERT_CONFIGURATION_SCOPE_TYPE_PER_USER" +_BLOCK = "BLOCK_USAGE" +_TIMEOUT = 30.0 +_POLL = 0.5 + + +@dataclass(frozen=True, slots=True) +class BudgetTarget: + workspace_id: str + account_host: str + account_id: str + budget_id: str + sonnet_model: str + sol_model: str + luna_model: str + account_token: str = field(repr=False) + + @classmethod + def from_environment(cls) -> BudgetTarget: + return cls( + os.environ["UG_BUDGET_WORKSPACE_ID"].strip(), + os.environ["UG_BUDGET_ACCOUNT_HOST"].strip(), + os.environ["UG_BUDGET_ACCOUNT_ID"].strip(), + os.environ["UG_BUDGET_ID"].strip(), + os.environ["UG_BUDGET_SONNET_MODEL"].strip(), + os.environ["UG_BUDGET_SOL_MODEL"].strip(), + os.environ["UG_BUDGET_LUNA_MODEL"].strip(), + os.environ["UG_BUDGET_ACCOUNT_TOKEN"].strip(), + ) + + +def _budget_update(budget: Mapping[str, Any]) -> dict[str, Any]: + """Keep the account API's writable budget fields and remove response metadata.""" + + def clean(value: Any) -> Any: + if isinstance(value, dict): + result = {} + for key, item in value.items(): + if key in {"create_time", "update_time"}: + continue + if key == "target" and value.get("action_type") == _BLOCK: + continue + cleaned = clean(item) + if cleaned is None or cleaned == []: + continue + result[key] = cleaned + return result + if isinstance(value, list): + return [clean(item) for item in value] + return value + + return { + key: clean(budget[key]) + for key in ( + "account_id", + "alert_configurations", + "budget_configuration_id", + "display_name", + "filter", + "resource_type", + ) + if key in budget + } + + +def _request_json( + method: str, + url: str, + token: str, + body: Mapping[str, Any] | None, + operation: str, +) -> Any: + encoded = json.dumps(body).encode("utf-8") if body is not None else None + headers = {"Accept": "application/json", "Authorization": f"Bearer {token}"} + if encoded is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=encoded, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=_TIMEOUT) as response: + raw = response.read() + except urllib.error.HTTPError as error: + raise AssertionError(f"{operation} failed: HTTP {error.code}") from None + return json.loads(raw) if raw else {} + + +class BudgetDefaults: + """Temporarily move one existing per-user hard-block threshold.""" + + def __init__(self, session: Any, workspace: str, target: BudgetTarget): + self.session = session + self.workspace = workspace.rstrip("/") + self.target = target + self.sonnet_model = target.sonnet_model + self.sol_model = target.sol_model + self.luna_model = target.luna_model + self._original_threshold: str | None = None + self._dirty = False + self._evidence = 0 + self._exact_boundary = False + + def __enter__(self) -> BudgetDefaults: + self._validate_config() + current = self._account_budget() + assert current["budget_configuration_id"] == self.target.budget_id + assert str(current["account_id"]) == self.target.account_id + workspace_filter = current["filter"]["workspace_id"] + assert workspace_filter["operator"] in ("IN", "BUDGET_CONFIGURATION_FILTER_OPERATOR_IN") + assert [str(value) for value in workspace_filter["values"]] == [self.target.workspace_id] + _, self._original_threshold = self._alert(current) + self._record("setup", threshold=self._original_threshold, status="captured") + return self + + def __exit__(self, exc_type, exc_value, traceback) -> bool: + if self._dirty: + self._restore() + return False + + def _validate_config(self) -> None: + payload = self._workspace_request("GET", _CONFIG_PATH, None, "managed config") + config = payload["coding_agent_configs"][0] + assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE" + + agents = {entry["agent"]: entry["config"] for entry in config["enabled_agents"]} + claude = agents["CODING_AGENT_CLAUDE_CODE"] + codex = agents["CODING_AGENT_CODEX"] + assert claude["models"]["model_services"] == [self.sonnet_model] + assert claude["default_models"]["default_model"] == self.sonnet_model + assert set(codex["models"]["model_services"]) == {self.sol_model, self.luna_model} + assert codex["default_models"]["default_model"] == self.sol_model + + smart_defaults = config["smart_defaults"] + assert smart_defaults["budget_id"] == self.target.budget_id + tiers = smart_defaults["tiers"] + assert [Decimal(str(tier["spending_percentage"])) for tier in tiers] == [ + Decimal("0.5"), + Decimal("0.8"), + ] + assert [(tier["recommended_agent"], tier["recommended_model"]) for tier in tiers] == [ + ("CODING_AGENT_CODEX", self.sol_model), + ("CODING_AGENT_CODEX", self.luna_model), + ] + + def recommendation(self, label: str = "recommendation") -> dict[str, Any]: + payload = self._workspace_request("POST", _RECOMMEND_PATH, {}, "recommendModel") + self._record( + "recommendation", + label=label, + current_spend=payload.get("current_spend"), + effective_threshold=payload.get("effective_threshold"), + recommended_agent=payload.get("recommended_agent"), + recommended_model=payload.get("recommended_model"), + ) + return payload + + def prepare(self, multiplier: Decimal, label: str = "prepare") -> dict[str, Any]: + before = self.recommendation(label + ":before") + spend = Decimal(str(before["current_spend"])) + assert spend > 0, "budget fixture requires existing positive spend" + self._exact_boundary = multiplier in {Decimal("2"), Decimal("1.25")} + with localcontext() as context: + context.prec = 64 + threshold = spend * multiplier + threshold_text = format(threshold, "f") + + current = self._account_budget() + index, _ = self._alert(current) + current["alert_configurations"][index]["quantity_threshold"] = threshold_text + self._put(current, label, threshold_text) + recommendation = self._wait_for_recommendation(threshold, label) + readback_spend = Decimal(str(recommendation["current_spend"])) + if readback_spend != spend: + detail = ( + "exact boundary is unverified" + if self._exact_boundary + else "recommendation spend changed" + ) + raise AssertionError(f"{label}: {detail} before inference") + return recommendation + + def assert_stable(self, before: Mapping[str, Any], label: str = "stable") -> dict[str, Any]: + after = self.recommendation(label + ":after") + for key in ( + "current_spend", + "effective_threshold", + "recommended_agent", + "recommended_model", + ): + old, new = before.get(key), after.get(key) + if key in {"current_spend", "effective_threshold"}: + old, new = Decimal(str(old)), Decimal(str(new)) + if old != new: + detail = "exact boundary is unverified; " if self._exact_boundary else "" + raise AssertionError(f"{label}: {detail}recommendation changed {key}") + return after + + def _restore(self) -> None: + assert self._original_threshold is not None + current = self._account_budget() + index, _ = self._alert(current) + current["alert_configurations"][index]["quantity_threshold"] = self._original_threshold + self._put(current, "restore", self._original_threshold) + _, restored_threshold = self._alert(self._account_budget()) + assert Decimal(restored_threshold) == Decimal(self._original_threshold) + self._record("restore", threshold=self._original_threshold, status="verified") + + def _wait_for_recommendation(self, threshold: Decimal, label: str) -> dict[str, Any]: + deadline = time.monotonic() + _TIMEOUT + while True: + result = self.recommendation(label + ":readback") + if Decimal(str(result["effective_threshold"])) == threshold: + return result + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AssertionError(f"{label}: recommendation threshold did not converge") + time.sleep(min(_POLL, remaining)) + + def _account_budget(self) -> dict[str, Any]: + payload = _request_json( + "GET", self._account_url(), self.target.account_token, None, "account budget GET" + ) + return payload.get("budget", payload) + + def _alert(self, budget: Mapping[str, Any]) -> tuple[int, str]: + matches = [] + for index, alert in enumerate(budget["alert_configurations"]): + actions = alert.get("action_configurations", []) + if alert.get("scope_type") == _PER_USER and any( + action.get("action_type") == _BLOCK for action in actions + ): + assert not alert.get("principal_overrides") + matches.append((index, str(alert["quantity_threshold"]))) + assert len(matches) == 1, "account budget must have one per-user BLOCK_USAGE alert" + return matches[0] + + def _put(self, budget: Mapping[str, Any], label: str, threshold: str) -> None: + self._dirty = True + self._record("mutation", label=label, threshold=threshold, status="write_started") + _request_json( + "PUT", + self._account_url(), + self.target.account_token, + {"budget": _budget_update(budget)}, + "account budget PUT", + ) + + def _account_url(self) -> str: + return self.target.account_host.rstrip("/") + _BUDGET_PATH.format( + account_id=urllib.parse.quote(self.target.account_id, safe=""), + budget_id=urllib.parse.quote(self.target.budget_id, safe=""), + ) + + def _workspace_request( + self, method: str, path: str, body: Mapping[str, Any] | None, operation: str + ) -> Any: + token = self.session.env["DATABRICKS_BEARER"] + return _request_json(method, self.workspace + path, token, body, operation) + + def _record(self, kind: str, **fields: object) -> None: + self._evidence += 1 + self.session.record( + f"budget-defaults-{self._evidence:03d}-{kind}.json", + {"kind": kind, **fields}, + ) diff --git a/tests/integration/utils/evidence.py b/tests/integration/utils/evidence.py index 4c813d671..b8074f023 100644 --- a/tests/integration/utils/evidence.py +++ b/tests/integration/utils/evidence.py @@ -2,6 +2,7 @@ import json import re +import subprocess import uuid from pathlib import Path @@ -273,3 +274,98 @@ def assert_subagent_routed( # Some agent versions omit the child's model from SubagentStart. The report # preserves that unknown value; this test claims decision + spawn + task, # not model-identity verification when the agent did not expose it. + + +def assert_task_model(session, agent: str, task, expected_model: str, label: str) -> str: + """Require the unique task's native completion to use the expected model.""" + + def normalize(model): + model = model.removeprefix("system.ai.").removesuffix("[1m]").removesuffix("[200k]") + return re.sub(r"^(gpt-\d+)-(\d+)", r"\1.\2", model) if agent == "codex" else model + + observed = [] + for path, records in agent_sessions(session, agent).items(): + if is_child_session(agent, path, records): + continue + contexts = {} + for record in records: + if agent == "claude": + if record.get("type") != "assistant": + continue + message = record.get("message", {}) + if message.get("role") != "assistant": + continue + content = message.get("content") + answer = ( + content + if isinstance(content, str) + else "\n".join(assistant_answers(agent, [record])) + ) + if task.value not in answer: + continue + model = message.get("model") + assert isinstance(model, str) and model, f"Claude task answer has no model: {path}" + else: + payload = record.get("payload", {}) + if record.get("type") == "turn_context" and payload.get("turn_id"): + contexts[payload["turn_id"]] = payload.get("model") + continue + if record.get("type") != "event_msg" or payload.get("type") != "task_complete": + continue + answer = payload.get("last_agent_message") + if not isinstance(answer, str) or task.value not in answer: + continue + model = contexts.get(payload.get("turn_id")) + assert isinstance(model, str) and model, ( + f"Codex task completion has no preceding model context: {path}" + ) + observed.append(model) + + assert observed, f"{label}: completed {agent} task exposed no native request model" + models = {normalize(model) for model in observed} + expected = normalize(expected_model) + session.record(f"{label}-task-model.json", {"agent": agent, "models": sorted(models)}) + assert models == {expected}, ( + f"{label}: completed {agent} task used models {sorted(models)!r}, expected {expected!r}" + ) + return observed[-1] + + +def assert_native_process(terminal, agent: str, label: str, session) -> None: + """Require a real Claude/Codex executable or its installed npm package in the PTY tree.""" + root_pid = int(terminal.child.pid) + result = subprocess.run( + ["ps", "-axo", "pid=,ppid=,comm=,args="], + check=True, + capture_output=True, + text=True, + timeout=5, + ) + rows = {} + for line in result.stdout.splitlines(): + match = re.match(r"\s*(\d+)\s+(\d+)\s+(\S+)(?:\s+(.*))?$", line) + if match: + rows[int(match.group(1))] = { + "pid": int(match.group(1)), + "ppid": int(match.group(2)), + "executable": match.group(3), + "command": (match.group(4) or "").strip(), + } + descendants = {root_pid} + while True: + new = {row["pid"] for row in rows.values() if row["ppid"] in descendants} - descendants + if not new: + break + descendants.update(new) + package = { + "claude": "/node_modules/@anthropic-ai/claude-code/", + "codex": "/node_modules/@openai/codex/", + }[agent] + native = [ + row + for pid, row in rows.items() + if pid in descendants + and (Path(row["executable"]).name == agent or package in row["command"]) + ] + assert native, f"{label}: no native {agent} process in PTY descendants" + session.record(f"{label}-process.json", {"root_pid": root_pid, "native": native}) diff --git a/tests/integration/utils/harness.py b/tests/integration/utils/harness.py index c522a2304..b3af73f47 100644 --- a/tests/integration/utils/harness.py +++ b/tests/integration/utils/harness.py @@ -108,7 +108,12 @@ def __init__(self, root: Path, project_root: Path, binary: Path, artifacts: Path def redact(self, text: str, *, strip_ansi: bool = True) -> str: # Also scrub the relayed launch's subscription OAuth token, not just the bearer. - for name in ("DATABRICKS_BEARER", "DATABRICKS_SECOND_BEARER", "CLAUDE_CODE_OAUTH_TOKEN"): + for name in ( + "DATABRICKS_BEARER", + "DATABRICKS_SECOND_BEARER", + "CLAUDE_CODE_OAUTH_TOKEN", + "UG_BUDGET_ACCOUNT_TOKEN", + ): for token in (os.environ.get(name), self.env.get(name)): if token: text = text.replace(token, "") diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 1e37cf160..fdb733f6b 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -71,7 +71,7 @@ def test_integration_ci_gate_requires_selected_managed_jobs(suite, managed_resul assert script is not None results = { job: {"result": "success"} - for job in ("installation", "workspace", "smoke", "full", "managed") + for job in ("installation", "workspace", "smoke", "full", "cuj5", "managed") } results["managed"]["result"] = managed_result for job in { From db633ce1bccef6cb634c1a5e6b0fe5f652f5b2bf Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Mon, 5 Oct 2026 15:05:44 +0000 Subject: [PATCH 02/19] Migrate CUJ5 to the dedicated SDK-backed test suite --- .github/workflows/ci.yml | 5 +- .github/workflows/integration.yml | 36 ++- scripts/run_integration.py | 107 ++++----- tests/README.md | 14 +- tests/conftest.py | 2 +- tests/e2e_cuj/AGENTS.md | 18 ++ tests/e2e_cuj/base.py | 29 +++ .../utils => e2e_cuj}/budget_defaults.py | 141 +++++------- tests/e2e_cuj/conftest.py | 21 ++ tests/e2e_cuj/pytest.ini | 7 + tests/e2e_cuj/test_ug_budget_defaults.py | 208 ++++++++++++++++++ tests/integration/README.md | 62 ++---- tests/integration/pytest.ini | 1 - tests/integration/test_ug_budget_defaults.py | 188 ---------------- tests/integration/utils/harness.py | 2 +- tests/test_integration_contract.py | 43 +++- tests/test_integration_runner.py | 31 +++ 17 files changed, 523 insertions(+), 392 deletions(-) create mode 100644 tests/e2e_cuj/AGENTS.md create mode 100644 tests/e2e_cuj/base.py rename tests/{integration/utils => e2e_cuj}/budget_defaults.py (71%) create mode 100644 tests/e2e_cuj/conftest.py create mode 100644 tests/e2e_cuj/pytest.ini create mode 100644 tests/e2e_cuj/test_ug_budget_defaults.py delete mode 100644 tests/integration/test_ug_budget_defaults.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bbb44c10..62142aed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,10 @@ permissions: concurrency: group: ci-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} - cancel-in-progress: true + # Integration includes a serialized CUJ that mutates a shared target; keep + # the enclosing workflow queued so cancellation cannot interrupt cleanup. + queue: max + cancel-in-progress: false jobs: lint: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index fcccb6b73..13f3b263b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,7 +44,10 @@ permissions: concurrency: group: integration-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} - cancel-in-progress: true + # Dedicated CUJs mutate shared targets. Keep runs queued so a newer + # commit cannot interrupt an active mutation before its cleanup runs. + queue: max + cancel-in-progress: false env: EXPECTED_WORKSPACE: https://eng-ml-inference-team-us-east-1.cloud.databricks.com @@ -212,6 +215,7 @@ jobs: - name: Run the selected live cases against the e2e workspace shell: bash run: | + suite="${TEST_SUITE:-integration}" case "$AGENT" in claude) args=(--claude-version "$CLAUDE_VERSION") ;; codex) args=(--codex-version "$CODEX_VERSION") ;; @@ -224,7 +228,7 @@ jobs: uv run --no-project --python 3.12 python scripts/run_integration.py \ --python 3.12 --ug-version "$UG_VERSION" --entry-point "$ENTRY_POINT" \ --default-index "$PACKAGE_INDEX" --output "$RUNNER_TEMP/ug-integration" \ - "${args[@]}" -- -m "$TEST_MARKER" + --suite "$suite" "${args[@]}" -- -m "$TEST_MARKER" - name: Upload live test evidence if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -347,29 +351,23 @@ jobs: ARTIFACT_NAME: integration-full-${{ matrix.agent }} steps: *live-steps - cuj5: - name: "CUJ5: budget smart defaults · Claude and Codex" + cuj: + name: "Dedicated CUJs · Claude and Codex" if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} runs-on: ubuntu-22.04 timeout-minutes: 70 concurrency: - group: cuj5-budget-defaults + group: e2e-cuj + queue: max cancel-in-progress: false env: - UCODE_TEST_WORKSPACE: ${{ secrets.UG_CUJ5_WORKSPACE }} - DATABRICKS_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} - DATABRICKS_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} - UG_BUDGET_WORKSPACE_ID: "7474645051570450" - UG_BUDGET_ACCOUNT_HOST: https://accounts.cloud.databricks.com - UG_BUDGET_ACCOUNT_ID: 2860b24d-05af-4fb5-83b4-6539492f8180 - UG_BUDGET_ID: 25c4ce5c-fcd6-4d00-8266-69030cc1a236 - UG_BUDGET_SONNET_MODEL: system.ai.claude-sonnet-4-6 - UG_BUDGET_SOL_MODEL: system.ai.gpt-5-6-sol - UG_BUDGET_LUNA_MODEL: system.ai.gpt-5-6-luna + UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} DEPENDENCY: ${{ inputs.dependency }} AGENT: both - TEST_MARKER: cuj5 - ARTIFACT_NAME: integration-cuj5 + TEST_SUITE: e2e-cuj + TEST_MARKER: cuj + ARTIFACT_NAME: integration-e2e-cuj steps: *live-steps # Non-blocking: continue-on-error and absent from the cujs `needs`, so a flaky OpenCode @@ -458,7 +456,7 @@ jobs: name: All integration tests if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} # installation-windows remains advisory until its initial platform issues are fixed. - needs: [installation, workspace, smoke, full, cuj5, managed] + needs: [installation, workspace, smoke, full, cuj, managed] runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -479,7 +477,7 @@ jobs: required.append("smoke") if suite != "smoke": required.append("full") - required.append("cuj5") + required.append("cuj") if suite in {"full", "live"}: required.append("managed") failed = [job for job in required if results[job]["result"] != "success"] diff --git a/scripts/run_integration.py b/scripts/run_integration.py index cc7070b93..f7750d5c3 100644 --- a/scripts/run_integration.py +++ b/scripts/run_integration.py @@ -220,27 +220,6 @@ def mint_m2m_token(workspace: str, client_id: str, client_secret: str) -> str: return token -def mint_account_m2m_token( - account_host: str, client_id: str, client_secret: str, *, account_id: str -) -> str: - """Mint a short-lived account token for the account-scoped budget API.""" - basic = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode() - body = urllib.parse.urlencode( - {"grant_type": "client_credentials", "scope": "all-apis"} - ).encode() - request = urllib.request.Request( - account_host.rstrip("/") - + f"/oidc/accounts/{urllib.parse.quote(account_id, safe='')}/v1/token", - data=body, - headers={ - "Authorization": f"Basic {basic}", - "Content-Type": "application/x-www-form-urlencoded", - }, - ) - with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 (https account URL) - return json.load(response)["access_token"] - - @contextlib.contextmanager def managed_process(command, *, interrupt=False, **kwargs): """Bound child lifetimes, including descendants that outlive their parent.""" @@ -300,6 +279,12 @@ def arguments( platform_name = os.name if platform_name is None else platform_name environment = os.environ if environment is None else environment parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--suite", + choices=["integration", "e2e-cuj"], + default="integration", + help="CUJ tests declare their own workspace and use shared service-principal credentials.", + ) source = parser.add_mutually_exclusive_group() source.add_argument("--ug-version", default="checkout", help="Exact ug release, or checkout.") source.add_argument( @@ -375,7 +360,7 @@ def arguments( ) parser.add_argument("--npm-registry", default="https://registry.npmjs.org") parser.add_argument("--profile", help="Explicit Databricks profile to mint the live bearer.") - parser.add_argument("--workspace", default=environment.get("UCODE_TEST_WORKSPACE")) + parser.add_argument("--workspace") parser.add_argument( "--second-workspace", default=environment.get("UCODE_TEST_SECOND_WORKSPACE"), @@ -394,6 +379,14 @@ def arguments( "pytest_args", nargs=argparse.REMAINDER, help="After --, pass pytest filters." ) args = parser.parse_args(argv) + if args.suite == "integration" and args.workspace is None: + args.workspace = environment.get("UCODE_TEST_WORKSPACE") + if args.suite == "e2e-cuj" and (args.workspace or args.profile): + parser.error( + "CUJ tests declare their workspace and auth; do not use --workspace/--profile." + ) + if args.suite == "e2e-cuj" and (args.installation_only or args.headless_only): + parser.error("--installation-only and --headless-only belong to the integration suite.") if platform_name != "posix" and not (args.installation_only or args.headless_only): parser.error( "Live agent/TUI integration requires POSIX PTY, managed-settings, and signal " @@ -408,7 +401,7 @@ def arguments( filters.add_argument("--maxfail", type=int) extra = args.pytest_args[1:] if args.pytest_args[:1] == ["--"] else args.pytest_args selected = filters.parse_args(extra) - marker = selected.m or "live" + marker = selected.m or ("cuj" if args.suite == "e2e-cuj" else "live") if args.installation_only: marker = f"installation and ({selected.m})" if selected.m else "installation" args.pytest_args = [] @@ -430,7 +423,7 @@ def arguments( for dependency in args.dependency: if not re.fullmatch(r"[A-Za-z0-9_.-]+==[A-Za-z0-9_.!+-]+", dependency): parser.error("--dependency requires an exact PACKAGE==VERSION constraint.") - if not args.installation_only: + if not args.installation_only and args.suite == "integration": if not args.workspace or not args.workspace.startswith("https://"): parser.error( "Set UCODE_TEST_WORKSPACE to the existing e2e workspace, or use --workspace." @@ -542,11 +535,9 @@ def terminate(signum, frame): second_bearer = os.environ.get("DATABRICKS_SECOND_BEARER", "").strip() oauth_token = os.environ.get("CLAUDE_CODE_OAUTH_TOKEN", "").strip() target_bearers: dict[str, str] = {} - budget_environment = { - key: value for key, value in os.environ.items() if key.startswith("UG_BUDGET_") - } client_secrets = ( os.environ.get("DATABRICKS_CLIENT_SECRET", ""), + os.environ.get("UG_CUJ_SP_CLIENT_SECRET", ""), os.environ.get("UG_MPS_DEFAULTS_CLIENT_SECRET", ""), os.environ.get("UG_PARENT_SCHEMA_DEFAULTS_CLIENT_SECRET", ""), ) @@ -559,7 +550,6 @@ def redact(value: str) -> str: second_bearer, oauth_token, *target_bearers.values(), - budget_environment.get("UG_BUDGET_ACCOUNT_TOKEN", ""), *client_secrets, *installer_secrets, ), @@ -616,6 +606,7 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "warehouse_id": args.warehouse_id, }, "platform": platform.platform(), + "suite": args.suite, "installation_only": args.installation_only, "headless_only": args.headless_only, } @@ -822,7 +813,7 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: raise RuntimeError(f"Expected {agent} {expected}, got {version!r}") report["agents"][agent] = version - if args.profile and not args.installation_only: + if args.profile and not args.installation_only and args.suite == "integration": # Never select a local profile implicitly. Do not persist auth output. with managed_process( [ @@ -850,19 +841,12 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: if not bearer: raise RuntimeError("Selected profile returned no access token.") - if not args.installation_only: + if not args.installation_only and args.suite == "integration": client_id = os.environ.get("DATABRICKS_CLIENT_ID", "").strip() client_secret = os.environ.get("DATABRICKS_CLIENT_SECRET", "").strip() if not bearer and not args.profile and client_id and client_secret: bearer = mint_m2m_token(args.workspace, client_id, client_secret) - if budget_environment: - budget_environment["UG_BUDGET_ACCOUNT_TOKEN"] = mint_account_m2m_token( - budget_environment["UG_BUDGET_ACCOUNT_HOST"], - client_id, - client_secret, - account_id=budget_environment["UG_BUDGET_ACCOUNT_ID"], - ) for bearer_env, target_workspace, client_id, secret_env in MANAGED_DEFAULTS_TARGETS: secret = os.environ.get(secret_env, "").strip() if args.workspace.rstrip("/") == target_workspace: @@ -871,6 +855,8 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: target_bearers[bearer_env] = mint_m2m_token(target_workspace, client_id, secret) test_dependencies = ["pytest==9.0.3"] + if args.suite == "e2e-cuj": + test_dependencies.append("databricks-sdk==0.135.0") if os.name == "posix": test_dependencies.extend(["pexpect==4.9.0", "pyte==0.8.2"]) run( @@ -896,7 +882,6 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "UG_INTEGRATION_AGENTS": ",".join(agents), "UG_INTEGRATION_CLAUDE_PROVIDER": args.claude_provider, "UG_INTEGRATION_CLAUDE_RELAYED_PROVIDER": args.claude_relayed_provider, - "UG_INTEGRATION_CLAUDE_OAUTH_TOKEN": oauth_token, "UG_INTEGRATION_CLAUDE_PROVIDER_MODEL": args.claude_provider_model, "UG_INTEGRATION_CLAUDE_BEDROCK_ALLOW_ALL_PROVIDER": args.claude_bedrock_allow_all_provider, "UG_INTEGRATION_CLAUDE_BEDROCK_ALLOW_ALL_MODEL": args.claude_bedrock_allow_all_model, @@ -905,24 +890,38 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "UG_INTEGRATION_PARENT_SCHEMA": args.parent_schema, "UG_INTEGRATION_CLAUDE_PARENT_MODEL": args.claude_parent_model, "UG_INTEGRATION_CODEX_PARENT_MODEL": args.codex_parent_model, - "UCODE_TEST_WORKSPACE": args.workspace or "", - "DATABRICKS_BEARER": bearer, - "UG_MPS_DEFAULTS_BEARER": target_bearers.get("UG_MPS_DEFAULTS_BEARER", ""), - "UG_PARENT_SCHEMA_DEFAULTS_BEARER": target_bearers.get( - "UG_PARENT_SCHEMA_DEFAULTS_BEARER", "" - ), - "UCODE_TEST_SECOND_WORKSPACE": args.second_workspace or "", - "DATABRICKS_SECOND_BEARER": second_bearer, - "UG_INTEGRATION_WAREHOUSE_ID": args.warehouse_id or "", } ) + if args.suite == "integration": + runtime_env.update( + { + "UCODE_TEST_WORKSPACE": args.workspace or "", + "DATABRICKS_BEARER": bearer, + "UG_MPS_DEFAULTS_BEARER": target_bearers.get("UG_MPS_DEFAULTS_BEARER", ""), + "UG_PARENT_SCHEMA_DEFAULTS_BEARER": target_bearers.get( + "UG_PARENT_SCHEMA_DEFAULTS_BEARER", "" + ), + "UCODE_TEST_SECOND_WORKSPACE": args.second_workspace or "", + "DATABRICKS_SECOND_BEARER": second_bearer, + "UG_INTEGRATION_CLAUDE_OAUTH_TOKEN": oauth_token, + "UG_INTEGRATION_WAREHOUSE_ID": args.warehouse_id or "", + } + ) for agent in agents: runtime_env[f"UG_INTEGRATION_{agent.upper()}_MODEL"] = ( getattr(args, f"{agent}_model") or "" ) - suite = ROOT / "tests/integration" + suite = ROOT / "tests" / args.suite.replace("-", "_") suite_hash = hashlib.sha256() - for path in [Path(__file__), *sorted(suite.rglob("*.py")), suite / "pytest.ini"]: + sources = [Path(__file__), *sorted(suite.rglob("*.py")), suite / "pytest.ini"] + if args.suite == "e2e-cuj": + sources.extend( + [ + ROOT / "tests/integration/conftest.py", + *sorted((ROOT / "tests/integration/utils").rglob("*.py")), + ] + ) + for path in sources: suite_hash.update(str(path.relative_to(ROOT)).encode() + b"\0" + path.read_bytes()) report["suite_sha256"] = suite_hash.hexdigest() extra = args.pytest_args @@ -936,6 +935,14 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: installation_only=args.installation_only, headless_only=args.headless_only, ) + test_env = dict(runtime_env) + if args.suite == "e2e-cuj": + test_env.update( + { + key: os.environ.get(key, "") + for key in ("UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET") + } + ) with managed_process( [ test_python, @@ -951,7 +958,7 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: f"--junitxml={output / 'junit.xml'}", *extra, ], - env={**runtime_env, **budget_environment}, + env=test_env, cwd=output, stdin=subprocess.DEVNULL, interrupt=True, diff --git a/tests/README.md b/tests/README.md index 0e0599e35..ed1d1b059 100644 --- a/tests/README.md +++ b/tests/README.md @@ -9,6 +9,7 @@ mocks, monkeypatching, fake binaries/services, or fabricated ug state. | Unit/component | Existing `test_*.py` files | Individual behavior; dependencies may be mocked | | Existing e2e | `test_e2e*.py` | Real workspace behavior with some patched setup/internal calls | | Integration CUJs | `integration/test_*.py` | Public configure, TUI, script, command, protocol, and lifecycle journeys | +| Dedicated-workspace CUJs | `e2e_cuj/test_*.py` | Mutating journeys against test-owned workspaces and targets | | Installation | `integration/test_installation.py` | Fresh installed package, CLI, and local helpers without credentials on Linux and advisory native Windows | `test_entry_points.py` also runs both installed console scripts (`ug` and `ucode`) @@ -147,7 +148,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `test_ug_budget_defaults.py` (CUJ5) | Five bare launches across the 50%/80% tiers and an explicit Claude launch at 80% | Real native processes and completed file tasks; model evidence from native session logs; backend spend/threshold agreement with `ug usage` and the launch panel; recommendation display; budget restoration. Requires existing positive spend. Select with `-m cuj5`; prerequisites and coverage limits are in the integration README. | +| `e2e_cuj/test_ug_budget_defaults.py` | Five bare launches across the 50%/80% tiers and an explicit Claude launch at 80% | Real native processes and completed file tasks; model evidence from native session logs; backend spend/threshold agreement with `ug usage` and the launch panel; recommendation display; budget restoration. Requires existing positive spend. Run with `--suite e2e-cuj -- -m cuj`; the test owns its target constants and details. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -173,8 +174,10 @@ executions. Thirteen additional managed-fixture cases cover focused model, MCP, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. -CUJ5 adds six live cases: five bare launches and one explicit Claude override at 80%. -Generated-file cleanup and strict app-server stdout assertions remain enforced. +The dedicated-workspace CUJ suite adds six budget cases: five bare launches and one +explicit Claude override at 80%. It runs separately from the 62-case integration +suite and owns its target constants in the test. Generated-file cleanup and strict +app-server stdout assertions remain enforced in the integration suite. Unmanaged discovery Cases 7–14 configure, list models, or open the picker without submitting inference prompts; separate task journeys still perform inference. They require a real workspace with no CodingAgentConfig; a read-only prerequisite @@ -215,7 +218,10 @@ journeys for both agents, in two parallel jobs. After smoke finishes, the full suite runs all 62 live cases across two parallel agent jobs: one Claude VM and one Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs -for the same agent overlap within a run. +for the same agent overlap within a run. The dedicated six-case budget suite runs +with both agents in one serial job and is queued without cancellation because it +mutates a shared target; only the shared `UG_CUJ_SP_CLIENT_ID` and +`UG_CUJ_SP_CLIENT_SECRET` secrets are forwarded. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites share workspace capacity; overlapping their requests can still encounter rate limits. diff --git a/tests/conftest.py b/tests/conftest.py index 4ae9e4362..2b0c9f090 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ # The integration suite has its own configuration and subprocess-only fixtures. # Run it through scripts/run_integration.py, outside this fixture hierarchy. -collect_ignore = ["integration"] +collect_ignore = ["integration", "e2e_cuj"] @pytest.fixture(autouse=True) diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md new file mode 100644 index 000000000..dcde5b8df --- /dev/null +++ b/tests/e2e_cuj/AGENTS.md @@ -0,0 +1,18 @@ +# Dedicated-workspace CUJs + +Each CUJ owns a separate workspace. Subclass `BaseCujTest` from `base.py` and set +`WORKSPACE_URL`. Setup provides `self.workspace`, a Databricks SDK client using +`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` with OAuth M2M authentication. +Do not share the workspace between concurrent runs. No stubbed configuration. + +Run from the repository root through the isolated integration runner: + +```bash +uv run --no-project --python 3.12 python scripts/run_integration.py \ + --suite e2e-cuj --ug-version checkout \ + --claude-version 2.1.280 --codex-version 0.154.0 -- -m cuj +``` + +The runner installs the pinned Databricks SDK and both agent CLIs, then invokes +pytest with this directory as its `--confcutdir`. The default marker for this +suite is `cuj`; keep the explicit `-m cuj` when reproducing CI. diff --git a/tests/e2e_cuj/base.py b/tests/e2e_cuj/base.py new file mode 100644 index 000000000..e006ba9e6 --- /dev/null +++ b/tests/e2e_cuj/base.py @@ -0,0 +1,29 @@ +"""Base class for CUJs with a dedicated workspace and service-principal auth.""" + +import os +from typing import ClassVar + +import pytest +from databricks.sdk import WorkspaceClient + + +class BaseCujTest: + WORKSPACE_URL: ClassVar[str] = "" + workspace: WorkspaceClient + + @pytest.fixture(autouse=True) + def setup_workspace(self): + if not self.WORKSPACE_URL: + pytest.fail("Set WORKSPACE_URL on your CUJ test class.", pytrace=False) + + client_id = os.environ.get("UG_CUJ_SP_CLIENT_ID", "") + client_secret = os.environ.get("UG_CUJ_SP_CLIENT_SECRET", "") + if not client_id or not client_secret: + pytest.fail("Set UG_CUJ_SP_CLIENT_ID and UG_CUJ_SP_CLIENT_SECRET.", pytrace=False) + + self.workspace = WorkspaceClient( + host=self.WORKSPACE_URL, + client_id=client_id, + client_secret=client_secret, + auth_type="oauth-m2m", + ) diff --git a/tests/integration/utils/budget_defaults.py b/tests/e2e_cuj/budget_defaults.py similarity index 71% rename from tests/integration/utils/budget_defaults.py rename to tests/e2e_cuj/budget_defaults.py index 01be2d883..e09b33463 100644 --- a/tests/integration/utils/budget_defaults.py +++ b/tests/e2e_cuj/budget_defaults.py @@ -1,52 +1,25 @@ -"""Small live fixture for the CUJ5 budget smart-default journeys.""" +"""Small live fixture for the CUJ budget smart-default journeys.""" from __future__ import annotations -import json -import os import time -import urllib.error import urllib.parse -import urllib.request from collections.abc import Mapping -from dataclasses import dataclass, field from decimal import Decimal, localcontext from typing import Any +from databricks.sdk import AccountClient, WorkspaceClient +from databricks.sdk.core import Config + _CONFIG_PATH = "/api/ai-gateway/v2/coding-agent-configs" _RECOMMEND_PATH = _CONFIG_PATH + ":recommendModel" _BUDGET_PATH = "/api/2.1/accounts/{account_id}/budgets/{budget_id}" _PER_USER = "ALERT_CONFIGURATION_SCOPE_TYPE_PER_USER" _BLOCK = "BLOCK_USAGE" -_TIMEOUT = 30.0 +_TIMEOUT = 30 _POLL = 0.5 -@dataclass(frozen=True, slots=True) -class BudgetTarget: - workspace_id: str - account_host: str - account_id: str - budget_id: str - sonnet_model: str - sol_model: str - luna_model: str - account_token: str = field(repr=False) - - @classmethod - def from_environment(cls) -> BudgetTarget: - return cls( - os.environ["UG_BUDGET_WORKSPACE_ID"].strip(), - os.environ["UG_BUDGET_ACCOUNT_HOST"].strip(), - os.environ["UG_BUDGET_ACCOUNT_ID"].strip(), - os.environ["UG_BUDGET_ID"].strip(), - os.environ["UG_BUDGET_SONNET_MODEL"].strip(), - os.environ["UG_BUDGET_SOL_MODEL"].strip(), - os.environ["UG_BUDGET_LUNA_MODEL"].strip(), - os.environ["UG_BUDGET_ACCOUNT_TOKEN"].strip(), - ) - - def _budget_update(budget: Mapping[str, Any]) -> dict[str, Any]: """Keep the account API's writable budget fields and remove response metadata.""" @@ -81,36 +54,43 @@ def clean(value: Any) -> Any: } -def _request_json( - method: str, - url: str, - token: str, - body: Mapping[str, Any] | None, - operation: str, -) -> Any: - encoded = json.dumps(body).encode("utf-8") if body is not None else None - headers = {"Accept": "application/json", "Authorization": f"Bearer {token}"} - if encoded is not None: - headers["Content-Type"] = "application/json" - request = urllib.request.Request(url, data=encoded, headers=headers, method=method) - try: - with urllib.request.urlopen(request, timeout=_TIMEOUT) as response: - raw = response.read() - except urllib.error.HTTPError as error: - raise AssertionError(f"{operation} failed: HTTP {error.code}") from None - return json.loads(raw) if raw else {} - - class BudgetDefaults: """Temporarily move one existing per-user hard-block threshold.""" - def __init__(self, session: Any, workspace: str, target: BudgetTarget): + def __init__( + self, + session: Any, + workspace: WorkspaceClient, + *, + workspace_id: str, + account_host: str, + account_id: str, + budget_id: str, + sonnet_model: str, + sol_model: str, + luna_model: str, + ): self.session = session - self.workspace = workspace.rstrip("/") - self.target = target - self.sonnet_model = target.sonnet_model - self.sol_model = target.sol_model - self.luna_model = target.luna_model + self.workspace_client = workspace + self.workspace = workspace.config.host.rstrip("/") + self.workspace_id = workspace_id + self.account_host = account_host.rstrip("/") + self.account_id = account_id + self.budget_id = budget_id + self.sonnet_model = sonnet_model + self.sol_model = sol_model + self.luna_model = luna_model + self.account = AccountClient( + config=Config( + host=self.account_host, + account_id=self.account_id, + client_id=workspace.config.client_id, + client_secret=workspace.config.client_secret, + auth_type="oauth-m2m", + http_timeout_seconds=_TIMEOUT, + retry_timeout_seconds=_TIMEOUT, + ) + ) self._original_threshold: str | None = None self._dirty = False self._evidence = 0 @@ -119,11 +99,11 @@ def __init__(self, session: Any, workspace: str, target: BudgetTarget): def __enter__(self) -> BudgetDefaults: self._validate_config() current = self._account_budget() - assert current["budget_configuration_id"] == self.target.budget_id - assert str(current["account_id"]) == self.target.account_id + assert current["budget_configuration_id"] == self.budget_id + assert str(current["account_id"]) == self.account_id workspace_filter = current["filter"]["workspace_id"] assert workspace_filter["operator"] in ("IN", "BUDGET_CONFIGURATION_FILTER_OPERATOR_IN") - assert [str(value) for value in workspace_filter["values"]] == [self.target.workspace_id] + assert [str(value) for value in workspace_filter["values"]] == [self.workspace_id] _, self._original_threshold = self._alert(current) self._record("setup", threshold=self._original_threshold, status="captured") return self @@ -134,7 +114,7 @@ def __exit__(self, exc_type, exc_value, traceback) -> bool: return False def _validate_config(self) -> None: - payload = self._workspace_request("GET", _CONFIG_PATH, None, "managed config") + payload: Any = self.workspace_client.api_client.do("GET", path=_CONFIG_PATH) config = payload["coding_agent_configs"][0] assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE" @@ -147,7 +127,7 @@ def _validate_config(self) -> None: assert codex["default_models"]["default_model"] == self.sol_model smart_defaults = config["smart_defaults"] - assert smart_defaults["budget_id"] == self.target.budget_id + assert smart_defaults["budget_id"] == self.budget_id tiers = smart_defaults["tiers"] assert [Decimal(str(tier["spending_percentage"])) for tier in tiers] == [ Decimal("0.5"), @@ -159,7 +139,7 @@ def _validate_config(self) -> None: ] def recommendation(self, label: str = "recommendation") -> dict[str, Any]: - payload = self._workspace_request("POST", _RECOMMEND_PATH, {}, "recommendModel") + payload: Any = self.workspace_client.api_client.do("POST", path=_RECOMMEND_PATH, body={}) self._record( "recommendation", label=label, @@ -233,9 +213,7 @@ def _wait_for_recommendation(self, threshold: Decimal, label: str) -> dict[str, time.sleep(min(_POLL, remaining)) def _account_budget(self) -> dict[str, Any]: - payload = _request_json( - "GET", self._account_url(), self.target.account_token, None, "account budget GET" - ) + payload = self._account_request("GET", None) return payload.get("budget", payload) def _alert(self, budget: Mapping[str, Any]) -> tuple[int, str]: @@ -253,26 +231,23 @@ def _alert(self, budget: Mapping[str, Any]) -> tuple[int, str]: def _put(self, budget: Mapping[str, Any], label: str, threshold: str) -> None: self._dirty = True self._record("mutation", label=label, threshold=threshold, status="write_started") - _request_json( - "PUT", - self._account_url(), - self.target.account_token, - {"budget": _budget_update(budget)}, - "account budget PUT", + self._account_request("PUT", {"budget": _budget_update(budget)}) + + def _account_request(self, method: str, body: dict[str, Any] | None) -> Any: + # AccountClient's ApiClient calls Config.authenticate for each request, allowing the + # restoration path to refresh an expired service-principal token. + return self.account.api_client.do( + method, + path=self._account_path(), + body=body, ) - def _account_url(self) -> str: - return self.target.account_host.rstrip("/") + _BUDGET_PATH.format( - account_id=urllib.parse.quote(self.target.account_id, safe=""), - budget_id=urllib.parse.quote(self.target.budget_id, safe=""), + def _account_path(self) -> str: + return _BUDGET_PATH.format( + account_id=urllib.parse.quote(self.account_id, safe=""), + budget_id=urllib.parse.quote(self.budget_id, safe=""), ) - def _workspace_request( - self, method: str, path: str, body: Mapping[str, Any] | None, operation: str - ) -> Any: - token = self.session.env["DATABRICKS_BEARER"] - return _request_json(method, self.workspace + path, token, body, operation) - def _record(self, kind: str, **fields: object) -> None: self._evidence += 1 self.session.record( diff --git a/tests/e2e_cuj/conftest.py b/tests/e2e_cuj/conftest.py new file mode 100644 index 000000000..07c407569 --- /dev/null +++ b/tests/e2e_cuj/conftest.py @@ -0,0 +1,21 @@ +"""Standalone fixtures for dedicated-workspace CUJs.""" + +from __future__ import annotations + +import shutil + +import pytest + +from tests.integration.conftest import installed_binary as installed_binary +from tests.integration.conftest import session as session + + +@pytest.fixture +def live_session(session, setup_workspace, request): + """Reuse the installed-product session with a fresh workspace bearer.""" + headers = request.instance.workspace.config.authenticate() + session.env["DATABRICKS_BEARER"] = headers["Authorization"].removeprefix("Bearer ") + for binary in ("databricks", "claude", "codex"): + if not shutil.which(binary, path=session.env["PATH"]): + pytest.fail(f"Required CUJ binary is missing: {binary}", pytrace=False) + return session diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini new file mode 100644 index 000000000..9102e6b80 --- /dev/null +++ b/tests/e2e_cuj/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +testpaths = . +pythonpath = ../.. ../integration +addopts = --strict-markers --tb=short +markers = + cuj: dedicated-workspace customer journey + tui: real interactive terminal boot, keyboard input, exit and reopen diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py new file mode 100644 index 000000000..bd9b8253e --- /dev/null +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -0,0 +1,208 @@ +"""Live customer journeys for workspace smart budget defaults.""" + +import re +from decimal import ROUND_HALF_UP, Decimal + +import pytest +from base import BaseCujTest +from budget_defaults import BudgetDefaults +from utils.evidence import ( + FileTask, + assert_native_process, + assert_task_model, +) +from utils.terminal import AgentTerminal + +pytestmark = [pytest.mark.cuj, pytest.mark.tui] + + +class TestCujBudgetDefaults(BaseCujTest): + WORKSPACE_URL = "https://dbc-497adeef-62c0.cloud.databricks.com" + WORKSPACE_ID = "7474645051570450" + ACCOUNT_HOST = "https://accounts.cloud.databricks.com" + ACCOUNT_ID = "2860b24d-05af-4fb5-83b4-6539492f8180" + BUDGET_ID = "25c4ce5c-fcd6-4d00-8266-69030cc1a236" + SONNET_MODEL = "system.ai.claude-sonnet-4-6" + SOL_MODEL = "system.ai.gpt-5-6-sol" + LUNA_MODEL = "system.ai.gpt-5-6-luna" + + @pytest.fixture + def budget_defaults(self, live_session): + """Supply the target; each journey opens and restores its own budget context.""" + return BudgetDefaults( + live_session, + self.workspace, + workspace_id=self.WORKSPACE_ID, + account_host=self.ACCOUNT_HOST, + account_id=self.ACCOUNT_ID, + budget_id=self.BUDGET_ID, + sonnet_model=self.SONNET_MODEL, + sol_model=self.SOL_MODEL, + luna_model=self.LUNA_MODEL, + ) + + @staticmethod + def _assert_spend(session, text, recommendation, *, usage=False): + """Compare displayed dollars and percentage with the independent backend response.""" + text = session.redact(text) + money = r"\$([\d,]+\.\d{2})" + pattern = ( + rf"Budget spend:\s*{money}\s+of\s+{money}\s+\((\d+)%\)" + if usage + else rf"{money}\s*/\s*{money}\s+(\d+)%\s+used" + ) + matches = re.findall(pattern, text) + assert matches, f"Budget spend was not displayed:\n{text}" + actual_spend, actual_threshold, actual_percent = matches[-1] + spend = Decimal(str(recommendation["current_spend"])) + threshold = Decimal(str(recommendation["effective_threshold"])) + if usage: + # The usage command formats Decimal dollars half-up and percent with .0%. + expected_money = [ + value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) + for value in (spend, threshold) + ] + expected_percent = int(format(float(spend / threshold), ".0%").removesuffix("%")) + else: + # The launch panel formats float dollars and rounds its percentage half-up. + expected_money = [Decimal(format(float(value), ".2f")) for value in (spend, threshold)] + expected_percent = int(float(spend) / float(threshold) * 100 + 0.5) + assert [ + Decimal(actual_spend.replace(",", "")), + Decimal(actual_threshold.replace(",", "")), + ] == expected_money, text + assert int(actual_percent) == expected_percent, text + + def _assert_launch(self, session, tui, budget, before, label): + assert_native_process(tui, tui.agent, label, session) + budget.assert_stable(before, label) + transcript = "".join(tui.output) + "\n" + tui.visible + self._assert_spend(session, transcript, before) + text = session.redact(transcript) + agent = before.get("recommended_agent") + model = before.get("recommended_model") + if agent: + name = {"CODING_AGENT_CODEX": "Codex", "CODING_AGENT_CLAUDE_CODE": "Claude Code"}[agent] + recommendation = f"Recommended agent is {name}" + if model: + recommendation += f" with model {model}" + assert recommendation in text, text + elif model: + assert f"Recommended model is {model}" in text, text + else: + assert "Recommended agent is" not in text, text + assert "Recommended model is" not in text, text + + def test_bare_ug_launches_claude_below_50_percent(self, live_session, budget_defaults): + """Scenario: configure the workspace and launch bare ``ug`` at 40% budget usage. + + Expected: Claude completes a file task on Sonnet; its real process and the + usage/launch displays agree with the stable backend budget response. + """ + session = live_session + label = "bare-40-percent" + with budget_defaults as budget: + session.run( + "configure", + "--workspace", + budget.workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + before = budget.prepare(Decimal("2.5"), label) + assert before.get("recommended_agent") in (None, "CODING_AGENT_CLAUDE_CODE"), before + assert before.get("recommended_model") in (None, budget.sonnet_model), before + usage = session.run("usage", timeout=120) + self._assert_spend(session, usage.stdout + usage.stderr, before, usage=True) + + task = FileTask(session) + with AgentTerminal(session, "claude", [str(session.binary)], label) as tui: + tui.boot(timeout=240) + self._assert_launch(session, tui, budget, before, label) + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + assert_task_model(session, "claude", task, budget.sonnet_model, label) + tui.exit_normally() + session.assert_not_routed() + + @pytest.mark.parametrize( + ("multiplier", "model_attribute"), + [ + pytest.param("2", "sol_model", id="exact-50"), + pytest.param("1.6", "sol_model", id="between-50-and-80"), + pytest.param("1.25", "luna_model", id="exact-80"), + pytest.param("1.125", "luna_model", id="between-80-and-100"), + ], + ) + def test_bare_ug_launches_codex_at_budget_tier( + self, live_session, budget_defaults, multiplier, model_attribute + ): + """Scenario: launch bare ``ug`` at 50%, 62.5%, 80%, or approximately 88.9% usage. + + Expected: the highest applicable tier selects Codex Sol or Luna, superseding + Codex's Sol default at 80%+. Check the backend response before launch, the native + process, usage display, and completed task's model. Unstable exact boundaries fail. + """ + session = live_session + label = f"bare-codex-{multiplier}" + with budget_defaults as budget: + session.run( + "configure", + "--workspace", + budget.workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + model = getattr(budget, model_attribute) + before = budget.prepare(Decimal(multiplier), label) + assert before.get("recommended_agent") == "CODING_AGENT_CODEX", before + assert before.get("recommended_model") == model, before + usage = session.run("usage", timeout=120) + self._assert_spend(session, usage.stdout + usage.stderr, before, usage=True) + + task = FileTask(session) + with AgentTerminal(session, "codex", [str(session.binary)], label) as tui: + tui.boot(timeout=240) + self._assert_launch(session, tui, budget, before, label) + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + assert_task_model(session, "codex", task, model, label) + tui.exit_normally() + session.assert_not_routed() + + def test_ug_claude_keeps_its_default_when_budget_recommends_codex( + self, live_session, budget_defaults + ): + """Scenario: run explicit ``ug claude`` at 80%, where the budget recommends Codex Luna. + + Expected: the real Claude TUI displays that recommendation but completes a file + task on its own Sonnet default. The Codex-only Luna model never reaches Claude. + """ + session = live_session + label = "explicit-claude-luna-tier" + with budget_defaults as budget: + session.run( + "configure", + "--workspace", + budget.workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + before = budget.prepare(Decimal("1.25"), label) + assert before.get("recommended_agent") == "CODING_AGENT_CODEX", before + assert before.get("recommended_model") == budget.luna_model, before + usage = session.run("usage", timeout=120) + self._assert_spend(session, usage.stdout + usage.stderr, before, usage=True) + + task = FileTask(session) + with AgentTerminal(session, "claude", [str(session.binary), "claude"], label) as tui: + tui.boot(timeout=240) + self._assert_launch(session, tui, budget, before, label) + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + assert_task_model(session, "claude", task, budget.sonnet_model, label) + tui.exit_normally() + session.assert_not_routed() diff --git a/tests/integration/README.md b/tests/integration/README.md index adde8007b..096a25d29 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -13,41 +13,28 @@ The `smart_defaults` wire schema, legacy `spend_tiers` cache reads, and recommen request gating are covered by unit/component tests listed in `../README.md`. This suite does not yet assert live `recommendModel` request counts for configs with and without tiers. -## CUJ5: Budget-driven smart defaults (AIGTWY-4880) +## Dedicated-workspace CUJs -`test_ug_budget_defaults.py` contains six live journeys: +The budget-default journeys live in +[`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py), +outside this existing-workspace integration suite. The test owns its workspace, +account, budget, and model identifiers, while CI forwards only the shared +`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets. -| Launch | Budget usage | Expected agent / model | -| --- | --- | --- | -| Bare `ug` | 40% | Claude / `system.ai.claude-sonnet-4-6` | -| Bare `ug` | Exactly 50% and 62.5% | Codex / `system.ai.gpt-5-6-sol` | -| Bare `ug` | Exactly 80% and approximately 88.9% | Codex / `system.ai.gpt-5-6-luna` | -| Explicit `ug claude` | Exactly 80% | Claude / Sonnet; display the Codex Luna recommendation | - -Each case verifies the real budget and recommendation, runs `ug usage`, checks the native -process and displayed spend/threshold/recommendation, and completes a unique file task on -the expected model. Model evidence comes from native session logs, not HTTP capture. -The fixture changes the dedicated budget's per-user BLOCK_USAGE threshold relative to existing -positive spend and restores it on exit. Setup uses no mocks or generated spend. Detected -instability at an exact boundary fails as "unverified"; usage is sampled, not locked. - -The CUJ5 job in `.github/workflows/integration.yml` owns all workspace/account/budget/model -values. It uses `UG_CUJ5_WORKSPACE` and shared `UG_CUJ_SP_CLIENT_ID` / -`UG_CUJ_SP_CLIENT_SECRET` secrets. The principal needs workspace/model access and account -budget read/update permission. CI runs both agents serially and serializes CUJ5 jobs because -they share one budget. The job is required for `full`, `live`, and `tui` suites. - -For a local run, export the job's `UCODE_TEST_WORKSPACE`, `DATABRICKS_CLIENT_ID`, -`DATABRICKS_CLIENT_SECRET`, and `UG_BUDGET_*` values, then run on a clean POSIX runner: +Run the six cases through the dedicated suite from a clean POSIX runner: ```bash +UG_CUJ_SP_CLIENT_ID=... UG_CUJ_SP_CLIENT_SECRET=... \ python3.12 scripts/run_integration.py \ - --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ - -- -m cuj5 + --suite e2e-cuj --ug-version checkout \ + --claude-version 2.1.280 --codex-version 0.154.0 -- -m cuj ``` -Do not overlap local and CI budget runs. A killed process or forced cancellation can prevent -restoration; verify the budget against its recorded original threshold before rerunning. +The runner installs `databricks-sdk==0.135.0` in the isolated test environment. +Workspace and account clients refresh OAuth M2M tokens for backend calls, including +budget restoration. Do not overlap a local run with +CI: the cases mutate a shared budget threshold and restore it during cleanup. If a +run is interrupted, verify the recorded original threshold before rerunning. Shared subprocess command resolution is covered by `../test_subprocess_cross_os.py` and enforced by Ruff. These component checks do not establish native Windows coverage @@ -215,7 +202,6 @@ test_ug_codex_managed_model_discovery.py # fetched/reused Codex MPS policy case test_ug_claude_model_discovery.py # unmanaged scenarios 7, 9, 11, 13 test_ug_codex_model_discovery.py # unmanaged scenarios 8, 10, 12, 14 test_ug_configure_managed.py # managed workspace: static model list/catalog pointer, no agent selector -test_ug_budget_defaults.py # live budget tiers, agent/model choice, explicit-agent override, usage test_ug_configure_managed_models.py # injected model sources, smart-routing banner, Codex fallback metadata test_ug_configure_managed_mcp.py # injected managed MCP list test_ug_configure_managed_skills.py # injected managed skills: download, coexist, reconcile away @@ -345,8 +331,8 @@ they only configure, list models, and open/close the picker. Other live CUJs per real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex; six CUJ5 budget cases are selected by `cuj5` -and run in their dedicated job. +checks** with Claude and Codex. The separate dedicated-workspace suite adds six +budget cases selected by `-m cuj` and run by the `cuj` job. Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** @@ -371,7 +357,7 @@ constants in the runner; CI only needs `UG_MPS_DEFAULTS_CLIENT_SECRET` for west- mints short-lived tokens and passes bearers to pytest; each test selects its target bearer for `ug configure` and Claude. The client secrets do not enter the pytest process. The 14 retained numbered scenarios comprise 24 explicit journeys: 12 managed and 12 unmanaged -executions; the complete integration suite collects 108 executions. See the named coverage and gaps matrix in +executions; the complete integration suite collects 102 executions. See the named coverage and gaps matrix in [../README.md](../README.md). ```bash @@ -515,7 +501,7 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs all 62 live cases, including those smoke cases, in two -disjoint agent lanes. The dedicated CUJ5 job runs its six budget cases with +disjoint agent lanes. The dedicated `cuj` job runs its six budget cases with both agent binaries on one runner: | Agent lane | Marker | Cases | @@ -538,11 +524,11 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, -both full lanes, both managed lanes, and CUJ5 to pass for full/live runs. TUI runs also require -CUJ5; managed jobs are intentionally not selected for TUI. Each tracing journey is +both full lanes, both managed lanes, and `cuj` to pass for full/live runs. TUI runs also require +`cuj`; managed jobs are intentionally not selected for TUI. Each tracing journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke -and installation subsets do not select CUJ5 or managed tests. +and installation subsets do not select `cuj` or managed tests. The advisory Windows installation and headless lanes are not yet included in that aggregate check. The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. @@ -662,7 +648,7 @@ For a manual run, use **Actions → Integration → Run workflow**, select the b and choose `full` (default), `smoke`, `tui`, or `installation`. `live` remains an alias for `full`. Manual subsets are explicit: `smoke` runs just the six smoke cases; `tui` adds `and tui` to each agent lane's marker and runs all 12 live TUI cases, -and also runs the dedicated CUJ5 job. Installation checks always run. Set the ug/agent +and also runs the dedicated `cuj` job. Installation checks always run. Set the ug/agent versions. From the CLI: ```bash @@ -691,7 +677,7 @@ gh run download RUN_ID -R databricks/unity-gateway \ ``` Use `integration-full-AGENT` for a full lane, `integration-smoke-AGENT` for -smoke, or `integration-cuj5` for the budget journeys. Use `integration-installation` for Linux package failures, or +smoke, or `integration-e2e-cuj` for the budget journeys. Use `integration-installation` for Linux package failures, or `integration-installation-windows` for native Windows package failures, or `integration-headless-windows-claude` for the Windows gateway journey. Older runs used `integration-full-AGENT-GROUP`, `integration-cujs`, or numbered `integration-live-*` diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 854c55e94..cc0d9b0ca 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -7,7 +7,6 @@ markers = managed: requires an admin CodingAgentConfig published on the selected workspace managed_fixture: real ug/TUI against a real workspace, but the managed CodingAgentConfig is injected via UCODE_MANAGED_CONFIG_STUB workspace_switch: requires two real workspace hosts and their respective credentials - cuj5: budget smart-defaults journeys using the selected workspace and shared CUJ service principal smoke: Databricks Hosted, custom OAuth CLI TUI, and headless prompt for each agent tui: real interactive terminal boot, keyboard input, exit and reopen claude: only runs when Claude Code is explicitly selected diff --git a/tests/integration/test_ug_budget_defaults.py b/tests/integration/test_ug_budget_defaults.py deleted file mode 100644 index d55c0a5ea..000000000 --- a/tests/integration/test_ug_budget_defaults.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Live customer journeys for workspace smart budget defaults.""" - -import re -from decimal import ROUND_HALF_UP, Decimal - -import pytest -from utils.budget_defaults import BudgetDefaults, BudgetTarget -from utils.evidence import ( - FileTask, - assert_native_process, - assert_task_model, -) -from utils.terminal import AgentTerminal - -pytestmark = [pytest.mark.tui, pytest.mark.cuj5] - - -@pytest.fixture -def budget_defaults(live_session, workspace): - """Supply the target; each journey opens and restores its own budget context.""" - return BudgetDefaults(live_session, workspace, BudgetTarget.from_environment()) - - -def _assert_spend(session, text, recommendation, *, usage=False): - """Compare displayed dollars and percentage with the independent backend response.""" - text = session.redact(text) - money = r"\$([\d,]+\.\d{2})" - pattern = ( - rf"Budget spend:\s*{money}\s+of\s+{money}\s+\((\d+)%\)" - if usage - else rf"{money}\s*/\s*{money}\s+(\d+)%\s+used" - ) - matches = re.findall(pattern, text) - assert matches, f"Budget spend was not displayed:\n{text}" - actual_spend, actual_threshold, actual_percent = matches[-1] - spend = Decimal(str(recommendation["current_spend"])) - threshold = Decimal(str(recommendation["effective_threshold"])) - if usage: - # The usage command formats Decimal dollars half-up and percent with .0%. - expected_money = [ - value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) for value in (spend, threshold) - ] - expected_percent = int(format(float(spend / threshold), ".0%").removesuffix("%")) - else: - # The launch panel formats float dollars and rounds its percentage half-up. - expected_money = [Decimal(format(float(value), ".2f")) for value in (spend, threshold)] - expected_percent = int(float(spend) / float(threshold) * 100 + 0.5) - assert [ - Decimal(actual_spend.replace(",", "")), - Decimal(actual_threshold.replace(",", "")), - ] == expected_money, text - assert int(actual_percent) == expected_percent, text - - -def _assert_launch(session, tui, budget, before, label): - assert_native_process(tui, tui.agent, label, session) - budget.assert_stable(before, label) - transcript = "".join(tui.output) + "\n" + tui.visible - _assert_spend(session, transcript, before) - text = session.redact(transcript) - agent = before.get("recommended_agent") - model = before.get("recommended_model") - if agent: - name = {"CODING_AGENT_CODEX": "Codex", "CODING_AGENT_CLAUDE_CODE": "Claude Code"}[agent] - recommendation = f"Recommended agent is {name}" - if model: - recommendation += f" with model {model}" - assert recommendation in text, text - elif model: - assert f"Recommended model is {model}" in text, text - else: - assert "Recommended agent is" not in text, text - assert "Recommended model is" not in text, text - - -def test_bare_ug_launches_claude_below_50_percent(live_session, budget_defaults): - """Scenario: configure the workspace and launch bare ``ug`` at 40% budget usage. - - Expected: Claude completes a file task on Sonnet; its real process and the - usage/launch displays agree with the stable backend budget response. - """ - session = live_session - label = "bare-40-percent" - with budget_defaults as budget: - session.run( - "configure", - "--workspace", - budget.workspace, - "--skip-upgrade", - "--disable-databricks-ai-tools", - timeout=240, - ) - before = budget.prepare(Decimal("2.5"), label) - assert before.get("recommended_agent") in (None, "CODING_AGENT_CLAUDE_CODE"), before - assert before.get("recommended_model") in (None, budget.sonnet_model), before - usage = session.run("usage", timeout=120) - _assert_spend(session, usage.stdout + usage.stderr, before, usage=True) - - task = FileTask(session) - with AgentTerminal(session, "claude", [str(session.binary)], label) as tui: - tui.boot(timeout=240) - _assert_launch(session, tui, budget, before, label) - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - assert_task_model(session, "claude", task, budget.sonnet_model, label) - tui.exit_normally() - session.assert_not_routed() - - -@pytest.mark.parametrize( - ("multiplier", "model_attribute"), - [ - pytest.param("2", "sol_model", id="exact-50"), - pytest.param("1.6", "sol_model", id="between-50-and-80"), - pytest.param("1.25", "luna_model", id="exact-80"), - pytest.param("1.125", "luna_model", id="between-80-and-100"), - ], -) -def test_bare_ug_launches_codex_at_budget_tier( - live_session, budget_defaults, multiplier, model_attribute -): - """Scenario: launch bare ``ug`` at 50%, 62.5%, 80%, or approximately 88.9% usage. - - Expected: the highest applicable tier selects Codex Sol or Luna, superseding - Codex's Sol default at 80%+. Check the backend response before launch, the native - process, usage display, and completed task's model. Unstable exact boundaries fail. - """ - session = live_session - label = f"bare-codex-{multiplier}" - with budget_defaults as budget: - session.run( - "configure", - "--workspace", - budget.workspace, - "--skip-upgrade", - "--disable-databricks-ai-tools", - timeout=240, - ) - model = getattr(budget, model_attribute) - before = budget.prepare(Decimal(multiplier), label) - assert before.get("recommended_agent") == "CODING_AGENT_CODEX", before - assert before.get("recommended_model") == model, before - usage = session.run("usage", timeout=120) - _assert_spend(session, usage.stdout + usage.stderr, before, usage=True) - - task = FileTask(session) - with AgentTerminal(session, "codex", [str(session.binary)], label) as tui: - tui.boot(timeout=240) - _assert_launch(session, tui, budget, before, label) - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - assert_task_model(session, "codex", task, model, label) - tui.exit_normally() - session.assert_not_routed() - - -def test_ug_claude_keeps_its_default_when_budget_recommends_codex(live_session, budget_defaults): - """Scenario: run explicit ``ug claude`` at 80%, where the budget recommends Codex Luna. - - Expected: the real Claude TUI displays that recommendation but completes a file - task on its own Sonnet default. The Codex-only Luna model never reaches Claude. - """ - session = live_session - label = "explicit-claude-luna-tier" - with budget_defaults as budget: - session.run( - "configure", - "--workspace", - budget.workspace, - "--skip-upgrade", - "--disable-databricks-ai-tools", - timeout=240, - ) - before = budget.prepare(Decimal("1.25"), label) - assert before.get("recommended_agent") == "CODING_AGENT_CODEX", before - assert before.get("recommended_model") == budget.luna_model, before - usage = session.run("usage", timeout=120) - _assert_spend(session, usage.stdout + usage.stderr, before, usage=True) - - task = FileTask(session) - with AgentTerminal(session, "claude", [str(session.binary), "claude"], label) as tui: - tui.boot(timeout=240) - _assert_launch(session, tui, budget, before, label) - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - assert_task_model(session, "claude", task, budget.sonnet_model, label) - tui.exit_normally() - session.assert_not_routed() diff --git a/tests/integration/utils/harness.py b/tests/integration/utils/harness.py index b3af73f47..7756f6827 100644 --- a/tests/integration/utils/harness.py +++ b/tests/integration/utils/harness.py @@ -112,7 +112,7 @@ def redact(self, text: str, *, strip_ansi: bool = True) -> str: "DATABRICKS_BEARER", "DATABRICKS_SECOND_BEARER", "CLAUDE_CODE_OAUTH_TOKEN", - "UG_BUDGET_ACCOUNT_TOKEN", + "UG_CUJ_SP_CLIENT_SECRET", ): for token in (os.environ.get(name), self.env.get(name)): if token: diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index fdb733f6b..3c8b7d374 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -62,18 +62,48 @@ def test_windows_integration_ci_uses_shared_claude_version(): assert contents.count('"--claude-version", $env:CLAUDE_VERSION,') == 2 +def test_dedicated_cuj_job_uses_shared_credentials_and_queued_suite(): + workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" + contents = workflow.read_text() + cuj = contents.split("\n cuj:\n", 1)[1].split("\n # Non-blocking:", 1)[0] + + assert "group: e2e-cuj" in cuj + assert "queue: max" in cuj + assert "cancel-in-progress: false" in cuj + assert "TEST_SUITE: e2e-cuj" in cuj + assert "TEST_MARKER: cuj" in cuj + assert "UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }}" in cuj + assert "UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }}" in cuj + for legacy in ( + "UG_CUJ5_WORKSPACE", + "DATABRICKS_CLIENT_ID", + "DATABRICKS_CLIENT_SECRET", + "UG_BUDGET_", + ): + assert legacy not in cuj + + +def test_cuj_workflow_ancestors_do_not_cancel_active_runs(): + root = Path(__file__).parent.parent + for path in (root / ".github/workflows/integration.yml", root / ".github/workflows/ci.yml"): + header = path.read_text().split("\njobs:\n", 1)[0] + assert "queue: max" in header, path + assert "cancel-in-progress: false" in header, path + + @pytest.mark.parametrize("suite", ["full", "live", "smoke", "tui", "installation"]) -@pytest.mark.parametrize("managed_result", ["success", "failure", "cancelled", "skipped"]) -def test_integration_ci_gate_requires_selected_managed_jobs(suite, managed_result): +@pytest.mark.parametrize("required_job", ["managed", "cuj"]) +@pytest.mark.parametrize("job_result", ["success", "failure", "cancelled", "skipped"]) +def test_integration_ci_gate_requires_selected_jobs(suite, required_job, job_result): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" gate = workflow.read_text().split("\n cujs:\n", 1)[1] script = re.search(r" python3 - <<'PY'\n(.*?) PY", gate, re.DOTALL) assert script is not None results = { job: {"result": "success"} - for job in ("installation", "workspace", "smoke", "full", "cuj5", "managed") + for job in ("installation", "workspace", "smoke", "full", "cuj", "managed") } - results["managed"]["result"] = managed_result + results[required_job]["result"] = job_result for job in { "installation": ("workspace", "smoke", "full"), "smoke": ("full",), @@ -87,9 +117,10 @@ def test_integration_ci_gate_requires_selected_managed_jobs(suite, managed_resul text=True, timeout=10, ) - if suite in {"full", "live"} and managed_result != "success": + selected = suite in {"full", "live"} or (required_job == "cuj" and suite == "tui") + if selected and job_result != "success": assert result.returncode != 0 - assert "Integration jobs did not pass: managed" in result.stderr + assert f"Integration jobs did not pass: {required_job}" in result.stderr else: assert result.returncode == 0, result.stderr assert "All selected integration jobs passed:" in result.stdout diff --git a/tests/test_integration_runner.py b/tests/test_integration_runner.py index 49f749c95..40cdde08f 100644 --- a/tests/test_integration_runner.py +++ b/tests/test_integration_runner.py @@ -122,9 +122,40 @@ def test_headless_only_is_allowed_on_windows_with_live_workspace_and_auth(): assert args.headless_only is True assert args.installation_only is False + assert args.suite == "integration" assert args.pytest_args == ["-m", "live"] +def test_cuj_suite_uses_test_owned_workspaces(): + args = runner.arguments( + ["--suite", "e2e-cuj", "--claude-version", "2.1.280", "--codex-version", "0.154.0"], + platform_name="posix", + environment={"UCODE_TEST_WORKSPACE": "https://unrelated-integration-workspace.test"}, + ) + + assert args.suite == "e2e-cuj" + assert args.workspace is None + assert args.pytest_args == ["-m", "cuj"] + + +@pytest.mark.parametrize( + "options", + [ + ["--installation-only"], + ["--headless-only"], + ["--workspace", "https://unrelated-integration-workspace.test"], + ["--profile", "unrelated-integration-profile"], + ], +) +def test_cuj_suite_rejects_integration_only_options(options): + with pytest.raises(SystemExit): + runner.arguments( + ["--suite", "e2e-cuj", "--claude-version", "2.1.280", *options], + platform_name="posix", + environment={}, + ) + + def test_windows_policy_paths_match_pinned_agent_locations(): paths = runner.managed_policy_paths( ["claude", "codex"], From 514589cba3b7248cc750a55917aae3afacf78e12 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Mon, 5 Oct 2026 15:46:59 +0000 Subject: [PATCH 03/19] Limit budget test serialization to CUJ5 --- .github/workflows/ci.yml | 7 ------- .github/workflows/integration.yml | 21 +++++++-------------- tests/README.md | 8 ++++---- tests/e2e_cuj/AGENTS.md | 2 +- tests/e2e_cuj/pytest.ini | 1 + tests/e2e_cuj/test_ug_budget_defaults.py | 2 +- tests/integration/README.md | 22 +++++++++++++--------- tests/test_integration_contract.py | 17 ++++++++--------- 8 files changed, 35 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62142aed4..08cf1279c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,13 +10,6 @@ on: permissions: contents: read -concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} - # Integration includes a serialized CUJ that mutates a shared target; keep - # the enclosing workflow queued so cancellation cannot interrupt cleanup. - queue: max - cancel-in-progress: false - jobs: lint: name: Lint diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 13f3b263b..392638423 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -42,13 +42,6 @@ on: permissions: contents: read -concurrency: - group: integration-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} - # Dedicated CUJs mutate shared targets. Keep runs queued so a newer - # commit cannot interrupt an active mutation before its cleanup runs. - queue: max - cancel-in-progress: false - env: EXPECTED_WORKSPACE: https://eng-ml-inference-team-us-east-1.cloud.databricks.com UG_VERSION: ${{ inputs.ug_version || 'checkout' }} @@ -351,13 +344,13 @@ jobs: ARTIFACT_NAME: integration-full-${{ matrix.agent }} steps: *live-steps - cuj: - name: "Dedicated CUJs · Claude and Codex" + cuj5: + name: "CUJ5: budget defaults · Claude and Codex" if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} runs-on: ubuntu-22.04 timeout-minutes: 70 concurrency: - group: e2e-cuj + group: e2e-cuj5 queue: max cancel-in-progress: false env: @@ -366,8 +359,8 @@ jobs: DEPENDENCY: ${{ inputs.dependency }} AGENT: both TEST_SUITE: e2e-cuj - TEST_MARKER: cuj - ARTIFACT_NAME: integration-e2e-cuj + TEST_MARKER: cuj5 + ARTIFACT_NAME: integration-cuj5 steps: *live-steps # Non-blocking: continue-on-error and absent from the cujs `needs`, so a flaky OpenCode @@ -456,7 +449,7 @@ jobs: name: All integration tests if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} # installation-windows remains advisory until its initial platform issues are fixed. - needs: [installation, workspace, smoke, full, cuj, managed] + needs: [installation, workspace, smoke, full, cuj5, managed] runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -477,7 +470,7 @@ jobs: required.append("smoke") if suite != "smoke": required.append("full") - required.append("cuj") + required.append("cuj5") if suite in {"full", "live"}: required.append("managed") failed = [job for job in required if results[job]["result"] != "success"] diff --git a/tests/README.md b/tests/README.md index ed1d1b059..a5d1f2c32 100644 --- a/tests/README.md +++ b/tests/README.md @@ -148,7 +148,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Five bare launches across the 50%/80% tiers and an explicit Claude launch at 80% | Real native processes and completed file tasks; model evidence from native session logs; backend spend/threshold agreement with `ug usage` and the launch panel; recommendation display; budget restoration. Requires existing positive spend. Run with `--suite e2e-cuj -- -m cuj`; the test owns its target constants and details. | +| `e2e_cuj/test_ug_budget_defaults.py` | Five bare launches across the 50%/80% tiers and an explicit Claude launch at 80% | Real native processes and completed file tasks; model evidence from native session logs; backend spend/threshold agreement with `ug usage` and the launch panel; recommendation display; budget restoration. Requires existing positive spend. Run with `--suite e2e-cuj -- -m cuj5`; the test owns its target constants and details. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -219,9 +219,9 @@ suite runs all 62 live cases across two parallel agent jobs: one Claude VM and o Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs for the same agent overlap within a run. The dedicated six-case budget suite runs -with both agents in one serial job and is queued without cancellation because it -mutates a shared target; only the shared `UG_CUJ_SP_CLIENT_ID` and -`UG_CUJ_SP_CLIENT_SECRET` secrets are forwarded. +with both agents in the `cuj5` job. Only this job queues across runs without +cancellation because it mutates a shared target; parent workflow runs can overlap. +Only the shared `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets are forwarded. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites share workspace capacity; overlapping their requests can still encounter rate limits. diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index dcde5b8df..f471d2c0c 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -15,4 +15,4 @@ uv run --no-project --python 3.12 python scripts/run_integration.py \ The runner installs the pinned Databricks SDK and both agent CLIs, then invokes pytest with this directory as its `--confcutdir`. The default marker for this -suite is `cuj`; keep the explicit `-m cuj` when reproducing CI. +suite is `cuj`; use `-m cuj5` to reproduce the serialized budget-default CI job. diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini index 9102e6b80..13ee7513a 100644 --- a/tests/e2e_cuj/pytest.ini +++ b/tests/e2e_cuj/pytest.ini @@ -4,4 +4,5 @@ pythonpath = ../.. ../integration addopts = --strict-markers --tb=short markers = cuj: dedicated-workspace customer journey + cuj5: budget-default journeys sharing one workspace and budget tui: real interactive terminal boot, keyboard input, exit and reopen diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index bd9b8253e..c9c37a571 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -13,7 +13,7 @@ ) from utils.terminal import AgentTerminal -pytestmark = [pytest.mark.cuj, pytest.mark.tui] +pytestmark = [pytest.mark.cuj, pytest.mark.cuj5, pytest.mark.tui] class TestCujBudgetDefaults(BaseCujTest): diff --git a/tests/integration/README.md b/tests/integration/README.md index 096a25d29..af4649da0 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -27,7 +27,7 @@ Run the six cases through the dedicated suite from a clean POSIX runner: UG_CUJ_SP_CLIENT_ID=... UG_CUJ_SP_CLIENT_SECRET=... \ python3.12 scripts/run_integration.py \ --suite e2e-cuj --ug-version checkout \ - --claude-version 2.1.280 --codex-version 0.154.0 -- -m cuj + --claude-version 2.1.280 --codex-version 0.154.0 -- -m cuj5 ``` The runner installs `databricks-sdk==0.135.0` in the isolated test environment. @@ -332,7 +332,7 @@ real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation checks** with Claude and Codex. The separate dedicated-workspace suite adds six -budget cases selected by `-m cuj` and run by the `cuj` job. +budget cases selected by `-m cuj5` and run by the `cuj5` job. Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** @@ -501,8 +501,12 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs all 62 live cases, including those smoke cases, in two -disjoint agent lanes. The dedicated `cuj` job runs its six budget cases with -both agent binaries on one runner: +disjoint agent lanes. The dedicated `cuj5` job runs its six budget cases with +both agent binaries on one runner. Its `e2e-cuj5` concurrency group serializes only +CUJ5 across runs. The parent workflows have no concurrency group, so other jobs +can overlap across runs and newer commits do not automatically cancel older runs. + +The full-suite agent lanes are: | Agent lane | Marker | Cases | | --- | --- | --- | @@ -524,11 +528,11 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, -both full lanes, both managed lanes, and `cuj` to pass for full/live runs. TUI runs also require -`cuj`; managed jobs are intentionally not selected for TUI. Each tracing journey is +both full lanes, both managed lanes, and `cuj5` to pass for full/live runs. TUI runs also require +`cuj5`; managed jobs are intentionally not selected for TUI. Each tracing journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke -and installation subsets do not select `cuj` or managed tests. +and installation subsets do not select `cuj5` or managed tests. The advisory Windows installation and headless lanes are not yet included in that aggregate check. The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. @@ -648,7 +652,7 @@ For a manual run, use **Actions → Integration → Run workflow**, select the b and choose `full` (default), `smoke`, `tui`, or `installation`. `live` remains an alias for `full`. Manual subsets are explicit: `smoke` runs just the six smoke cases; `tui` adds `and tui` to each agent lane's marker and runs all 12 live TUI cases, -and also runs the dedicated `cuj` job. Installation checks always run. Set the ug/agent +and also runs the dedicated `cuj5` job. Installation checks always run. Set the ug/agent versions. From the CLI: ```bash @@ -677,7 +681,7 @@ gh run download RUN_ID -R databricks/unity-gateway \ ``` Use `integration-full-AGENT` for a full lane, `integration-smoke-AGENT` for -smoke, or `integration-e2e-cuj` for the budget journeys. Use `integration-installation` for Linux package failures, or +smoke, or `integration-cuj5` for the budget journeys. Use `integration-installation` for Linux package failures, or `integration-installation-windows` for native Windows package failures, or `integration-headless-windows-claude` for the Windows gateway journey. Older runs used `integration-full-AGENT-GROUP`, `integration-cujs`, or numbered `integration-live-*` diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 3c8b7d374..3acaad9f9 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -65,13 +65,13 @@ def test_windows_integration_ci_uses_shared_claude_version(): def test_dedicated_cuj_job_uses_shared_credentials_and_queued_suite(): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" contents = workflow.read_text() - cuj = contents.split("\n cuj:\n", 1)[1].split("\n # Non-blocking:", 1)[0] + cuj = contents.split("\n cuj5:\n", 1)[1].split("\n # Non-blocking:", 1)[0] - assert "group: e2e-cuj" in cuj + assert "group: e2e-cuj5" in cuj assert "queue: max" in cuj assert "cancel-in-progress: false" in cuj assert "TEST_SUITE: e2e-cuj" in cuj - assert "TEST_MARKER: cuj" in cuj + assert "TEST_MARKER: cuj5" in cuj assert "UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }}" in cuj assert "UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }}" in cuj for legacy in ( @@ -83,16 +83,15 @@ def test_dedicated_cuj_job_uses_shared_credentials_and_queued_suite(): assert legacy not in cuj -def test_cuj_workflow_ancestors_do_not_cancel_active_runs(): +def test_integration_workflows_do_not_define_workflow_level_concurrency(): root = Path(__file__).parent.parent for path in (root / ".github/workflows/integration.yml", root / ".github/workflows/ci.yml"): header = path.read_text().split("\njobs:\n", 1)[0] - assert "queue: max" in header, path - assert "cancel-in-progress: false" in header, path + assert "\nconcurrency:" not in header, path @pytest.mark.parametrize("suite", ["full", "live", "smoke", "tui", "installation"]) -@pytest.mark.parametrize("required_job", ["managed", "cuj"]) +@pytest.mark.parametrize("required_job", ["managed", "cuj5"]) @pytest.mark.parametrize("job_result", ["success", "failure", "cancelled", "skipped"]) def test_integration_ci_gate_requires_selected_jobs(suite, required_job, job_result): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" @@ -101,7 +100,7 @@ def test_integration_ci_gate_requires_selected_jobs(suite, required_job, job_res assert script is not None results = { job: {"result": "success"} - for job in ("installation", "workspace", "smoke", "full", "cuj", "managed") + for job in ("installation", "workspace", "smoke", "full", "cuj5", "managed") } results[required_job]["result"] = job_result for job in { @@ -117,7 +116,7 @@ def test_integration_ci_gate_requires_selected_jobs(suite, required_job, job_res text=True, timeout=10, ) - selected = suite in {"full", "live"} or (required_job == "cuj" and suite == "tui") + selected = suite in {"full", "live"} or (required_job == "cuj5" and suite == "tui") if selected and job_result != "success": assert result.returncode != 0 assert f"Integration jobs did not pass: {required_job}" in result.stderr From 22fa3c10a4863dfb710f932196f83798edafa27a Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Mon, 5 Oct 2026 16:08:56 +0000 Subject: [PATCH 04/19] Revert "Limit budget test serialization to CUJ5" This reverts commit 514589cba3b7248cc750a55917aae3afacf78e12. --- .github/workflows/ci.yml | 7 +++++++ .github/workflows/integration.yml | 21 ++++++++++++++------- tests/README.md | 8 ++++---- tests/e2e_cuj/AGENTS.md | 2 +- tests/e2e_cuj/pytest.ini | 1 - tests/e2e_cuj/test_ug_budget_defaults.py | 2 +- tests/integration/README.md | 22 +++++++++------------- tests/test_integration_contract.py | 17 +++++++++-------- 8 files changed, 45 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08cf1279c..62142aed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,13 @@ on: permissions: contents: read +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + # Integration includes a serialized CUJ that mutates a shared target; keep + # the enclosing workflow queued so cancellation cannot interrupt cleanup. + queue: max + cancel-in-progress: false + jobs: lint: name: Lint diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 392638423..13f3b263b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -42,6 +42,13 @@ on: permissions: contents: read +concurrency: + group: integration-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + # Dedicated CUJs mutate shared targets. Keep runs queued so a newer + # commit cannot interrupt an active mutation before its cleanup runs. + queue: max + cancel-in-progress: false + env: EXPECTED_WORKSPACE: https://eng-ml-inference-team-us-east-1.cloud.databricks.com UG_VERSION: ${{ inputs.ug_version || 'checkout' }} @@ -344,13 +351,13 @@ jobs: ARTIFACT_NAME: integration-full-${{ matrix.agent }} steps: *live-steps - cuj5: - name: "CUJ5: budget defaults · Claude and Codex" + cuj: + name: "Dedicated CUJs · Claude and Codex" if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} runs-on: ubuntu-22.04 timeout-minutes: 70 concurrency: - group: e2e-cuj5 + group: e2e-cuj queue: max cancel-in-progress: false env: @@ -359,8 +366,8 @@ jobs: DEPENDENCY: ${{ inputs.dependency }} AGENT: both TEST_SUITE: e2e-cuj - TEST_MARKER: cuj5 - ARTIFACT_NAME: integration-cuj5 + TEST_MARKER: cuj + ARTIFACT_NAME: integration-e2e-cuj steps: *live-steps # Non-blocking: continue-on-error and absent from the cujs `needs`, so a flaky OpenCode @@ -449,7 +456,7 @@ jobs: name: All integration tests if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} # installation-windows remains advisory until its initial platform issues are fixed. - needs: [installation, workspace, smoke, full, cuj5, managed] + needs: [installation, workspace, smoke, full, cuj, managed] runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -470,7 +477,7 @@ jobs: required.append("smoke") if suite != "smoke": required.append("full") - required.append("cuj5") + required.append("cuj") if suite in {"full", "live"}: required.append("managed") failed = [job for job in required if results[job]["result"] != "success"] diff --git a/tests/README.md b/tests/README.md index a5d1f2c32..ed1d1b059 100644 --- a/tests/README.md +++ b/tests/README.md @@ -148,7 +148,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Five bare launches across the 50%/80% tiers and an explicit Claude launch at 80% | Real native processes and completed file tasks; model evidence from native session logs; backend spend/threshold agreement with `ug usage` and the launch panel; recommendation display; budget restoration. Requires existing positive spend. Run with `--suite e2e-cuj -- -m cuj5`; the test owns its target constants and details. | +| `e2e_cuj/test_ug_budget_defaults.py` | Five bare launches across the 50%/80% tiers and an explicit Claude launch at 80% | Real native processes and completed file tasks; model evidence from native session logs; backend spend/threshold agreement with `ug usage` and the launch panel; recommendation display; budget restoration. Requires existing positive spend. Run with `--suite e2e-cuj -- -m cuj`; the test owns its target constants and details. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -219,9 +219,9 @@ suite runs all 62 live cases across two parallel agent jobs: one Claude VM and o Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs for the same agent overlap within a run. The dedicated six-case budget suite runs -with both agents in the `cuj5` job. Only this job queues across runs without -cancellation because it mutates a shared target; parent workflow runs can overlap. -Only the shared `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets are forwarded. +with both agents in one serial job and is queued without cancellation because it +mutates a shared target; only the shared `UG_CUJ_SP_CLIENT_ID` and +`UG_CUJ_SP_CLIENT_SECRET` secrets are forwarded. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites share workspace capacity; overlapping their requests can still encounter rate limits. diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index f471d2c0c..dcde5b8df 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -15,4 +15,4 @@ uv run --no-project --python 3.12 python scripts/run_integration.py \ The runner installs the pinned Databricks SDK and both agent CLIs, then invokes pytest with this directory as its `--confcutdir`. The default marker for this -suite is `cuj`; use `-m cuj5` to reproduce the serialized budget-default CI job. +suite is `cuj`; keep the explicit `-m cuj` when reproducing CI. diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini index 13ee7513a..9102e6b80 100644 --- a/tests/e2e_cuj/pytest.ini +++ b/tests/e2e_cuj/pytest.ini @@ -4,5 +4,4 @@ pythonpath = ../.. ../integration addopts = --strict-markers --tb=short markers = cuj: dedicated-workspace customer journey - cuj5: budget-default journeys sharing one workspace and budget tui: real interactive terminal boot, keyboard input, exit and reopen diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index c9c37a571..bd9b8253e 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -13,7 +13,7 @@ ) from utils.terminal import AgentTerminal -pytestmark = [pytest.mark.cuj, pytest.mark.cuj5, pytest.mark.tui] +pytestmark = [pytest.mark.cuj, pytest.mark.tui] class TestCujBudgetDefaults(BaseCujTest): diff --git a/tests/integration/README.md b/tests/integration/README.md index af4649da0..096a25d29 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -27,7 +27,7 @@ Run the six cases through the dedicated suite from a clean POSIX runner: UG_CUJ_SP_CLIENT_ID=... UG_CUJ_SP_CLIENT_SECRET=... \ python3.12 scripts/run_integration.py \ --suite e2e-cuj --ug-version checkout \ - --claude-version 2.1.280 --codex-version 0.154.0 -- -m cuj5 + --claude-version 2.1.280 --codex-version 0.154.0 -- -m cuj ``` The runner installs `databricks-sdk==0.135.0` in the isolated test environment. @@ -332,7 +332,7 @@ real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation checks** with Claude and Codex. The separate dedicated-workspace suite adds six -budget cases selected by `-m cuj5` and run by the `cuj5` job. +budget cases selected by `-m cuj` and run by the `cuj` job. Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** @@ -501,12 +501,8 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs all 62 live cases, including those smoke cases, in two -disjoint agent lanes. The dedicated `cuj5` job runs its six budget cases with -both agent binaries on one runner. Its `e2e-cuj5` concurrency group serializes only -CUJ5 across runs. The parent workflows have no concurrency group, so other jobs -can overlap across runs and newer commits do not automatically cancel older runs. - -The full-suite agent lanes are: +disjoint agent lanes. The dedicated `cuj` job runs its six budget cases with +both agent binaries on one runner: | Agent lane | Marker | Cases | | --- | --- | --- | @@ -528,11 +524,11 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, -both full lanes, both managed lanes, and `cuj5` to pass for full/live runs. TUI runs also require -`cuj5`; managed jobs are intentionally not selected for TUI. Each tracing journey is +both full lanes, both managed lanes, and `cuj` to pass for full/live runs. TUI runs also require +`cuj`; managed jobs are intentionally not selected for TUI. Each tracing journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke -and installation subsets do not select `cuj5` or managed tests. +and installation subsets do not select `cuj` or managed tests. The advisory Windows installation and headless lanes are not yet included in that aggregate check. The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. @@ -652,7 +648,7 @@ For a manual run, use **Actions → Integration → Run workflow**, select the b and choose `full` (default), `smoke`, `tui`, or `installation`. `live` remains an alias for `full`. Manual subsets are explicit: `smoke` runs just the six smoke cases; `tui` adds `and tui` to each agent lane's marker and runs all 12 live TUI cases, -and also runs the dedicated `cuj5` job. Installation checks always run. Set the ug/agent +and also runs the dedicated `cuj` job. Installation checks always run. Set the ug/agent versions. From the CLI: ```bash @@ -681,7 +677,7 @@ gh run download RUN_ID -R databricks/unity-gateway \ ``` Use `integration-full-AGENT` for a full lane, `integration-smoke-AGENT` for -smoke, or `integration-cuj5` for the budget journeys. Use `integration-installation` for Linux package failures, or +smoke, or `integration-e2e-cuj` for the budget journeys. Use `integration-installation` for Linux package failures, or `integration-installation-windows` for native Windows package failures, or `integration-headless-windows-claude` for the Windows gateway journey. Older runs used `integration-full-AGENT-GROUP`, `integration-cujs`, or numbered `integration-live-*` diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 3acaad9f9..3c8b7d374 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -65,13 +65,13 @@ def test_windows_integration_ci_uses_shared_claude_version(): def test_dedicated_cuj_job_uses_shared_credentials_and_queued_suite(): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" contents = workflow.read_text() - cuj = contents.split("\n cuj5:\n", 1)[1].split("\n # Non-blocking:", 1)[0] + cuj = contents.split("\n cuj:\n", 1)[1].split("\n # Non-blocking:", 1)[0] - assert "group: e2e-cuj5" in cuj + assert "group: e2e-cuj" in cuj assert "queue: max" in cuj assert "cancel-in-progress: false" in cuj assert "TEST_SUITE: e2e-cuj" in cuj - assert "TEST_MARKER: cuj5" in cuj + assert "TEST_MARKER: cuj" in cuj assert "UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }}" in cuj assert "UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }}" in cuj for legacy in ( @@ -83,15 +83,16 @@ def test_dedicated_cuj_job_uses_shared_credentials_and_queued_suite(): assert legacy not in cuj -def test_integration_workflows_do_not_define_workflow_level_concurrency(): +def test_cuj_workflow_ancestors_do_not_cancel_active_runs(): root = Path(__file__).parent.parent for path in (root / ".github/workflows/integration.yml", root / ".github/workflows/ci.yml"): header = path.read_text().split("\njobs:\n", 1)[0] - assert "\nconcurrency:" not in header, path + assert "queue: max" in header, path + assert "cancel-in-progress: false" in header, path @pytest.mark.parametrize("suite", ["full", "live", "smoke", "tui", "installation"]) -@pytest.mark.parametrize("required_job", ["managed", "cuj5"]) +@pytest.mark.parametrize("required_job", ["managed", "cuj"]) @pytest.mark.parametrize("job_result", ["success", "failure", "cancelled", "skipped"]) def test_integration_ci_gate_requires_selected_jobs(suite, required_job, job_result): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" @@ -100,7 +101,7 @@ def test_integration_ci_gate_requires_selected_jobs(suite, required_job, job_res assert script is not None results = { job: {"result": "success"} - for job in ("installation", "workspace", "smoke", "full", "cuj5", "managed") + for job in ("installation", "workspace", "smoke", "full", "cuj", "managed") } results[required_job]["result"] = job_result for job in { @@ -116,7 +117,7 @@ def test_integration_ci_gate_requires_selected_jobs(suite, required_job, job_res text=True, timeout=10, ) - selected = suite in {"full", "live"} or (required_job == "cuj5" and suite == "tui") + selected = suite in {"full", "live"} or (required_job == "cuj" and suite == "tui") if selected and job_result != "success": assert result.returncode != 0 assert f"Integration jobs did not pass: {required_job}" in result.stderr From 38bb84687d7b8bac9142645666c6ee371c4de3a6 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Mon, 5 Oct 2026 20:40:23 +0000 Subject: [PATCH 05/19] Simplify CUJ5 to read-only Luna model selection --- .github/workflows/ci.yml | 5 +- .github/workflows/integration.yml | 9 +- tests/README.md | 18 +- tests/e2e_cuj/AGENTS.md | 4 +- tests/e2e_cuj/budget_defaults.py | 256 ----------------------- tests/e2e_cuj/test_ug_budget_defaults.py | 229 ++++---------------- tests/integration/README.md | 29 +-- tests/integration/utils/evidence.py | 96 --------- tests/test_integration_contract.py | 21 +- 9 files changed, 76 insertions(+), 591 deletions(-) delete mode 100644 tests/e2e_cuj/budget_defaults.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62142aed4..2bbb44c10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,10 +12,7 @@ permissions: concurrency: group: ci-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} - # Integration includes a serialized CUJ that mutates a shared target; keep - # the enclosing workflow queued so cancellation cannot interrupt cleanup. - queue: max - cancel-in-progress: false + cancel-in-progress: true jobs: lint: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 13f3b263b..6202cf3e7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,10 +44,7 @@ permissions: concurrency: group: integration-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} - # Dedicated CUJs mutate shared targets. Keep runs queued so a newer - # commit cannot interrupt an active mutation before its cleanup runs. - queue: max - cancel-in-progress: false + cancel-in-progress: true env: EXPECTED_WORKSPACE: https://eng-ml-inference-team-us-east-1.cloud.databricks.com @@ -356,10 +353,6 @@ jobs: if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} runs-on: ubuntu-22.04 timeout-minutes: 70 - concurrency: - group: e2e-cuj - queue: max - cancel-in-progress: false env: UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} diff --git a/tests/README.md b/tests/README.md index ed1d1b059..5ab5d9af5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -9,7 +9,7 @@ mocks, monkeypatching, fake binaries/services, or fabricated ug state. | Unit/component | Existing `test_*.py` files | Individual behavior; dependencies may be mocked | | Existing e2e | `test_e2e*.py` | Real workspace behavior with some patched setup/internal calls | | Integration CUJs | `integration/test_*.py` | Public configure, TUI, script, command, protocol, and lifecycle journeys | -| Dedicated-workspace CUJs | `e2e_cuj/test_*.py` | Mutating journeys against test-owned workspaces and targets | +| Dedicated-workspace CUJs | `e2e_cuj/test_*.py` | Journeys against test-owned workspaces and targets | | Installation | `integration/test_installation.py` | Fresh installed package, CLI, and local helpers without credentials on Linux and advisory native Windows | `test_entry_points.py` also runs both installed console scripts (`ug` and `ucode`) @@ -148,7 +148,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Five bare launches across the 50%/80% tiers and an explicit Claude launch at 80% | Real native processes and completed file tasks; model evidence from native session logs; backend spend/threshold agreement with `ug usage` and the launch panel; recommendation display; budget restoration. Requires existing positive spend. Run with `--suite e2e-cuj -- -m cuj`; the test owns its target constants and details. | +| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with the published 2% budget tier recommending Codex/Luna | Real backend recommendation and Codex model selection; Luna overrides the managed Sol default. No budget writes or inference task. Run with `--suite e2e-cuj -- -m cuj`; the test owns its workspace and expected config. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -174,9 +174,10 @@ executions. Thirteen additional managed-fixture cases cover focused model, MCP, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. -The dedicated-workspace CUJ suite adds six budget cases: five bare launches and one -explicit Claude override at 80%. It runs separately from the 62-case integration -suite and owns its target constants in the test. Generated-file cleanup and strict +The dedicated-workspace CUJ suite adds one budget model-selection case for the +fixed 2% Luna tier. It runs separately from the 62-case integration suite and owns +its target constants in the test. It does not cover spend boundaries, selection +between multiple tiers, explicit Claude overrides, or completed inference tasks. Generated-file cleanup and strict app-server stdout assertions remain enforced in the integration suite. Unmanaged discovery Cases 7–14 configure, list models, or open the picker without submitting inference prompts; separate task journeys still perform inference. @@ -218,9 +219,10 @@ journeys for both agents, in two parallel jobs. After smoke finishes, the full suite runs all 62 live cases across two parallel agent jobs: one Claude VM and one Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs -for the same agent overlap within a run. The dedicated six-case budget suite runs -with both agents in one serial job and is queued without cancellation because it -mutates a shared target; only the shared `UG_CUJ_SP_CLIENT_ID` and +for the same agent overlap within a run. The dedicated budget model-selection test +runs with both agent binaries installed because managed configure enables both. +It reads shared workspace configuration without changing the budget, so its `cuj` +job has no concurrency lock. Only the shared `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets are forwarded. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index dcde5b8df..be7123522 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -3,7 +3,9 @@ Each CUJ owns a separate workspace. Subclass `BaseCujTest` from `base.py` and set `WORKSPACE_URL`. Setup provides `self.workspace`, a Databricks SDK client using `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` with OAuth M2M authentication. -Do not share the workspace between concurrent runs. No stubbed configuration. +CUJs that mutate workspace or budget configuration must not share their target +between concurrent runs. Read-only CUJs may share a workspace on separate runners. +No stubbed configuration. Run from the repository root through the isolated integration runner: diff --git a/tests/e2e_cuj/budget_defaults.py b/tests/e2e_cuj/budget_defaults.py deleted file mode 100644 index e09b33463..000000000 --- a/tests/e2e_cuj/budget_defaults.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Small live fixture for the CUJ budget smart-default journeys.""" - -from __future__ import annotations - -import time -import urllib.parse -from collections.abc import Mapping -from decimal import Decimal, localcontext -from typing import Any - -from databricks.sdk import AccountClient, WorkspaceClient -from databricks.sdk.core import Config - -_CONFIG_PATH = "/api/ai-gateway/v2/coding-agent-configs" -_RECOMMEND_PATH = _CONFIG_PATH + ":recommendModel" -_BUDGET_PATH = "/api/2.1/accounts/{account_id}/budgets/{budget_id}" -_PER_USER = "ALERT_CONFIGURATION_SCOPE_TYPE_PER_USER" -_BLOCK = "BLOCK_USAGE" -_TIMEOUT = 30 -_POLL = 0.5 - - -def _budget_update(budget: Mapping[str, Any]) -> dict[str, Any]: - """Keep the account API's writable budget fields and remove response metadata.""" - - def clean(value: Any) -> Any: - if isinstance(value, dict): - result = {} - for key, item in value.items(): - if key in {"create_time", "update_time"}: - continue - if key == "target" and value.get("action_type") == _BLOCK: - continue - cleaned = clean(item) - if cleaned is None or cleaned == []: - continue - result[key] = cleaned - return result - if isinstance(value, list): - return [clean(item) for item in value] - return value - - return { - key: clean(budget[key]) - for key in ( - "account_id", - "alert_configurations", - "budget_configuration_id", - "display_name", - "filter", - "resource_type", - ) - if key in budget - } - - -class BudgetDefaults: - """Temporarily move one existing per-user hard-block threshold.""" - - def __init__( - self, - session: Any, - workspace: WorkspaceClient, - *, - workspace_id: str, - account_host: str, - account_id: str, - budget_id: str, - sonnet_model: str, - sol_model: str, - luna_model: str, - ): - self.session = session - self.workspace_client = workspace - self.workspace = workspace.config.host.rstrip("/") - self.workspace_id = workspace_id - self.account_host = account_host.rstrip("/") - self.account_id = account_id - self.budget_id = budget_id - self.sonnet_model = sonnet_model - self.sol_model = sol_model - self.luna_model = luna_model - self.account = AccountClient( - config=Config( - host=self.account_host, - account_id=self.account_id, - client_id=workspace.config.client_id, - client_secret=workspace.config.client_secret, - auth_type="oauth-m2m", - http_timeout_seconds=_TIMEOUT, - retry_timeout_seconds=_TIMEOUT, - ) - ) - self._original_threshold: str | None = None - self._dirty = False - self._evidence = 0 - self._exact_boundary = False - - def __enter__(self) -> BudgetDefaults: - self._validate_config() - current = self._account_budget() - assert current["budget_configuration_id"] == self.budget_id - assert str(current["account_id"]) == self.account_id - workspace_filter = current["filter"]["workspace_id"] - assert workspace_filter["operator"] in ("IN", "BUDGET_CONFIGURATION_FILTER_OPERATOR_IN") - assert [str(value) for value in workspace_filter["values"]] == [self.workspace_id] - _, self._original_threshold = self._alert(current) - self._record("setup", threshold=self._original_threshold, status="captured") - return self - - def __exit__(self, exc_type, exc_value, traceback) -> bool: - if self._dirty: - self._restore() - return False - - def _validate_config(self) -> None: - payload: Any = self.workspace_client.api_client.do("GET", path=_CONFIG_PATH) - config = payload["coding_agent_configs"][0] - assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE" - - agents = {entry["agent"]: entry["config"] for entry in config["enabled_agents"]} - claude = agents["CODING_AGENT_CLAUDE_CODE"] - codex = agents["CODING_AGENT_CODEX"] - assert claude["models"]["model_services"] == [self.sonnet_model] - assert claude["default_models"]["default_model"] == self.sonnet_model - assert set(codex["models"]["model_services"]) == {self.sol_model, self.luna_model} - assert codex["default_models"]["default_model"] == self.sol_model - - smart_defaults = config["smart_defaults"] - assert smart_defaults["budget_id"] == self.budget_id - tiers = smart_defaults["tiers"] - assert [Decimal(str(tier["spending_percentage"])) for tier in tiers] == [ - Decimal("0.5"), - Decimal("0.8"), - ] - assert [(tier["recommended_agent"], tier["recommended_model"]) for tier in tiers] == [ - ("CODING_AGENT_CODEX", self.sol_model), - ("CODING_AGENT_CODEX", self.luna_model), - ] - - def recommendation(self, label: str = "recommendation") -> dict[str, Any]: - payload: Any = self.workspace_client.api_client.do("POST", path=_RECOMMEND_PATH, body={}) - self._record( - "recommendation", - label=label, - current_spend=payload.get("current_spend"), - effective_threshold=payload.get("effective_threshold"), - recommended_agent=payload.get("recommended_agent"), - recommended_model=payload.get("recommended_model"), - ) - return payload - - def prepare(self, multiplier: Decimal, label: str = "prepare") -> dict[str, Any]: - before = self.recommendation(label + ":before") - spend = Decimal(str(before["current_spend"])) - assert spend > 0, "budget fixture requires existing positive spend" - self._exact_boundary = multiplier in {Decimal("2"), Decimal("1.25")} - with localcontext() as context: - context.prec = 64 - threshold = spend * multiplier - threshold_text = format(threshold, "f") - - current = self._account_budget() - index, _ = self._alert(current) - current["alert_configurations"][index]["quantity_threshold"] = threshold_text - self._put(current, label, threshold_text) - recommendation = self._wait_for_recommendation(threshold, label) - readback_spend = Decimal(str(recommendation["current_spend"])) - if readback_spend != spend: - detail = ( - "exact boundary is unverified" - if self._exact_boundary - else "recommendation spend changed" - ) - raise AssertionError(f"{label}: {detail} before inference") - return recommendation - - def assert_stable(self, before: Mapping[str, Any], label: str = "stable") -> dict[str, Any]: - after = self.recommendation(label + ":after") - for key in ( - "current_spend", - "effective_threshold", - "recommended_agent", - "recommended_model", - ): - old, new = before.get(key), after.get(key) - if key in {"current_spend", "effective_threshold"}: - old, new = Decimal(str(old)), Decimal(str(new)) - if old != new: - detail = "exact boundary is unverified; " if self._exact_boundary else "" - raise AssertionError(f"{label}: {detail}recommendation changed {key}") - return after - - def _restore(self) -> None: - assert self._original_threshold is not None - current = self._account_budget() - index, _ = self._alert(current) - current["alert_configurations"][index]["quantity_threshold"] = self._original_threshold - self._put(current, "restore", self._original_threshold) - _, restored_threshold = self._alert(self._account_budget()) - assert Decimal(restored_threshold) == Decimal(self._original_threshold) - self._record("restore", threshold=self._original_threshold, status="verified") - - def _wait_for_recommendation(self, threshold: Decimal, label: str) -> dict[str, Any]: - deadline = time.monotonic() + _TIMEOUT - while True: - result = self.recommendation(label + ":readback") - if Decimal(str(result["effective_threshold"])) == threshold: - return result - remaining = deadline - time.monotonic() - if remaining <= 0: - raise AssertionError(f"{label}: recommendation threshold did not converge") - time.sleep(min(_POLL, remaining)) - - def _account_budget(self) -> dict[str, Any]: - payload = self._account_request("GET", None) - return payload.get("budget", payload) - - def _alert(self, budget: Mapping[str, Any]) -> tuple[int, str]: - matches = [] - for index, alert in enumerate(budget["alert_configurations"]): - actions = alert.get("action_configurations", []) - if alert.get("scope_type") == _PER_USER and any( - action.get("action_type") == _BLOCK for action in actions - ): - assert not alert.get("principal_overrides") - matches.append((index, str(alert["quantity_threshold"]))) - assert len(matches) == 1, "account budget must have one per-user BLOCK_USAGE alert" - return matches[0] - - def _put(self, budget: Mapping[str, Any], label: str, threshold: str) -> None: - self._dirty = True - self._record("mutation", label=label, threshold=threshold, status="write_started") - self._account_request("PUT", {"budget": _budget_update(budget)}) - - def _account_request(self, method: str, body: dict[str, Any] | None) -> Any: - # AccountClient's ApiClient calls Config.authenticate for each request, allowing the - # restoration path to refresh an expired service-principal token. - return self.account.api_client.do( - method, - path=self._account_path(), - body=body, - ) - - def _account_path(self) -> str: - return _BUDGET_PATH.format( - account_id=urllib.parse.quote(self.account_id, safe=""), - budget_id=urllib.parse.quote(self.budget_id, safe=""), - ) - - def _record(self, kind: str, **fields: object) -> None: - self._evidence += 1 - self.session.record( - f"budget-defaults-{self._evidence:03d}-{kind}.json", - {"kind": kind, **fields}, - ) diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index bd9b8253e..f19e46db7 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -1,16 +1,10 @@ -"""Live customer journeys for workspace smart budget defaults.""" +"""Live model-selection check for a fixed workspace budget recommendation.""" import re -from decimal import ROUND_HALF_UP, Decimal +import tomllib import pytest from base import BaseCujTest -from budget_defaults import BudgetDefaults -from utils.evidence import ( - FileTask, - assert_native_process, - assert_task_model, -) from utils.terminal import AgentTerminal pytestmark = [pytest.mark.cuj, pytest.mark.tui] @@ -18,191 +12,52 @@ class TestCujBudgetDefaults(BaseCujTest): WORKSPACE_URL = "https://dbc-497adeef-62c0.cloud.databricks.com" - WORKSPACE_ID = "7474645051570450" - ACCOUNT_HOST = "https://accounts.cloud.databricks.com" - ACCOUNT_ID = "2860b24d-05af-4fb5-83b4-6539492f8180" BUDGET_ID = "25c4ce5c-fcd6-4d00-8266-69030cc1a236" - SONNET_MODEL = "system.ai.claude-sonnet-4-6" SOL_MODEL = "system.ai.gpt-5-6-sol" LUNA_MODEL = "system.ai.gpt-5-6-luna" - @pytest.fixture - def budget_defaults(self, live_session): - """Supply the target; each journey opens and restores its own budget context.""" - return BudgetDefaults( - live_session, - self.workspace, - workspace_id=self.WORKSPACE_ID, - account_host=self.ACCOUNT_HOST, - account_id=self.ACCOUNT_ID, - budget_id=self.BUDGET_ID, - sonnet_model=self.SONNET_MODEL, - sol_model=self.SOL_MODEL, - luna_model=self.LUNA_MODEL, - ) - - @staticmethod - def _assert_spend(session, text, recommendation, *, usage=False): - """Compare displayed dollars and percentage with the independent backend response.""" - text = session.redact(text) - money = r"\$([\d,]+\.\d{2})" - pattern = ( - rf"Budget spend:\s*{money}\s+of\s+{money}\s+\((\d+)%\)" - if usage - else rf"{money}\s*/\s*{money}\s+(\d+)%\s+used" - ) - matches = re.findall(pattern, text) - assert matches, f"Budget spend was not displayed:\n{text}" - actual_spend, actual_threshold, actual_percent = matches[-1] - spend = Decimal(str(recommendation["current_spend"])) - threshold = Decimal(str(recommendation["effective_threshold"])) - if usage: - # The usage command formats Decimal dollars half-up and percent with .0%. - expected_money = [ - value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) - for value in (spend, threshold) - ] - expected_percent = int(format(float(spend / threshold), ".0%").removesuffix("%")) - else: - # The launch panel formats float dollars and rounds its percentage half-up. - expected_money = [Decimal(format(float(value), ".2f")) for value in (spend, threshold)] - expected_percent = int(float(spend) / float(threshold) * 100 + 0.5) - assert [ - Decimal(actual_spend.replace(",", "")), - Decimal(actual_threshold.replace(",", "")), - ] == expected_money, text - assert int(actual_percent) == expected_percent, text - - def _assert_launch(self, session, tui, budget, before, label): - assert_native_process(tui, tui.agent, label, session) - budget.assert_stable(before, label) - transcript = "".join(tui.output) + "\n" + tui.visible - self._assert_spend(session, transcript, before) - text = session.redact(transcript) - agent = before.get("recommended_agent") - model = before.get("recommended_model") - if agent: - name = {"CODING_AGENT_CODEX": "Codex", "CODING_AGENT_CLAUDE_CODE": "Claude Code"}[agent] - recommendation = f"Recommended agent is {name}" - if model: - recommendation += f" with model {model}" - assert recommendation in text, text - elif model: - assert f"Recommended model is {model}" in text, text - else: - assert "Recommended agent is" not in text, text - assert "Recommended model is" not in text, text - - def test_bare_ug_launches_claude_below_50_percent(self, live_session, budget_defaults): - """Scenario: configure the workspace and launch bare ``ug`` at 40% budget usage. + def test_bare_ug_uses_luna_budget_recommendation(self, live_session): + """Scenario: launch bare ``ug`` with the fixed 2% Luna budget tier active. - Expected: Claude completes a file task on Sonnet; its real process and the - usage/launch displays agree with the stable backend budget response. + Expected: the real backend recommends Codex/Luna and the Codex TUI selects + Luna over its managed Sol default, without budget writes or an inference task. """ session = live_session - label = "bare-40-percent" - with budget_defaults as budget: - session.run( - "configure", - "--workspace", - budget.workspace, - "--skip-upgrade", - "--disable-databricks-ai-tools", - timeout=240, - ) - before = budget.prepare(Decimal("2.5"), label) - assert before.get("recommended_agent") in (None, "CODING_AGENT_CLAUDE_CODE"), before - assert before.get("recommended_model") in (None, budget.sonnet_model), before - usage = session.run("usage", timeout=120) - self._assert_spend(session, usage.stdout + usage.stderr, before, usage=True) - - task = FileTask(session) - with AgentTerminal(session, "claude", [str(session.binary)], label) as tui: - tui.boot(timeout=240) - self._assert_launch(session, tui, budget, before, label) - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - assert_task_model(session, "claude", task, budget.sonnet_model, label) - tui.exit_normally() - session.assert_not_routed() - - @pytest.mark.parametrize( - ("multiplier", "model_attribute"), - [ - pytest.param("2", "sol_model", id="exact-50"), - pytest.param("1.6", "sol_model", id="between-50-and-80"), - pytest.param("1.25", "luna_model", id="exact-80"), - pytest.param("1.125", "luna_model", id="between-80-and-100"), - ], - ) - def test_bare_ug_launches_codex_at_budget_tier( - self, live_session, budget_defaults, multiplier, model_attribute - ): - """Scenario: launch bare ``ug`` at 50%, 62.5%, 80%, or approximately 88.9% usage. - - Expected: the highest applicable tier selects Codex Sol or Luna, superseding - Codex's Sol default at 80%+. Check the backend response before launch, the native - process, usage display, and completed task's model. Unstable exact boundaries fail. - """ - session = live_session - label = f"bare-codex-{multiplier}" - with budget_defaults as budget: - session.run( - "configure", - "--workspace", - budget.workspace, - "--skip-upgrade", - "--disable-databricks-ai-tools", - timeout=240, - ) - model = getattr(budget, model_attribute) - before = budget.prepare(Decimal(multiplier), label) - assert before.get("recommended_agent") == "CODING_AGENT_CODEX", before - assert before.get("recommended_model") == model, before - usage = session.run("usage", timeout=120) - self._assert_spend(session, usage.stdout + usage.stderr, before, usage=True) - - task = FileTask(session) - with AgentTerminal(session, "codex", [str(session.binary)], label) as tui: - tui.boot(timeout=240) - self._assert_launch(session, tui, budget, before, label) - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - assert_task_model(session, "codex", task, model, label) - tui.exit_normally() - session.assert_not_routed() - - def test_ug_claude_keeps_its_default_when_budget_recommends_codex( - self, live_session, budget_defaults - ): - """Scenario: run explicit ``ug claude`` at 80%, where the budget recommends Codex Luna. + config_path = "/api/ai-gateway/v2/coding-agent-configs" + payload = self.workspace.api_client.do("GET", path=config_path) + config = payload["coding_agent_configs"][0] + assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE", config + assert config["smart_defaults"]["budget_id"] == self.BUDGET_ID, config + assert config["smart_defaults"]["tiers"] == [ + { + "spending_percentage": 0.02, + "recommended_agent": "CODING_AGENT_CODEX", + "recommended_model": self.LUNA_MODEL, + } + ], config + + session.run( + "configure", + "--workspace", + self.WORKSPACE_URL, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + model_config = session.home / ".codex" / "ucode.config.toml" + assert tomllib.loads(model_config.read_text())["model"] == self.SOL_MODEL - Expected: the real Claude TUI displays that recommendation but completes a file - task on its own Sonnet default. The Codex-only Luna model never reaches Claude. - """ - session = live_session - label = "explicit-claude-luna-tier" - with budget_defaults as budget: - session.run( - "configure", - "--workspace", - budget.workspace, - "--skip-upgrade", - "--disable-databricks-ai-tools", - timeout=240, + recommendation = self.workspace.api_client.do( + "POST", path=config_path + ":recommendModel", body={} + ) + assert recommendation.get("recommended_agent") == "CODING_AGENT_CODEX", recommendation + assert recommendation.get("recommended_model") == self.LUNA_MODEL, recommendation + + with AgentTerminal(session, "codex", [str(session.binary)], "budget-luna") as tui: + tui.boot(timeout=240) + assert tomllib.loads(model_config.read_text())["model"] == self.LUNA_MODEL + tui.wait_for( + lambda text: re.search(rf"model:\s+{re.escape(self.LUNA_MODEL)}\s", text), + "Codex's selected model to be Luna", ) - before = budget.prepare(Decimal("1.25"), label) - assert before.get("recommended_agent") == "CODING_AGENT_CODEX", before - assert before.get("recommended_model") == budget.luna_model, before - usage = session.run("usage", timeout=120) - self._assert_spend(session, usage.stdout + usage.stderr, before, usage=True) - - task = FileTask(session) - with AgentTerminal(session, "claude", [str(session.binary), "claude"], label) as tui: - tui.boot(timeout=240) - self._assert_launch(session, tui, budget, before, label) - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - assert_task_model(session, "claude", task, budget.sonnet_model, label) - tui.exit_normally() - session.assert_not_routed() + tui.exit_normally() diff --git a/tests/integration/README.md b/tests/integration/README.md index 096a25d29..7a8e19655 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -15,13 +15,13 @@ does not yet assert live `recommendModel` request counts for configs with and wi ## Dedicated-workspace CUJs -The budget-default journeys live in +The budget model-selection test lives in [`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py), outside this existing-workspace integration suite. The test owns its workspace, -account, budget, and model identifiers, while CI forwards only the shared +budget and model identifiers, while CI forwards only the shared `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets. -Run the six cases through the dedicated suite from a clean POSIX runner: +Run the case through the dedicated suite from a clean POSIX runner: ```bash UG_CUJ_SP_CLIENT_ID=... UG_CUJ_SP_CLIENT_SECRET=... \ @@ -31,10 +31,12 @@ python3.12 scripts/run_integration.py \ ``` The runner installs `databricks-sdk==0.135.0` in the isolated test environment. -Workspace and account clients refresh OAuth M2M tokens for backend calls, including -budget restoration. Do not overlap a local run with -CI: the cases mutate a shared budget threshold and restore it during cleanup. If a -run is interrupted, verify the recorded original threshold before rerunning. +The workspace client reads the published config and real recommendation using +OAuth M2M. The single 2% tier must recommend Codex on `system.ai.gpt-5-6-luna` +for the test principal. The test launches bare `ug` and verifies Codex selects +Luna over its managed Sol default; it submits no inference task. It neither writes +the budget nor requires account-level authentication. Independent runners can +overlap; each still needs isolated machine-level agent settings. Shared subprocess command resolution is covered by `../test_subprocess_cross_os.py` and enforced by Ruff. These component checks do not establish native Windows coverage @@ -331,8 +333,8 @@ they only configure, list models, and open/close the picker. Other live CUJs per real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex. The separate dedicated-workspace suite adds six -budget cases selected by `-m cuj` and run by the `cuj` job. +checks** with Claude and Codex. The separate dedicated-workspace suite adds one +budget model-selection case selected by `-m cuj` and run by the `cuj` job. Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** @@ -501,8 +503,11 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs all 62 live cases, including those smoke cases, in two -disjoint agent lanes. The dedicated `cuj` job runs its six budget cases with -both agent binaries on one runner: +disjoint agent lanes. The dedicated `cuj` job runs one budget model-selection case +with both agent binaries installed and no cross-run concurrency lock. The parent +workflows retain their normal cancellation of superseded runs. + +The full-suite agent lanes are: | Agent lane | Marker | Cases | | --- | --- | --- | @@ -677,7 +682,7 @@ gh run download RUN_ID -R databricks/unity-gateway \ ``` Use `integration-full-AGENT` for a full lane, `integration-smoke-AGENT` for -smoke, or `integration-e2e-cuj` for the budget journeys. Use `integration-installation` for Linux package failures, or +smoke, or `integration-e2e-cuj` for the budget model-selection test. Use `integration-installation` for Linux package failures, or `integration-installation-windows` for native Windows package failures, or `integration-headless-windows-claude` for the Windows gateway journey. Older runs used `integration-full-AGENT-GROUP`, `integration-cujs`, or numbered `integration-live-*` diff --git a/tests/integration/utils/evidence.py b/tests/integration/utils/evidence.py index b8074f023..4c813d671 100644 --- a/tests/integration/utils/evidence.py +++ b/tests/integration/utils/evidence.py @@ -2,7 +2,6 @@ import json import re -import subprocess import uuid from pathlib import Path @@ -274,98 +273,3 @@ def assert_subagent_routed( # Some agent versions omit the child's model from SubagentStart. The report # preserves that unknown value; this test claims decision + spawn + task, # not model-identity verification when the agent did not expose it. - - -def assert_task_model(session, agent: str, task, expected_model: str, label: str) -> str: - """Require the unique task's native completion to use the expected model.""" - - def normalize(model): - model = model.removeprefix("system.ai.").removesuffix("[1m]").removesuffix("[200k]") - return re.sub(r"^(gpt-\d+)-(\d+)", r"\1.\2", model) if agent == "codex" else model - - observed = [] - for path, records in agent_sessions(session, agent).items(): - if is_child_session(agent, path, records): - continue - contexts = {} - for record in records: - if agent == "claude": - if record.get("type") != "assistant": - continue - message = record.get("message", {}) - if message.get("role") != "assistant": - continue - content = message.get("content") - answer = ( - content - if isinstance(content, str) - else "\n".join(assistant_answers(agent, [record])) - ) - if task.value not in answer: - continue - model = message.get("model") - assert isinstance(model, str) and model, f"Claude task answer has no model: {path}" - else: - payload = record.get("payload", {}) - if record.get("type") == "turn_context" and payload.get("turn_id"): - contexts[payload["turn_id"]] = payload.get("model") - continue - if record.get("type") != "event_msg" or payload.get("type") != "task_complete": - continue - answer = payload.get("last_agent_message") - if not isinstance(answer, str) or task.value not in answer: - continue - model = contexts.get(payload.get("turn_id")) - assert isinstance(model, str) and model, ( - f"Codex task completion has no preceding model context: {path}" - ) - observed.append(model) - - assert observed, f"{label}: completed {agent} task exposed no native request model" - models = {normalize(model) for model in observed} - expected = normalize(expected_model) - session.record(f"{label}-task-model.json", {"agent": agent, "models": sorted(models)}) - assert models == {expected}, ( - f"{label}: completed {agent} task used models {sorted(models)!r}, expected {expected!r}" - ) - return observed[-1] - - -def assert_native_process(terminal, agent: str, label: str, session) -> None: - """Require a real Claude/Codex executable or its installed npm package in the PTY tree.""" - root_pid = int(terminal.child.pid) - result = subprocess.run( - ["ps", "-axo", "pid=,ppid=,comm=,args="], - check=True, - capture_output=True, - text=True, - timeout=5, - ) - rows = {} - for line in result.stdout.splitlines(): - match = re.match(r"\s*(\d+)\s+(\d+)\s+(\S+)(?:\s+(.*))?$", line) - if match: - rows[int(match.group(1))] = { - "pid": int(match.group(1)), - "ppid": int(match.group(2)), - "executable": match.group(3), - "command": (match.group(4) or "").strip(), - } - descendants = {root_pid} - while True: - new = {row["pid"] for row in rows.values() if row["ppid"] in descendants} - descendants - if not new: - break - descendants.update(new) - package = { - "claude": "/node_modules/@anthropic-ai/claude-code/", - "codex": "/node_modules/@openai/codex/", - }[agent] - native = [ - row - for pid, row in rows.items() - if pid in descendants - and (Path(row["executable"]).name == agent or package in row["command"]) - ] - assert native, f"{label}: no native {agent} process in PTY descendants" - session.record(f"{label}-process.json", {"root_pid": root_pid, "native": native}) diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 3c8b7d374..d98843146 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -62,33 +62,16 @@ def test_windows_integration_ci_uses_shared_claude_version(): assert contents.count('"--claude-version", $env:CLAUDE_VERSION,') == 2 -def test_dedicated_cuj_job_uses_shared_credentials_and_queued_suite(): +def test_dedicated_cuj_job_uses_shared_credentials_without_serialization(): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" contents = workflow.read_text() cuj = contents.split("\n cuj:\n", 1)[1].split("\n # Non-blocking:", 1)[0] - assert "group: e2e-cuj" in cuj - assert "queue: max" in cuj - assert "cancel-in-progress: false" in cuj + assert "concurrency:" not in cuj assert "TEST_SUITE: e2e-cuj" in cuj assert "TEST_MARKER: cuj" in cuj assert "UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }}" in cuj assert "UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }}" in cuj - for legacy in ( - "UG_CUJ5_WORKSPACE", - "DATABRICKS_CLIENT_ID", - "DATABRICKS_CLIENT_SECRET", - "UG_BUDGET_", - ): - assert legacy not in cuj - - -def test_cuj_workflow_ancestors_do_not_cancel_active_runs(): - root = Path(__file__).parent.parent - for path in (root / ".github/workflows/integration.yml", root / ".github/workflows/ci.yml"): - header = path.read_text().split("\njobs:\n", 1)[0] - assert "queue: max" in header, path - assert "cancel-in-progress: false" in header, path @pytest.mark.parametrize("suite", ["full", "live", "smoke", "tui", "installation"]) From e0f2081b88fa5e3e3afd32f8d4199d0a2384ff32 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Mon, 5 Oct 2026 20:50:04 +0000 Subject: [PATCH 06/19] Restore read-only budget usage checks in CUJ5 --- tests/README.md | 4 +-- tests/e2e_cuj/test_ug_budget_defaults.py | 45 +++++++++++++++++++++--- tests/integration/README.md | 6 ++-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/tests/README.md b/tests/README.md index 5ab5d9af5..e7b933a25 100644 --- a/tests/README.md +++ b/tests/README.md @@ -148,7 +148,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with the published 2% budget tier recommending Codex/Luna | Real backend recommendation and Codex model selection; Luna overrides the managed Sol default. No budget writes or inference task. Run with `--suite e2e-cuj -- -m cuj`; the test owns its workspace and expected config. | +| `e2e_cuj/test_ug_budget_defaults.py` | Run `ug usage` and launch bare `ug` with the published 2% budget tier recommending Codex/Luna | Usage spend, threshold, and percentage agree with real backend reads; the backend recommends Luna and Codex selects it over the managed Sol default. No budget writes or inference task. Run with `--suite e2e-cuj -- -m cuj`; the test owns its workspace and expected config. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -174,7 +174,7 @@ executions. Thirteen additional managed-fixture cases cover focused model, MCP, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. -The dedicated-workspace CUJ suite adds one budget model-selection case for the +The dedicated-workspace CUJ suite adds one usage and model-selection case for the fixed 2% Luna tier. It runs separately from the 62-case integration suite and owns its target constants in the test. It does not cover spend boundaries, selection between multiple tiers, explicit Claude overrides, or completed inference tasks. Generated-file cleanup and strict diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index f19e46db7..a76226bbd 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -2,6 +2,7 @@ import re import tomllib +from decimal import ROUND_HALF_UP, Decimal import pytest from base import BaseCujTest @@ -19,8 +20,9 @@ class TestCujBudgetDefaults(BaseCujTest): def test_bare_ug_uses_luna_budget_recommendation(self, live_session): """Scenario: launch bare ``ug`` with the fixed 2% Luna budget tier active. - Expected: the real backend recommends Codex/Luna and the Codex TUI selects - Luna over its managed Sol default, without budget writes or an inference task. + Expected: the read-only `usage` command reports the backend's spend and threshold, then + the real backend recommends Codex/Luna and the Codex TUI selects Luna over its managed Sol + default, without budget writes or an inference task. """ session = live_session config_path = "/api/ai-gateway/v2/coding-agent-configs" @@ -47,11 +49,44 @@ def test_bare_ug_uses_luna_budget_recommendation(self, live_session): model_config = session.home / ".codex" / "ucode.config.toml" assert tomllib.loads(model_config.read_text())["model"] == self.SOL_MODEL - recommendation = self.workspace.api_client.do( + before_usage = self.workspace.api_client.do( "POST", path=config_path + ":recommendModel", body={} ) - assert recommendation.get("recommended_agent") == "CODING_AGENT_CODEX", recommendation - assert recommendation.get("recommended_model") == self.LUNA_MODEL, recommendation + usage = session.run("usage", timeout=120) + after_usage = self.workspace.api_client.do( + "POST", path=config_path + ":recommendModel", body={} + ) + for recommendation in (before_usage, after_usage): + assert recommendation.get("recommended_agent") == "CODING_AGENT_CODEX", recommendation + assert recommendation.get("recommended_model") == self.LUNA_MODEL, recommendation + + match = re.search( + r"Budget spend:\s*\$([\d,]+\.\d{2})\s+of\s+\$([\d,]+\.\d{2})\s+\((\d+)%\)", + session.redact(usage.stdout + usage.stderr), + ) + assert match, usage.stdout + usage.stderr + displayed_spend, displayed_threshold, displayed_percent = match.groups() + displayed_spend = Decimal(displayed_spend.replace(",", "")) + displayed_threshold = Decimal(displayed_threshold.replace(",", "")) + cent = Decimal("0.01") + + before_spend = Decimal(str(before_usage["current_spend"])) + after_spend = Decimal(str(after_usage["current_spend"])) + before_threshold = Decimal(str(before_usage["effective_threshold"])) + after_threshold = Decimal(str(after_usage["effective_threshold"])) + assert before_spend <= after_spend, (before_usage, after_usage) + assert before_threshold == after_threshold, (before_usage, after_usage) + assert displayed_threshold == before_threshold.quantize(cent, rounding=ROUND_HALF_UP) + assert ( + before_spend.quantize(cent, rounding=ROUND_HALF_UP) + <= displayed_spend + <= after_spend.quantize(cent, rounding=ROUND_HALF_UP) + ), (before_usage, after_usage, displayed_spend) + assert ( + int(format(float(before_spend / before_threshold), ".0%").removesuffix("%")) + <= int(displayed_percent) + <= int(format(float(after_spend / after_threshold), ".0%").removesuffix("%")) + ), (before_usage, after_usage, usage) with AgentTerminal(session, "codex", [str(session.binary)], "budget-luna") as tui: tui.boot(timeout=240) diff --git a/tests/integration/README.md b/tests/integration/README.md index 7a8e19655..7e5f4d50f 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -15,7 +15,7 @@ does not yet assert live `recommendModel` request counts for configs with and wi ## Dedicated-workspace CUJs -The budget model-selection test lives in +The budget usage and model-selection test lives in [`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py), outside this existing-workspace integration suite. The test owns its workspace, budget and model identifiers, while CI forwards only the shared @@ -33,7 +33,9 @@ python3.12 scripts/run_integration.py \ The runner installs `databricks-sdk==0.135.0` in the isolated test environment. The workspace client reads the published config and real recommendation using OAuth M2M. The single 2% tier must recommend Codex on `system.ai.gpt-5-6-luna` -for the test principal. The test launches bare `ug` and verifies Codex selects +for the test principal. The test checks `ug usage` spend, threshold, and percentage +against backend reads, allowing spend to increase while the command runs. +It then launches bare `ug` and verifies Codex selects Luna over its managed Sol default; it submits no inference task. It neither writes the budget nor requires account-level authentication. Independent runners can overlap; each still needs isolated machine-level agent settings. From 8e7f87610f2397007eacbb9cdd8ba31543181417 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Mon, 5 Oct 2026 21:34:40 +0000 Subject: [PATCH 07/19] Add CUJ5 default-model check with a dedicated low-spend principal --- .github/workflows/integration.yml | 24 ++++++-- tests/README.md | 18 +++--- tests/e2e_cuj/AGENTS.md | 8 ++- tests/e2e_cuj/pytest.ini | 1 + tests/e2e_cuj/test_ug_budget_defaults.py | 74 ++++++++++++++++++++++++ tests/integration/README.md | 34 +++++++---- tests/test_integration_contract.py | 15 +++-- 7 files changed, 143 insertions(+), 31 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6202cf3e7..d5177cfc7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -349,18 +349,32 @@ jobs: steps: *live-steps cuj: - name: "Dedicated CUJs · Claude and Codex" + name: ${{ matrix.name }} if: ${{ inputs.suite != 'installation' && inputs.suite != 'smoke' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} runs-on: ubuntu-22.04 timeout-minutes: 70 + strategy: + fail-fast: false + matrix: + include: + - name: "Dedicated CUJs · shared principal" + test_marker: cuj and not cuj5_below_tier + client_id_secret: UG_CUJ_SP_CLIENT_ID + client_secret_secret: UG_CUJ_SP_CLIENT_SECRET + artifact_name: integration-e2e-cuj + - name: "CUJ5: below-tier default · Claude" + test_marker: cuj5_below_tier + client_id_secret: UG_BUDGET_CUJ_SP_CLIENT_ID + client_secret_secret: UG_BUDGET_CUJ_SP_CLIENT_SECRET + artifact_name: integration-cuj5-below-tier env: - UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} - UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + UG_CUJ_SP_CLIENT_ID: ${{ secrets[matrix.client_id_secret] }} + UG_CUJ_SP_CLIENT_SECRET: ${{ secrets[matrix.client_secret_secret] }} DEPENDENCY: ${{ inputs.dependency }} AGENT: both TEST_SUITE: e2e-cuj - TEST_MARKER: cuj - ARTIFACT_NAME: integration-e2e-cuj + TEST_MARKER: ${{ matrix.test_marker }} + ARTIFACT_NAME: ${{ matrix.artifact_name }} steps: *live-steps # Non-blocking: continue-on-error and absent from the cujs `needs`, so a flaky OpenCode diff --git a/tests/README.md b/tests/README.md index e7b933a25..167ec3412 100644 --- a/tests/README.md +++ b/tests/README.md @@ -148,7 +148,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Run `ug usage` and launch bare `ug` with the published 2% budget tier recommending Codex/Luna | Usage spend, threshold, and percentage agree with real backend reads; the backend recommends Luna and Codex selects it over the managed Sol default. No budget writes or inference task. Run with `--suite e2e-cuj -- -m cuj`; the test owns its workspace and expected config. | +| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 2% tier | Low-spend principal selects the default Claude/Sonnet; above-tier principal selects Codex/Luna over Sol and `ug usage` agrees with backend spend, threshold, and percentage. No budget writes or inference tasks. Run with `--suite e2e-cuj` and the credential-specific marker documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -174,8 +174,9 @@ executions. Thirteen additional managed-fixture cases cover focused model, MCP, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. -The dedicated-workspace CUJ suite adds one usage and model-selection case for the -fixed 2% Luna tier. It runs separately from the 62-case integration suite and owns +The dedicated-workspace CUJ suite adds two cases around the fixed 2% Luna tier: +default Claude/Sonnet selection with a low-spend principal, and usage plus Codex/Luna +selection with the above-tier principal. It runs separately from the 62-case integration suite and owns its target constants in the test. It does not cover spend boundaries, selection between multiple tiers, explicit Claude overrides, or completed inference tasks. Generated-file cleanup and strict app-server stdout assertions remain enforced in the integration suite. @@ -219,11 +220,12 @@ journeys for both agents, in two parallel jobs. After smoke finishes, the full suite runs all 62 live cases across two parallel agent jobs: one Claude VM and one Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs -for the same agent overlap within a run. The dedicated budget model-selection test -runs with both agent binaries installed because managed configure enables both. -It reads shared workspace configuration without changing the budget, so its `cuj` -job has no concurrency lock. Only the shared `UG_CUJ_SP_CLIENT_ID` and -`UG_CUJ_SP_CLIENT_SECRET` secrets are forwarded. +for the same agent overlap within a run. The dedicated budget cases run on separate +runners with both agent binaries installed because managed configure enables both. +They read shared workspace configuration without changing the budget, so the `cuj` +matrix has no concurrency lock. The above-tier case uses `UG_CUJ_SP_CLIENT_ID` and +`UG_CUJ_SP_CLIENT_SECRET`; the below-tier case uses `UG_BUDGET_CUJ_SP_CLIENT_ID` and +`UG_BUDGET_CUJ_SP_CLIENT_SECRET`, mapped to the same fixture environment variables. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites share workspace capacity; overlapping their requests can still encounter rate limits. diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index be7123522..399f71468 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -12,9 +12,11 @@ Run from the repository root through the isolated integration runner: ```bash uv run --no-project --python 3.12 python scripts/run_integration.py \ --suite e2e-cuj --ug-version checkout \ - --claude-version 2.1.280 --codex-version 0.154.0 -- -m cuj + --claude-version 2.1.280 --codex-version 0.154.0 -- -m 'cuj and not cuj5_below_tier' ``` The runner installs the pinned Databricks SDK and both agent CLIs, then invokes -pytest with this directory as its `--confcutdir`. The default marker for this -suite is `cuj`; keep the explicit `-m cuj` when reproducing CI. +pytest with this directory as its `--confcutdir`. CUJ5's low-spend principal runs +separately with `-m cuj5_below_tier`; CI maps its `UG_BUDGET_CUJ_SP_CLIENT_ID` and +`UG_BUDGET_CUJ_SP_CLIENT_SECRET` secrets into the standard fixture variables. +Keep the credential-specific marker when reproducing CI. diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini index 9102e6b80..e83520808 100644 --- a/tests/e2e_cuj/pytest.ini +++ b/tests/e2e_cuj/pytest.ini @@ -4,4 +4,5 @@ pythonpath = ../.. ../integration addopts = --strict-markers --tb=short markers = cuj: dedicated-workspace customer journey + cuj5_below_tier: CUJ5 default selection with the dedicated low-spend principal tui: real interactive terminal boot, keyboard input, exit and reopen diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index a76226bbd..9f1052d87 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -1,5 +1,6 @@ """Live model-selection check for a fixed workspace budget recommendation.""" +import json import re import tomllib from decimal import ROUND_HALF_UP, Decimal @@ -14,9 +15,82 @@ class TestCujBudgetDefaults(BaseCujTest): WORKSPACE_URL = "https://dbc-497adeef-62c0.cloud.databricks.com" BUDGET_ID = "25c4ce5c-fcd6-4d00-8266-69030cc1a236" + SONNET_MODEL = "system.ai.claude-sonnet-4-6" SOL_MODEL = "system.ai.gpt-5-6-sol" LUNA_MODEL = "system.ai.gpt-5-6-luna" + @pytest.mark.cuj5_below_tier + def test_bare_ug_uses_claude_default_below_budget_tier(self, live_session): + """Scenario: launch bare ``ug`` as a fresh principal below the fixed 2% tier. + + Expected: the published config keeps Claude/Sonnet as the default, the real recommendation + also selects Claude/Sonnet, and the real Claude TUI starts on that model without a prompt, + inference request, budget write, or account-login flow. A fresh principal may have no + spend counters yet; when the backend returns an effective threshold, omitted spend is + treated as zero and must remain below the 2% tier. If both figures are absent, this case + verifies default selection without a numeric spend assertion. + """ + session = live_session + config_path = "/api/ai-gateway/v2/coding-agent-configs" + payload = self.workspace.api_client.do("GET", path=config_path) + config = payload["coding_agent_configs"][0] + assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE", config + claude = next( + agent + for agent in config["enabled_agents"] + if agent["agent"] == "CODING_AGENT_CLAUDE_CODE" + ) + claude_defaults = claude["config"]["default_models"] + assert claude_defaults["default_model"] == self.SONNET_MODEL, config + assert config["smart_defaults"]["budget_id"] == self.BUDGET_ID, config + assert config["smart_defaults"]["tiers"] == [ + { + "spending_percentage": 0.02, + "recommended_agent": "CODING_AGENT_CODEX", + "recommended_model": self.LUNA_MODEL, + } + ], config + + session.run( + "configure", + "--workspace", + self.WORKSPACE_URL, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + + recommendation = self.workspace.api_client.do( + "POST", path=config_path + ":recommendModel", body={} + ) + assert recommendation.get("recommended_agent") == "CODING_AGENT_CLAUDE_CODE", recommendation + assert recommendation.get("recommended_model") == self.SONNET_MODEL, recommendation + + threshold = recommendation.get("effective_threshold") + spend = recommendation.get("current_spend") + if threshold is not None: + threshold = Decimal(str(threshold)) + assert threshold > 0, recommendation + spend = Decimal("0") if spend is None else Decimal(str(spend)) + assert spend >= 0, recommendation + assert spend / threshold < Decimal("0.02"), recommendation + else: + # A fresh principal can have no usage counter; that does not establish a numeric ratio. + assert spend is None, recommendation + + with AgentTerminal( + session, "claude", [str(session.binary)], "budget-default-claude" + ) as tui: + tui.boot(timeout=240) + claude_settings = session.home / ".claude" / "ucode-settings.json" + settings = json.loads(claude_settings.read_text()) + assert settings["env"]["ANTHROPIC_MODEL"] == self.SONNET_MODEL, settings + tui.wait_for( + lambda text: re.search(r"Claude Sonnet 4\.6\s*·", text), + "Claude's native Sonnet model header", + ) + tui.exit_normally() + def test_bare_ug_uses_luna_budget_recommendation(self, live_session): """Scenario: launch bare ``ug`` with the fixed 2% Luna budget tier active. diff --git a/tests/integration/README.md b/tests/integration/README.md index 7e5f4d50f..8497f453a 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -15,21 +15,27 @@ does not yet assert live `recommendModel` request counts for configs with and wi ## Dedicated-workspace CUJs -The budget usage and model-selection test lives in +The budget usage and model-selection tests live in [`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py), -outside this existing-workspace integration suite. The test owns its workspace, -budget and model identifiers, while CI forwards only the shared -`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets. +outside this existing-workspace integration suite. The tests own their workspace, +budget and model identifiers. CI runs them on separate runners: the above-tier case +uses the shared `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets, while the +below-tier case uses `UG_BUDGET_CUJ_SP_CLIENT_ID` and `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. +Both pairs map to the fixture's existing `UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET` +environment variables; each runner receives only its selected identity. -Run the case through the dedicated suite from a clean POSIX runner: +Run the above-tier case through the dedicated suite from a clean POSIX runner: ```bash UG_CUJ_SP_CLIENT_ID=... UG_CUJ_SP_CLIENT_SECRET=... \ python3.12 scripts/run_integration.py \ --suite e2e-cuj --ug-version checkout \ - --claude-version 2.1.280 --codex-version 0.154.0 -- -m cuj + --claude-version 2.1.280 --codex-version 0.154.0 -- -m 'cuj and not cuj5_below_tier' ``` +For the below-tier case, use the same command with `-m cuj5_below_tier` and set the +two environment variables to the dedicated low-spend principal's credentials. + The runner installs `databricks-sdk==0.135.0` in the isolated test environment. The workspace client reads the published config and real recommendation using OAuth M2M. The single 2% tier must recommend Codex on `system.ai.gpt-5-6-luna` @@ -40,6 +46,11 @@ Luna over its managed Sol default; it submits no inference task. It neither writ the budget nor requires account-level authentication. Independent runners can overlap; each still needs isolated machine-level agent settings. +The below-tier case verifies the real default recommendation and bare `ug` selecting +Claude Code on `system.ai.claude-sonnet-4-6`, without submitting a prompt. A fresh +principal may have no spend counter, so this case establishes default model selection; +numeric spend reporting remains covered by the above-tier case. + Shared subprocess command resolution is covered by `../test_subprocess_cross_os.py` and enforced by Ruff. These component checks do not establish native Windows coverage for every agent; the Windows journeys below validate their explicitly selected paths. @@ -335,8 +346,8 @@ they only configure, list models, and open/close the picker. Other live CUJs per real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex. The separate dedicated-workspace suite adds one -budget model-selection case selected by `-m cuj` and run by the `cuj` job. +checks** with Claude and Codex. The separate dedicated-workspace suite adds two +budget model-selection cases run by the `cuj` matrix with separate credentials and markers. Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** @@ -505,8 +516,8 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs all 62 live cases, including those smoke cases, in two -disjoint agent lanes. The dedicated `cuj` job runs one budget model-selection case -with both agent binaries installed and no cross-run concurrency lock. The parent +disjoint agent lanes. The dedicated `cuj` matrix runs the above-tier and below-tier +budget cases on separate runners with both agent binaries installed and no cross-run concurrency lock. The parent workflows retain their normal cancellation of superseded runs. The full-suite agent lanes are: @@ -684,7 +695,8 @@ gh run download RUN_ID -R databricks/unity-gateway \ ``` Use `integration-full-AGENT` for a full lane, `integration-smoke-AGENT` for -smoke, or `integration-e2e-cuj` for the budget model-selection test. Use `integration-installation` for Linux package failures, or +smoke, `integration-e2e-cuj` for the above-tier budget test, or `integration-cuj5-below-tier` +for the below-tier case. Use `integration-installation` for Linux package failures, or `integration-installation-windows` for native Windows package failures, or `integration-headless-windows-claude` for the Windows gateway journey. Older runs used `integration-full-AGENT-GROUP`, `integration-cujs`, or numbered `integration-live-*` diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index d98843146..a48705a7a 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -62,16 +62,23 @@ def test_windows_integration_ci_uses_shared_claude_version(): assert contents.count('"--claude-version", $env:CLAUDE_VERSION,') == 2 -def test_dedicated_cuj_job_uses_shared_credentials_without_serialization(): +def test_dedicated_cuj_job_selects_credentials_without_serialization(): workflow = Path(__file__).parent.parent / ".github/workflows/integration.yml" contents = workflow.read_text() cuj = contents.split("\n cuj:\n", 1)[1].split("\n # Non-blocking:", 1)[0] assert "concurrency:" not in cuj assert "TEST_SUITE: e2e-cuj" in cuj - assert "TEST_MARKER: cuj" in cuj - assert "UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }}" in cuj - assert "UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }}" in cuj + assert "test_marker: cuj and not cuj5_below_tier" in cuj + assert "test_marker: cuj5_below_tier" in cuj + assert "client_id_secret: UG_CUJ_SP_CLIENT_ID" in cuj + assert "client_secret_secret: UG_CUJ_SP_CLIENT_SECRET" in cuj + assert "client_id_secret: UG_BUDGET_CUJ_SP_CLIENT_ID" in cuj + assert "client_secret_secret: UG_BUDGET_CUJ_SP_CLIENT_SECRET" in cuj + assert "UG_CUJ_SP_CLIENT_ID: ${{ secrets[matrix.client_id_secret] }}" in cuj + assert "UG_CUJ_SP_CLIENT_SECRET: ${{ secrets[matrix.client_secret_secret] }}" in cuj + assert "TEST_MARKER: ${{ matrix.test_marker }}" in cuj + assert "ARTIFACT_NAME: ${{ matrix.artifact_name }}" in cuj @pytest.mark.parametrize("suite", ["full", "live", "smoke", "tui", "installation"]) From bbde97881f800dfc788db098bfdff06a4280174e Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 15:06:10 +0000 Subject: [PATCH 08/19] Update CUJ5 to the 1% budget tier and clarify model failures --- tests/README.md | 6 ++--- tests/e2e_cuj/AGENTS.md | 2 +- tests/e2e_cuj/test_ug_budget_defaults.py | 32 ++++++++++++++++++------ tests/integration/README.md | 2 +- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/tests/README.md b/tests/README.md index fd35d0baf..36509e741 100644 --- a/tests/README.md +++ b/tests/README.md @@ -161,7 +161,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 0.01% tier (`spending_percentage=0.0001`) | Low-spend principal selects the default Claude/Sonnet; above-tier principal selects Codex/Luna over Sol and read-only `ug usage` agrees with backend spend, threshold, and percentage. No budget writes or inference tasks. Run in the `dedicated-cuj` workflow with the credential-specific marker documented in `integration/README.md`. | +| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`) | Low-spend principal selects the default Claude/Sonnet; above-tier principal selects Codex/Luna over Sol and read-only `ug usage` agrees with backend spend, threshold, and percentage. No budget writes or inference tasks. Run in the `dedicated-cuj` workflow with the credential-specific marker documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -187,8 +187,8 @@ executions. Thirteen additional managed-fixture cases cover focused model, MCP, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. -The dedicated-workspace CUJ suite adds two cases around the fixed 0.01% Luna tier -(`spending_percentage=0.0001`): +The dedicated-workspace CUJ suite adds two cases around the fixed 1% Luna tier +(`spending_percentage=0.01`): default Claude/Sonnet selection with a low-spend principal, and usage plus Codex/Luna selection with the above-tier principal. It runs outside the 62-case integration suite in the `dedicated-cuj` workflow's two matrix entries: the shared principal selects diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index 0837ad4b2..4aa9065e2 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -11,7 +11,7 @@ Keep local homes, task files, and artifacts isolated per run. No stubbed configu Use `CLAUDE` / `CODEX` from `helpers/constants.py`; reject unsupported agents explicitly. The budget-default CUJ uses two marker-selected runs against the fixed tier with -`spending_percentage=0.0001` (0.01%). The shared-principal run uses +`spending_percentage=0.01` (1%). The shared-principal run uses `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` with `-m 'not cuj5_below_tier'`; the below-tier run uses `UG_BUDGET_CUJ_SP_CLIENT_ID` and `UG_BUDGET_CUJ_SP_CLIENT_SECRET` with `-m cuj5_below_tier`. CI maps the selected diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index ab7a9fac7..190df5a2d 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -24,13 +24,13 @@ class TestCujBudgetDefaults(BaseCujTest): @pytest.mark.cuj5_below_tier def test_bare_ug_uses_claude_default_below_budget_tier(self, cuj): - """Scenario: launch bare ``ug`` as a fresh principal below the fixed 0.01% tier. + """Scenario: launch bare ``ug`` as a fresh principal below the fixed 1% tier. Expected: the published config keeps Claude/Sonnet as the default, the real recommendation also selects Claude/Sonnet, and the real Claude TUI starts on that model without a prompt, inference request, budget write, or account-login flow. A fresh principal may have no spend counters yet; when the backend returns an effective threshold, omitted spend is - treated as zero and must remain below the 0.01% tier. If both figures are absent, this case + treated as zero and must remain below the 1% tier. If both figures are absent, this case verifies default selection without a numeric spend assertion. """ session, workspace, _ = cuj @@ -46,7 +46,7 @@ def test_bare_ug_uses_claude_default_below_budget_tier(self, cuj): assert config["smart_defaults"]["budget_id"] == self.BUDGET_ID, config assert config["smart_defaults"]["tiers"] == [ { - "spending_percentage": 0.0001, + "spending_percentage": 0.01, "recommended_agent": CodingAgent.CODEX, "recommended_model": self.LUNA_MODEL, } @@ -75,7 +75,7 @@ def test_bare_ug_uses_claude_default_below_budget_tier(self, cuj): assert threshold > 0, recommendation spend = Decimal("0") if spend is None else Decimal(str(spend)) assert spend >= 0, recommendation - assert spend / threshold < Decimal("0.0001"), recommendation + assert spend / threshold < Decimal("0.01"), recommendation else: # A fresh principal can have no usage counter; that does not establish a numeric ratio. assert spend is None, recommendation @@ -92,7 +92,7 @@ def test_bare_ug_uses_claude_default_below_budget_tier(self, cuj): tui.exit_normally() def test_bare_ug_uses_luna_budget_recommendation(self, cuj): - """Scenario: launch bare ``ug`` with the fixed 0.01% Luna budget tier active. + """Scenario: launch bare ``ug`` with the fixed 1% Luna budget tier active. Expected: the read-only `usage` command reports the backend's spend and threshold, then the real backend recommends Codex/Luna and the Codex TUI selects Luna over its managed Sol @@ -103,10 +103,20 @@ def test_bare_ug_uses_luna_budget_recommendation(self, cuj): payload = workspace.client.api_client.do("GET", path=config_path) config = payload["coding_agent_configs"][0] assert config["default_agent"] == CodingAgent.CLAUDE_CODE, config + codex_agent = next( + agent for agent in config["enabled_agents"] if agent["agent"] == CodingAgent.CODEX + ) + live_default_model = ( + codex_agent.get("config", {}).get("default_models", {}).get("default_model") + ) + assert live_default_model == self.SOL_MODEL, ( + f"Live Codex config model mismatch: expected {self.SOL_MODEL!r}, " + f"got {live_default_model!r}" + ) assert config["smart_defaults"]["budget_id"] == self.BUDGET_ID, config assert config["smart_defaults"]["tiers"] == [ { - "spending_percentage": 0.0001, + "spending_percentage": 0.01, "recommended_agent": CodingAgent.CODEX, "recommended_model": self.LUNA_MODEL, } @@ -122,7 +132,10 @@ def test_bare_ug_uses_luna_budget_recommendation(self, cuj): ] ) model_config = session.home / ".codex" / "ucode.config.toml" - assert tomllib.loads(model_config.read_text())["model"] == self.SOL_MODEL + actual_model = tomllib.loads(model_config.read_text()).get("model") + assert actual_model == self.SOL_MODEL, ( + f"Codex config model mismatch: expected {self.SOL_MODEL!r}, got {actual_model!r}" + ) before_usage = workspace.client.api_client.do( "POST", path=config_path + ":recommendModel", body={} @@ -165,7 +178,10 @@ def test_bare_ug_uses_luna_budget_recommendation(self, cuj): with AgentTerminal(session, CODEX, [str(session.binary)], "budget-luna") as tui: tui.boot(timeout=240) - assert tomllib.loads(model_config.read_text())["model"] == self.LUNA_MODEL + actual_model = tomllib.loads(model_config.read_text()).get("model") + assert actual_model == self.LUNA_MODEL, ( + f"Codex config model mismatch: expected {self.LUNA_MODEL!r}, got {actual_model!r}" + ) tui.wait_for( lambda text: re.search(rf"model:\s+{re.escape(self.LUNA_MODEL)}\s", text), "Codex's selected model to be Luna", diff --git a/tests/integration/README.md b/tests/integration/README.md index 58283263b..9fcdbbaa5 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -26,7 +26,7 @@ The budget usage and model-selection tests live in [`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py), outside this existing-workspace integration suite. The tests read their dedicated workspace's published budget and model identifiers. The fixed tier has -`spending_percentage=0.0001` (0.01%). The `dedicated-cuj` workflow runs the suite +`spending_percentage=0.01` (1%). The `dedicated-cuj` workflow runs the suite in two matrix entries on isolated runners: the above-tier case uses the shared `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets, while the below-tier case uses `UG_BUDGET_CUJ_SP_CLIENT_ID` and `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. From 1b04347fac3ec8c02186f8a515e090822c046868 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 15:13:11 +0000 Subject: [PATCH 09/19] Preserve managed Codex models when saving state --- src/ucode/agents/codex.py | 8 ++++++-- tests/README.md | 2 ++ tests/test_agent_codex.py | 37 ++++++++++++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/ucode/agents/codex.py b/src/ucode/agents/codex.py index 9fcd62527..51838969f 100644 --- a/src/ucode/agents/codex.py +++ b/src/ucode/agents/codex.py @@ -786,12 +786,16 @@ def read_managed_mcp_urls() -> dict[str, str]: def default_model(state: dict) -> str | None: - """Return a managed Codex model, or leave selection to Codex.""" + """Return a managed Codex model, or leave selection to Codex. + + Model cleanup is handled by the config writer and launch path. Keeping this lookup pure is + required because state hydration calls it while persisting configuration; clearing the profile + there would remove a managed model immediately after the writer saved it. + """ if isinstance(state.get("codex_default_model"), str): return state["codex_default_model"] if smart_routing_v2.smart_routing_enabled(): return _smart_routing_config_model(state) - clear_model_preferences(state) return None diff --git a/tests/README.md b/tests/README.md index 36509e741..21ee0dd8d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -73,6 +73,8 @@ Managed smart defaults are covered by `test_managed_config.py`, `test_cli.py`, `smart_defaults` wire field, reading older `spend_tiers` caches, skipping recommendations without tier rules, applying recommendations when tiers exist, and serializing the current API field. These are unit/component checks; live request-count coverage is not included. +Codex configuration regressions also check that saving managed state preserves the generated +default or budget-selected model and that model lookup leaves configuration files unchanged. `test_codex_smart_routing_v2.py` checks that the remote Codex TUI receives the gateway provider on Windows while Unix launch arguments stay unchanged and routing diff --git a/tests/test_agent_codex.py b/tests/test_agent_codex.py index c810a1b6f..7591e935f 100644 --- a/tests/test_agent_codex.py +++ b/tests/test_agent_codex.py @@ -226,6 +226,37 @@ def test_writes_ucode_profile_config_file(self, tmp_path, monkeypatch): assert "model_reasoning_effort" not in doc assert "profiles" not in doc + @pytest.mark.parametrize("model", ["system.ai.gpt-5-6-sol", "system.ai.gpt-5-6-luna"]) + def test_managed_model_survives_state_hydration(self, tmp_path, monkeypatch, model): + import ucode.state as state_mod + + config_path = tmp_path / ".codex" / "ucode.config.toml" + monkeypatch.setattr(codex, "CODEX_CONFIG_PATH", config_path) + monkeypatch.setattr(codex, "CODEX_BACKUP_PATH", tmp_path / "codex-ucode-config.backup.toml") + monkeypatch.setattr(codex, "agent_version", lambda _binary: "0.154.0") + monkeypatch.setattr(codex, "_reconcile_managed_config", lambda *args, **kwargs: None) + + app_dir = tmp_path / ".ucode" + monkeypatch.setattr(state_mod, "APP_DIR", app_dir) + monkeypatch.setattr(state_mod, "STATE_PATH", app_dir / "state.json") + monkeypatch.setattr(codex, "save_state", state_mod.save_state) + + from ucode.managed_resolve import resolve_state + + managed = { + "enabled_agents": { + "codex": {"model_config": {"default_model": "system.ai.gpt-5-6-sol"}} + } + } + state = resolve_state(managed, {"workspace": WS}, "codex") + codex.write_tool_config(state, model=model) + + assert read_toml_safe(config_path)["model"] == model + persisted = json.loads(state_mod.STATE_PATH.read_text()) + persisted_state = persisted["workspaces"][WS] + assert "codex_default_model" not in persisted_state + assert "_managed_overlay" not in persisted_state + def test_smart_routing_preserves_configured_startup_model(self, tmp_path, monkeypatch): config_path = tmp_path / "ucode.config.toml" config_path.write_text('model = "gpt-5.6-sol"\n') @@ -909,15 +940,15 @@ def _isolate_profile_config(self, tmp_path, monkeypatch): monkeypatch.setattr(codex, "CODEX_CONFIG_PATH", tmp_path / "ucode.config.toml") monkeypatch.setattr(codex, "CODEX_BACKUP_PATH", tmp_path / "backup.toml") - def test_clears_profile_model_preferences(self, tmp_path): + def test_default_model_lookup_leaves_profile_model_preferences(self, tmp_path): codex.CODEX_CONFIG_PATH.write_text( 'model = "gpt-5.6-sol"\nmodel_reasoning_effort = "medium"\n', encoding="utf-8" ) assert codex.default_model({"codex_models": ["system.ai.gpt-5-6-luna"]}) is None doc = read_toml_safe(codex.CODEX_CONFIG_PATH) - assert "model" not in doc - assert "model_reasoning_effort" not in doc + assert doc["model"] == "gpt-5.6-sol" + assert doc["model_reasoning_effort"] == "medium" def test_none_when_no_configured_model(self): assert codex.default_model({}) is None From a3a51e75fe73eb1051c6a57d22206cc47817e5e5 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 15:29:41 +0000 Subject: [PATCH 10/19] Revert "Preserve managed Codex models when saving state" This reverts commit 1b04347fac3ec8c02186f8a515e090822c046868. --- src/ucode/agents/codex.py | 8 ++------ tests/README.md | 2 -- tests/test_agent_codex.py | 37 +++---------------------------------- 3 files changed, 5 insertions(+), 42 deletions(-) diff --git a/src/ucode/agents/codex.py b/src/ucode/agents/codex.py index 51838969f..9fcd62527 100644 --- a/src/ucode/agents/codex.py +++ b/src/ucode/agents/codex.py @@ -786,16 +786,12 @@ def read_managed_mcp_urls() -> dict[str, str]: def default_model(state: dict) -> str | None: - """Return a managed Codex model, or leave selection to Codex. - - Model cleanup is handled by the config writer and launch path. Keeping this lookup pure is - required because state hydration calls it while persisting configuration; clearing the profile - there would remove a managed model immediately after the writer saved it. - """ + """Return a managed Codex model, or leave selection to Codex.""" if isinstance(state.get("codex_default_model"), str): return state["codex_default_model"] if smart_routing_v2.smart_routing_enabled(): return _smart_routing_config_model(state) + clear_model_preferences(state) return None diff --git a/tests/README.md b/tests/README.md index 21ee0dd8d..36509e741 100644 --- a/tests/README.md +++ b/tests/README.md @@ -73,8 +73,6 @@ Managed smart defaults are covered by `test_managed_config.py`, `test_cli.py`, `smart_defaults` wire field, reading older `spend_tiers` caches, skipping recommendations without tier rules, applying recommendations when tiers exist, and serializing the current API field. These are unit/component checks; live request-count coverage is not included. -Codex configuration regressions also check that saving managed state preserves the generated -default or budget-selected model and that model lookup leaves configuration files unchanged. `test_codex_smart_routing_v2.py` checks that the remote Codex TUI receives the gateway provider on Windows while Unix launch arguments stay unchanged and routing diff --git a/tests/test_agent_codex.py b/tests/test_agent_codex.py index 7591e935f..c810a1b6f 100644 --- a/tests/test_agent_codex.py +++ b/tests/test_agent_codex.py @@ -226,37 +226,6 @@ def test_writes_ucode_profile_config_file(self, tmp_path, monkeypatch): assert "model_reasoning_effort" not in doc assert "profiles" not in doc - @pytest.mark.parametrize("model", ["system.ai.gpt-5-6-sol", "system.ai.gpt-5-6-luna"]) - def test_managed_model_survives_state_hydration(self, tmp_path, monkeypatch, model): - import ucode.state as state_mod - - config_path = tmp_path / ".codex" / "ucode.config.toml" - monkeypatch.setattr(codex, "CODEX_CONFIG_PATH", config_path) - monkeypatch.setattr(codex, "CODEX_BACKUP_PATH", tmp_path / "codex-ucode-config.backup.toml") - monkeypatch.setattr(codex, "agent_version", lambda _binary: "0.154.0") - monkeypatch.setattr(codex, "_reconcile_managed_config", lambda *args, **kwargs: None) - - app_dir = tmp_path / ".ucode" - monkeypatch.setattr(state_mod, "APP_DIR", app_dir) - monkeypatch.setattr(state_mod, "STATE_PATH", app_dir / "state.json") - monkeypatch.setattr(codex, "save_state", state_mod.save_state) - - from ucode.managed_resolve import resolve_state - - managed = { - "enabled_agents": { - "codex": {"model_config": {"default_model": "system.ai.gpt-5-6-sol"}} - } - } - state = resolve_state(managed, {"workspace": WS}, "codex") - codex.write_tool_config(state, model=model) - - assert read_toml_safe(config_path)["model"] == model - persisted = json.loads(state_mod.STATE_PATH.read_text()) - persisted_state = persisted["workspaces"][WS] - assert "codex_default_model" not in persisted_state - assert "_managed_overlay" not in persisted_state - def test_smart_routing_preserves_configured_startup_model(self, tmp_path, monkeypatch): config_path = tmp_path / "ucode.config.toml" config_path.write_text('model = "gpt-5.6-sol"\n') @@ -940,15 +909,15 @@ def _isolate_profile_config(self, tmp_path, monkeypatch): monkeypatch.setattr(codex, "CODEX_CONFIG_PATH", tmp_path / "ucode.config.toml") monkeypatch.setattr(codex, "CODEX_BACKUP_PATH", tmp_path / "backup.toml") - def test_default_model_lookup_leaves_profile_model_preferences(self, tmp_path): + def test_clears_profile_model_preferences(self, tmp_path): codex.CODEX_CONFIG_PATH.write_text( 'model = "gpt-5.6-sol"\nmodel_reasoning_effort = "medium"\n', encoding="utf-8" ) assert codex.default_model({"codex_models": ["system.ai.gpt-5-6-luna"]}) is None doc = read_toml_safe(codex.CODEX_CONFIG_PATH) - assert doc["model"] == "gpt-5.6-sol" - assert doc["model_reasoning_effort"] == "medium" + assert "model" not in doc + assert "model_reasoning_effort" not in doc def test_none_when_no_configured_model(self): assert codex.default_model({}) is None From a2d9b5f4d1e351220119918a2b262806bfdbcbeb Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 15:34:42 +0000 Subject: [PATCH 11/19] Remove remaining Codex production change from CUJ5 PR --- src/ucode/agents/codex.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/ucode/agents/codex.py b/src/ucode/agents/codex.py index 9fcd62527..75606811d 100644 --- a/src/ucode/agents/codex.py +++ b/src/ucode/agents/codex.py @@ -405,12 +405,9 @@ def write_tool_config( workspace = state["workspace"] # Leave model selection to Codex. The gateway still receives the configured # provider and authentication settings, while Codex uses its own default. - # A managed launch model is the sole exception: it may be the configured - # default or a budget recommendation that supersedes that default. + # A managed default is the sole exception. managed_model = state.get("codex_default_model") chosen_model = managed_model if isinstance(managed_model, str) else None - if chosen_model is not None and isinstance(model, str): - chosen_model = model databricks_profile = state.get("profile") static_models = state.get("codex_static_models") static_models = static_models if isinstance(static_models, list) and static_models else None From b7db0e166db28e483f5de5a64286e283742780ef Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 15:47:56 +0000 Subject: [PATCH 12/19] Remove intermediate CUJ5 Codex profile assertion --- tests/README.md | 2 ++ tests/e2e_cuj/test_ug_budget_defaults.py | 4 ---- tests/integration/README.md | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/README.md b/tests/README.md index 36509e741..f3cf1fc60 100644 --- a/tests/README.md +++ b/tests/README.md @@ -197,6 +197,8 @@ Both entries read the preconfigured workspace and make no budget writes or infer requests. It does not cover spend boundaries, selection between multiple tiers, or explicit Claude overrides. Generated-file cleanup and strict app-server stdout assertions remain enforced in the integration suite. +The published config verifies Codex's managed Sol default; the generated local model is +checked only after launch, with no intermediate after-configure local-profile assertion. Unmanaged discovery Cases 7–14 configure, list models, or open the picker without submitting inference prompts; separate task journeys still perform inference. They require a real workspace with no CodingAgentConfig; a read-only prerequisite diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index 190df5a2d..9a337d23a 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -132,10 +132,6 @@ def test_bare_ug_uses_luna_budget_recommendation(self, cuj): ] ) model_config = session.home / ".codex" / "ucode.config.toml" - actual_model = tomllib.loads(model_config.read_text()).get("model") - assert actual_model == self.SOL_MODEL, ( - f"Codex config model mismatch: expected {self.SOL_MODEL!r}, got {actual_model!r}" - ) before_usage = workspace.client.api_client.do( "POST", path=config_path + ":recommendModel", body={} diff --git a/tests/integration/README.md b/tests/integration/README.md index 9fcdbbaa5..c9787ac0f 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -52,6 +52,8 @@ Luna over its managed Sol default. The below-tier case launches bare `ug` and verifies Claude selects Sonnet. Neither case submits an inference task or writes the budget; the dedicated workspace configuration remains read-only. Independent matrix entries can overlap because their local homes and artifacts are isolated. +The published config verifies Codex's managed Sol default; the generated local model is +checked only after launch, with no intermediate after-configure local-profile assertion. A fresh below-tier principal may have no spend counter, so that case establishes default model selection; numeric spend reporting remains covered by the above-tier case. From 16655030bee33b901d6f164cd4aabdc0084b6b75 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 16:11:28 +0000 Subject: [PATCH 13/19] Verify CUJ5 Codex model through its native TUI --- tests/README.md | 4 ++-- tests/e2e_cuj/test_ug_budget_defaults.py | 6 ------ tests/integration/README.md | 4 ++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/README.md b/tests/README.md index f3cf1fc60..41c0d93ac 100644 --- a/tests/README.md +++ b/tests/README.md @@ -197,8 +197,8 @@ Both entries read the preconfigured workspace and make no budget writes or infer requests. It does not cover spend boundaries, selection between multiple tiers, or explicit Claude overrides. Generated-file cleanup and strict app-server stdout assertions remain enforced in the integration suite. -The published config verifies Codex's managed Sol default; the generated local model is -checked only after launch, with no intermediate after-configure local-profile assertion. +The published config verifies Codex's managed Sol default, and the native Codex TUI verifies +the above-tier Luna selection over Sol. Unmanaged discovery Cases 7–14 configure, list models, or open the picker without submitting inference prompts; separate task journeys still perform inference. They require a real workspace with no CodingAgentConfig; a read-only prerequisite diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index 9a337d23a..0a58ef29b 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -2,7 +2,6 @@ import json import re -import tomllib from decimal import ROUND_HALF_UP, Decimal import pytest @@ -131,7 +130,6 @@ def test_bare_ug_uses_luna_budget_recommendation(self, cuj): "--disable-databricks-ai-tools", ] ) - model_config = session.home / ".codex" / "ucode.config.toml" before_usage = workspace.client.api_client.do( "POST", path=config_path + ":recommendModel", body={} @@ -174,10 +172,6 @@ def test_bare_ug_uses_luna_budget_recommendation(self, cuj): with AgentTerminal(session, CODEX, [str(session.binary)], "budget-luna") as tui: tui.boot(timeout=240) - actual_model = tomllib.loads(model_config.read_text()).get("model") - assert actual_model == self.LUNA_MODEL, ( - f"Codex config model mismatch: expected {self.LUNA_MODEL!r}, got {actual_model!r}" - ) tui.wait_for( lambda text: re.search(rf"model:\s+{re.escape(self.LUNA_MODEL)}\s", text), "Codex's selected model to be Luna", diff --git a/tests/integration/README.md b/tests/integration/README.md index c9787ac0f..77c7116d1 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -52,8 +52,8 @@ Luna over its managed Sol default. The below-tier case launches bare `ug` and verifies Claude selects Sonnet. Neither case submits an inference task or writes the budget; the dedicated workspace configuration remains read-only. Independent matrix entries can overlap because their local homes and artifacts are isolated. -The published config verifies Codex's managed Sol default; the generated local model is -checked only after launch, with no intermediate after-configure local-profile assertion. +The published config verifies Codex's managed Sol default, and the native Codex TUI verifies +the above-tier Luna selection over Sol. A fresh below-tier principal may have no spend counter, so that case establishes default model selection; numeric spend reporting remains covered by the above-tier case. From c24944898d9c1fb2abdb88fe2efa7561ce990364 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 16:21:52 +0000 Subject: [PATCH 14/19] Restore CUJ5 recommendation and startup coverage --- tests/README.md | 9 +++++---- tests/e2e_cuj/AGENTS.md | 6 ++++-- tests/e2e_cuj/test_ug_budget_defaults.py | 22 +++++++++++++++------- tests/integration/README.md | 12 +++++++----- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/tests/README.md b/tests/README.md index 41c0d93ac..766ee968d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -161,7 +161,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`) | Low-spend principal selects the default Claude/Sonnet; above-tier principal selects Codex/Luna over Sol and read-only `ug usage` agrees with backend spend, threshold, and percentage. No budget writes or inference tasks. Run in the `dedicated-cuj` workflow with the credential-specific marker documented in `integration/README.md`. | +| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`) | Low-spend principal selects the default Claude/Sonnet; above-tier principal receives and displays a Codex/Luna recommendation, starts native Codex, and read-only `ug usage` agrees with backend spend, threshold, and percentage. No budget writes or inference tasks. Run in the `dedicated-cuj` workflow with the credential-specific marker documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -187,10 +187,10 @@ executions. Thirteen additional managed-fixture cases cover focused model, MCP, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. -The dedicated-workspace CUJ suite adds two cases around the fixed 1% Luna tier +The dedicated-workspace CUJ suite adds two cases around the fixed 1% Luna recommendation tier (`spending_percentage=0.01`): default Claude/Sonnet selection with a low-spend principal, and usage plus Codex/Luna -selection with the above-tier principal. It runs outside the 62-case integration suite +recommendation display and native Codex startup with the above-tier principal. It runs outside the 62-case integration suite in the `dedicated-cuj` workflow's two matrix entries: the shared principal selects `not cuj5_below_tier`, while the low-spend principal selects `cuj5_below_tier`. Both entries read the preconfigured workspace and make no budget writes or inference @@ -198,7 +198,8 @@ requests. It does not cover spend boundaries, selection between multiple tiers, explicit Claude overrides. Generated-file cleanup and strict app-server stdout assertions remain enforced in the integration suite. The published config verifies Codex's managed Sol default, and the native Codex TUI verifies -the above-tier Luna selection over Sol. +the above-tier Codex launch and its displayed Luna recommendation. Applying Luna over Sol is +outside this coverage. Unmanaged discovery Cases 7–14 configure, list models, or open the picker without submitting inference prompts; separate task journeys still perform inference. They require a real workspace with no CodingAgentConfig; a read-only prerequisite diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index 4aa9065e2..7d39e6710 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -19,8 +19,10 @@ identity into the standard `UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET` fixture variables. Both cases read the workspace and budget recommendation; below-tier bare `ug` selects Claude/Sonnet, while above-tier read-only `ug usage` agrees with backend spend, -threshold, and percentage and bare `ug` selects Codex/Luna over Sol. Neither case -writes workspace or budget configuration or submits an inference task. +threshold, and percentage, the launch panel displays a Codex/Luna recommendation, +and bare `ug` starts the native Codex TUI. Neither case writes workspace or budget +configuration or submits an inference task. The above-tier case does not assert that +Codex applies Luna over its managed Sol default. Before adding any helper, search the existing test utilities and installed SDK for equivalent functionality. Reuse them directly; never build a parallel HTTP/auth diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index 0a58ef29b..dcbb4c6b6 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -1,4 +1,4 @@ -"""Live model-selection check for a fixed workspace budget recommendation.""" +"""Live budget recommendation and default-model checks for a fixed workspace tier.""" import json import re @@ -90,12 +90,13 @@ def test_bare_ug_uses_claude_default_below_budget_tier(self, cuj): ) tui.exit_normally() - def test_bare_ug_uses_luna_budget_recommendation(self, cuj): + def test_bare_ug_shows_luna_recommendation_and_starts_codex(self, cuj): """Scenario: launch bare ``ug`` with the fixed 1% Luna budget tier active. Expected: the read-only `usage` command reports the backend's spend and threshold, then - the real backend recommends Codex/Luna and the Codex TUI selects Luna over its managed Sol - default, without budget writes or an inference task. + the real backend recommends Codex/Luna, the launch panel displays that recommendation, and + the native Codex TUI starts and exits normally, without budget writes or an inference task. + Applying the recommended Luna model over the managed Sol default is outside this coverage. """ session, workspace, _ = cuj config_path = "/api/ai-gateway/v2/coding-agent-configs" @@ -172,8 +173,15 @@ def test_bare_ug_uses_luna_budget_recommendation(self, cuj): with AgentTerminal(session, CODEX, [str(session.binary)], "budget-luna") as tui: tui.boot(timeout=240) - tui.wait_for( - lambda text: re.search(rf"model:\s+{re.escape(self.LUNA_MODEL)}\s", text), - "Codex's selected model to be Luna", + transcript = session.redact( + "".join(tui.output) + "\n" + tui.visible, + strip_ansi=True, + ) + assert re.search( + rf"Recommended agent is Codex with model {re.escape(self.LUNA_MODEL)}\.", + transcript, + ), ( + "Codex launch did not display the backend recommendation " + f"for {self.LUNA_MODEL!r}:\n{transcript}" ) tui.exit_normally() diff --git a/tests/integration/README.md b/tests/integration/README.md index 77c7116d1..eba9243c1 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -22,7 +22,7 @@ does not yet assert live `recommendModel` request counts for configs with and wi ## Dedicated-workspace CUJs -The budget usage and model-selection tests live in +The budget usage and model recommendation/default tests live in [`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py), outside this existing-workspace integration suite. The tests read their dedicated workspace's published budget and model identifiers. The fixed tier has @@ -47,13 +47,15 @@ low-spend principal's credentials and use `-m cuj5_below_tier` with the same com The workspace client reads the published config and real recommendation using OAuth M2M. The above-tier case checks read-only `ug usage` spend, threshold, and percentage against backend reads, allowing spend to increase while the command runs. -It then launches bare `ug` and verifies Codex selects -Luna over its managed Sol default. The below-tier case launches bare `ug` and +It then launches bare `ug`, verifies the launch panel displays the backend's +Codex/Luna recommendation, and confirms the native Codex TUI starts and exits +normally. The below-tier case launches bare `ug` and verifies Claude selects Sonnet. Neither case submits an inference task or writes the budget; the dedicated workspace configuration remains read-only. Independent matrix entries can overlap because their local homes and artifacts are isolated. The published config verifies Codex's managed Sol default, and the native Codex TUI verifies -the above-tier Luna selection over Sol. +the above-tier Codex launch and its displayed Luna recommendation. Applying Luna over Sol is +outside this coverage. A fresh below-tier principal may have no spend counter, so that case establishes default model selection; numeric spend reporting remains covered by the above-tier case. @@ -354,7 +356,7 @@ real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation checks** with Claude and Codex. The separate dedicated-workspace suite adds two -budget model-selection cases run by the `dedicated-cuj` matrix with separate credentials +budget recommendation/default cases run by the `dedicated-cuj` matrix with separate credentials and markers. Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against From d841655dd4faaa0ee1428234f6a33f8e5fde03ad Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 16:31:29 +0000 Subject: [PATCH 15/19] Run both CUJ5 identities in the shared E2E job --- .github/workflows/integration.yml | 27 ++++----------- tests/README.md | 18 +++++----- tests/e2e_cuj/AGENTS.md | 20 +++++------ tests/e2e_cuj/base.py | 10 ++++-- tests/e2e_cuj/conftest.py | 19 +++++++++- tests/e2e_cuj/pytest.ini | 1 - tests/e2e_cuj/test_ug_budget_defaults.py | 10 ++++-- tests/integration/README.md | 36 ++++++++----------- tests/test_cuj_session.py | 4 +++ tests/test_cuj_workspace.py | 44 ++++++++++++++++++++++++ tests/test_integration_contract.py | 24 ++++++------- 11 files changed, 132 insertions(+), 81 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 6bea03d77..1c4b71b11 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -422,30 +422,17 @@ jobs: ${{ runner.temp }}/ug-integration/wheels/ dedicated-cuj: - name: ${{ matrix.name }} + name: E2E CUJs # CUJ definitions: https://docs.google.com/document/d/1WKd1fdWD0Y4tAV1H9Si-SGx2UZFL7iZ2S3HtBjidmS0/edit?usp=drive_web&ouid=117446678071138243519 if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} runs-on: ubuntu-22.04 timeout-minutes: 70 - strategy: - fail-fast: false - matrix: - include: - - name: "Dedicated CUJs · shared principal" - test_marker: not cuj5_below_tier - client_id_secret: UG_CUJ_SP_CLIENT_ID - client_secret_secret: UG_CUJ_SP_CLIENT_SECRET - artifact_name: integration-e2e-cuj - - name: "CUJ5: below-tier default · Claude" - test_marker: cuj5_below_tier - client_id_secret: UG_BUDGET_CUJ_SP_CLIENT_ID - client_secret_secret: UG_BUDGET_CUJ_SP_CLIENT_SECRET - artifact_name: integration-cuj5-below-tier env: UV_INDEX_URL: https://pypi.org/simple - UG_CUJ_SP_CLIENT_ID: ${{ secrets[matrix.client_id_secret] }} - UG_CUJ_SP_CLIENT_SECRET: ${{ secrets[matrix.client_secret_secret] }} - TEST_MARKER: ${{ matrix.test_marker }} + UG_CUJ_SP_CLIENT_ID: ${{ secrets.UG_CUJ_SP_CLIENT_ID }} + UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} + UG_BUDGET_CUJ_SP_CLIENT_ID: ${{ secrets.UG_BUDGET_CUJ_SP_CLIENT_ID }} + UG_BUDGET_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_BUDGET_CUJ_SP_CLIENT_SECRET }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -471,13 +458,13 @@ jobs: exit 0 fi uv run --with pexpect==4.9.0 --with pyte==0.8.2 \ - pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v -m "$TEST_MARKER" \ + pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v \ --basetemp "$RUNNER_TEMP/ug-cuj" --junitxml "$RUNNER_TEMP/ug-cuj-results.xml" - name: Upload CUJ evidence if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: ${{ matrix.artifact_name }} + name: integration-e2e-cuj path: | ${{ runner.temp }}/ug-cuj/**/artifacts/ ${{ runner.temp }}/ug-cuj-results.xml diff --git a/tests/README.md b/tests/README.md index 766ee968d..0c5e84aa6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -161,7 +161,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`) | Low-spend principal selects the default Claude/Sonnet; above-tier principal receives and displays a Codex/Luna recommendation, starts native Codex, and read-only `ug usage` agrees with backend spend, threshold, and percentage. No budget writes or inference tasks. Run in the `dedicated-cuj` workflow with the credential-specific marker documented in `integration/README.md`. | +| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`) | Low-spend principal selects the default Claude/Sonnet; above-tier principal receives and displays a Codex/Luna recommendation, starts native Codex, and read-only `ug usage` agrees with backend spend, threshold, and percentage. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | @@ -191,8 +191,8 @@ The dedicated-workspace CUJ suite adds two cases around the fixed 1% Luna recomm (`spending_percentage=0.01`): default Claude/Sonnet selection with a low-spend principal, and usage plus Codex/Luna recommendation display and native Codex startup with the above-tier principal. It runs outside the 62-case integration suite -in the `dedicated-cuj` workflow's two matrix entries: the shared principal selects -`not cuj5_below_tier`, while the low-spend principal selects `cuj5_below_tier`. +in the shared `E2E CUJs` job. Both cases are collected together; separate test classes +select the shared and low-spend credentials without changing process-wide authentication. Both entries read the preconfigured workspace and make no budget writes or inference requests. It does not cover spend boundaries, selection between multiple tiers, or explicit Claude overrides. Generated-file cleanup and strict app-server stdout @@ -240,13 +240,11 @@ journeys for both agents, in two parallel jobs. After smoke finishes, the full suite runs all 62 live cases across two parallel agent jobs: one Claude VM and one Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs -for the same agent overlap within a run. The dedicated budget cases run in the -`dedicated-cuj` matrix on separate runners with both agent binaries installed. -They read shared workspace configuration without changing the budget, so the matrix -needs no concurrency lock. The above-tier entry selects `not cuj5_below_tier` and -uses `UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET`; the below-tier entry selects -`cuj5_below_tier` and maps `UG_BUDGET_CUJ_SP_CLIENT_ID` / -`UG_BUDGET_CUJ_SP_CLIENT_SECRET` into the same fixture environment variables. +for the same agent overlap within a run. The dedicated budget cases run together +with all dedicated-workspace CUJs in the `E2E CUJs` job with both agent binaries +installed. Each class selects the shared or low-spend credentials and gets its own +local home and artifacts. `ug revert` cleans up local agent configuration between +classes. Workspace and budget configuration remain read-only, with no cross-run lock. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites share workspace capacity; overlapping their requests can still encounter rate limits. diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index 7d39e6710..888b8e70a 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -3,6 +3,8 @@ Each CUJ owns a separate workspace. Subclass `BaseCujTest` from `base.py` and set `WORKSPACE_URL`. Setup provides `self.workspace`, a Databricks SDK client using `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` with OAuth M2M authentication. +A CUJ that needs another identity overrides `CLIENT_ID_ENV` and `CLIENT_SECRET_ENV` +with its credential environment-variable names. All workspace configurations are READ ONLY. Never create, publish, update, delete, or restore workspace configuration, including during setup or teardown. Validate the preconfigured workspace and fail on a mismatch; never repair it. Concurrent @@ -10,13 +12,11 @@ runs may read the same CUJ workspace. Do not add reservations or remote lock fil Keep local homes, task files, and artifacts isolated per run. No stubbed configuration. Use `CLAUDE` / `CODEX` from `helpers/constants.py`; reject unsupported agents explicitly. -The budget-default CUJ uses two marker-selected runs against the fixed tier with -`spending_percentage=0.01` (1%). The shared-principal run uses -`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` with `-m 'not cuj5_below_tier'`; -the below-tier run uses `UG_BUDGET_CUJ_SP_CLIENT_ID` and -`UG_BUDGET_CUJ_SP_CLIENT_SECRET` with `-m cuj5_below_tier`. CI maps the selected -identity into the standard -`UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET` fixture variables. Both cases +The budget-default CUJ runs both cases in the shared `E2E CUJs` job against the fixed +`spending_percentage=0.01` (1%) tier. The above-tier class uses the standard +`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` credentials; the below-tier class +selects `UG_BUDGET_CUJ_SP_CLIENT_ID` and `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. +Each class gets its own SDK client, local home, and artifacts. Both cases read the workspace and budget recommendation; below-tier bare `ug` selects Claude/Sonnet, while above-tier read-only `ug usage` agrees with backend spend, threshold, and percentage, the launch panel displays a Codex/Luna recommendation, @@ -62,10 +62,8 @@ Use a clean disposable POSIX runner without existing machine-wide agent settings Install the intended versions of `ug`, `claude`, `codex`, and `databricks` on PATH. The terminal helpers also require `pexpect==4.9.0` and `pyte==0.8.2`: `uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v`. -Append `-m 'not cuj5_below_tier'` for the shared-principal run or -`-m cuj5_below_tier` for the dedicated low-spend run, after setting the standard -credential variables to the selected identity. Collection with `--collect-only` -does not authenticate or contact a workspace. Only the short-lived bearer is +Set both credential pairs to run CUJ5 together with the other CUJs. Collection with +`--collect-only` does not authenticate or contact a workspace. Only the short-lived bearer is forwarded to agent processes, never the SP secret. Artifacts are written beneath pytest's per-scenario temporary directory, printed during setup. diff --git a/tests/e2e_cuj/base.py b/tests/e2e_cuj/base.py index a0061649f..cbffca794 100644 --- a/tests/e2e_cuj/base.py +++ b/tests/e2e_cuj/base.py @@ -9,6 +9,8 @@ class BaseCujTest: WORKSPACE_URL: ClassVar[str] = "" + CLIENT_ID_ENV: ClassVar[str] = "UG_CUJ_SP_CLIENT_ID" + CLIENT_SECRET_ENV: ClassVar[str] = "UG_CUJ_SP_CLIENT_SECRET" workspace: WorkspaceClient @pytest.fixture(scope="class", autouse=True) @@ -17,10 +19,12 @@ def setup_workspace(self, request): if not workspace_url: pytest.fail("Set WORKSPACE_URL on your CUJ test class.", pytrace=False) - client_id = os.environ.get("UG_CUJ_SP_CLIENT_ID", "") - client_secret = os.environ.get("UG_CUJ_SP_CLIENT_SECRET", "") + client_id_env = request.cls.CLIENT_ID_ENV + client_secret_env = request.cls.CLIENT_SECRET_ENV + client_id = os.environ.get(client_id_env, "") + client_secret = os.environ.get(client_secret_env, "") if not client_id or not client_secret: - pytest.fail("Set UG_CUJ_SP_CLIENT_ID and UG_CUJ_SP_CLIENT_SECRET.", pytrace=False) + pytest.fail(f"Set {client_id_env} and {client_secret_env}.", pytrace=False) request.cls.workspace = WorkspaceClient( host=workspace_url, diff --git a/tests/e2e_cuj/conftest.py b/tests/e2e_cuj/conftest.py index 2306b8c12..3a14c2768 100644 --- a/tests/e2e_cuj/conftest.py +++ b/tests/e2e_cuj/conftest.py @@ -45,7 +45,24 @@ def cuj(request, setup_workspace, tmp_path_factory): try: yield session, workspace, recorder finally: - workspace.assert_unchanged(published) + try: + workspace.assert_unchanged(published) + finally: + # A TUI launch can reconcile machine-wide settings. Restore them through + # the public interactive command before the next class starts. + if any( + (session.home / ".ucode" / name).is_file() + for name in ("state.json", "managed-backups/manifest.json") + ): + from tests.integration.utils.terminal import TerminalProcess + + with TerminalProcess( + session, + "ug", + [str(session.binary), "revert"], + "cleanup-revert", + ) as terminal: + terminal.finish() except DatabricksError as error: # Server messages may echo credentials; retain only the SDK error type. raise RuntimeError(f"Workspace API failed: {type(error).__name__}") from None diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini index ec945e56f..11f9e73be 100644 --- a/tests/e2e_cuj/pytest.ini +++ b/tests/e2e_cuj/pytest.ini @@ -4,5 +4,4 @@ pythonpath = ../.. addopts = --strict-markers --tb=short markers = cuj: dedicated-workspace customer journey - cuj5_below_tier: CUJ5 default selection with the dedicated low-spend principal tui: real interactive terminal boot, keyboard input, exit and reopen diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index dcbb4c6b6..5e46dd87d 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -14,14 +14,18 @@ pytestmark = [pytest.mark.cuj, pytest.mark.tui] -class TestCujBudgetDefaults(BaseCujTest): +class _BudgetDefaultsBase(BaseCujTest): WORKSPACE_URL = "https://dbc-497adeef-62c0.cloud.databricks.com" BUDGET_ID = "25c4ce5c-fcd6-4d00-8266-69030cc1a236" SONNET_MODEL = "system.ai.claude-sonnet-4-6" SOL_MODEL = "system.ai.gpt-5-6-sol" LUNA_MODEL = "system.ai.gpt-5-6-luna" - @pytest.mark.cuj5_below_tier + +class TestCujBudgetDefaultsBelowTier(_BudgetDefaultsBase): + CLIENT_ID_ENV = "UG_BUDGET_CUJ_SP_CLIENT_ID" + CLIENT_SECRET_ENV = "UG_BUDGET_CUJ_SP_CLIENT_SECRET" + def test_bare_ug_uses_claude_default_below_budget_tier(self, cuj): """Scenario: launch bare ``ug`` as a fresh principal below the fixed 1% tier. @@ -90,6 +94,8 @@ def test_bare_ug_uses_claude_default_below_budget_tier(self, cuj): ) tui.exit_normally() + +class TestCujBudgetDefaults(_BudgetDefaultsBase): def test_bare_ug_shows_luna_recommendation_and_starts_codex(self, cuj): """Scenario: launch bare ``ug`` with the fixed 1% Luna budget tier active. diff --git a/tests/integration/README.md b/tests/integration/README.md index eba9243c1..cbb56befd 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -26,24 +26,21 @@ The budget usage and model recommendation/default tests live in [`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py), outside this existing-workspace integration suite. The tests read their dedicated workspace's published budget and model identifiers. The fixed tier has -`spending_percentage=0.01` (1%). The `dedicated-cuj` workflow runs the suite -in two matrix entries on isolated runners: the above-tier case uses the shared -`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` secrets, while the below-tier -case uses `UG_BUDGET_CUJ_SP_CLIENT_ID` and `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. -Each entry maps its selected identity into the fixture's standard -`UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET` environment variables. +`spending_percentage=0.01` (1%). The shared `E2E CUJs` job runs all tests in one +pytest collection. The above-tier case uses `UG_CUJ_SP_CLIENT_ID` and +`UG_CUJ_SP_CLIENT_SECRET`; the below-tier class selects `UG_BUDGET_CUJ_SP_CLIENT_ID` +and `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. Each class gets its own authenticated SDK +client, local home, and artifacts. -Run the above-tier case directly from a clean POSIX runner: +Run the suite directly from a clean POSIX runner with both credential pairs set: ```bash UG_CUJ_SP_CLIENT_ID=... UG_CUJ_SP_CLIENT_SECRET=... \ +UG_BUDGET_CUJ_SP_CLIENT_ID=... UG_BUDGET_CUJ_SP_CLIENT_SECRET=... \ uv run --with pexpect==4.9.0 --with pyte==0.8.2 \ - pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v -m 'not cuj5_below_tier' + pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v ``` -For the below-tier case, set those standard environment variables to the dedicated -low-spend principal's credentials and use `-m cuj5_below_tier` with the same command. - The workspace client reads the published config and real recommendation using OAuth M2M. The above-tier case checks read-only `ug usage` spend, threshold, and percentage against backend reads, allowing spend to increase while the command runs. @@ -51,8 +48,7 @@ It then launches bare `ug`, verifies the launch panel displays the backend's Codex/Luna recommendation, and confirms the native Codex TUI starts and exits normally. The below-tier case launches bare `ug` and verifies Claude selects Sonnet. Neither case submits an inference task or writes -the budget; the dedicated workspace configuration remains read-only. Independent -matrix entries can overlap because their local homes and artifacts are isolated. +the budget; the dedicated workspace configuration remains read-only. The published config verifies Codex's managed Sol default, and the native Codex TUI verifies the above-tier Codex launch and its displayed Luna recommendation. Applying Luna over Sol is outside this coverage. @@ -356,8 +352,7 @@ real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation checks** with Claude and Codex. The separate dedicated-workspace suite adds two -budget recommendation/default cases run by the `dedicated-cuj` matrix with separate credentials -and markers. +budget recommendation/default cases run in the shared `E2E CUJs` job with separate credentials. Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** @@ -527,12 +522,11 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs all 62 live cases, including those smoke cases, in two -disjoint agent lanes. The dedicated-cuj matrix runs the above-tier and below-tier -budget cases on separate runners with both agent binaries installed. Its entries -select `not cuj5_below_tier` with the shared credentials and `cuj5_below_tier` -with the dedicated low-spend credentials; each runner keeps its own home and -pytest artifacts. The workspace and budget configuration remain read-only, so -these entries need no remote reservation or cross-run lock. The parent workflows +disjoint agent lanes. The `E2E CUJs` job runs both budget cases alongside all other +dedicated-workspace CUJs. Each class selects its credentials and gets its own home +and artifacts; `ug revert` cleans up local agent configuration between classes. +Workspace and budget configuration remain read-only, with no remote reservation or +cross-run lock. The parent workflows retain their normal cancellation of superseded runs. The full-suite agent lanes are: diff --git a/tests/test_cuj_session.py b/tests/test_cuj_session.py index d3beff12f..6da893156 100644 --- a/tests/test_cuj_session.py +++ b/tests/test_cuj_session.py @@ -16,6 +16,8 @@ def test_cuj_session_does_not_inherit_agent_credentials_or_configuration(tmp_pat "CODEX_HOME", "UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET", + "UG_BUDGET_CUJ_SP_CLIENT_ID", + "UG_BUDGET_CUJ_SP_CLIENT_SECRET", ): monkeypatch.setenv(name, "developer-value") session = UserSession(tmp_path, Path("/installed/ug"), tmp_path / "artifacts", "test-token") @@ -27,6 +29,8 @@ def test_cuj_session_does_not_inherit_agent_credentials_or_configuration(tmp_pat assert "UCODE_MANAGED_CONFIG_STUB" not in session.env assert "UG_CUJ_SP_CLIENT_ID" not in session.env assert "UG_CUJ_SP_CLIENT_SECRET" not in session.env + assert "UG_BUDGET_CUJ_SP_CLIENT_ID" not in session.env + assert "UG_BUDGET_CUJ_SP_CLIENT_SECRET" not in session.env assert session.env["CODEX_HOME"] == str(session.home / ".codex") assert not list(session.home.iterdir()), "Fixtures must not manufacture ug/agent state" diff --git a/tests/test_cuj_workspace.py b/tests/test_cuj_workspace.py index 244b75571..fc08baa4f 100644 --- a/tests/test_cuj_workspace.py +++ b/tests/test_cuj_workspace.py @@ -1,11 +1,14 @@ """Offline checks for read-only CUJ workspace configuration and catalog access.""" import copy +from types import SimpleNamespace import pytest from databricks.sdk import WorkspaceClient from databricks.sdk.core import Config +from tests.e2e_cuj import base +from tests.e2e_cuj.base import BaseCujTest from tests.e2e_cuj.helpers.constants import CLAUDE, CODEX from tests.e2e_cuj.helpers.workspace import Workspace @@ -16,6 +19,47 @@ def client(monkeypatch): return WorkspaceClient(host="https://example.test", token="test-token", auth_type="pat") +def test_cuj_workspace_auth_uses_each_class_selected_credentials(monkeypatch): + calls = [] + + class CapturingClient: + def __init__(self, **kwargs): + calls.append(kwargs) + + class Shared(BaseCujTest): + WORKSPACE_URL = "https://shared.example" + + class Budget(BaseCujTest): + WORKSPACE_URL = "https://budget.example" + CLIENT_ID_ENV = "UG_BUDGET_CUJ_SP_CLIENT_ID" + CLIENT_SECRET_ENV = "UG_BUDGET_CUJ_SP_CLIENT_SECRET" + + monkeypatch.setattr(base, "WorkspaceClient", CapturingClient) + monkeypatch.setenv("UG_CUJ_SP_CLIENT_ID", "shared-id") + monkeypatch.setenv("UG_CUJ_SP_CLIENT_SECRET", "shared-secret") + monkeypatch.setenv("UG_BUDGET_CUJ_SP_CLIENT_ID", "budget-id") + monkeypatch.setenv("UG_BUDGET_CUJ_SP_CLIENT_SECRET", "budget-secret") + + setup_workspace = BaseCujTest.setup_workspace.__wrapped__ + setup_workspace(None, SimpleNamespace(cls=Shared)) + setup_workspace(None, SimpleNamespace(cls=Budget)) + + assert calls == [ + { + "host": "https://shared.example", + "client_id": "shared-id", + "client_secret": "shared-secret", + "auth_type": "oauth-m2m", + }, + { + "host": "https://budget.example", + "client_id": "budget-id", + "client_secret": "budget-secret", + "auth_type": "oauth-m2m", + }, + ] + + def test_cuj_catalog_rejects_unknown_agent_before_network_access(client, monkeypatch): workspace = Workspace(client) diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index ccc6b64a2..18f7d5959 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -74,22 +74,22 @@ def test_windows_integration_ci_uses_shared_claude_version(): assert contents.count('"--claude-version", $env:CLAUDE_VERSION,') == 2 -def test_dedicated_cuj_job_selects_credentials_without_serialization(): +def test_dedicated_cuj_job_runs_both_principals_in_one_collection(): workflow = (Path(__file__).parent.parent / ".github/workflows/integration.yml").read_text() cuj = workflow.split("\n dedicated-cuj:\n", 1)[1].split("\n cujs:\n", 1)[0] assert "concurrency:" not in cuj - assert "test_marker: not cuj5_below_tier" in cuj - assert "test_marker: cuj5_below_tier" in cuj - assert "client_id_secret: UG_CUJ_SP_CLIENT_ID" in cuj - assert "client_secret_secret: UG_CUJ_SP_CLIENT_SECRET" in cuj - assert "client_id_secret: UG_BUDGET_CUJ_SP_CLIENT_ID" in cuj - assert "client_secret_secret: UG_BUDGET_CUJ_SP_CLIENT_SECRET" in cuj - assert "UG_CUJ_SP_CLIENT_ID: ${{ secrets[matrix.client_id_secret] }}" in cuj - assert "UG_CUJ_SP_CLIENT_SECRET: ${{ secrets[matrix.client_secret_secret] }}" in cuj - assert 'tests/e2e_cuj -v -m "$TEST_MARKER"' in cuj - assert "TEST_MARKER: ${{ matrix.test_marker }}" in cuj - assert "name: ${{ matrix.artifact_name }}" in cuj + assert "matrix:" not in cuj + for key in ( + "UG_CUJ_SP_CLIENT_ID", + "UG_CUJ_SP_CLIENT_SECRET", + "UG_BUDGET_CUJ_SP_CLIENT_ID", + "UG_BUDGET_CUJ_SP_CLIENT_SECRET", + ): + assert f"{key}: ${{{{ secrets.{key} }}}}" in cuj + assert "tests/e2e_cuj -v" in cuj + assert "TEST_MARKER" not in cuj + assert "name: integration-e2e-cuj" in cuj @pytest.mark.parametrize( From 0b7f8a19c9842da7737dc441cdc581465d2a0146 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 18:17:21 +0000 Subject: [PATCH 16/19] Remove optional CUJ CI artifact collection --- .github/workflows/integration.yml | 11 +---------- tests/integration/README.md | 6 ++---- tests/test_integration_contract.py | 1 - 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1c4b71b11..ac50d32aa 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -458,16 +458,7 @@ jobs: exit 0 fi uv run --with pexpect==4.9.0 --with pyte==0.8.2 \ - pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v \ - --basetemp "$RUNNER_TEMP/ug-cuj" --junitxml "$RUNNER_TEMP/ug-cuj-results.xml" - - name: Upload CUJ evidence - if: ${{ !cancelled() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: integration-e2e-cuj - path: | - ${{ runner.temp }}/ug-cuj/**/artifacts/ - ${{ runner.temp }}/ug-cuj-results.xml + pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v cujs: name: All integration tests diff --git a/tests/integration/README.md b/tests/integration/README.md index a981dbc83..dc1436edb 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -556,8 +556,7 @@ The **All integration tests** check requires installation, workspace validation, both full lanes, both managed lanes, and `dedicated-cuj` to pass. Each tracing journey is included in its agent's Full lane. The managed and dedicated-cuj lanes do not use `continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate -check. The dedicated-cuj artifacts are named `integration-e2e-cuj` and -`integration-cuj5-below-tier`. +check. The advisory Windows installation and headless lanes are not yet included in that aggregate check. The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. @@ -703,8 +702,7 @@ gh run download RUN_ID -R databricks/unity-gateway \ ``` Use `integration-full-AGENT` for a full lane, `integration-smoke-AGENT` for -smoke, `integration-e2e-cuj` for the above-tier budget test, or `integration-cuj5-below-tier` -for the below-tier case. Use `integration-installation` for Linux package failures, or +smoke. Use `integration-installation` for Linux package failures, or `integration-installation-windows` for native Windows package failures, or `integration-headless-windows-claude` for the Windows gateway journey. Older runs used `integration-full-AGENT-GROUP`, `integration-cujs`, or numbered `integration-live-*` diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 18f7d5959..37fb9aa81 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -89,7 +89,6 @@ def test_dedicated_cuj_job_runs_both_principals_in_one_collection(): assert f"{key}: ${{{{ secrets.{key} }}}}" in cuj assert "tests/e2e_cuj -v" in cuj assert "TEST_MARKER" not in cuj - assert "name: integration-e2e-cuj" in cuj @pytest.mark.parametrize( From 03f257d0601c2e0e2027b7fe47ce655898c60908 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 18:28:12 +0000 Subject: [PATCH 17/19] Minimize CUJ5 supporting changes and documentation --- tests/README.md | 20 +------- tests/e2e_cuj/AGENTS.md | 17 +------ tests/e2e_cuj/pytest.ini | 7 --- tests/e2e_cuj/test_ug_budget_defaults.py | 4 -- tests/integration/README.md | 58 +++++++----------------- tests/integration/utils/harness.py | 7 +-- tests/test_integration_contract.py | 17 ------- 7 files changed, 22 insertions(+), 108 deletions(-) delete mode 100644 tests/e2e_cuj/pytest.ini diff --git a/tests/README.md b/tests/README.md index aa4bb56a3..f89bef523 100644 --- a/tests/README.md +++ b/tests/README.md @@ -190,19 +190,7 @@ executions. Thirteen additional managed-fixture cases cover focused model, MCP, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. -The dedicated-workspace CUJ suite adds two cases around the fixed 1% Luna recommendation tier -(`spending_percentage=0.01`): -default Claude/Sonnet selection with a low-spend principal, and usage plus Codex/Luna -recommendation display and native Codex startup with the above-tier principal. It runs outside the 62-case integration suite -in the shared `E2E CUJs` job. Both cases are collected together; separate test classes -select the shared and low-spend credentials without changing process-wide authentication. -Both entries read the preconfigured workspace and make no budget writes or inference -requests. It does not cover spend boundaries, selection between multiple tiers, or -explicit Claude overrides. Generated-file cleanup and strict app-server stdout -assertions remain enforced in the integration suite. -The published config verifies Codex's managed Sol default, and the native Codex TUI verifies -the above-tier Codex launch and its displayed Luna recommendation. Applying Luna over Sol is -outside this coverage. +Generated-file cleanup and strict app-server stdout assertions remain enforced. Unmanaged discovery Cases 7–14 configure, list models, or open the picker without submitting inference prompts; separate task journeys still perform inference. They require a real workspace with no CodingAgentConfig; a read-only prerequisite @@ -243,11 +231,7 @@ journeys for both agents, in two parallel jobs. After smoke finishes, the full suite runs all 62 live cases across two parallel agent jobs: one Claude VM and one Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs -for the same agent overlap within a run. The dedicated budget cases run together -with all dedicated-workspace CUJs in the `E2E CUJs` job with both agent binaries -installed. Each class selects the shared or low-spend credentials and gets its own -local home and artifacts. `ug revert` cleans up local agent configuration between -classes. Workspace and budget configuration remain read-only, with no cross-run lock. +for the same agent overlap within a run. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites share workspace capacity; overlapping their requests can still encounter rate limits. diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index 399cf17e6..ba5800ae4 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -12,18 +12,6 @@ runs may read the same CUJ workspace. Do not add reservations or remote lock fil Keep local homes, task files, and artifacts isolated per run. No stubbed configuration. Use `CLAUDE` / `CODEX` from `helpers/constants.py`; reject unsupported agents explicitly. -The budget-default CUJ runs both cases in the shared `E2E CUJs` job against the fixed -`spending_percentage=0.01` (1%) tier. The above-tier class uses the standard -`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` credentials; the below-tier class -selects `UG_BUDGET_CUJ_SP_CLIENT_ID` and `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. -Each class gets its own SDK client, local home, and artifacts. Both cases -read the workspace and budget recommendation; below-tier bare `ug` selects -Claude/Sonnet, while above-tier read-only `ug usage` agrees with backend spend, -threshold, and percentage, the launch panel displays a Codex/Luna recommendation, -and bare `ug` starts the native Codex TUI. Neither case writes workspace or budget -configuration or submits an inference task. The above-tier case does not assert that -Codex applies Luna over its managed Sol default. - Before adding any helper, search the existing test utilities and installed SDK for equivalent functionality. Reuse them directly; never build a parallel HTTP/auth client, session harness, terminal driver, or workspace CRUD layer. Workspace @@ -67,8 +55,7 @@ Use a clean disposable POSIX runner without existing machine-wide agent settings Install the intended versions of `ug`, `claude`, `codex`, and `databricks` on PATH. The terminal helpers also require `pexpect==4.9.0` and `pyte==0.8.2`: `uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v`. -Set both credential pairs to run CUJ5 together with the other CUJs. Collection with -`--collect-only` does not authenticate or contact a workspace. Only the short-lived bearer is -forwarded to agent processes, never the SP secret. +Collection with `--collect-only` does not authenticate or contact a workspace. +Only the short-lived bearer is forwarded to agent processes, never the SP secret. Artifacts are written beneath pytest's per-scenario temporary directory, printed during setup. diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini deleted file mode 100644 index 11f9e73be..000000000 --- a/tests/e2e_cuj/pytest.ini +++ /dev/null @@ -1,7 +0,0 @@ -[pytest] -testpaths = . -pythonpath = ../.. -addopts = --strict-markers --tb=short -markers = - cuj: dedicated-workspace customer journey - tui: real interactive terminal boot, keyboard input, exit and reopen diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index 5e46dd87d..1078d011d 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -4,15 +4,11 @@ import re from decimal import ROUND_HALF_UP, Decimal -import pytest - from tests.integration.utils.terminal import AgentTerminal from .base import BaseCujTest from .helpers.constants import CLAUDE, CODEX, CodingAgent -pytestmark = [pytest.mark.cuj, pytest.mark.tui] - class _BudgetDefaultsBase(BaseCujTest): WORKSPACE_URL = "https://dbc-497adeef-62c0.cloud.databricks.com" diff --git a/tests/integration/README.md b/tests/integration/README.md index dc1436edb..121ee7f0e 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -22,19 +22,14 @@ The `smart_defaults` wire schema, legacy `spend_tiers` cache reads, and recommen request gating are covered by unit/component tests listed in `../README.md`. This suite does not yet assert live `recommendModel` request counts for configs with and without tiers. -## Dedicated-workspace CUJs +## Dedicated-workspace CUJ5 -The budget usage and model recommendation/default tests live in -[`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py), -outside this existing-workspace integration suite. The tests read their dedicated -workspace's published budget and model identifiers. The fixed tier has -`spending_percentage=0.01` (1%). The shared `E2E CUJs` job runs all tests in one -pytest collection. The above-tier case uses `UG_CUJ_SP_CLIENT_ID` and -`UG_CUJ_SP_CLIENT_SECRET`; the below-tier class selects `UG_BUDGET_CUJ_SP_CLIENT_ID` -and `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. Each class gets its own authenticated SDK -client, local home, and artifacts. +[`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py) runs +both budget cases in the shared `E2E CUJs` job against the fixed 1% tier. The above-tier +case uses `UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET`; the below-tier case uses +`UG_BUDGET_CUJ_SP_CLIENT_ID` / `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. -Run the suite directly from a clean POSIX runner with both credential pairs set: +Run from a clean POSIX runner with both credential pairs set: ```bash UG_CUJ_SP_CLIENT_ID=... UG_CUJ_SP_CLIENT_SECRET=... \ @@ -43,20 +38,10 @@ uv run --with pexpect==4.9.0 --with pyte==0.8.2 \ pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v ``` -The workspace client reads the published config and real recommendation using -OAuth M2M. The above-tier case checks read-only `ug usage` spend, threshold, and -percentage against backend reads, allowing spend to increase while the command runs. -It then launches bare `ug`, verifies the launch panel displays the backend's -Codex/Luna recommendation, and confirms the native Codex TUI starts and exits -normally. The below-tier case launches bare `ug` and -verifies Claude selects Sonnet. Neither case submits an inference task or writes -the budget; the dedicated workspace configuration remains read-only. -The published config verifies Codex's managed Sol default, and the native Codex TUI verifies -the above-tier Codex launch and its displayed Luna recommendation. Applying Luna over Sol is -outside this coverage. - -A fresh below-tier principal may have no spend counter, so that case establishes -default model selection; numeric spend reporting remains covered by the above-tier case. +Both cases are read-only and submit no inference tasks. The above-tier case checks +`ug usage`, the Luna recommendation, and Codex startup; it does not assert that Codex +applies Luna. The below-tier case verifies Claude/Sonnet selection and checks the +spend percentage when the backend supplies budget figures. Shared subprocess command resolution is covered by `../test_subprocess_cross_os.py` and enforced by Ruff. These component checks do not establish native Windows coverage @@ -353,9 +338,7 @@ they only configure, list models, and open/close the picker. Other live CUJs per real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex. The separate dedicated-workspace suite adds two -budget recommendation/default cases run in the shared `E2E CUJs` job with separate credentials. -Selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent +checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** uses two real workspaces and checks skills MCP cleanup and a completed Claude task. @@ -524,14 +507,7 @@ Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, two agent jobs). Full runs all 62 live cases, including those smoke cases, in two -disjoint agent lanes. The `E2E CUJs` job runs both budget cases alongside all other -dedicated-workspace CUJs. Each class selects its credentials and gets its own home -and artifacts; `ug revert` cleans up local agent configuration between classes. -Workspace and budget configuration remain read-only, with no remote reservation or -cross-run lock. The parent workflows -retain their normal cancellation of superseded runs. - -The full-suite agent lanes are: +disjoint agent lanes: | Agent lane | Marker | Cases | | --- | --- | --- | @@ -553,10 +529,10 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, -both full lanes, both managed lanes, and `dedicated-cuj` to pass. Each tracing journey is -included in its agent's Full lane. The managed and dedicated-cuj lanes do not use -`continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate -check. +both full lanes, and both **Managed config** lanes to pass for full/live runs. Each tracing +journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: +a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke, TUI, +and installation subsets do not select managed tests and do not require them. The advisory Windows installation and headless lanes are not yet included in that aggregate check. The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. @@ -702,7 +678,7 @@ gh run download RUN_ID -R databricks/unity-gateway \ ``` Use `integration-full-AGENT` for a full lane, `integration-smoke-AGENT` for -smoke. Use `integration-installation` for Linux package failures, or +smoke, `integration-installation` for Linux package failures, or `integration-installation-windows` for native Windows package failures, or `integration-headless-windows-claude` for the Windows gateway journey. Older runs used `integration-full-AGENT-GROUP`, `integration-cujs`, or numbered `integration-live-*` diff --git a/tests/integration/utils/harness.py b/tests/integration/utils/harness.py index 7756f6827..c522a2304 100644 --- a/tests/integration/utils/harness.py +++ b/tests/integration/utils/harness.py @@ -108,12 +108,7 @@ def __init__(self, root: Path, project_root: Path, binary: Path, artifacts: Path def redact(self, text: str, *, strip_ansi: bool = True) -> str: # Also scrub the relayed launch's subscription OAuth token, not just the bearer. - for name in ( - "DATABRICKS_BEARER", - "DATABRICKS_SECOND_BEARER", - "CLAUDE_CODE_OAUTH_TOKEN", - "UG_CUJ_SP_CLIENT_SECRET", - ): + for name in ("DATABRICKS_BEARER", "DATABRICKS_SECOND_BEARER", "CLAUDE_CODE_OAUTH_TOKEN"): for token in (os.environ.get(name), self.env.get(name)): if token: text = text.replace(token, "") diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index 37fb9aa81..34c5f7159 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -74,23 +74,6 @@ def test_windows_integration_ci_uses_shared_claude_version(): assert contents.count('"--claude-version", $env:CLAUDE_VERSION,') == 2 -def test_dedicated_cuj_job_runs_both_principals_in_one_collection(): - workflow = (Path(__file__).parent.parent / ".github/workflows/integration.yml").read_text() - cuj = workflow.split("\n dedicated-cuj:\n", 1)[1].split("\n cujs:\n", 1)[0] - - assert "concurrency:" not in cuj - assert "matrix:" not in cuj - for key in ( - "UG_CUJ_SP_CLIENT_ID", - "UG_CUJ_SP_CLIENT_SECRET", - "UG_BUDGET_CUJ_SP_CLIENT_ID", - "UG_BUDGET_CUJ_SP_CLIENT_SECRET", - ): - assert f"{key}: ${{{{ secrets.{key} }}}}" in cuj - assert "tests/e2e_cuj -v" in cuj - assert "TEST_MARKER" not in cuj - - @pytest.mark.parametrize( "failed_job", ["installation", "workspace", "smoke", "full", "managed", "dedicated-cuj"] ) From 5144e3feaaab6825d9bd8b9ef69e84b34de8d3af Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 22:12:48 +0000 Subject: [PATCH 18/19] Test CUJ5 recommendation display for explicit Claude launches --- tests/README.md | 2 +- tests/e2e_cuj/test_ug_budget_defaults.py | 42 ++++++++++++++++++++++++ tests/integration/README.md | 10 ++++-- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/tests/README.md b/tests/README.md index f89bef523..07a847e81 100644 --- a/tests/README.md +++ b/tests/README.md @@ -164,7 +164,7 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`) | Low-spend principal selects the default Claude/Sonnet; above-tier principal receives and displays a Codex/Luna recommendation, starts native Codex, and read-only `ug usage` agrees with backend spend, threshold, and percentage. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | +| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`); explicitly launch `ug claude` above the tier | Bare launches check Claude/Sonnet below the tier and the Codex/Luna recommendation above it; `ug usage` agrees with backend spend, threshold, and percentage. Explicit `ug claude` also displays the backend's Codex/Luna recommendation. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index 1078d011d..9361f3c1f 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -187,3 +187,45 @@ def test_bare_ug_shows_luna_recommendation_and_starts_codex(self, cuj): f"for {self.LUNA_MODEL!r}:\n{transcript}" ) tui.exit_normally() + + +class TestCujBudgetDefaultsExplicitClaude(_BudgetDefaultsBase): + def test_ug_claude_displays_codex_luna_recommendation(self, cuj): + """Scenario: launch explicit ``ug claude`` for an above-tier principal. + + Expected: the live backend recommends Codex/Luna, and the Claude launch displays + that recommendation before exiting normally. No task is submitted; this checks + recommendation display, not which model Claude uses for inference. + """ + session, workspace, _ = cuj + session.configure( + [ + "configure", + "--workspace", + self.WORKSPACE_URL, + "--skip-upgrade", + "--disable-databricks-ai-tools", + ] + ) + recommendation = workspace.client.api_client.do( + "POST", path="/api/ai-gateway/v2/coding-agent-configs:recommendModel", body={} + ) + assert recommendation.get("recommended_agent") == CodingAgent.CODEX, recommendation + assert recommendation.get("recommended_model") == self.LUNA_MODEL, recommendation + + with AgentTerminal( + session, CLAUDE, [str(session.binary), CLAUDE], "explicit-claude" + ) as tui: + tui.boot(timeout=240) + transcript = session.redact( + "".join(tui.output) + "\n" + tui.visible, + strip_ansi=True, + ) + assert re.search( + rf"Recommended agent is Codex with model {re.escape(self.LUNA_MODEL)}\.", + transcript, + ), ( + "Claude launch did not display the backend recommendation " + f"for {self.LUNA_MODEL!r}:\n{transcript}" + ) + tui.exit_normally() diff --git a/tests/integration/README.md b/tests/integration/README.md index 121ee7f0e..ecef0b8be 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -25,8 +25,8 @@ does not yet assert live `recommendModel` request counts for configs with and wi ## Dedicated-workspace CUJ5 [`../e2e_cuj/test_ug_budget_defaults.py`](../e2e_cuj/test_ug_budget_defaults.py) runs -both budget cases in the shared `E2E CUJs` job against the fixed 1% tier. The above-tier -case uses `UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET`; the below-tier case uses +three budget cases in the shared `E2E CUJs` job against the fixed 1% tier. The above-tier +cases use `UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET`; the below-tier case uses `UG_BUDGET_CUJ_SP_CLIENT_ID` / `UG_BUDGET_CUJ_SP_CLIENT_SECRET`. Run from a clean POSIX runner with both credential pairs set: @@ -38,11 +38,15 @@ uv run --with pexpect==4.9.0 --with pyte==0.8.2 \ pytest --confcutdir=tests/e2e_cuj tests/e2e_cuj -v ``` -Both cases are read-only and submit no inference tasks. The above-tier case checks +All three cases submit no inference tasks. The above-tier bare-launch case checks `ug usage`, the Luna recommendation, and Codex startup; it does not assert that Codex applies Luna. The below-tier case verifies Claude/Sonnet selection and checks the spend percentage when the backend supplies budget figures. +The explicit `ug claude` case verifies that startup displays the backend's Codex/Luna +recommendation, then exits. It does not verify Claude's inference model. All three +cases leave workspace and budget configuration unchanged. + Shared subprocess command resolution is covered by `../test_subprocess_cross_os.py` and enforced by Ruff. These component checks do not establish native Windows coverage for every agent; the Windows journeys below validate their explicitly selected paths. From eb1cdacfbe7203003d478d0fcae84eb2d1ee833f Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Tue, 6 Oct 2026 23:04:21 +0000 Subject: [PATCH 19/19] Verify explicit Claude startup keeps Sonnet under Codex recommendation --- tests/README.md | 2 +- tests/e2e_cuj/test_ug_budget_defaults.py | 13 ++++++++++--- tests/integration/README.md | 5 +++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/README.md b/tests/README.md index 170ba49bf..dcfc7497f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -181,7 +181,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_configure_claude_cleans_stale_skills_mcp_on_workspace_switch` | Configure the first workspace, register its skills MCP, switch to a second real workspace, and use Claude | Old registration removed from Claude and the new workspace state; old workspace bucket preserved; repeat configure stays clean; real file task completes on the second workspace | | `test_ug_configure_claude_rejects_invalid_credentials`, `test_ug_configure_codex_rejects_invalid_credentials` | Configure with a rejected bearer against the real workspace | Authentication failure; no successful saved setup | | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | -| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`); explicitly launch `ug claude` above the tier | Bare launches check Claude/Sonnet below the tier and the Codex/Luna recommendation above it; `ug usage` agrees with backend spend, threshold, and percentage. Explicit `ug claude` also displays the backend's Codex/Luna recommendation. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | +| `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`); explicitly launch `ug claude` above the tier | Bare launches check Claude/Sonnet below the tier and the Codex/Luna recommendation above it; `ug usage` agrees with backend spend, threshold, and percentage. Explicit `ug claude` displays the backend's Codex/Luna recommendation while its generated model setting and native header select Sonnet. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | diff --git a/tests/e2e_cuj/test_ug_budget_defaults.py b/tests/e2e_cuj/test_ug_budget_defaults.py index 9361f3c1f..54b6a63fe 100644 --- a/tests/e2e_cuj/test_ug_budget_defaults.py +++ b/tests/e2e_cuj/test_ug_budget_defaults.py @@ -190,12 +190,12 @@ def test_bare_ug_shows_luna_recommendation_and_starts_codex(self, cuj): class TestCujBudgetDefaultsExplicitClaude(_BudgetDefaultsBase): - def test_ug_claude_displays_codex_luna_recommendation(self, cuj): + def test_ug_claude_starts_on_sonnet_despite_codex_luna_recommendation(self, cuj): """Scenario: launch explicit ``ug claude`` for an above-tier principal. Expected: the live backend recommends Codex/Luna, and the Claude launch displays - that recommendation before exiting normally. No task is submitted; this checks - recommendation display, not which model Claude uses for inference. + that recommendation while its generated model setting and native header select Sonnet. + No task is submitted; this checks startup selection, not the model used for inference. """ session, workspace, _ = cuj session.configure( @@ -228,4 +228,11 @@ def test_ug_claude_displays_codex_luna_recommendation(self, cuj): "Claude launch did not display the backend recommendation " f"for {self.LUNA_MODEL!r}:\n{transcript}" ) + claude_settings = session.home / ".claude" / "ucode-settings.json" + settings = json.loads(claude_settings.read_text()) + assert settings["env"]["ANTHROPIC_MODEL"] == self.SONNET_MODEL, settings + tui.wait_for( + lambda text: re.search(r"Claude Sonnet 4\.6\s*·", text), + "Claude's native Sonnet model header", + ) tui.exit_normally() diff --git a/tests/integration/README.md b/tests/integration/README.md index cca04d5c2..c6497d503 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -54,8 +54,9 @@ applies Luna. The below-tier case verifies Claude/Sonnet selection and checks th spend percentage when the backend supplies budget figures. The explicit `ug claude` case verifies that startup displays the backend's Codex/Luna -recommendation, then exits. It does not verify Claude's inference model. All three -cases leave workspace and budget configuration unchanged. +recommendation while its generated `ANTHROPIC_MODEL` setting and native header select +Sonnet, then exits. It does not verify Claude's inference model. All three cases leave +workspace and budget configuration unchanged. Shared subprocess command resolution is covered by `../test_subprocess_cross_os.py` and enforced by Ruff. These component checks do not establish native Windows coverage