From d0be289f7b597d6a87069be0e79c62b91bd75513 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Thu, 1 Oct 2026 14:43:34 +0000 Subject: [PATCH 01/18] Add fresh-state agent launch integration journeys for AIGTWY-4882 --- scripts/run_integration.py | 11 + tests/README.md | 4 + tests/integration/README.md | 27 +- tests/integration/conftest.py | 11 + tests/integration/test_ug_claude_headless.py | 104 ++++++++ tests/integration/test_ug_codex_headless.py | 103 ++++++++ tests/integration/utils/mps_fixture.py | 253 +++++++++++++++++++ 7 files changed, 510 insertions(+), 3 deletions(-) create mode 100644 tests/integration/utils/mps_fixture.py diff --git a/scripts/run_integration.py b/scripts/run_integration.py index 8300e237c..bb67df5cc 100644 --- a/scripts/run_integration.py +++ b/scripts/run_integration.py @@ -372,6 +372,11 @@ def arguments( parser.add_argument("--npm-registry", default="https://registry.npmjs.org") parser.add_argument("--profile", help="Explicit Databricks profile to mint the live bearer.") parser.add_argument("--workspace", default=environment.get("UCODE_TEST_WORKSPACE")) + parser.add_argument( + "--mps-fixture-schema", + default=environment.get("UG_INTEGRATION_MPS_FIXTURE_SCHEMA"), + help="Catalog.schema for reusable Anthropic/OpenAI MPS fixtures in fresh-provider tests.", + ) parser.add_argument( "--second-workspace", default=environment.get("UCODE_TEST_SECOND_WORKSPACE"), @@ -421,6 +426,10 @@ def arguments( for dependency in args.dependency: if not re.fullmatch(r"[A-Za-z0-9_.-]+==[A-Za-z0-9_.!+-]+", dependency): parser.error("--dependency requires an exact PACKAGE==VERSION constraint.") + if args.mps_fixture_schema and not re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*", args.mps_fixture_schema + ): + parser.error("--mps-fixture-schema requires catalog.schema.") if not args.installation_only: if not args.workspace or not args.workspace.startswith("https://"): parser.error( @@ -589,6 +598,7 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "claude_bedrock_allow_all_model": args.claude_bedrock_allow_all_model, "codex_provider": args.codex_provider, "codex_provider_model": args.codex_provider_model, + "mps_fixture_schema": args.mps_fixture_schema, "parent_schema": args.parent_schema, "claude_parent_model": args.claude_parent_model, "codex_parent_model": args.codex_parent_model, @@ -871,6 +881,7 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "UG_INTEGRATION_CLAUDE_BEDROCK_ALLOW_ALL_MODEL": args.claude_bedrock_allow_all_model, "UG_INTEGRATION_CODEX_PROVIDER": args.codex_provider, "UG_INTEGRATION_CODEX_PROVIDER_MODEL": args.codex_provider_model, + "UG_INTEGRATION_MPS_FIXTURE_SCHEMA": args.mps_fixture_schema or "", "UG_INTEGRATION_PARENT_SCHEMA": args.parent_schema, "UG_INTEGRATION_CLAUDE_PARENT_MODEL": args.claude_parent_model, "UG_INTEGRATION_CODEX_PARENT_MODEL": args.codex_parent_model, diff --git a/tests/README.md b/tests/README.md index d479c8110..ab29f412d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -243,6 +243,10 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_opencode_headless_prompt_argument` | Run OpenCode from a script (`run --format json --auto`) with an argument prompt | Completed Read tool call; final text answer contains the file value; exit zero (non-blocking CI lane) | | `test_ug_agents_self_managed_opencode_journey` | Under the injected `managed_workspace_default` config (enables Claude/Codex, not OpenCode): bare configure, `ug agents list`, refused OpenCode launch, `ug agents add opencode`, real headless task, `ug agents remove opencode` (marker `managed_fixture and opencode`, non-blocking CI lane) | OpenCode absent from the list and launch fails with "doesn't enable OpenCode" / `ug agents add opencode`; after add it is listed self-managed and the headless Read task returns the fixture value; after remove it is hidden and refused again | | `test_ug_agents_admin_managed_guardrails` | `ug agents add` / `remove` on an agent the admin config enables | Add is a no-op noting the admin manages it; remove is rejected with nonzero exit | +| `test_ug_claude_headless_fresh_workspace`, `test_ug_codex_headless_fresh_workspace` | From fresh state, launch an agent with `--workspace` against a workspace with no managed config | Claude Haiku 4.5 and Codex GPT-5.4 Nano read an unpredictable file value through the gateway and return it in a structured completed answer; exit zero; no routing | +| Fresh Claude/Codex `--provider` journeys | From fresh state, launch each real agent CLI with `--workspace` and an explicit provider, optionally obtained from a reusable dummy MPS fixture | Claude's provider header and all-targets catalog, plus Codex's app-server model catalog, match the selected services; both CLIs exit successfully without routing or inference | +| `test_ug_claude_headless_fresh_model_location` | From fresh state, launch Claude with `--workspace` and `--model-location system.ai` | Haiku 4.5 reads an unpredictable file value and returns it in a structured completed answer; exit zero; no routing | +| `test_ug_codex_headless_fresh_model_location` | From fresh state, launch Codex with `--workspace` and `--model-location` | Codex uses a model in the parent schema to read an unpredictable file value and returns it in a structured completed answer; exit zero; no routing | | `test_ug_claude_exports_trace_to_configured_table`, `test_ug_codex_exports_trace_to_configured_table` | Configure tracing, complete a headless task carrying a unique trace marker, then wait for ingestion | The configured trace table contains an agent span with the same trace-safe marker and requested model | | `test_ug_claude_headless_explicit_model_bypasses_routing` | Pass `--model VALUE` / `--model=VALUE` before and after ug's separator with smart routing enabled, without workspace policy | Real file task completes; JSON `modelUsage` reports the requested model with output tokens; no routing wrapper | | `test_ug_codex_headless_explicit_model_bypasses_routing` | Pass `--model VALUE` / `--model=VALUE` / `-m VALUE` with smart routing enabled | Real file task completes; no routing wrapper | diff --git a/tests/integration/README.md b/tests/integration/README.md index cd4809d9c..1dc07b08b 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -385,9 +385,27 @@ MPS CUJs select the existing services already used by e2e: Use `--claude-provider` / `--claude-relayed-provider` / `--codex-provider` to reproduce another existing service. Use `--claude-provider-model` / `--codex-provider-model` when it allows a different model. Those choices are recorded in `versions.json`. -No service is created or modified. A missing service, permission, or OAuth token +By default, no service is created or modified. A missing service, permission, or OAuth token fails the selected CUJ, rather than skipping it. +The two fresh `--provider` setup journeys can get or create named MPS fixtures as part of test +setup. Pass `--mps-fixture-schema ug_e2e.provider` in the integration workspace. The fixtures +reuse existing Anthropic and OpenAI services after validating their provider types, +all-targets setting, and declared model metadata, or create them with literal dummy provider +keys if absent. The Anthropic fixture allows all provider models; the OpenAI fixture allows only +its declared target. The schema must already exist; the MPS resources remain for later runs. +These journeys check ug's provider selection and real agent CLI launch; they do not send +inference requests or claim that the dummy keys can serve model traffic. This option does not +change the other MPS journeys. + +```bash +python3.12 scripts/run_integration.py \ + --ug-version checkout --claude-version 2.1.268 --codex-version 0.154.0 \ + --workspace https://dbc-14e376e8-6541.cloud.databricks.com --profile YOUR_PROFILE \ + --mps-fixture-schema ug_e2e.provider \ + -- -k fresh_provider +``` + The tracing journeys are part of their respective Full agent lanes and use the existing e2e workspace and bearer. Because that workspace deliberately has no published managed configuration, each journey injects only a tracing-enabled CodingAgentConfig input (the @@ -441,8 +459,11 @@ provider or parent replacement picker can render the raw gateway ID/display name the assertion accepts either numbered-row form. Custom Model Services must still appear by their gateway IDs or display names in a numbered picker row; startup banners and footer text cannot satisfy discovery assertions. Cases 7–14 send no inference prompts; -they only configure, list models, and open/close the picker. Other live CUJs perform -real model tasks. +they only configure, list models, and open/close the picker. Separate fresh-state headless +CUJs complete real file tasks through `--workspace` and `--model-location` when no managed +config is present. The fresh workspace tasks pin Claude Haiku 4.5 and Codex GPT-5.4 Nano; the +fresh Claude model-location task also pins Haiku 4.5 to keep inference inexpensive. The two +fresh `--provider` journeys check setup and launch with dummy MPS credentials, without inference. There are **80 live cases** (including 14 marked TUI journeys) and **7 installation checks** with Claude and Codex; selecting OpenCode adds one live headless case. One diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7354ffdd2..f79b4ca52 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -17,6 +17,7 @@ import pytest from utils.harness import UserSession from utils.managed import MANAGED_CONFIGS_PATH, assert_no_managed_config +from utils.mps_fixture import MpsFixture def pytest_collection_modifyitems(config, items): @@ -52,6 +53,16 @@ def workspace(): return value +@pytest.fixture +def mps_fixture(workspace): + """Provision optional direct MPSes for a single fresh-provider journey.""" + yield MpsFixture( + workspace, + os.environ.get("UG_INTEGRATION_MPS_FIXTURE_SCHEMA", ""), + os.environ.get("DATABRICKS_BEARER", "").strip(), + ) + + @pytest.fixture(scope="session") def unmanaged_workspace(workspace): """Require a real no-config workspace, without injecting or changing its policy.""" diff --git a/tests/integration/test_ug_claude_headless.py b/tests/integration/test_ug_claude_headless.py index f94a4a991..d0bacf75b 100644 --- a/tests/integration/test_ug_claude_headless.py +++ b/tests/integration/test_ug_claude_headless.py @@ -1,9 +1,12 @@ """CUJs for using claude from scripts through installed ug.""" import json +import os import pytest +from utils.constants import CLAUDE_TEST_MODEL from utils.evidence import FileTask +from utils.provider_catalog import fetch_anthropic_provider_catalog pytestmark = [pytest.mark.live, pytest.mark.claude] @@ -114,6 +117,107 @@ def test_ug_claude_headless_prompt_after_separator(live_session, workspace): session.assert_not_routed() +@pytest.mark.usefixtures("unmanaged_workspace") +def test_ug_claude_headless_fresh_workspace(live_session, workspace): + """Scenario: launch Claude headlessly against an unmanaged workspace from fresh state. + + Expected: ``ug claude --workspace`` starts the real installed Claude CLI without a + configure step, and its structured result contains the unpredictable fixture value after + using the Read tool with the inexpensive Haiku model; the command exits without routing. + """ + session = live_session + task = FileTask(session) + + result = session.run( + "claude", + "--workspace", + workspace, + "--", + "--model", + CLAUDE_TEST_MODEL, + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + timeout=180, + ) + task.assert_headless_answer("claude", result) + session.assert_not_routed() + + +@pytest.mark.usefixtures("unmanaged_workspace") +def test_ug_claude_headless_fresh_model_location(live_session, workspace): + """Scenario: launch Claude from fresh state with the system.ai model location. + + Expected: the real Claude CLI uses the inexpensive Haiku model to read an unpredictable + file value through the selected location and returns it in a completed structured answer; + the command exits without routing. + """ + session = live_session + task = FileTask(session) + + result = session.run( + "claude", + "--workspace", + workspace, + "--model-location", + "system.ai", + "--", + "--model", + CLAUDE_TEST_MODEL, + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + timeout=180, + ) + task.assert_headless_answer("claude", result) + session.assert_not_routed() + + +@pytest.mark.usefixtures("unmanaged_workspace") +def test_ug_claude_fresh_provider_launch( + live_session, workspace, claude_provider, claude_provider_model, mps_fixture +): + """Scenario: launch Claude from fresh state with an explicit provider service. + + Expected: ``ug claude --workspace`` with ``--provider`` starts the real installed Claude + CLI without a configure step, reports the selected provider, and writes the provider header + to its generated settings. The dummy all-targets MPS exposes more than its declared model; + ``--version`` exits successfully without inference or routing. + """ + session = live_session + provider = mps_fixture.provider_for( + "anthropic", fallback_provider=claude_provider, model=claude_provider_model + ) + + result = session.run( + "claude", + "--workspace", + workspace, + "--provider", + provider.provider, + "--", + "--version", + timeout=180, + ) + assert provider.provider in result.stdout + settings = json.loads((session.home / ".claude/ucode-settings.json").read_text()) + headers = (settings.get("env") or {}).get("ANTHROPIC_CUSTOM_HEADERS", "").splitlines() + assert f"Databricks-Model-Provider-Service: {provider.provider}" in headers + if provider.allow_all_targets: + catalog = fetch_anthropic_provider_catalog( + workspace, os.environ["DATABRICKS_BEARER"], provider.provider + ) + assert provider.model in catalog.model_ids + assert any(model != provider.model for model in catalog.model_ids) + session.assert_not_routed() + + @pytest.mark.parametrize("model_form", ["separate", "equals"]) @pytest.mark.parametrize("model_owner", ["ug", "claude"]) def test_ug_claude_headless_explicit_model_bypasses_routing( diff --git a/tests/integration/test_ug_codex_headless.py b/tests/integration/test_ug_codex_headless.py index 289c138eb..cbab21f5e 100644 --- a/tests/integration/test_ug_codex_headless.py +++ b/tests/integration/test_ug_codex_headless.py @@ -158,3 +158,106 @@ def test_ug_codex_headless_explicit_model_bypasses_routing(live_session, workspa ) task.assert_headless_answer("codex", result) session.assert_not_routed() + + +@pytest.mark.usefixtures("unmanaged_workspace") +def test_ug_codex_headless_fresh_workspace(live_session, workspace): + """Scenario: launch Codex headlessly against an unmanaged workspace from fresh state. + + Expected: ``ug codex --workspace`` starts the real installed Codex CLI without a + configure step, and its structured completed answer contains the unpredictable fixture + value after using the workspace model; the command exits successfully without routing. + """ + session = live_session + task = FileTask(session) + + result = session.run( + "codex", + "--workspace", + workspace, + "--", + "exec", + "--skip-git-repo-check", + "--json", + "--model", + CODEX_TEST_MODEL, + task.prompt, + timeout=180, + ) + task.assert_headless_answer("codex", result) + session.assert_not_routed() + + +@pytest.mark.usefixtures("unmanaged_workspace") +def test_ug_codex_fresh_provider_launch( + live_session, workspace, codex_provider, codex_provider_model, mps_fixture +): + """Scenario: launch Codex from fresh state with an explicit provider service. + + Expected: ``ug codex --workspace`` with ``--provider`` starts the real installed Codex + CLI without a configure step, reports the selected provider, and exposes exactly its model + through the real app-server; ``--version`` exits without inference or routing. + """ + session = live_session + provider = mps_fixture.provider_for( + "openai", fallback_provider=codex_provider, model=codex_provider_model + ) + + result = session.run( + "codex", + "--workspace", + workspace, + "--provider", + provider.provider, + "--", + "--version", + timeout=180, + ) + assert provider.provider in result.stdout + models = session.codex_model_ids( + [ + "--workspace", + workspace, + "--provider", + provider.provider, + "--", + "app-server", + "--listen", + "stdio://", + ] + ) + assert models == [provider.model] + session.assert_not_routed() + + +@pytest.mark.usefixtures("unmanaged_workspace") +def test_ug_codex_headless_fresh_model_location( + live_session, workspace, parent_schema, codex_parent_model +): + """Scenario: launch Codex headlessly from fresh state with a model location. + + Expected: ``ug codex --workspace`` with ``--model-location`` starts the real installed + Codex CLI without a configure step, and its structured completed answer contains the + unpredictable fixture value after using the parent-schema model; the command exits + successfully without routing. + """ + session = live_session + task = FileTask(session) + + result = session.run( + "codex", + "--workspace", + workspace, + "--model-location", + parent_schema, + "--", + "exec", + "--skip-git-repo-check", + "--json", + "--model", + codex_parent_model, + task.prompt, + timeout=180, + ) + task.assert_headless_answer("codex", result) + session.assert_not_routed() diff --git a/tests/integration/utils/mps_fixture.py b/tests/integration/utils/mps_fixture.py new file mode 100644 index 000000000..af577ece4 --- /dev/null +++ b/tests/integration/utils/mps_fixture.py @@ -0,0 +1,253 @@ +"""Stable Unity Catalog Model Provider Service fixtures for live CUJs.""" + +from __future__ import annotations + +import json +import re +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Literal + +_MODEL_PROVIDER_SERVICES_PATH = "/api/2.1/unity-catalog/model-provider-services" +_UNITY_CATALOG_PATH = "/api/2.1/unity-catalog" +_REQUEST_TIMEOUT_SECONDS = 30 +_SCHEMA_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*\Z") +_DUMMY_API_KEYS = { + "anthropic": "dummy-anthropic-api-key", + "openai": "dummy-openai-api-key", +} +_STABLE_LEAVES = { + "anthropic": "ug_4882_claude_anthropic_all", + "openai": "ug_4882_codex_openai", +} + +ProviderKind = Literal["anthropic", "openai"] + + +class MpsFixtureError(AssertionError): + """A bounded MPS fixture request or validation failed.""" + + def __init__(self, message: str, *, status: int | None = None): + super().__init__(message) + self.status = status + + +@dataclass(frozen=True) +class MpsProvider: + """The safe provider inputs a test passes to ug.""" + + provider: str + model: str + allow_all_targets: bool = False + + +class MpsFixture: + """Get or create stable direct Anthropic/OpenAI MPS fixtures. + + An empty schema deliberately leaves the existing runner-selected provider untouched. + Configured resources use stable names and remain in the test schema so repeated runs can + reuse them without requiring provider keys. Existing resources are validated before reuse; + this helper never updates or deletes a resource owned by the integration schema. + """ + + def __init__(self, workspace: str, schema: str | None, bearer: str): + self._workspace = workspace.rstrip("/") + self._schema = schema.strip() if schema else "" + self._bearer = bearer + if self._schema and not _SCHEMA_PATTERN.fullmatch(self._schema): + raise MpsFixtureError( + "UG_INTEGRATION_MPS_FIXTURE_SCHEMA must be a catalog.schema identifier." + ) + if self._schema and not self._bearer: + raise MpsFixtureError("MPS fixture provisioning requires DATABRICKS_BEARER.") + + def provider_for( + self, + kind: ProviderKind, + *, + fallback_provider: str, + model: str, + ) -> MpsProvider: + """Return the configured provider, creating it only when the stable MPS is absent.""" + if not self._schema: + return MpsProvider(provider=fallback_provider, model=model) + if kind not in _STABLE_LEAVES: + raise MpsFixtureError(f"Unsupported MPS fixture provider kind: {kind!r}") + if not model.strip(): + raise MpsFixtureError( + f"UG_INTEGRATION_{kind.upper()}_PROVIDER_MODEL is required for MPS provisioning." + ) + + catalog, schema = self._schema.split(".", maxsplit=1) + leaf = _STABLE_LEAVES[kind] + fqn = f"{catalog}.{schema}.{leaf}" + resource = f"model-provider-services/{fqn}" + try: + existing = self._request("GET", self._resource_url(resource)) + except MpsFixtureError as error: + if error.status != 404: + raise + api_key = _DUMMY_API_KEYS[kind] + try: + self._request( + "POST", + self._create_url(catalog, schema, leaf), + body=self._body(kind, model, api_key), + redactions=(api_key,), + ) + except MpsFixtureError as create_error: + if create_error.status != 409: + raise + # Another runner may have created the stable fixture between our GET and POST. + # Read the authoritative resource and validate it instead of updating it. + existing = self._request("GET", self._resource_url(resource)) + self._validate_resource(existing, resource, kind, model) + else: + self._validate_resource(existing, resource, kind, model) + return MpsProvider(provider=fqn, model=model, allow_all_targets=kind == "anthropic") + + def _create_url(self, catalog: str, schema: str, leaf: str) -> str: + query = urllib.parse.urlencode( + { + "parent": f"schemas/{catalog}.{schema}", + "model_provider_service_id": leaf, + } + ) + return f"{self._workspace}{_MODEL_PROVIDER_SERVICES_PATH}?{query}" + + def _resource_url(self, resource: str) -> str: + escaped = urllib.parse.quote(resource, safe="/.") + return f"{self._workspace}{_UNITY_CATALOG_PATH}/{escaped}" + + @staticmethod + def _body(kind: ProviderKind, model: str, api_key: str) -> dict: + if kind == "anthropic": + provider_type = "EXTERNAL_MODEL_PROVIDER_TYPE_ANTHROPIC" + provider = {"anthropic": {"direct": {"apiKey": {"plaintext": api_key}}}} + native_api_type = "anthropic/v1/messages" + else: + provider_type = "EXTERNAL_MODEL_PROVIDER_TYPE_OPENAI" + provider = {"openai": {"direct": {"apiKey": {"plaintext": api_key}}}} + native_api_type = "openai/v1/responses" + return { + "config": { + "providerType": provider_type, + **provider, + "allowAllTargets": kind == "anthropic", + "targets": [{"model": model, "nativeApiTypes": [native_api_type]}], + }, + "comment": "Unity Gateway AIGTWY-4882 integration MPS", + } + + @staticmethod + def _value(mapping: dict, snake: str, camel: str, default=None): + if snake in mapping: + return mapping[snake] + return mapping.get(camel, default) + + def _validate_resource( + self, payload: dict, resource: str, kind: ProviderKind, model: str + ) -> None: + if not isinstance(payload, dict): + raise MpsFixtureError(f"MPS response for {resource} was not an object.") + if payload.get("name") != resource: + raise MpsFixtureError( + f"MPS response returned an unexpected resource name: {payload.get('name')!r}" + ) + config = payload.get("config") + if not isinstance(config, dict): + raise MpsFixtureError(f"MPS {resource} has no provider config.") + + expected_type = ( + "EXTERNAL_MODEL_PROVIDER_TYPE_ANTHROPIC" + if kind == "anthropic" + else "EXTERNAL_MODEL_PROVIDER_TYPE_OPENAI" + ) + provider_type = self._value(config, "provider_type", "providerType") + if provider_type != expected_type: + raise MpsFixtureError( + f"MPS {resource} has provider type {provider_type!r}; expected {expected_type!r}." + ) + provider_config = config.get(kind) + if not isinstance(provider_config, dict) or not isinstance( + provider_config.get("direct"), dict + ): + raise MpsFixtureError(f"MPS {resource} is not a direct {kind} provider.") + allow_all_targets = self._value(config, "allow_all_targets", "allowAllTargets", False) + if allow_all_targets is not (kind == "anthropic"): + raise MpsFixtureError( + f"MPS {resource} has allow_all_targets={allow_all_targets!r}; " + f"expected {kind == 'anthropic'}." + ) + + targets = config.get("targets") + if not isinstance(targets, list): + raise MpsFixtureError(f"MPS {resource} has no target catalog.") + native_api_type = "anthropic/v1/messages" if kind == "anthropic" else "openai/v1/responses" + matching = [] + for target in targets: + if not isinstance(target, dict): + continue + if self._value(target, "model", "model") != model: + continue + native_api_types = self._value(target, "native_api_types", "nativeApiTypes", []) + if isinstance(native_api_types, list): + matching.extend(native_api_types) + if native_api_type not in matching: + raise MpsFixtureError( + f"MPS {resource} does not expose model {model!r} with native API " + f"{native_api_type!r}." + ) + + def _request( + self, + method: Literal["GET", "POST"], + url: str, + *, + body: dict | None = None, + redactions: tuple[str, ...] = (), + ) -> dict: + data = json.dumps(body).encode("utf-8") if body is not None else None + headers = { + "Authorization": f"Bearer {self._bearer}", + "Accept": "application/json", + } + if body is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=_REQUEST_TIMEOUT_SECONDS) as response: + status = getattr(response, "status", None) or response.getcode() + raw_body = response.read(16 * 1024) + except urllib.error.HTTPError as error: + raw_body = error.read(16 * 1024) + detail = _redacted_preview(raw_body, redactions) + raise MpsFixtureError( + f"{method} {url} returned HTTP {error.code}: {detail}", status=error.code + ) from error + except (urllib.error.URLError, TimeoutError, OSError) as error: + raise MpsFixtureError(f"{method} {url} failed: {error}") from error + + if status < 200 or status >= 300: + detail = _redacted_preview(raw_body, redactions) + raise MpsFixtureError(f"{method} {url} returned HTTP {status}: {detail}", status=status) + if not raw_body: + return {} + try: + payload = json.loads(raw_body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise MpsFixtureError(f"{method} {url} returned invalid JSON") from error + if not isinstance(payload, dict): + raise MpsFixtureError(f"{method} {url} returned a non-object JSON response") + return payload + + +def _redacted_preview(raw_body: bytes, redactions: tuple[str, ...]) -> str: + """Return bounded diagnostics with provider credentials removed.""" + preview = raw_body.decode("utf-8", errors="replace") + for secret in redactions: + if secret: + preview = preview.replace(secret, "") + return preview[:1000] From 297094839450eae31ee60459a12c396525f09983 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Tue, 6 Oct 2026 14:17:28 +0000 Subject: [PATCH 02/18] Migrate CUJ7 discovery to dedicated-workspace scaffolding --- .github/workflows/integration.yml | 1 + tests/README.md | 2 + tests/e2e_cuj/pytest.ini | 2 + tests/e2e_cuj/test_ug_cuj7_model_discovery.py | 116 ++++++++++++++++++ tests/integration/README.md | 8 ++ .../test_ug_claude_model_discovery.py | 55 +-------- .../test_ug_codex_model_discovery.py | 41 +------ tests/integration/utils/model_discovery.py | 29 +++++ tests/test_integration_contract.py | 16 ++- 9 files changed, 176 insertions(+), 94 deletions(-) create mode 100644 tests/e2e_cuj/test_ug_cuj7_model_discovery.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9c3fbff68..8c388142c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -563,6 +563,7 @@ jobs: UG_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_CUJ_SP_CLIENT_SECRET }} UG_BUDGET_CUJ_SP_CLIENT_ID: ${{ secrets.UG_BUDGET_CUJ_SP_CLIENT_ID }} UG_BUDGET_CUJ_SP_CLIENT_SECRET: ${{ secrets.UG_BUDGET_CUJ_SP_CLIENT_SECRET }} + UG_CUJ7_WORKSPACE: ${{ secrets.UG_CUJ7_WORKSPACE }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 diff --git a/tests/README.md b/tests/README.md index ab29f412d..cbf764763 100644 --- a/tests/README.md +++ b/tests/README.md @@ -346,6 +346,8 @@ suite runs all 80 live cases across two parallel agent jobs: one Claude VM and o Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs for the same agent overlap within a run. +CUJ7's two configured discovery cases run in the required `dedicated-cuj` job, +using the shared CUJ scaffolding and `UG_CUJ7_WORKSPACE` repository secret. CI starts integration alongside unit tests and the existing e2e shards. Integration does not wait for agent e2e or get skipped when an agent shard fails. These suites share workspace capacity; overlapping their requests can still encounter rate limits. diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini index ae59c16cd..0af591819 100644 --- a/tests/e2e_cuj/pytest.ini +++ b/tests/e2e_cuj/pytest.ini @@ -8,5 +8,7 @@ markers = mcp_registration: managed MCP registration, connection, and tool use workspace_isolated: reads preconfigured CUJ workspace state with isolated local sessions tui: real interactive terminal and completed task + live: requires the CUJ's real workspace + cuj7: unmanaged configured model discovery claude: only runs when Claude Code is explicitly selected codex: only runs when Codex is explicitly selected diff --git a/tests/e2e_cuj/test_ug_cuj7_model_discovery.py b/tests/e2e_cuj/test_ug_cuj7_model_discovery.py new file mode 100644 index 000000000..27f5a5271 --- /dev/null +++ b/tests/e2e_cuj/test_ug_cuj7_model_discovery.py @@ -0,0 +1,116 @@ +"""CUJ7: configured model discovery in a dedicated unmanaged workspace.""" + +import os +import shutil +from pathlib import Path + +import pytest +from databricks.sdk.errors import DatabricksError, NotFound + +from tests.integration.utils.managed import MANAGED_CONFIGS_PATH, assert_no_managed_config +from tests.integration.utils.model_discovery import ( + assert_claude_system_models_in_picker, + assert_codex_default_models, +) + +from .base import BaseCujTest +from .helpers.constants import CLAUDE, CODEX, MANAGED_PATHS +from .helpers.session import UserSession +from .helpers.terminal import Terminal + +pytestmark = [pytest.mark.live, pytest.mark.cuj7, pytest.mark.workspace_isolated] + + +class TestUnmanagedModelDiscovery(BaseCujTest): + WORKSPACE_URL = os.environ.get("UG_CUJ7_WORKSPACE", "").strip().rstrip("/") + + @pytest.fixture(autouse=True) + def unmanaged_workspace(self, setup_workspace): + self._assert_unmanaged() + yield + self._assert_unmanaged() + + def _assert_unmanaged(self): + try: + payload = self.workspace.api_client.do("GET", MANAGED_CONFIGS_PATH) + except NotFound: + return + except DatabricksError as error: + raise RuntimeError(f"Workspace API failed: {type(error).__name__}") from None + assert_no_managed_config(payload) + + @pytest.fixture + def live_session(self, unmanaged_workspace, tmp_path): + """Reuse the session helper; the shared cuj fixture requires a managed config.""" + assert os.name == "posix", "CUJ7 requires a disposable POSIX runner" + assert not any(path.exists() for path in MANAGED_PATHS), ( + "Existing machine-wide agent settings; use a clean disposable runner" + ) + binary = shutil.which("ug") + assert binary, "Install ug before running CUJ7" + for tool in (CLAUDE, CODEX, "databricks"): + assert shutil.which(tool), f"Install the required CLI before running CUJ7: {tool}" + try: + authorization = self.workspace.config.authenticate().get("Authorization", "") + except DatabricksError as error: + raise RuntimeError(f"Workspace authentication failed: {type(error).__name__}") from None + assert authorization.startswith("Bearer ") and authorization.removeprefix("Bearer "), ( + "Service-principal authentication did not return a bearer token" + ) + return UserSession( + tmp_path, + Path(binary), + tmp_path / "artifacts", + authorization.removeprefix("Bearer "), + ) + + @pytest.mark.claude + @pytest.mark.tui + def test_case_07_configured_claude_discovers_system_models(self, live_session): + """Scenario: configure Claude, then launch without source overrides or discovery flags. + + Expected: native discovery caches system.ai models as raw IDs or recognized Claude + gateway aliases and shows a discovered picker entry. + """ + session = live_session + session.configure( + [ + "configure", + "--agents", + CLAUDE, + "--workspace", + self.workspace.config.host, + "--skip-upgrade", + "--disable-databricks-ai-tools", + ] + ) + with Terminal(session, "case-07-system-models", [CLAUDE]) as tui: + tui.boot() + screen = tui.open_model_picker( + model_visible=lambda _text: session.claude_gateway_cache_ready() + ) + tui.exit_normally() + assert_claude_system_models_in_picker(session, screen) + + @pytest.mark.codex + def test_case_08_configured_codex_uses_default_models(self, live_session): + """Scenario: configure Codex, then launch without source overrides. + + Expected: unmanaged configuration leaves model selection to Codex's native default; + ug records system.ai discovery while model and reasoning preferences remain unset; + app-server exposes native GPT entries without a generated provider/parent-scoped catalog. + """ + session = live_session + session.configure( + [ + "configure", + "--agents", + CODEX, + "--workspace", + self.workspace.config.host, + "--skip-upgrade", + "--disable-databricks-ai-tools", + ] + ) + models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) + assert_codex_default_models(session, models) diff --git a/tests/integration/README.md b/tests/integration/README.md index 1dc07b08b..155afb3cb 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -618,6 +618,14 @@ Claude lane also passes `CLAUDE_CODE_OAUTH_TOKEN` (the same secret the e2e workf uses) for the relayed hybrid CUJ. Fork PRs run installation checks only because they cannot receive those secrets. +CUJ7's configured Claude picker and Codex app-server discovery cases live in +`tests/e2e_cuj/test_ug_cuj7_model_discovery.py`. The existing required `dedicated-cuj` +job runs them with the other CUJs, using `UG_CUJ7_WORKSPACE` and the shared +`UG_CUJ_SP_CLIENT_ID` / `UG_CUJ_SP_CLIENT_SECRET` credentials. The workspace must +publish no CodingAgentConfig and expose discoverable `system.ai` models. Both +cases reuse the CUJ session/terminal helpers and check that the workspace remains +unmanaged after the test; they never change remote configuration. + The workspace check requires the secret to match `https://eng-ml-inference-team-us-east-1.cloud.databricks.com` (a trailing slash is accepted). It never changes the secret or switches workspaces. There is no diff --git a/tests/integration/test_ug_claude_model_discovery.py b/tests/integration/test_ug_claude_model_discovery.py index 2289da231..5c513a27b 100644 --- a/tests/integration/test_ug_claude_model_discovery.py +++ b/tests/integration/test_ug_claude_model_discovery.py @@ -1,10 +1,13 @@ -"""Claude model-discovery CUJs for repository scenarios 7, 9, 11, and 13.""" +"""Claude model-discovery CUJs for repository scenarios 9, 11, and 13.""" import json import re import pytest -from utils.model_discovery import claude_model_in_picker, claude_system_model_ids +from utils.model_discovery import ( + assert_claude_system_models_in_picker as _assert_system_models_in_picker, +) +from utils.model_discovery import claude_model_in_picker from utils.terminal import AgentTerminal pytestmark = [pytest.mark.claude, pytest.mark.usefixtures("unmanaged_workspace")] @@ -53,25 +56,6 @@ def _system_models_visible(session): return lambda _text: session.claude_gateway_cache_ready() -def _assert_system_models_in_picker(session, screen): - models = session.claude_gateway_models() - ids = claude_system_model_ids(models) - discovered = session.workspace_state()["claude_models"] - assert discovered, "ug configure found no Claude system.ai models" - # ug caches the newest system.ai model per family from UC model-services, which can list a - # just-released model (e.g. a new opus tier) before Claude Code's own gateway-model discovery - # surfaces it in the picker cache. Require the cached defaults to be well-formed system.ai - # Claude ids that overlap Claude Code's catalog, rather than a strict subset of a catalog that - # can legitimately lag behind UC model-services. - assert all(model_id.startswith("system.ai.claude-") for model_id in discovered.values()), ( - discovered - ) - assert set(discovered.values()) & set(ids), (discovered, models) - assert any( - claude_model_in_picker(screen, model["id"], model.get("display_name")) for model in models - ), screen - - def _assert_replacement_picker(session, expected_ids): settings = json.loads((session.home / ".claude" / "ucode-settings.json").read_text()) assert not {"availableModels", "enforceAvailableModels"} & settings.keys(), settings @@ -80,35 +64,6 @@ def _assert_replacement_picker(session, expected_ids): assert [option["model"] for option in picker["options"]] == expected_ids, picker -@pytest.mark.live -@pytest.mark.tui -def test_case_07_configured_claude_discovers_system_models(live_session, workspace): - """Scenario: configure Claude, then launch without source overrides or discovery flags. - - Expected: native discovery caches system.ai models as raw IDs or recognized Claude - gateway aliases and shows a discovered picker entry. - """ - session = live_session - session.run( - "configure", - "--agents", - "claude", - "--workspace", - workspace, - "--skip-upgrade", - "--disable-databricks-ai-tools", - timeout=240, - ) - - command = [str(session.binary), "claude"] - with AgentTerminal(session, "claude", command, "case-07-system-models") as tui: - tui.boot() - screen = tui.open_model_picker(model_visible=_system_models_visible(session)) - tui.exit_normally() - - _assert_system_models_in_picker(session, screen) - - @pytest.mark.live @pytest.mark.tui def test_case_09_fresh_claude_discovers_system_models(live_session, workspace): diff --git a/tests/integration/test_ug_codex_model_discovery.py b/tests/integration/test_ug_codex_model_discovery.py index 178e10657..8f27bdf9e 100644 --- a/tests/integration/test_ug_codex_model_discovery.py +++ b/tests/integration/test_ug_codex_model_discovery.py @@ -1,9 +1,9 @@ -"""Codex model-discovery CUJs for repository scenarios 8, 10, 12, and 14.""" +"""Codex model-discovery CUJs for repository scenarios 10, 12, and 14.""" import os -import tomllib import pytest +from utils.model_discovery import assert_codex_default_models as _assert_default_models from utils.provider_catalog import fetch_codex_parent_catalog pytestmark = [pytest.mark.codex, pytest.mark.usefixtures("unmanaged_workspace")] @@ -19,43 +19,6 @@ def _codex_parent_catalog(unmanaged_workspace, parent_schema, codex_parent_model return catalog -def _assert_default_models(session, models): - discovered = session.workspace_state()["codex_models"] - assert discovered and all(model.startswith("system.ai.") for model in discovered) - config = tomllib.loads((session.home / ".codex/ucode.config.toml").read_text()) - assert "model" not in config, config - assert "model_reasoning_effort" not in config, config - assert "model_catalog_json" not in config, config - assert models and len(models) == len(set(models)), models - assert any(model.startswith("gpt-") for model in models), models - assert not list((session.home / ".ucode").glob("codex-model-catalog-*.json")) - - -@pytest.mark.live -def test_case_08_configured_codex_uses_default_models(live_session, workspace): - """Scenario: configure Codex, then launch without source overrides. - - Expected: unmanaged configuration leaves model selection to Codex's native default; - ug records system.ai discovery while model and reasoning preferences remain unset; - app-server exposes native GPT entries without a generated provider/parent-scoped catalog. - """ - session = live_session - session.run( - "configure", - "--agents", - "codex", - "--workspace", - workspace, - "--skip-upgrade", - "--disable-databricks-ai-tools", - timeout=240, - ) - - models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) - - _assert_default_models(session, models) - - @pytest.mark.live def test_case_10_fresh_codex_uses_default_models(live_session, workspace): """Scenario: launch fresh Codex with --workspace and no source overrides. diff --git a/tests/integration/utils/model_discovery.py b/tests/integration/utils/model_discovery.py index 926dc063b..ae805cbba 100644 --- a/tests/integration/utils/model_discovery.py +++ b/tests/integration/utils/model_discovery.py @@ -2,6 +2,7 @@ import hashlib import re +import tomllib _CLAUDE_GATEWAY_ALIAS = re.compile(r"^anthropic-aigw-[0-9a-fA-F]{8}-(?P.*)$") _CLAUDE_PICKER_ROW = re.compile(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+\.[ \t]+(?P