From 9d720ecafc08515556561da88fbaaf20a04d31a8 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 17:17:11 +0000 Subject: [PATCH 01/19] AIGTWY-4878 Add standalone CUJ3 model-service fixtures --- fixtures/cuj/models/README.md | 145 +++++++++ fixtures/cuj/models/cuj3-managed-config.json | 27 ++ fixtures/cuj/models/provision.py | 257 +++++++++++++++ tests/test_cuj3_model_fixtures.py | 322 +++++++++++++++++++ 4 files changed, 751 insertions(+) create mode 100644 fixtures/cuj/models/README.md create mode 100644 fixtures/cuj/models/cuj3-managed-config.json create mode 100644 fixtures/cuj/models/provision.py create mode 100644 tests/test_cuj3_model_fixtures.py diff --git a/fixtures/cuj/models/README.md b/fixtures/cuj/models/README.md new file mode 100644 index 000000000..8600d68cd --- /dev/null +++ b/fixtures/cuj/models/README.md @@ -0,0 +1,145 @@ +# CUJ3 model-only fixtures + +This standalone layer adapts the full CUJ3 provisioner and managed config for +model discovery only. It owns seven UC model services and, when missing, the +`ug_e2e` catalog and its `models` and `other_models` schemas. It does not require, +inspect, create, or configure MCP services, skills, Databricks Apps, connections, +agent installations, credentials, grants, or managed-config publication. Later +MCP/skill CUJ stacks are separate layers, not prerequisites for this layer. + +| Schema | Services | Purpose | +| --- | --- | --- | +| `ug_e2e.models` | `gpt_luna` | Codex default; GPT family | +| `ug_e2e.models` | `claude_haiku`, `claude_sonnet` | Claude family; Sonnet is the Claude default | +| `ug_e2e.models` | `kimi` | Shared Claude/Codex compatible model | +| `ug_e2e.models` | `gemini_flash` | Gemini family; intended exclusion from both agents | +| `ug_e2e.other_models` | `claude_decoy`, `codex_decoy` | Compatible, accessible, out-of-scope decoys | + +Service identities are stable aliases, not inferred source model names. `gpt_luna` +replaces the earlier `gpt_terra` choice. The provisioner maps every leaf to its +schema explicitly; inventory ordering does not determine scope. + +## Prerequisites and setup order + +1. Use Python 3.12+ and select the exact dedicated CUJ3 HTTPS workspace origin. + No SDK, CLI profile, or implicit workspace/authentication fallback is used. +2. An operator must explicitly select seven existing, canonical `system.ai.` + registered-model FQNs in that workspace/region, including both decoys. These are + backing UC models, not friendly model-service aliases. Live GET checks verified + that `system.ai.gpt-6-luna` is not a registered model, while + `system.ai.databricks-gpt-6-luna` exists; supply the latter for that chosen version. + Before any writes, both `--validate` and `--apply` GET every unique source at + `/api/2.1/unity-catalog/models/{source}` and require its `full_name` to exactly + match the explicit source. Missing or mismatched sources fail without writes; + the script never resolves aliases or chooses a replacement. Choose the intended + GPT, Claude Haiku, Claude Sonnet, Kimi, and Gemini Flash families for their aliases. + Haiku, Sonnet, Kimi, and the Claude decoy must support Claude's wire protocol; + GPT Luna, Kimi, and the Codex decoy must support Codex's wire protocol. + The script never guesses versions, discovers sources, or supplies defaults. + Decoys may reuse compatible in-scope sources; their UC identities remain distinct. +3. For `--validate` or `--apply` only, supply a workspace bearer through + `DATABRICKS_BEARER` (or an explicitly chosen + `--bearer-env` variable) using your approved secret mechanism. Never put tokens + in arguments, source, shell history, logs, or captured output; disable shell + tracing. The script prints only resource names, source names, and sanitized + errors, never tokens or server response bodies. Redirects are refused. +4. The provisioning principal needs permission to read the existing inventory, + create missing catalog/schemas/services, and use the chosen source models. + If it cannot create the catalog, have an authorized operator create it first. + Separately grant the CUJ execution principal `USE_CATALOG`, `USE_SCHEMA`, and + the required model-service invocation/discovery permissions for **both** schemas. + Decoys must be accessible so exclusion proves scope filtering, not missing grants. +5. Run the default offline plan (no bearer or network needed), review the workspace, + all seven mappings, permissions, and request format. Use `--validate` for an + authenticated read-only inventory check; missing resources cause a nonzero exit. + After review, explicitly add `--apply` to create missing resources. Rerun + `--validate`: it should report zero missing resources and exit successfully. +6. Review `cuj3-managed-config.json` before an authorized administrator publishes + it separately. Publishing changes workspace policy; this script never does it. + The config points both agents to `ug_e2e.models`, defaults the agent to Claude, + defaults Claude to Sonnet and Codex to GPT Luna, disables + smart routing and tracing, and contains no MCP or skill scope. Those omissions + do not promise to remove unrelated pre-existing local MCP/skill configuration. + +From the repository root, without needing a bearer for the default offline plan: + +```bash +python3 fixtures/cuj/models/provision.py \ + --workspace "$UG_CUJ3_WORKSPACE" \ + --bearer-env DATABRICKS_BEARER \ + --gpt-luna-source "$GPT_LUNA_SOURCE" \ + --claude-haiku-source "$CLAUDE_HAIKU_SOURCE" \ + --claude-sonnet-source "$CLAUDE_SONNET_SOURCE" \ + --kimi-source "$KIMI_SOURCE" \ + --gemini-flash-source "$GEMINI_FLASH_SOURCE" \ + --claude-decoy-source "$CLAUDE_DECOY_SOURCE" \ + --codex-decoy-source "$CODEX_DECOY_SOURCE" +``` + +Each source variable must contain a concrete `system.ai.` FQN. The default +plan validates only argument syntax and prints the intended inventory; it performs +no network requests, reads no bearer, and cannot determine what already exists. +`--validate` requires a bearer, performs GETs, validates every existing service, +and fails if any resource is missing, without writing. `--validate` and `--apply` +are mutually exclusive. `--apply` performs the full preflight **before any POST**, then +creates only missing resources in catalog → schemas → services order and reads back +all seven services for validation. Existing routing must exactly match the expected +single destination, destination type, and source-model reference. Mismatches abort; +there is no overwrite, update, delete, or cleanup path. Concurrent create conflicts +and partial failures stop the run; inspect the inventory with `--validate` +before retrying. This is not an atomic transaction, and created resources persist. + +## Compatibility and validation boundary + +The intended agent-specific catalog contract is: + +| Agent | Exact `ug_e2e.models` leaves | Default | +| --- | --- | --- | +| Claude | `claude_haiku`, `claude_sonnet`, `kimi` | `claude_sonnet` | +| Codex | `gpt_luna`, `kimi` | `gpt_luna` | + +`gemini_flash` is intentionally excluded from both catalogs. This is an expected +contract, **not proven by provisioning**. Both agents share the same schema pointer; +API-specific compatibility filtering must produce these sets, not a fixture-side +static allowlist. The compatible decoy for each agent must be discoverable under +`ug_e2e.other_models` but absent from the `ug_e2e.models` catalog. + +Discovery is not validated inference: a catalog entry, generated configuration, +picker row, or successful startup alone does not prove a callable model. Each +expected in-scope model must complete a real file task with native model identity +evidence in the live CUJ. Provisioning validation checks routing only and sends no +inference requests; it cannot prove source availability or agent compatibility. + +The model-service API uses `/api/2.1/unity-catalog/model-services`, a schema +`parent`, and canonical snake_case fields matching the local Python SDK sample +`experimental/eng-ai-governance/api-examples/python-sdk/model_service.py` in Universe. +Creation sets `pay_per_token_config.model = models/system.ai.` and +`traffic_percentage = 100`. Usage tracking is always on; its removed/reserved +configuration field is omitted, as specified by +`managed-catalog/atlas/modules/model_service/model_service.proto` in Universe. +Readback validation requires exactly one destination with the selected source name, +pay-per-token destination type, and matching model target. It accepts server-added +metadata and defaults: `is_deleted = false`, empty fallback routing, or omitted +traffic percentage (the SDK documents that a single primary receives all traffic). +Deleted/disabled destinations, explicit traffic percentages other than 100, +fallback destinations, and source/type/target mismatches fail without overwriting +existing services. Usage-tracking metadata is not part of routing validation. +Actual workspace API availability, source existence, and +Claude/Codex inference wire compatibility are **not verified by offline checks**. +Operator confirmation against the intended workspace is required before apply; +service metadata validation alone cannot prove completed inference. + +After separate publication and agent setup, root-owned CUJ tests should verify each +agent's compatible in-scope default/extra models completes inference with the +expected service identity, its compatible decoy is independently discoverable and +callable outside the managed scope, and managed discovery excludes that decoy. +This layer neither runs inference nor installs/configures either agent. Tests, CI, +and broader CUJ documentation remain outside this directory's ownership. + +Suggested focused local checks (no workspace or bearer needed): + +```bash +uv run ruff check fixtures/cuj/models/provision.py +uv run ruff format --check fixtures/cuj/models/provision.py +python3 fixtures/cuj/models/provision.py --help +``` diff --git a/fixtures/cuj/models/cuj3-managed-config.json b/fixtures/cuj/models/cuj3-managed-config.json new file mode 100644 index 000000000..1c1beb448 --- /dev/null +++ b/fixtures/cuj/models/cuj3-managed-config.json @@ -0,0 +1,27 @@ +{ + "spec_version": 1, + "default_agent": "CODING_AGENT_CLAUDE_CODE", + "enabled_agents": [ + { + "agent": "CODING_AGENT_CLAUDE_CODE", + "config": { + "models": { "unity_catalog_location": "ug_e2e.models" }, + "default_models": { + "default_model": "ug_e2e.models.claude_sonnet", + "default_sonnet_model": "ug_e2e.models.claude_sonnet" + }, + "smart_routing": { "enabled": false }, + "tracing": { "enabled": false } + } + }, + { + "agent": "CODING_AGENT_CODEX", + "config": { + "models": { "unity_catalog_location": "ug_e2e.models" }, + "default_models": { "default_model": "ug_e2e.models.gpt_luna" }, + "smart_routing": { "enabled": false }, + "tracing": { "enabled": false } + } + } + ] +} diff --git a/fixtures/cuj/models/provision.py b/fixtures/cuj/models/provision.py new file mode 100644 index 000000000..ef03c7cd2 --- /dev/null +++ b/fixtures/cuj/models/provision.py @@ -0,0 +1,257 @@ +"""Plan seven CUJ3 models offline; authenticate only for --validate or --apply.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + +UC = "/api/2.1/unity-catalog" +SCHEMAS = ("models", "other_models") +MODEL_SCHEMAS = { + "gpt_luna": "models", + "claude_haiku": "models", + "claude_sonnet": "models", + "kimi": "models", + "gemini_flash": "models", + "claude_decoy": "other_models", + "codex_decoy": "other_models", +} +MODEL_LEAVES = tuple(MODEL_SCHEMAS) +SOURCE = re.compile(r"system\.ai\.[A-Za-z0-9_-]+\Z") +DESTINATION_TYPE = "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL" + + +class ApiError(RuntimeError): + def __init__(self, method: str, path: str, status: int): + super().__init__(f"{method} {path.split('?', 1)[0]} returned HTTP {status}") + self.status = status + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def workspace_origin(workspace: str) -> str: + parsed = urllib.parse.urlsplit(workspace) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + ): + raise ValueError("--workspace must be an HTTPS workspace origin without credentials") + return f"https://{parsed.netloc}" + + +class Client: + def __init__(self, workspace: str, bearer: str): + self.origin = workspace_origin(workspace) + self.bearer = bearer + self.opener = urllib.request.build_opener(NoRedirect()) + + def request(self, method: str, path: str, *, params=None, body=None): + query = f"?{urllib.parse.urlencode(params)}" if params else "" + data = json.dumps(body).encode() if body is not None else None + headers = {"Authorization": f"Bearer {self.bearer}"} + if data is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + self.origin + path + query, data=data, headers=headers, method=method + ) + try: + with self.opener.open(request, timeout=60) as response: + payload = response.read() + except urllib.error.HTTPError as error: + raise ApiError(method, path, error.code) from None + except (urllib.error.URLError, TimeoutError, OSError): + raise RuntimeError("Workspace request failed; check connectivity and TLS") from None + try: + return json.loads(payload) if payload else None + except (ValueError, UnicodeError): + raise RuntimeError("Workspace returned an invalid JSON response") from None + + def get_optional(self, path: str): + try: + return self.request("GET", path) + except ApiError as error: + if error.status == 404: + return None + raise + + +def require(condition: bool, message: str): + if not condition: + raise RuntimeError(message) + + +def model_body(source: str): + return { + "config": { + "routing": { + "destinations": [ + { + "name": source, + "destination_type": DESTINATION_TYPE, + "pay_per_token_config": {"model": f"models/{source}"}, + "traffic_percentage": 100, + } + ] + }, + } + } + + +def validate_model(existing: dict, fqn: str, source: str): + message = f"Model service {fqn} has unexpected routing; refusing to overwrite it" + require(isinstance(existing, dict) and isinstance(existing.get("config"), dict), message) + routing = existing["config"].get("routing") + require(isinstance(routing, dict), message) + require(routing.get("fallback") in (None, {}, {"destinations": []}), message) + destinations = routing.get("destinations") + require(isinstance(destinations, list) and len(destinations) == 1, message) + destination = destinations[0] + require(isinstance(destination, dict), message) + target = destination.get("pay_per_token_config") + traffic = destination.get("traffic_percentage", 100) + require( + destination.get("name") == source + and destination.get("destination_type") == DESTINATION_TYPE + and isinstance(target, dict) + and target.get("model") == f"models/{source}" + and type(traffic) is int + and traffic == 100 + and all( + destination.get(field, False) is False + for field in ("is_deleted", "is_disabled", "disabled") + ), + message, + ) + + +def print_plan(sources: dict[str, str]): + print("catalog ug_e2e: ensure only if missing") + for schema in SCHEMAS: + print(f"schema ug_e2e.{schema}: ensure only if missing") + for leaf in MODEL_LEAVES: + schema = MODEL_SCHEMAS[leaf] + print(f"model service ug_e2e.{schema}.{leaf}: planned -> {sources[leaf]}") + print( + "Offline dry plan: no network or writes; no bearer required; existing inventory not checked" + ) + + +def provision(client: Client, sources: dict[str, str], apply: bool): + for source in sorted(set(sources.values())): + registered_model = client.get_optional(f"{UC}/models/{source}") + require( + isinstance(registered_model, dict) and registered_model.get("full_name") == source, + f"Source registered model {source} is missing or has an unexpected full_name; " + "supply an exact canonical registered-model FQN, not a model-service alias; " + "no writes performed", + ) + print(f"source registered model {source}: validated") + missing = [] + if client.get_optional(f"{UC}/catalogs/ug_e2e") is None: + missing.append((f"{UC}/catalogs", None, {"name": "ug_e2e", "comment": "UG CUJ fixtures"})) + print("catalog ug_e2e: would create") + else: + print("catalog ug_e2e: existing") + for schema in SCHEMAS: + fqn = f"ug_e2e.{schema}" + if client.get_optional(f"{UC}/schemas/{fqn}") is None: + missing.append( + ( + f"{UC}/schemas", + None, + {"name": schema, "catalog_name": "ug_e2e", "comment": "UG model fixture scope"}, + ) + ) + print(f"schema {fqn}: would create") + else: + print(f"schema {fqn}: existing") + for leaf in MODEL_LEAVES: + schema = MODEL_SCHEMAS[leaf] + fqn = f"ug_e2e.{schema}.{leaf}" + source = sources[leaf] + existing = client.get_optional(f"{UC}/model-services/{fqn}") + if existing is None: + missing.append( + ( + f"{UC}/model-services", + {"parent": f"schemas/ug_e2e.{schema}", "model_service_id": leaf}, + model_body(source), + ) + ) + print(f"model service {fqn}: would create -> {source}") + else: + validate_model(existing, fqn, source) + print(f"model service {fqn}: validated -> {source}") + if not apply: + print(f"Validation: {len(missing)} missing resources; no writes performed") + require(not missing, "Inventory is incomplete; review the offline plan before --apply") + return + for path, params, body in missing: + client.request("POST", path, params=params, body=body) + for leaf in MODEL_LEAVES: + schema = MODEL_SCHEMAS[leaf] + fqn = f"ug_e2e.{schema}.{leaf}" + validate_model(client.request("GET", f"{UC}/model-services/{fqn}"), fqn, sources[leaf]) + print("Model-only fixture inventory is ready; managed config was not published") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workspace", required=True, help="Exact HTTPS workspace origin") + parser.add_argument( + "--bearer-env", + default="DATABRICKS_BEARER", + help="Environment variable containing an explicit workspace bearer token", + ) + for leaf in MODEL_LEAVES: + parser.add_argument( + f"--{leaf.replace('_', '-')}-source", + required=True, + help="Canonical existing system.ai registered-model FQN, not a model-service alias", + ) + modes = parser.add_mutually_exclusive_group() + modes.add_argument( + "--validate", + action="store_true", + help="Authenticate and validate existing resources; GET only", + ) + modes.add_argument("--apply", action="store_true", help="Create missing resources after review") + args = parser.parse_args() + origin = workspace_origin(args.workspace) + sources = {leaf: getattr(args, f"{leaf}_source") for leaf in MODEL_LEAVES} + require( + all(SOURCE.fullmatch(source) for source in sources.values()), + "Every model source must be an existing system.ai model FQN", + ) + print(f"workspace: {origin}") + if not args.validate and not args.apply: + print_plan(sources) + return + bearer = os.environ.get(args.bearer_env, "") + require(bool(bearer.strip()), "Bearer environment variable must contain a workspace token") + require( + not any(character.isspace() for character in bearer), "Bearer must not contain whitespace" + ) + provision(Client(origin, bearer), sources, args.apply) + + +if __name__ == "__main__": + try: + main() + except (ApiError, RuntimeError, ValueError) as error: + print(f"model fixture provisioning failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tests/test_cuj3_model_fixtures.py b/tests/test_cuj3_model_fixtures.py new file mode 100644 index 000000000..580980487 --- /dev/null +++ b/tests/test_cuj3_model_fixtures.py @@ -0,0 +1,322 @@ +"""Component safety checks for the model-only workspace provisioning script.""" + +import json +import os +import runpy +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).parent.parent +SCRIPT = ROOT / "fixtures/cuj/models/provision.py" + + +@pytest.fixture +def provisioner(): + return SimpleNamespace(**runpy.run_path(str(SCRIPT))) + + +def model_sources(provisioner): + return {leaf: f"system.ai.{leaf}" for leaf in provisioner.MODEL_LEAVES} + + +class RecordingClient: + def __init__(self, inventory): + self.inventory = inventory + self.requests = [] + + def get_optional(self, path): + self.requests.append(("GET", path, None, None)) + return self.inventory.get(path) + + def request(self, method, path, *, params=None, body=None): + self.requests.append((method, path, params, body)) + if method == "GET": + return self.inventory[path] + if path.endswith("model-services"): + schema = params["parent"].removeprefix("schemas/") + self.inventory[f"{path}/{schema}.{params['model_service_id']}"] = body + return body + + +def source_inventory(provisioner, sources=None): + sources = model_sources(provisioner) if sources is None else sources + return { + f"{provisioner.UC}/models/{source}": {"full_name": source} + for source in set(sources.values()) + } + + +def complete_inventory(provisioner): + inventory = source_inventory(provisioner) + inventory[f"{provisioner.UC}/catalogs/ug_e2e"] = {} + inventory.update( + {f"{provisioner.UC}/schemas/ug_e2e.{schema}": {} for schema in provisioner.SCHEMAS} + ) + for leaf, source in model_sources(provisioner).items(): + schema = "other_models" if leaf.endswith("decoy") else "models" + inventory[f"{provisioner.UC}/model-services/ug_e2e.{schema}.{leaf}"] = ( + provisioner.model_body(source) + ) + return inventory + + +def model_readback(*, server_defaults): + destination = { + "name": "system.ai.databricks-gpt-6-luna", + "destination_type": "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", + "pay_per_token_config": {"model": "models/system.ai.databricks-gpt-6-luna"}, + } + routing = {"destinations": [destination]} + if server_defaults: + destination.update(traffic_percentage=100, is_deleted=False) + routing["fallback"] = {"destinations": []} + return { + "name": "model-services/ug_e2e.models.gpt_luna", + "config": {"routing": routing, "usage_tracking": {"enabled": True}}, + } + + +INVALID_DESTINATIONS = [ + {"name": "system.ai.wrong_source"}, + {"name": "other.ai.fixture"}, + {"destination_type": "DESTINATION_TYPE_EXTERNAL_MODEL"}, + {"pay_per_token_config": {"model": "models/system.ai.wrong_source"}}, + {"pay_per_token_config": None}, + {"traffic_percentage": 0}, + {"traffic_percentage": 50}, + {"traffic_percentage": "100"}, + {"traffic_percentage": None}, + {"traffic_percentage": True}, + {"is_deleted": True}, + {"is_deleted": "false"}, + {"is_disabled": True}, + {"disabled": True}, +] + + +def test_model_fixture_create_body_uses_canonical_sdk_fields(provisioner): + assert provisioner.model_body("system.ai.databricks-gpt-6-luna") == { + "config": { + "routing": { + "destinations": [ + { + "name": "system.ai.databricks-gpt-6-luna", + "destination_type": "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", + "pay_per_token_config": {"model": "models/system.ai.databricks-gpt-6-luna"}, + "traffic_percentage": 100, + } + ] + }, + } + } + + +@pytest.mark.parametrize("server_defaults", [False, True]) +def test_model_fixture_accepts_canonical_readback(provisioner, server_defaults): + provisioner.validate_model( + model_readback(server_defaults=server_defaults), + "ug_e2e.models.gpt_luna", + "system.ai.databricks-gpt-6-luna", + ) + + +@pytest.mark.parametrize("destination_updates", INVALID_DESTINATIONS) +def test_model_fixture_rejects_invalid_readback_before_writes(provisioner, destination_updates): + inventory = complete_inventory(provisioner) + inventory.pop(f"{provisioner.UC}/catalogs/ug_e2e") + payload = model_readback(server_defaults=True) + payload["config"]["routing"]["destinations"][0].update(destination_updates) + inventory[f"{provisioner.UC}/model-services/ug_e2e.models.gpt_luna"] = payload + sources = model_sources(provisioner) + sources["gpt_luna"] = "system.ai.databricks-gpt-6-luna" + inventory.update(source_inventory(provisioner, sources)) + client = RecordingClient(inventory) + with pytest.raises(RuntimeError, match="refusing to overwrite"): + provisioner.provision(client, sources, apply=True) + assert all(method == "GET" for method, *_ in client.requests) + + +@pytest.mark.parametrize("fallback", [False, True]) +def test_model_fixture_rejects_multiple_sources_before_writes(provisioner, fallback): + inventory = complete_inventory(provisioner) + inventory.pop(f"{provisioner.UC}/catalogs/ug_e2e") + payload = provisioner.model_body("system.ai.gpt_luna") + routing = payload["config"]["routing"] + extra = provisioner.model_body("system.ai.other")["config"]["routing"]["destinations"] + if fallback: + routing["fallback"] = {"destinations": extra} + else: + routing["destinations"].extend(extra) + inventory[f"{provisioner.UC}/model-services/ug_e2e.models.gpt_luna"] = payload + client = RecordingClient(inventory) + with pytest.raises(RuntimeError, match="refusing to overwrite"): + provisioner.provision(client, model_sources(provisioner), apply=True) + assert all(method == "GET" for method, *_ in client.requests) + + +def test_model_fixture_dry_plan_requires_no_credentials_or_network(): + arguments = [sys.executable, str(SCRIPT), "--workspace", "https://workspace.invalid"] + for leaf in ( + "gpt-luna", + "claude-haiku", + "claude-sonnet", + "kimi", + "gemini-flash", + "claude-decoy", + "codex-decoy", + ): + arguments.extend([f"--{leaf}-source", f"system.ai.{leaf}"]) + environment = os.environ.copy() + environment.pop("DATABRICKS_BEARER", None) + result = subprocess.run(arguments, env=environment, capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr + assert "no network or writes; no bearer required" in result.stdout + assert result.stdout.count("model service ug_e2e.") == 7 + assert "model service ug_e2e.models.gemini_flash" in result.stdout + assert "model service ug_e2e.other_models.claude_decoy" in result.stdout + assert "model service ug_e2e.other_models.codex_decoy" in result.stdout + assert "mcp-services" not in result.stdout + assert "skills" not in result.stdout + + +def test_model_fixture_config_is_model_only(): + config = json.loads(SCRIPT.with_name("cuj3-managed-config.json").read_text()) + assert "mcp_servers" not in config + assert "skills" not in config + agents = {entry["agent"]: entry["config"] for entry in config["enabled_agents"]} + assert set(agents) == {"CODING_AGENT_CLAUDE_CODE", "CODING_AGENT_CODEX"} + assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE" + assert agents["CODING_AGENT_CLAUDE_CODE"]["default_models"] == { + "default_model": "ug_e2e.models.claude_sonnet", + "default_sonnet_model": "ug_e2e.models.claude_sonnet", + } + assert agents["CODING_AGENT_CODEX"]["default_models"] == { + "default_model": "ug_e2e.models.gpt_luna" + } + for agent_config in agents.values(): + assert agent_config["models"] == {"unity_catalog_location": "ug_e2e.models"} + assert agent_config["smart_routing"] == {"enabled": False} + assert agent_config["tracing"] == {"enabled": False} + + +def test_model_fixture_validate_missing_inventory_performs_no_writes(provisioner): + client = RecordingClient(source_inventory(provisioner)) + with pytest.raises(RuntimeError, match="Inventory is incomplete"): + provisioner.provision(client, model_sources(provisioner), apply=False) + assert len(client.requests) == 17 + assert all(method == "GET" for method, *_ in client.requests) + + +@pytest.mark.parametrize("apply", [False, True]) +@pytest.mark.parametrize("registered_model", [None, {}, {"full_name": "system.ai.other"}, []]) +def test_model_fixture_source_preflight_failure_performs_no_writes( + provisioner, apply, registered_model +): + sources = model_sources(provisioner) + inventory = source_inventory(provisioner) + source = sorted(sources.values())[-1] + inventory[f"{provisioner.UC}/models/{source}"] = registered_model + client = RecordingClient(inventory) + with pytest.raises(RuntimeError, match="canonical registered-model FQN.*model-service alias"): + provisioner.provision(client, sources, apply=apply) + assert len(client.requests) == len(set(sources.values())) + assert all(method == "GET" and "/models/" in path for method, path, *_ in client.requests) + + +def test_model_fixture_source_preflight_rejects_friendly_alias_without_resolving(provisioner): + sources = model_sources(provisioner) + inventory = source_inventory(provisioner) + sources["gpt_luna"] = "system.ai.gpt-6-luna" + inventory[f"{provisioner.UC}/models/system.ai.databricks-gpt-6-luna"] = { + "full_name": "system.ai.databricks-gpt-6-luna" + } + inventory[f"{provisioner.UC}/model-services/system.ai.gpt-6-luna"] = model_readback( + server_defaults=True + ) + client = RecordingClient(inventory) + with pytest.raises(RuntimeError, match="system.ai.gpt-6-luna.*model-service alias"): + provisioner.provision(client, sources, apply=True) + assert all(method == "GET" and "/models/" in path for method, path, *_ in client.requests) + assert not any("databricks-gpt-6-luna" in path for _, path, *_ in client.requests) + + +def test_model_fixture_source_preflight_checks_each_unique_source_once(provisioner): + sources = dict.fromkeys(provisioner.MODEL_LEAVES, "system.ai.databricks-gpt-6-luna") + client = RecordingClient(source_inventory(provisioner, sources)) + provisioner.provision(client, sources, apply=True) + source_reads = [request for request in client.requests if "/models/" in request[1]] + assert source_reads == [ + ("GET", f"{provisioner.UC}/models/system.ai.databricks-gpt-6-luna", None, None) + ] + assert client.requests[0] == source_reads[0] + + +@pytest.mark.parametrize("apply", [False, True]) +def test_model_fixture_existing_inventory_is_not_rewritten(provisioner, apply): + inventory = complete_inventory(provisioner) + for path, payload in inventory.items(): + if "/model-services/" in path: + payload["config"]["routing"]["destinations"][0]["is_deleted"] = False + client = RecordingClient(inventory) + provisioner.provision(client, model_sources(provisioner), apply=apply) + assert all(method == "GET" for method, *_ in client.requests) + + +def test_model_fixture_mismatch_aborts_before_any_creation(provisioner): + inventory = complete_inventory(provisioner) + inventory.pop(f"{provisioner.UC}/catalogs/ug_e2e") + inventory[f"{provisioner.UC}/model-services/ug_e2e.other_models.codex_decoy"] = ( + provisioner.model_body("system.ai.wrong_model") + ) + client = RecordingClient(inventory) + with pytest.raises(RuntimeError, match="refusing to overwrite"): + provisioner.provision(client, model_sources(provisioner), apply=True) + assert all(method == "GET" for method, *_ in client.requests) + + +def test_model_fixture_apply_only_creates_models_in_dependency_order(provisioner): + client = RecordingClient(source_inventory(provisioner)) + provisioner.provision(client, model_sources(provisioner), apply=True) + writes = [request for request in client.requests if request[0] != "GET"] + assert [request[1] for request in writes] == [ + f"{provisioner.UC}/catalogs", + *[f"{provisioner.UC}/schemas"] * 2, + *[f"{provisioner.UC}/model-services"] * 7, + ] + assert all(request[0] == "POST" for request in writes) + assert all(request[0] == "GET" for request in client.requests[:17]) + assert {request[1] for request in client.requests[:7]} == set(source_inventory(provisioner)) + model_writes = [request for request in writes if request[1].endswith("model-services")] + expected_parents = { + "gpt_luna": "schemas/ug_e2e.models", + "claude_haiku": "schemas/ug_e2e.models", + "claude_sonnet": "schemas/ug_e2e.models", + "kimi": "schemas/ug_e2e.models", + "gemini_flash": "schemas/ug_e2e.models", + "claude_decoy": "schemas/ug_e2e.other_models", + "codex_decoy": "schemas/ug_e2e.other_models", + } + assert { + request[2]["model_service_id"]: request[2]["parent"] for request in model_writes + } == expected_parents + for request in model_writes: + leaf = request[2]["model_service_id"] + assert request[3] == provisioner.model_body(f"system.ai.{leaf}") + + +@pytest.mark.parametrize( + "workspace", + [ + "http://workspace.invalid", + "https://user:secret@workspace.invalid", + "https://workspace.invalid/path", + "https://workspace.invalid/?token=secret", + ], +) +def test_model_fixture_rejects_non_origin_workspaces(provisioner, workspace): + with pytest.raises(ValueError, match="HTTPS workspace origin"): + provisioner.workspace_origin(workspace) From 7188e2b6cba64bd05b9ae0fbf560e308da6f857b Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 17:18:45 +0000 Subject: [PATCH 02/19] AIGTWY-4878 Test schema-scoped agent model discovery and defaults --- tests/README.md | 11 +- tests/integration/README.md | 87 ++++- tests/integration/pytest.ini | 2 + .../test_ug_claude_managed_model_discovery.py | 4 +- .../test_ug_codex_managed_model_discovery.py | 4 +- .../test_ug_cuj3_model_discovery.py | 308 +++++++++++++++ tests/integration/utils/cuj3.py | 323 ++++++++++++++++ tests/integration/utils/terminal.py | 25 ++ tests/test_integration_cuj3.py | 357 ++++++++++++++++++ 9 files changed, 1106 insertions(+), 15 deletions(-) create mode 100644 tests/integration/test_ug_cuj3_model_discovery.py create mode 100644 tests/integration/utils/cuj3.py create mode 100644 tests/test_integration_cuj3.py diff --git a/tests/README.md b/tests/README.md index 66d24fa4f..facdd6a51 100644 --- a/tests/README.md +++ b/tests/README.md @@ -187,9 +187,10 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | | `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`); explicitly launch `ug claude` above the tier | Bare launches check Claude/Sonnet below the tier and the Codex/Luna recommendation above it; `ug usage` agrees with backend spend, threshold, and percentage. Explicit `ug claude` displays the backend's Codex/Luna recommendation while its generated model setting and native header select Sonnet. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | -| `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | +| CUJ3 model discovery (`test_case_03_*`, `test_case_04_*`) | Configure both agents in the dedicated workspace using the published `ug_e2e.models` schema pointer | Claude exposes Sonnet/Haiku/Kimi, Codex exposes GPT Luna/Kimi, and Gemini plus all other-schema entries are absent from exact native picker inventories. Bare `ug` and explicit `ug claude` select Claude/Sonnet for separate TUI tasks; Claude print mode, Codex TUI, and Codex exec retain their configured defaults. Every additional compatible model completes a headless task with native model identity. Native picker and task coverage still require a live journey pass; MCP and skills are deferred | +| `test_managed_fixture_claude_rejects_provider_override`, `test_managed_fixture_fresh_claude_rejects_provider_override`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | -| `test_case_04_*`, `test_case_06_*` | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state | +| `test_managed_fixture_codex_rejects_provider_override_after_configure`, `test_managed_fixture_codex_rejects_provider_override_from_fresh_state`, `test_case_06_*` | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state | | `test_ug_configure_managed_codex_catalog_fallback` | Configure from an injected managed response containing a GPT model absent from Codex's bundled catalog | Actionable metadata warning; conservative catalog entry for the unknown model; real Codex prompt on the valid default model | | `test_managed_fixture_codex_http_headers_in_managed_file` | Interactive PTY configure with injected managed `http_headers` for Codex | The specified header (`x-databricks-workspace`) lands in `model_providers.Databricks.http_headers` in `/etc/codex/managed_config.toml` with the exact admin value | | `test_managed_claude_mps_defaults_accompany_discovery`, `test_managed_claude_parent_schema_defaults_accompany_discovery` | Configure from the published admin config and launch Claude with MPS on `eng-ml-inference-batch-inference-us-west-2` and Unity Catalog discovery on `eng-ml-inference-ap-northeast-2`, respectively | Both generated settings files retain every admin-authored default alongside the source header and every independently fetched catalog model with its label; MPS pickers keep family shortcut rows separate from catalog entries; only UC Opus/Sonnet family ids gain `[1m]` | @@ -202,12 +203,12 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_and_ucode_web_search_helpers_preserve_mcp_stdio` | Initialize and list tools through both web-search helper commands | Exactly the MCP JSON-RPC responses; no text/ANSI contamination; existing server/tool identities preserved; no model request | With Claude and Codex selected there are **62 live cases** (12 marked TUI cases), -**6 managed-workspace cases** (marker `managed`, run against workspaces that +**8 managed-workspace cases** (marker `managed`, including two dedicated CUJ3 model journeys, run against workspaces that publish a CodingAgentConfig), **1 two-workspace case** (marker `workspace_switch`), **25 managed-fixture cases** (marker `managed_fixture`, with only the CodingAgentConfig input injected), and **7 installation checks**. The 14 retained numbered scenarios -comprise **24 explicit journeys**: 12 managed configured/fresh executions and 12 unmanaged -executions. Thirteen additional managed-fixture cases cover focused model, MCP, skills, +comprise **22 explicit journeys**: 8 managed-fixture configured/fresh executions, 2 CUJ3 model journeys, and 12 unmanaged +executions. Four provider-override journeys retain their assertions under descriptive names. Thirteen additional managed-fixture cases cover focused model, MCP, skills, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. diff --git a/tests/integration/README.md b/tests/integration/README.md index 8cdd60186..cb8744b61 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -10,7 +10,8 @@ CUJ2 adds three separately collected cases for exact MPS/MCP configuration, Code and Claude inference. This suite runs the **installed product** through subprocesses, against the same -`UCODE_TEST_WORKSPACE` used by the existing e2e tests. It does not import `ucode`, +`UCODE_TEST_WORKSPACE` used by the existing e2e tests; CUJ3 uses a separate managed +workspace selected explicitly for its model-only journeys. It does not import `ucode`, patch application functions, substitute agent executables, run a fake gateway, or construct ug state files. The normal test suite checks these boundaries. @@ -359,8 +360,8 @@ they only configure, list models, and open/close the picker. Other live CUJs per real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent -re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against +checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **8 managed-workspace cases** (one per agent, an idempotent +re-configure, a cache-TTL journey, two Claude defaults cases, and two CUJ3 model journeys; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** uses two real workspaces and checks skills MCP cleanup and a completed Claude task. A further **25 `managed_fixture` @@ -383,8 +384,9 @@ constants in the runner; CI only needs `UG_MPS_DEFAULTS_CLIENT_SECRET` for west- `UG_PARENT_SCHEMA_DEFAULTS_CLIENT_SECRET` for northeast-2. As with the base workspace, the runner mints short-lived tokens and passes bearers to pytest; each test selects its target bearer for `ug configure` and Claude. The client secrets do not enter the pytest process. -The 14 retained numbered scenarios comprise 24 explicit journeys: 12 managed and 12 unmanaged -executions; the complete integration suite collects 101 executions. See the named coverage and gaps matrix in +The 14 retained numbered scenarios comprise 22 explicit journeys: 8 managed-fixture, +2 CUJ3 model, and 12 unmanaged executions. Four provider-override journeys retain their +assertions under descriptive names; the complete integration suite collects 103 executions. See the named coverage and gaps matrix in [../README.md](../README.md). ```bash @@ -550,7 +552,7 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, -both full lanes, and both **Managed config** lanes to pass for full/live runs. Each tracing +both full lanes, both CUJ3 model-discovery lanes, and both **Managed config** lanes to pass for full/live runs. Each tracing journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke, TUI, and installation subsets do not select managed tests and do not require them. @@ -558,6 +560,79 @@ The advisory Windows installation and headless lanes are not yet included in tha The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. +### CUJ3 model-discovery stack + +The first CUJ3 layer covers only models, for both Claude and Codex. Provision +`ug_e2e.models` and `ug_e2e.other_models` using `fixtures/cuj/models/provision.py`, +then review and publish `fixtures/cuj/models/cuj3-managed-config.json` in the +dedicated workspace. The model-only fixtures do not require an App, UC connection, +MCP service, or skill bundle. Resource provisioning and config publication are +explicit admin setup, not work performed by the integration tests. + +Each journey calls the real `ug configure` before checking its persisted schema +pointer and defaults. A separate UC inventory check proves that all five in-scope +services exist, including Gemini. It independently requests each agent's scoped catalog, +proves the compatible decoy is accessible in another schema, and compares the +generated catalog and native picker/model-list with that oracle. Native +pickers must contain exactly the expected numbered model rows, with no unmatched, +ambiguous, or duplicate entries; friendly labels cannot hide an excluded service. +The configured default and every additional in-scope compatible model must complete a file task +whose contents are withheld from the prompt, with native model identity evidence. +Discovery or startup alone cannot pass. + +The version-agnostic services under `ug_e2e.models` and expected discovery sets are: + +| Service | Claude Code | Codex | +| --- | --- | --- | +| `gpt_luna` | Excluded | Included; default | +| `claude_haiku` | Included | Excluded | +| `claude_sonnet` | Included; default | Excluded | +| `kimi` | Included | Included | +| `gemini_flash` | Excluded | Excluded | + +Gemini is inside the managed schema: its exclusion tests agent compatibility, +not schema scoping. The accessible compatible services +`ug_e2e.other_models.claude_decoy` and `ug_e2e.other_models.codex_decoy` separately +test schema exclusion. Exact backing `system.ai` source versions are supplied +explicitly during provisioning. These expected subsets are an intended contract, +not a claim of a live pass: gateway API-type metadata, feature flags, and harness +compatibility must be verified in the dedicated workspace. Unexpected Gemini +visibility fails the tests; no local filtering hides it. + +When the gateway's Claude discovery-alias feature is enabled, non-Claude IDs such +as Kimi are returned as `anthropic-aigw-<8-character SHA-256 prefix>-`. +The journey verifies the checksum and exact service inventory without dropping +entries. Generated settings, the native cache, picker, explicit tasks, and native +transcript identity must retain the actual returned wire ID; Codex keeps raw FQNs. +The dedicated workspace enables this feature: its Claude journey requires the +exact encoded wire inventory as well as service identities and the coding-agent +mode header needed for inference to reverse the alias. + +With the published policy selecting Claude as the default agent, both bare `ug` +and explicit `ug claude` must launch Claude/Sonnet and complete separate real TUI +tasks. `ug claude -p "task"` must +complete a print-mode task on Sonnet without a model override. Codex's default +must complete both a TUI task and `ug codex -- exec --skip-git-repo-check --json "task"` +on `gpt_luna`, also without a model override. Native Codex `-p` selects a profile, +not a headless prompt; `ug codex -p "task"` is not the supported task command. + +CI runs `managed and cuj3 and workspace_isolated` in a two-agent matrix using +`UG_CUJ3_WORKSPACE` and the shared CUJ SP secrets. The shared managed lanes exclude +`workspace_isolated`; both CUJ3 legs are required for full/live runs. Local runs +must provide an explicit workspace and authenticated profile or bearer: + +```bash +python3.12 scripts/run_integration.py \ + --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ + --workspace "$CUJ3_WORKSPACE" --profile YOUR_CUJ3_PROFILE \ + -- -m 'managed and cuj3 and workspace_isolated' +``` + +MCP discovery/execution and skills discovery/invocation remain separate later +layers. The existing full CUJ3 and fixture branches are retained until those +layers are extracted and restacked. The model-only fixture PR is based directly +on main; the discovery-test PR is stacked only on that fixture PR, not CUJ7. + ### Managed-workspace journeys `test_ug_configure_managed.py` (marker `managed`, not `live`) runs in its own per-agent diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index cc0d9b0ca..7cdb26711 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -9,6 +9,8 @@ markers = workspace_switch: requires two real workspace hosts and their respective credentials smoke: Databricks Hosted, custom OAuth CLI TUI, and headless prompt for each agent tui: real interactive terminal boot, keyboard input, exit and reopen + workspace_isolated: requires a dedicated workspace instead of the shared integration workspace + cuj3: managed schema-scoped model discovery and inference in the dedicated CUJ3 workspace claude: only runs when Claude Code is explicitly selected codex: only runs when Codex is explicitly selected opencode: only runs when OpenCode is explicitly selected diff --git a/tests/integration/test_ug_claude_managed_model_discovery.py b/tests/integration/test_ug_claude_managed_model_discovery.py index f674d225e..e759823e2 100644 --- a/tests/integration/test_ug_claude_managed_model_discovery.py +++ b/tests/integration/test_ug_claude_managed_model_discovery.py @@ -186,7 +186,7 @@ def test_case_01_fresh_managed_claude_uses_admin_discovery( _assert_managed_provider_in_picker(session, workspace, screen, _managed_claude_provider_catalog) -def test_case_03_managed_claude_rejects_provider_override(live_session, workspace, claude_provider): +def test_managed_fixture_claude_rejects_provider_override(live_session, workspace, claude_provider): """Scenario: configure managed Claude, then pass --provider. Expected: ug rejects the override without changing agent-owned state/files. @@ -215,7 +215,7 @@ def test_case_03_managed_claude_rejects_provider_override(live_session, workspac _assert_rejected_before_claude_started(session, result, before) -def test_case_03_fresh_managed_claude_rejects_provider_override( +def test_managed_fixture_fresh_claude_rejects_provider_override( live_session, workspace, claude_provider ): """Scenario: pass --provider while launching managed Claude from fresh state. diff --git a/tests/integration/test_ug_codex_managed_model_discovery.py b/tests/integration/test_ug_codex_managed_model_discovery.py index 980f69085..f7484759b 100644 --- a/tests/integration/test_ug_codex_managed_model_discovery.py +++ b/tests/integration/test_ug_codex_managed_model_discovery.py @@ -193,7 +193,7 @@ def test_case_02_managed_codex_uses_admin_discovery_from_fresh_state( assert app_models == models, (app_models, models) -def test_case_04_managed_codex_rejects_provider_override_after_configure( +def test_managed_fixture_codex_rejects_provider_override_after_configure( live_session, workspace, codex_provider ): """Scenario: configure managed Codex, then pass a --provider override. @@ -225,7 +225,7 @@ def test_case_04_managed_codex_rejects_provider_override_after_configure( _assert_rejected_before_codex_started(session, result, before) -def test_case_04_managed_codex_rejects_provider_override_from_fresh_state( +def test_managed_fixture_codex_rejects_provider_override_from_fresh_state( live_session, workspace, codex_provider ): """Scenario: pass --workspace and a --provider override from fresh state. diff --git a/tests/integration/test_ug_cuj3_model_discovery.py b/tests/integration/test_ug_cuj3_model_discovery.py new file mode 100644 index 000000000..6eadeb458 --- /dev/null +++ b/tests/integration/test_ug_cuj3_model_discovery.py @@ -0,0 +1,308 @@ +"""CUJ3: complete managed, schema-scoped model discovery journeys.""" + +import json +import os +import tomllib + +import pytest +from utils.cuj3 import ( + CLAUDE_DECOY, + CLAUDE_DEFAULT, + CLAUDE_MODELS, + CODEX_DECOY, + CODEX_DEFAULT, + CODEX_MODELS, + GEMINI_MODEL, + MODEL_HEADER, + MODEL_SCHEMA, + MODEL_SERVICES, + OTHER_MODEL_SCHEMA, + assert_native_model_identity, + assert_persisted_config, + assert_picker_inventory, + claude_discovery_model_id, + claude_model_service_id, + codex_model_in_picker, + fetch_claude_parent_catalog, + fetch_codex_parent_catalog, + fetch_model_service_inventory, +) +from utils.evidence import FileTask +from utils.model_discovery import claude_model_in_picker +from utils.provider_catalog import parse_codex_provider_catalog +from utils.terminal import AgentTerminal + +pytestmark = [pytest.mark.managed, pytest.mark.cuj3, pytest.mark.workspace_isolated] + + +@pytest.mark.claude +@pytest.mark.tui +def test_case_03_managed_schema_pointers_claude(live_session, workspace): + """Scenario: configure Claude from the published CUJ3 model schema, then use every model. + + Expected: public configure persists the exact managed schema/defaults. An independent + UC GETs prove all five in-scope services, including Gemini, and both decoys exist. + An independent compatible inventory proves scoped models and the accessible decoy; + generated settings, native gateway cache, and exact numbered picker inventory match the scoped + inventory. Bare ug and explicit ug claude each complete a separate real Claude/Sonnet + TUI file task with native exact default identity and normal exit, without model overrides; + ug claude -p completes a Sonnet print task; every extra model completes a task with native identity. In-scope + Gemini and both out-of-scope decoys are excluded. These intended compatibility assertions + fail if the live backend differs; they are not a claim of live verification. + """ + session = live_session + configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + assert "Select coding agents to configure:" not in configured.stdout, configured.stdout + assert_persisted_config(session, workspace) + + bearer = os.environ["DATABRICKS_BEARER"] + inventory = fetch_model_service_inventory(workspace, bearer) + session.record("cuj3-model-service-sources.json", inventory) + assert set(inventory) == MODEL_SERVICES | {CLAUDE_DECOY, CODEX_DECOY}, inventory + parent_catalog = fetch_claude_parent_catalog(workspace, bearer, MODEL_SCHEMA) + decoy_catalog = fetch_claude_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) + session.record( + "cuj3-parent-inventory.json", + { + "scoped": parent_catalog.model_ids, + "decoy": decoy_catalog.model_ids, + "display_names": parent_catalog.display_names, + "pages": parent_catalog.payloads, + "decoy_pages": decoy_catalog.payloads, + }, + ) + service_ids = [claude_model_service_id(model) for model in parent_catalog.model_ids] + assert len(service_ids) == len(set(service_ids)), parent_catalog + assert set(service_ids) == CLAUDE_MODELS, parent_catalog + assert set(parent_catalog.model_ids) == { + claude_discovery_model_id(model) for model in CLAUDE_MODELS + }, parent_catalog + decoy_ids = [claude_model_service_id(model) for model in decoy_catalog.model_ids] + assert len(decoy_ids) == len(set(decoy_ids)), decoy_catalog + assert set(decoy_ids) == {CLAUDE_DECOY}, decoy_catalog + assert set(decoy_catalog.model_ids) == {claude_discovery_model_id(CLAUDE_DECOY)}, decoy_catalog + + default_task = FileTask(session) + with AgentTerminal( + session, "claude", [str(session.binary)], "cuj3-bare-ug-claude-default-and-picker" + ) as tui: + tui.boot() + settings = json.loads((session.home / ".claude/ucode-settings.json").read_text()) + env = settings["env"] + assert env.get("ANTHROPIC_MODEL") == CLAUDE_DEFAULT, settings + assert env.get("ANTHROPIC_DEFAULT_SONNET_MODEL") == CLAUDE_DEFAULT, settings + assert ( + "x-databricks-use-coding-agent-mode: true" + in env.get("ANTHROPIC_CUSTOM_HEADERS", "").splitlines() + ), settings + assert ( + env.get("ANTHROPIC_CUSTOM_HEADERS", "") + .splitlines() + .count(f"{MODEL_HEADER}: {MODEL_SCHEMA}") + == 1 + ), settings + assert not {"availableModels", "enforceAvailableModels"} & settings.keys(), settings + picker = settings["modelPicker"] + assert picker.get("replaceBuiltInOptions") is True, picker + options = picker["options"] + assert isinstance(options, list) and len(options) == len(parent_catalog.model_ids), picker + picker_ids = [option["model"] for option in options] + assert len(picker_ids) == len(set(picker_ids)), picker + assert set(picker_ids) == set(parent_catalog.model_ids), picker + for option in options: + if display_name := parent_catalog.display_names[option["model"]]: + assert option["label"] == display_name, option + session.record("cuj3-claude-settings.json", settings) + picker_screen = tui.open_model_picker( + model_visible=lambda screen: all( + claude_model_in_picker(screen, model, parent_catalog.display_names[model]) + for model in parent_catalog.model_ids + ) + ) + assert_picker_inventory(picker_screen, "claude", parent_catalog.display_names) + for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): + assert excluded not in picker_screen, picker_screen + assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen + tui.submit(default_task.prompt) + tui.wait_for_task(default_task, timeout=240) + tui.exit_normally() + default_task.assert_completed(session, "claude") + assert_native_model_identity(session, "claude", default_task.value, CLAUDE_DEFAULT) + + explicit_task = FileTask(session) + with AgentTerminal( + session, "claude", [str(session.binary), "claude"], "cuj3-explicit-ug-claude-default" + ) as tui: + tui.boot() + tui.submit(explicit_task.prompt) + tui.wait_for_task(explicit_task, timeout=240) + tui.exit_normally() + explicit_task.assert_completed(session, "claude") + assert_native_model_identity(session, "claude", explicit_task.value, CLAUDE_DEFAULT) + + gateway_ids = session.claude_gateway_model_ids() + assert len(gateway_ids) == len(set(gateway_ids)), gateway_ids + assert set(gateway_ids) == set(parent_catalog.model_ids), gateway_ids + assert not {GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY} & set(gateway_ids), gateway_ids + + print_task = FileTask(session) + result = session.run( + "claude", + "-p", + print_task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + timeout=240, + ) + print_task.assert_headless_answer("claude", result) + print_task.assert_completed(session, "claude") + assert_native_model_identity(session, "claude", print_task.value, CLAUDE_DEFAULT) + + for model in parent_catalog.model_ids: + if model == CLAUDE_DEFAULT: + continue + task = FileTask(session) + result = session.run( + "claude", + "--", + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + "--model", + model, + timeout=240, + ) + task.assert_headless_answer("claude", result) + task.assert_completed(session, "claude") + assert_native_model_identity(session, "claude", task.value, model) + session.assert_not_routed() + + +@pytest.mark.codex +@pytest.mark.tui +def test_case_04_managed_schema_pointers_codex(live_session, workspace): + """Scenario: configure Codex from the published CUJ3 model schema, then use every model. + + Expected: public configure persists the exact managed schema/defaults. An independent + UC GETs prove all five in-scope services, including Gemini, and both decoys exist. + Independent compatible inventories, generated catalogs, real model/list protocol results, + and the exact numbered native /model picker inventory agree. A default TUI task, a default exec task, + and every extra-model task complete with native identity. Codex -p is a native profile + option, not print mode: headless tasks use ug codex -- exec. In-scope Gemini and both + out-of-scope decoys are excluded. Intended compatibility fails if the backend differs; + these expectations are not a claim of live verification. + """ + session = live_session + configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + assert "Select coding agents to configure:" not in configured.stdout, configured.stdout + assert_persisted_config(session, workspace) + + bearer = os.environ["DATABRICKS_BEARER"] + inventory = fetch_model_service_inventory(workspace, bearer) + session.record("cuj3-model-service-sources.json", inventory) + assert set(inventory) == MODEL_SERVICES | {CLAUDE_DECOY, CODEX_DECOY}, inventory + parent_catalog = fetch_codex_parent_catalog(workspace, bearer, MODEL_SCHEMA) + decoy_catalog = fetch_codex_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) + session.record( + "cuj3-parent-inventory.json", + { + "scoped": parent_catalog.model_ids, + "decoy": decoy_catalog.model_ids, + "pages": parent_catalog.payloads, + "decoy_pages": decoy_catalog.payloads, + }, + ) + assert set(parent_catalog.model_ids) == CODEX_MODELS, parent_catalog + assert set(decoy_catalog.model_ids) == {CODEX_DECOY}, decoy_catalog + provider_ids = { + entry.get("slug") for page in parent_catalog.payloads for entry in page["models"] + } + assert not {GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY} & provider_ids, parent_catalog + decoy_provider_ids = { + entry.get("slug") for page in decoy_catalog.payloads for entry in page["models"] + } + assert not {GEMINI_MODEL, CLAUDE_DECOY} & decoy_provider_ids, decoy_catalog + + models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) + assert len(models) == len(set(models)), models + assert set(models) == set(parent_catalog.model_ids), models + assert not {GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY} & set(models), models + catalog_paths = list((session.home / ".ucode").glob("codex-model-catalog-*.json")) + assert len(catalog_paths) == 1, catalog_paths + catalog = json.loads(catalog_paths[0].read_text()) + assert json.loads((session.home / ".ucode/codex-model-catalog.json").read_text()) == catalog + catalog_ids = parse_codex_provider_catalog(catalog) + assert set(catalog_ids) == set(parent_catalog.model_ids), catalog + assert models == list(catalog_ids), (models, catalog_ids) + all_catalog_ids = {entry["slug"] for entry in catalog["models"]} + assert not {GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY} & all_catalog_ids, catalog + display_names = {} + for entry in catalog["models"]: + if entry["slug"] in CODEX_MODELS: + label = entry.get("display_name") + assert isinstance(label, str) and label.strip(), entry + display_names[entry["slug"]] = label + session.record("cuj3-codex-catalog.json", catalog) + + default_task = FileTask(session) + with AgentTerminal( + session, "codex", [str(session.binary), "codex"], "cuj3-codex-default" + ) as tui: + tui.boot() + config = tomllib.loads((session.home / ".codex/ucode.config.toml").read_text()) + assert config.get("model") == CODEX_DEFAULT, config + session.record("cuj3-codex-config.json", config) + picker_screen = tui.open_codex_model_picker( + model_visible=lambda screen: all( + codex_model_in_picker(screen, model, display_names[model]) + for model in parent_catalog.model_ids + ) + ) + assert_picker_inventory(picker_screen, "codex", display_names) + for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): + assert excluded not in picker_screen, picker_screen + assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen + tui.submit(default_task.prompt) + tui.wait_for_task(default_task, timeout=240) + tui.exit_normally() + default_task.assert_completed(session, "codex") + assert_native_model_identity(session, "codex", default_task.value, CODEX_DEFAULT) + + exec_task = FileTask(session) + result = session.run( + "codex", + "--", + "exec", + "--skip-git-repo-check", + "--json", + exec_task.prompt, + timeout=240, + ) + exec_task.assert_headless_answer("codex", result) + exec_task.assert_completed(session, "codex") + assert_native_model_identity(session, "codex", exec_task.value, CODEX_DEFAULT) + + for model in parent_catalog.model_ids: + if model == CODEX_DEFAULT: + continue + task = FileTask(session) + result = session.run( + "codex", + "--", + "exec", + "--skip-git-repo-check", + "--json", + "--model", + model, + task.prompt, + timeout=240, + ) + task.assert_headless_answer("codex", result) + task.assert_completed(session, "codex") + assert_native_model_identity(session, "codex", task.value, model) + session.assert_not_routed() diff --git a/tests/integration/utils/cuj3.py b/tests/integration/utils/cuj3.py new file mode 100644 index 000000000..63ce7cae7 --- /dev/null +++ b/tests/integration/utils/cuj3.py @@ -0,0 +1,323 @@ +"""Black-box model-discovery evidence for the dedicated CUJ3 workspace.""" + +from __future__ import annotations + +import hashlib +import json +import re +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + +from .evidence import agent_sessions, assistant_answers +from .provider_catalog import parse_anthropic_provider_page, parse_codex_provider_catalog + +MODEL_SCHEMA = "ug_e2e.models" +OTHER_MODEL_SCHEMA = "ug_e2e.other_models" +MODEL_HEADER = "Databricks-Model-Service-Parent-Schema" +CLAUDE_DEFAULT = f"{MODEL_SCHEMA}.claude_sonnet" +CODEX_DEFAULT = f"{MODEL_SCHEMA}.gpt_luna" +GEMINI_MODEL = f"{MODEL_SCHEMA}.gemini_flash" +CLAUDE_MODELS = frozenset({CLAUDE_DEFAULT, f"{MODEL_SCHEMA}.claude_haiku", f"{MODEL_SCHEMA}.kimi"}) +CODEX_MODELS = frozenset({CODEX_DEFAULT, f"{MODEL_SCHEMA}.kimi"}) +MODEL_SERVICES = CLAUDE_MODELS | CODEX_MODELS | {GEMINI_MODEL} +CLAUDE_DECOY = f"{OTHER_MODEL_SCHEMA}.claude_decoy" +CODEX_DECOY = f"{OTHER_MODEL_SCHEMA}.codex_decoy" + + +def claude_discovery_model_id(model_id: str) -> str: + """Encode the exact wire ID expected by this workspace's Claude discovery feature.""" + if "claude" in model_id.lower() or "anthropic" in model_id.lower(): + return model_id + checksum = hashlib.sha256(model_id.encode("utf-8")).hexdigest()[:8] + return f"anthropic-aigw-{checksum}-{model_id}" + + +def claude_model_service_id(model_id: str) -> str: + """Recover service identity only from the gateway's checksum-valid discovery alias.""" + alias = re.fullmatch(r"anthropic-aigw-([0-9a-f]{8})-(.+)", model_id) + if alias is None: + return model_id + checksum, original = alias.groups() + if hashlib.sha256(original.encode("utf-8")).hexdigest()[:8] != checksum: + return model_id + return original + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def fetch_model_service_inventory(workspace: str, token: str) -> dict[str, tuple[str, ...]]: + """Prove every fixture exists independently of agent-compatible model lists.""" + origin = urllib.parse.urlsplit(workspace) + assert ( + origin.scheme == "https" + and origin.hostname + and origin.username is None + and origin.password is None + and origin.path in {"", "/"} + and not origin.query + and not origin.fragment + ), "Expected an HTTPS workspace origin without credentials" + assert token, "Expected an explicit workspace bearer" + opener = urllib.request.build_opener(NoRedirect()) + inventory: dict[str, tuple[str, ...]] = {} + for model in sorted(MODEL_SERVICES | {CLAUDE_DECOY, CODEX_DECOY}): + url = f"https://{origin.netloc}/api/2.1/unity-catalog/model-services/{model}" + request = urllib.request.Request( + url, + headers={"Authorization": f"Bearer {token}", "Accept": "application/json"}, + method="GET", + ) + try: + with opener.open(request, timeout=30) as response: + payload = json.load(response) + except urllib.error.HTTPError as error: + raise AssertionError(f"Model service GET {model} returned HTTP {error.code}") from None + except (urllib.error.URLError, OSError, ValueError): + raise AssertionError( + f"Model service GET {model} failed or returned invalid JSON" + ) from None + assert isinstance(payload, dict), f"Model service {model} must be an object" + config = payload.get("config") + assert isinstance(config, dict), f"Model service {model} must have a config" + routing = config.get("routing") + assert isinstance(routing, dict), f"Model service {model} must have routing" + assert routing.get("fallback") in (None, {}, {"destinations": []}), ( + f"Model service {model} must not have fallback destinations" + ) + destinations = routing.get("destinations") + assert isinstance(destinations, list) and len(destinations) == 1, ( + f"Model service {model} must have exactly one routing destination" + ) + sources: list[str] = [] + for destination in destinations: + assert isinstance(destination, dict), f"Invalid destination for {model}" + source = destination.get("name") + assert isinstance(source, str) and re.fullmatch( + r"system\.ai\.[A-Za-z0-9_-]+", source + ), f"Model service {model} must target an existing system.ai source" + assert destination.get("destination_type") == ( + "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL" + ), f"Unexpected destination type for {model}" + target = destination.get("pay_per_token_config") + assert isinstance(target, dict) and target.get("model") == f"models/{source}", ( + f"Model service {model} has inconsistent source routing" + ) + traffic = destination.get("traffic_percentage", 100) + assert type(traffic) is int and traffic == 100, ( + f"Model service {model} must route 100 percent of traffic to its source" + ) + assert all( + destination.get(field, False) is False + for field in ("is_deleted", "is_disabled", "disabled") + ), f"Model service {model} has a disabled or deleted destination" + sources.append(source) + inventory[model] = tuple(sources) + return inventory + + +def codex_model_in_picker(screen: str, model_id: str, display_name: str) -> bool: + """Match native numbered picker rows, not banners, prompt text, or footers.""" + rows = re.findall(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)][ \t]+([^\n]+)$", screen) + return any(_picker_row_matches(row, model_id, display_name) for row in rows) + + +def _picker_row_matches(label: str, model_id: str, display_name: str | None) -> bool: + return any( + re.search(rf"(? None: + """Require exactly one unambiguous numbered native row per expected model.""" + assert agent in {"claude", "codex"}, agent + assert display_names, "Expected a nonempty provider model inventory" + rows = re.findall(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)](?:[ \t]+([^\n]*))?$", screen) + assert rows, f"No numbered {agent} picker rows:\n{screen}" + observed: list[str] = [] + for label in rows: + matched = { + model + for model, display_name in display_names.items() + if _picker_row_matches(label, model, display_name) + } + assert len(matched) == 1, ( + f"Unmatched or ambiguous {agent} picker row {label!r}: {sorted(matched)}" + ) + model = matched.pop() + assert model not in observed, f"Duplicate {agent} picker model row: {model}" + observed.append(model) + assert set(observed) == set(display_names), { + "agent": agent, + "expected": sorted(display_names), + "observed": observed, + } + + +@dataclass(frozen=True) +class ParentCatalog: + """Independently observed provider inventory, labels, and unfiltered payloads.""" + + model_ids: tuple[str, ...] + display_names: dict[str, str | None] + payloads: tuple[dict, ...] + + +def fetch_codex_parent_catalog(workspace: str, token: str, parent_schema: str) -> ParentCatalog: + """Keep the raw Codex catalog so hidden entries cannot conceal exclusions.""" + assert workspace.startswith("https://") and token and parent_schema + url = f"{workspace.rstrip('/')}/ai-gateway/codex/v1/models" + request = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + MODEL_HEADER: parent_schema, + }, + ) + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response: + payload = json.load(response) + except urllib.error.HTTPError as error: + raise AssertionError(f"Codex catalog GET returned HTTP {error.code}") from None + except (urllib.error.URLError, OSError, ValueError): + raise AssertionError("Codex catalog GET failed or returned invalid JSON") from None + model_ids = parse_codex_provider_catalog(payload) + display_names = {} + for entry in payload["models"]: + if entry.get("slug") in model_ids: + label = entry.get("display_name") + assert isinstance(label, str) and label.strip(), "Invalid Codex catalog display name" + display_names[entry["slug"]] = label + return ParentCatalog(model_ids, display_names, (payload,)) + + +def fetch_claude_parent_catalog(workspace: str, token: str, parent_schema: str) -> ParentCatalog: + """Read the real parent-scoped Anthropic catalog with bounded pagination.""" + assert workspace.startswith("https://") and token and parent_schema + model_ids: list[str] = [] + display_names: dict[str, str | None] = {} + pages: list[dict] = [] + cursor: str | None = None + seen_cursors: set[str] = set() + for _page_number in range(20): + query = {"limit": "1000"} + if cursor is not None: + query["after_id"] = cursor + url = ( + f"{workspace.rstrip('/')}/ai-gateway/anthropic/v1/models?" + f"{urllib.parse.urlencode(query)}" + ) + request = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "Anthropic-Version": "2023-06-01", + MODEL_HEADER: parent_schema, + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + payload = json.load(response) + except urllib.error.HTTPError as error: + raise AssertionError(f"GET {url} returned HTTP {error.code}: {error.reason}") from error + except (urllib.error.URLError, OSError) as error: + raise AssertionError(f"GET {url} failed: {error}") from error + page = parse_anthropic_provider_page(payload) + pages.append(payload) + for model_id, display_name in page.models: + assert model_id not in display_names, f"duplicate Claude model id: {model_id}" + model_ids.append(model_id) + display_names[model_id] = display_name + if not page.has_more: + return ParentCatalog(tuple(model_ids), display_names, tuple(pages)) + assert page.last_id is not None and page.last_id not in seen_cursors, payload + seen_cursors.add(page.last_id) + cursor = page.last_id + raise AssertionError("Claude parent catalog exceeded 20 pages") + + +def assert_cuj3_config(config: dict) -> None: + """Require exact model policy while allowing unrelated resource fields.""" + assert config["spec_version"] == 1, config + assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE", config + entries = config["enabled_agents"] + assert isinstance(entries, list) and len(entries) == 2, config + assert all(isinstance(entry, dict) for entry in entries), entries + by_agent = {entry.get("agent"): entry.get("config") for entry in entries} + expected_defaults = { + "CODING_AGENT_CLAUDE_CODE": { + "default_model": CLAUDE_DEFAULT, + "default_sonnet_model": CLAUDE_DEFAULT, + }, + "CODING_AGENT_CODEX": {"default_model": CODEX_DEFAULT}, + } + assert set(by_agent) == set(expected_defaults), by_agent + for agent, defaults in expected_defaults.items(): + agent_config = by_agent[agent] + assert isinstance(agent_config, dict), agent_config + assert agent_config.get("models") == {"unity_catalog_location": MODEL_SCHEMA}, agent_config + assert agent_config.get("default_models") == defaults, agent_config + for key in ("smart_routing", "tracing"): + settings = agent_config.get(key) + assert isinstance(settings, dict) and settings.get("enabled") is False, agent_config + + +def assert_persisted_config(session, workspace: str) -> dict: + """Validate the policy persisted by public configure, never a setup precheck.""" + path = session.home / ".ucode" / "managed-config.json" + assert path.is_file(), path + persisted = json.loads(path.read_text()) + assert isinstance(persisted, dict) and persisted.get("workspace") == workspace, persisted + config = persisted.get("config") + assert isinstance(config, dict), persisted + assert_cuj3_config(config) + session.record("cuj3-managed-config.json", persisted) + return config + + +def native_model_ids(session, agent: str, answer_value: str) -> set[str]: + """Correlate a completed file answer with its native transcript model identity.""" + assert agent in {"claude", "codex"}, agent + found: set[str] = set() + for records in agent_sessions(session, agent).values(): + if agent == "claude": + for record in records: + if not any(answer_value in answer for answer in assistant_answers(agent, [record])): + continue + message = record.get("message") or {} + model = message.get("model") + if isinstance(model, str) and model: + found.add(model.removesuffix("[1m]")) + else: + completed = { + (record.get("payload") or {}).get("turn_id") + for record in records + if record.get("type") == "event_msg" + and (record.get("payload") or {}).get("type") == "task_complete" + and answer_value in ((record.get("payload") or {}).get("last_agent_message") or "") + } + assert all(isinstance(turn_id, str) and turn_id for turn_id in completed), completed + for record in records: + if record.get("type") != "turn_context": + continue + payload = record.get("payload") or {} + if payload.get("turn_id") in completed and isinstance(payload.get("model"), str): + found.add(payload["model"]) + return found + + +def assert_native_model_identity(session, agent: str, answer_value: str, expected: str) -> None: + observed = native_model_ids(session, agent, answer_value) + session.record( + f"native-model-{agent}-{answer_value[:12]}.json", + {"expected": expected, "observed": sorted(observed)}, + ) + assert observed == {expected}, {"expected": expected, "observed": sorted(observed)} diff --git a/tests/integration/utils/terminal.py b/tests/integration/utils/terminal.py index d9336c341..85240dd59 100644 --- a/tests/integration/utils/terminal.py +++ b/tests/integration/utils/terminal.py @@ -339,6 +339,31 @@ def open_model_picker(self, *, model_visible=None): ) return screen + def open_codex_model_picker(self, *, model_visible): + """Capture Codex's native numbered /model menu, then dismiss it with Escape.""" + assert self.agent == "codex", self.agent + self.submit("/model") + self.wait_for( + lambda text: ( + "select model" in text.lower() + and re.search(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)][ \t]+\S", text) + ), + "Codex's numbered model picker", + timeout=60, + ) + self.wait_for(model_visible, "all scoped models in Codex's picker", timeout=60) + screen = self.visible + self.actions.append({"reason": "codex-model-picker-visible", "screen": screen}) + self.send("\x1b", "close Codex's model picker without changing its default") + self.wait_for( + lambda text: ( + "select model" not in text.lower() + and re.search(r"(?m)^\s*[❯›>]\s*(?!\d+[.)])", text) + ), + "the Codex prompt after closing the model picker", + ) + return screen + def wait_for_task(self, task, timeout=180): permission_in_progress = False diff --git a/tests/test_integration_cuj3.py b/tests/test_integration_cuj3.py new file mode 100644 index 000000000..610c39d78 --- /dev/null +++ b/tests/test_integration_cuj3.py @@ -0,0 +1,357 @@ +"""Component checks for CUJ3 discovery helpers and native model evidence.""" + +import io +import json +import runpy +import urllib.error +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.integration.utils import cuj3 + +ROOT = Path(__file__).parent.parent +SCRIPT = ROOT / "fixtures/cuj/models/provision.py" + + +@pytest.fixture +def provisioner(): + return SimpleNamespace(**runpy.run_path(str(SCRIPT))) + + +def model_readback(*, server_defaults): + destination = { + "name": "system.ai.databricks-gpt-6-luna", + "destination_type": "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", + "pay_per_token_config": {"model": "models/system.ai.databricks-gpt-6-luna"}, + } + routing = {"destinations": [destination]} + if server_defaults: + destination.update(traffic_percentage=100, is_deleted=False) + routing["fallback"] = {"destinations": []} + return { + "name": "model-services/ug_e2e.models.gpt_luna", + "config": {"routing": routing, "usage_tracking": {"enabled": True}}, + } + + +INVALID_DESTINATIONS = [ + {"name": "system.ai.wrong_source"}, + {"name": "other.ai.fixture"}, + {"destination_type": "DESTINATION_TYPE_EXTERNAL_MODEL"}, + {"pay_per_token_config": {"model": "models/system.ai.wrong_source"}}, + {"pay_per_token_config": None}, + {"traffic_percentage": 0}, + {"traffic_percentage": 50}, + {"traffic_percentage": "100"}, + {"traffic_percentage": None}, + {"traffic_percentage": True}, + {"is_deleted": True}, + {"is_deleted": "false"}, + {"is_disabled": True}, + {"disabled": True}, +] + + +@pytest.mark.parametrize( + ("wire_id", "expected"), + [ + ("ug_e2e.models.kimi", "ug_e2e.models.kimi"), + ("anthropic-aigw-7bf7897e-ug_e2e.models.kimi", "ug_e2e.models.kimi"), + ("ug_e2e.models.claude_sonnet", "ug_e2e.models.claude_sonnet"), + ( + "anthropic-aigw-00000000-ug_e2e.models.kimi", + "anthropic-aigw-00000000-ug_e2e.models.kimi", + ), + ( + "anthropic-aigw-7bf7897e-ug_e2e.models.kimi_v2", + "anthropic-aigw-7bf7897e-ug_e2e.models.kimi_v2", + ), + ( + "anthropic-aigw-7bf7897e-ug_e2e.other_models.kimi", + "anthropic-aigw-7bf7897e-ug_e2e.other_models.kimi", + ), + ( + "anthropic-aigw-7BF7897E-ug_e2e.models.kimi", + "anthropic-aigw-7BF7897E-ug_e2e.models.kimi", + ), + ], +) +def test_claude_service_identity_requires_exact_gateway_alias_checksum(wire_id, expected): + assert cuj3.claude_model_service_id(wire_id) == expected + + +def test_claude_discovery_requires_exact_gateway_wire_ids(): + assert {cuj3.claude_discovery_model_id(model) for model in cuj3.CLAUDE_MODELS} == { + "ug_e2e.models.claude_haiku", + "ug_e2e.models.claude_sonnet", + "anthropic-aigw-7bf7897e-ug_e2e.models.kimi", + } + assert cuj3.claude_discovery_model_id(cuj3.CLAUDE_DECOY) == cuj3.CLAUDE_DECOY + + +def test_model_discovery_matrix_filters_families_without_changing_schema_scope(provisioner): + expected_models = { + "ug_e2e.models.gpt_luna", + "ug_e2e.models.claude_haiku", + "ug_e2e.models.claude_sonnet", + "ug_e2e.models.kimi", + "ug_e2e.models.gemini_flash", + } + assert cuj3.MODEL_SERVICES == expected_models + assert cuj3.CLAUDE_MODELS == { + "ug_e2e.models.claude_haiku", + "ug_e2e.models.claude_sonnet", + "ug_e2e.models.kimi", + } + assert cuj3.CODEX_MODELS == {"ug_e2e.models.gpt_luna", "ug_e2e.models.kimi"} + assert cuj3.GEMINI_MODEL in cuj3.MODEL_SERVICES + assert cuj3.GEMINI_MODEL not in cuj3.CLAUDE_MODELS | cuj3.CODEX_MODELS + assert cuj3.CLAUDE_DEFAULT in cuj3.CLAUDE_MODELS + assert cuj3.CODEX_DEFAULT in cuj3.CODEX_MODELS + assert cuj3.CLAUDE_DECOY not in cuj3.MODEL_SERVICES + assert cuj3.CODEX_DECOY not in cuj3.MODEL_SERVICES + assert { + f"ug_e2e.models.{leaf}" for leaf in provisioner.MODEL_LEAVES if not leaf.endswith("decoy") + } == expected_models + + +@pytest.mark.parametrize("server_defaults", [False, True]) +def test_live_inventory_proves_gemini_exists_without_using_agent_discovery( + monkeypatch, server_defaults +): + requests = [] + handlers = [] + + class InventoryOpener: + def open(self, request, *, timeout): + requests.append((request, timeout)) + return io.BytesIO(json.dumps(model_readback(server_defaults=server_defaults)).encode()) + + def build_opener(handler): + handlers.append(handler) + return InventoryOpener() + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", build_opener) + inventory = cuj3.fetch_model_service_inventory("https://workspace.invalid/", "test-bearer") + expected = cuj3.MODEL_SERVICES | {cuj3.CLAUDE_DECOY, cuj3.CODEX_DECOY} + assert set(inventory) == expected + assert inventory[cuj3.GEMINI_MODEL] == ("system.ai.databricks-gpt-6-luna",) + assert len(requests) == len(expected) + assert len(handlers) == 1 and isinstance(handlers[0], cuj3.NoRedirect) + for request, timeout in requests: + assert request.get_method() == "GET" + assert request.full_url.startswith( + "https://workspace.invalid/api/2.1/unity-catalog/model-services/" + ) + assert request.data is None + assert request.get_header("Authorization") == "Bearer test-bearer" + assert timeout == 30 + + +@pytest.mark.parametrize("destination_updates", INVALID_DESTINATIONS) +def test_live_inventory_rejects_invalid_source_routing(monkeypatch, destination_updates): + payload = model_readback(server_defaults=True) + payload["config"]["routing"]["destinations"][0].update(destination_updates) + + class InvalidOpener: + def open(self, request, *, timeout): + return io.BytesIO(json.dumps(payload).encode()) + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: InvalidOpener()) + with pytest.raises(AssertionError): + cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") + + +@pytest.mark.parametrize( + "payload", + [ + None, + {}, + {"config": {}}, + {"config": {"routing": {"destinations": []}}}, + {"config": {"routing": {"destinations": [None]}}}, + {"config": {"routing": {"destinations": [{"name": "system.ai.fixture"}]}}}, + ], +) +def test_live_inventory_rejects_missing_or_malformed_services(monkeypatch, payload): + class InvalidOpener: + def open(self, request, *, timeout): + return io.BytesIO(json.dumps(payload).encode()) + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: InvalidOpener()) + with pytest.raises(AssertionError): + cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") + + +def test_live_inventory_fails_on_missing_gemini_even_when_other_models_exist( + provisioner, monkeypatch +): + class MissingGeminiOpener: + def open(self, request, *, timeout): + if request.full_url.endswith(cuj3.GEMINI_MODEL): + raise urllib.error.HTTPError(request.full_url, 404, "Not found", {}, None) + return io.BytesIO(json.dumps(provisioner.model_body("system.ai.fixture")).encode()) + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: MissingGeminiOpener()) + with pytest.raises(AssertionError, match=r"gemini_flash.*HTTP 404"): + cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") + + +@pytest.mark.parametrize("fallback", [False, True]) +def test_live_inventory_rejects_multiple_backing_models(provisioner, monkeypatch, fallback): + payload = provisioner.model_body("system.ai.fixture") + routing = payload["config"]["routing"] + if fallback: + routing["fallback"] = {"destinations": routing["destinations"]} + else: + routing["destinations"] *= 2 + + class MultipleSourceOpener: + def open(self, request, *, timeout): + return io.BytesIO(json.dumps(payload).encode()) + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: MultipleSourceOpener()) + with pytest.raises(AssertionError): + cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") + + +@pytest.mark.parametrize("cursor", ["", "❯ ", "› ", "> "]) +@pytest.mark.parametrize("label", ["ug_e2e.models.gpt_luna", "GPT Luna"]) +def test_codex_picker_matches_model_only_in_numbered_rows(cursor, label): + screen = f"Select Model and Effort\n {cursor}1. {label} (current)\nEnter to confirm" + assert cuj3.codex_model_in_picker(screen, cuj3.CODEX_DEFAULT, "GPT Luna") + + +@pytest.mark.parametrize( + "screen", + [ + "GPT Luna\nSelect Model and Effort\n 1. Other model", + "Select Model and Effort\n 1. Other model\nCurrent model: GPT Luna", + " 1. ug_e2e.models.gpt_luna_v2", + " 1.\nGPT Luna", + "", + ], +) +def test_codex_picker_rejects_nonrows_and_different_model_ids(screen): + assert not cuj3.codex_model_in_picker(screen, cuj3.CODEX_DEFAULT, "GPT Luna") + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_picker_inventory_accepts_exact_rows_and_ignores_banner_text(agent): + models = cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS + labels = {model: model.rsplit(".", 1)[-1].replace("_", " ").title() for model in models} + rows = "\n".join( + f" {position}. {label}" for position, label in enumerate(labels.values(), start=1) + ) + screen = f"Banner: Gemini Flash\nSelect model\n{rows}\nEnter to confirm" + cuj3.assert_picker_inventory(screen, agent, labels) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("excluded_label", ["Gemini Flash", "Friendly out-of-scope model"]) +def test_picker_inventory_rejects_extra_friendly_label_rows(agent, excluded_label): + models = cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"Select model\n{rows}\n {len(models) + 1}. {excluded_label}" + with pytest.raises(AssertionError): + cuj3.assert_picker_inventory(screen, agent, labels) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_picker_inventory_rejects_duplicate_and_missing_rows(agent): + models = sorted(cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + with pytest.raises(AssertionError): + cuj3.assert_picker_inventory(f"{rows}\n {len(models) + 1}. {models[0]}", agent, labels) + with pytest.raises(AssertionError): + cuj3.assert_picker_inventory(f" 1. {models[0]}", agent, labels) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_picker_inventory_rejects_ambiguous_labels_and_split_line_evidence(agent): + models = sorted(cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS) + with pytest.raises(AssertionError): + cuj3.assert_picker_inventory( + " 1. Shared label", agent, dict.fromkeys(models, "Shared label") + ) + with pytest.raises(AssertionError): + cuj3.assert_picker_inventory(f" 1.\n{models[0]}", agent, dict.fromkeys(models, None)) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("suffix", ["_v2", "-decoy", ".other"]) +def test_picker_inventory_rejects_model_id_prefix_matches(agent, suffix): + models = sorted(cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS) + rows = "\n".join( + f" {position}. {model}{suffix if position == 1 else ''}" + for position, model in enumerate(models, start=1) + ) + with pytest.raises(AssertionError): + cuj3.assert_picker_inventory(rows, agent, dict.fromkeys(models, None)) + + +def test_model_policy_allows_server_metadata_and_later_stack_fields(): + config = json.loads(SCRIPT.with_name("cuj3-managed-config.json").read_text()) + config["update_time"] = "2026-10-02T00:00:00Z" + config["name"] = "coding-agent-configs/cuj3" + config["mcp_servers"] = {"unity_catalog_location": "ug_e2e.tools"} + cuj3.assert_cuj3_config(config) + + +def test_model_policy_rejects_a_default_outside_the_managed_scope(): + config = json.loads(SCRIPT.with_name("cuj3-managed-config.json").read_text()) + config["enabled_agents"][1]["config"]["default_models"]["default_model"] = cuj3.CODEX_DECOY + with pytest.raises(AssertionError): + cuj3.assert_cuj3_config(config) + + +def test_codex_model_identity_uses_only_the_completed_answer_turn(monkeypatch): + records = [ + {"type": "turn_context", "payload": {"turn_id": "other", "model": cuj3.CODEX_DECOY}}, + {"type": "turn_context", "payload": {"turn_id": "matching", "model": cuj3.CODEX_DEFAULT}}, + { + "type": "event_msg", + "payload": { + "type": "task_complete", + "turn_id": "matching", + "last_agent_message": "withheld-file-value", + }, + }, + ] + monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) + assert cuj3.native_model_ids(None, "codex", "withheld-file-value") == {cuj3.CODEX_DEFAULT} + + +def test_codex_model_identity_rejects_prompt_only_evidence(monkeypatch): + records = [ + {"type": "turn_context", "payload": {"turn_id": "matching", "model": cuj3.CODEX_DEFAULT}}, + { + "type": "response_item", + "payload": {"type": "message", "role": "user", "content": "withheld-file-value"}, + }, + ] + monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) + assert cuj3.native_model_ids(None, "codex", "withheld-file-value") == set() + + +def test_claude_model_identity_uses_assistant_answer_not_tool_output(monkeypatch): + records = [ + { + "type": "user", + "message": {"model": cuj3.CLAUDE_DECOY, "content": [{"type": "text", "text": "value"}]}, + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "model": cuj3.CLAUDE_DEFAULT, + "content": [{"type": "text", "text": "value"}], + }, + }, + ] + monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) + assert cuj3.native_model_ids(None, "claude", "value") == {cuj3.CLAUDE_DEFAULT} From 3a8548b26cf2b5fed40f117dc836fde801a68271 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 17:59:32 +0000 Subject: [PATCH 03/19] AIGTWY-4878 Harden CUJ3 model evidence and simplify test helpers --- .../managed-config.json} | 0 fixtures/cuj/models/README.md | 129 +++--- tests/README.md | 7 +- tests/cuj3_fixture_data.py | 35 ++ tests/integration/README.md | 78 ++-- .../test_ug_cuj3_model_discovery.py | 51 +-- tests/integration/utils/cuj3.py | 218 +++++---- tests/test_cuj3_model_fixtures.py | 38 +- tests/test_integration_cuj3.py | 420 ++++++++++++++++-- 9 files changed, 665 insertions(+), 311 deletions(-) rename fixtures/{cuj/models/cuj3-managed-config.json => cuj-3/managed-config.json} (100%) create mode 100644 tests/cuj3_fixture_data.py diff --git a/fixtures/cuj/models/cuj3-managed-config.json b/fixtures/cuj-3/managed-config.json similarity index 100% rename from fixtures/cuj/models/cuj3-managed-config.json rename to fixtures/cuj-3/managed-config.json diff --git a/fixtures/cuj/models/README.md b/fixtures/cuj/models/README.md index 8600d68cd..1d94ace1a 100644 --- a/fixtures/cuj/models/README.md +++ b/fixtures/cuj/models/README.md @@ -1,11 +1,9 @@ # CUJ3 model-only fixtures -This standalone layer adapts the full CUJ3 provisioner and managed config for -model discovery only. It owns seven UC model services and, when missing, the -`ug_e2e` catalog and its `models` and `other_models` schemas. It does not require, -inspect, create, or configure MCP services, skills, Databricks Apps, connections, -agent installations, credentials, grants, or managed-config publication. Later -MCP/skill CUJ stacks are separate layers, not prerequisites for this layer. +This layer provisions seven UC model services and, when missing, the `ug_e2e` +catalog and its `models` and `other_models` schemas. Apps, connections, MCP, and +skills are not prerequisites. Agent installation, credentials, grants, and +managed-config publication are separate operator setup. | Schema | Services | Purpose | | --- | --- | --- | @@ -15,27 +13,22 @@ MCP/skill CUJ stacks are separate layers, not prerequisites for this layer. | `ug_e2e.models` | `gemini_flash` | Gemini family; intended exclusion from both agents | | `ug_e2e.other_models` | `claude_decoy`, `codex_decoy` | Compatible, accessible, out-of-scope decoys | -Service identities are stable aliases, not inferred source model names. `gpt_luna` -replaces the earlier `gpt_terra` choice. The provisioner maps every leaf to its -schema explicitly; inventory ordering does not determine scope. +Service identities are stable aliases, not source model names. Each leaf maps +explicitly to its schema; inventory ordering does not determine scope. ## Prerequisites and setup order 1. Use Python 3.12+ and select the exact dedicated CUJ3 HTTPS workspace origin. No SDK, CLI profile, or implicit workspace/authentication fallback is used. 2. An operator must explicitly select seven existing, canonical `system.ai.` - registered-model FQNs in that workspace/region, including both decoys. These are - backing UC models, not friendly model-service aliases. Live GET checks verified - that `system.ai.gpt-6-luna` is not a registered model, while - `system.ai.databricks-gpt-6-luna` exists; supply the latter for that chosen version. - Before any writes, both `--validate` and `--apply` GET every unique source at + registered-model FQNs in that workspace/region, including both decoys, not + friendly model-service aliases. Both `--validate` and `--apply` GET every unique source at `/api/2.1/unity-catalog/models/{source}` and require its `full_name` to exactly - match the explicit source. Missing or mismatched sources fail without writes; - the script never resolves aliases or chooses a replacement. Choose the intended - GPT, Claude Haiku, Claude Sonnet, Kimi, and Gemini Flash families for their aliases. + match the explicit source before any writes. Missing or mismatched sources fail; + the script never resolves aliases, discovers/replaces sources, or guesses versions/defaults. + Choose GPT, Claude Haiku, Claude Sonnet, Kimi, and Gemini Flash sources for their aliases. Haiku, Sonnet, Kimi, and the Claude decoy must support Claude's wire protocol; GPT Luna, Kimi, and the Codex decoy must support Codex's wire protocol. - The script never guesses versions, discovers sources, or supplies defaults. Decoys may reuse compatible in-scope sources; their UC identities remain distinct. 3. For `--validate` or `--apply` only, supply a workspace bearer through `DATABRICKS_BEARER` (or an explicitly chosen @@ -49,17 +42,15 @@ schema explicitly; inventory ordering does not determine scope. Separately grant the CUJ execution principal `USE_CATALOG`, `USE_SCHEMA`, and the required model-service invocation/discovery permissions for **both** schemas. Decoys must be accessible so exclusion proves scope filtering, not missing grants. -5. Run the default offline plan (no bearer or network needed), review the workspace, - all seven mappings, permissions, and request format. Use `--validate` for an - authenticated read-only inventory check; missing resources cause a nonzero exit. - After review, explicitly add `--apply` to create missing resources. Rerun - `--validate`: it should report zero missing resources and exit successfully. -6. Review `cuj3-managed-config.json` before an authorized administrator publishes - it separately. Publishing changes workspace policy; this script never does it. - The config points both agents to `ug_e2e.models`, defaults the agent to Claude, - defaults Claude to Sonnet and Codex to GPT Luna, disables - smart routing and tracing, and contains no MCP or skill scope. Those omissions - do not promise to remove unrelated pre-existing local MCP/skill configuration. +5. Review the offline plan's workspace, seven mappings, permissions, and request + format. Run `--validate`, then explicitly use `--apply` for missing resources. + Rerun `--validate` and require zero missing resources and a successful exit. +6. An authorized administrator separately reviews and publishes + `fixtures/cuj-3/managed-config.json`; this script never changes workspace policy. + It points both agents to `ug_e2e.models`, selects Claude as the default agent, + Sonnet for Claude and GPT Luna for Codex, disables smart routing/tracing, + and has no MCP/skill scope. + Those omissions do not remove unrelated pre-existing local MCP/skill config. From the repository root, without needing a bearer for the default offline plan: @@ -76,18 +67,15 @@ python3 fixtures/cuj/models/provision.py \ --codex-decoy-source "$CODEX_DECOY_SOURCE" ``` -Each source variable must contain a concrete `system.ai.` FQN. The default -plan validates only argument syntax and prints the intended inventory; it performs -no network requests, reads no bearer, and cannot determine what already exists. -`--validate` requires a bearer, performs GETs, validates every existing service, -and fails if any resource is missing, without writing. `--validate` and `--apply` -are mutually exclusive. `--apply` performs the full preflight **before any POST**, then -creates only missing resources in catalog → schemas → services order and reads back -all seven services for validation. Existing routing must exactly match the expected -single destination, destination type, and source-model reference. Mismatches abort; -there is no overwrite, update, delete, or cleanup path. Concurrent create conflicts -and partial failures stop the run; inspect the inventory with `--validate` -before retrying. This is not an atomic transaction, and created resources persist. +Each source variable is a concrete `system.ai.` FQN. The default plan checks +syntax and prints inventory without network access or reading a bearer; it cannot +check existing resources. `--validate` performs authenticated read-only GETs and +fails on missing resources or invalid services. It is mutually exclusive with +`--apply`, which completes full source/inventory/routing preflight **before any POST**, +creates only missing catalog → schemas → services, and reads back all seven services. +Mismatches abort: there is no overwrite, update, delete, or cleanup path. Concurrent +create conflicts and partial failures stop the non-atomic run; created resources +persist. Inspect with `--validate` before retrying. ## Compatibility and validation boundary @@ -98,43 +86,30 @@ The intended agent-specific catalog contract is: | Claude | `claude_haiku`, `claude_sonnet`, `kimi` | `claude_sonnet` | | Codex | `gpt_luna`, `kimi` | `gpt_luna` | -`gemini_flash` is intentionally excluded from both catalogs. This is an expected -contract, **not proven by provisioning**. Both agents share the same schema pointer; -API-specific compatibility filtering must produce these sets, not a fixture-side -static allowlist. The compatible decoy for each agent must be discoverable under -`ug_e2e.other_models` but absent from the `ug_e2e.models` catalog. - -Discovery is not validated inference: a catalog entry, generated configuration, -picker row, or successful startup alone does not prove a callable model. Each -expected in-scope model must complete a real file task with native model identity -evidence in the live CUJ. Provisioning validation checks routing only and sends no -inference requests; it cannot prove source availability or agent compatibility. - -The model-service API uses `/api/2.1/unity-catalog/model-services`, a schema -`parent`, and canonical snake_case fields matching the local Python SDK sample -`experimental/eng-ai-governance/api-examples/python-sdk/model_service.py` in Universe. +`gemini_flash` is excluded from both catalogs. API-specific filtering of the shared +schema pointer must produce these exact sets, without a fixture-side allowlist. +Each compatible decoy must be discoverable under `ug_e2e.other_models` but absent +from the managed catalog. + +The API uses `/api/2.1/unity-catalog/model-services`, a schema `parent`, and +canonical snake_case fields from Universe's Python SDK model-service sample. Creation sets `pay_per_token_config.model = models/system.ai.` and -`traffic_percentage = 100`. Usage tracking is always on; its removed/reserved -configuration field is omitted, as specified by -`managed-catalog/atlas/modules/model_service/model_service.proto` in Universe. -Readback validation requires exactly one destination with the selected source name, -pay-per-token destination type, and matching model target. It accepts server-added -metadata and defaults: `is_deleted = false`, empty fallback routing, or omitted -traffic percentage (the SDK documents that a single primary receives all traffic). -Deleted/disabled destinations, explicit traffic percentages other than 100, -fallback destinations, and source/type/target mismatches fail without overwriting -existing services. Usage-tracking metadata is not part of routing validation. -Actual workspace API availability, source existence, and -Claude/Codex inference wire compatibility are **not verified by offline checks**. -Operator confirmation against the intended workspace is required before apply; -service metadata validation alone cannot prove completed inference. - -After separate publication and agent setup, root-owned CUJ tests should verify each -agent's compatible in-scope default/extra models completes inference with the -expected service identity, its compatible decoy is independently discoverable and -callable outside the managed scope, and managed discovery excludes that decoy. -This layer neither runs inference nor installs/configures either agent. Tests, CI, -and broader CUJ documentation remain outside this directory's ownership. +`traffic_percentage = 100`; usage tracking is always on and its removed/reserved +configuration field is omitted. Preflight/readback require one destination with +the selected source name, pay-per-token type, and matching model target. Server +metadata/defaults (`is_deleted = false`, empty fallback routing, omitted traffic +percentage for a single primary) are accepted. Deleted/disabled destinations, +explicit traffic other than 100, fallbacks, or source/type/target mismatches fail. +Usage-tracking metadata is not a routing check. + +Offline checks do not verify workspace API availability, source existence, or wire +compatibility; confirm these in the intended workspace before apply. Provisioning +checks source metadata and routing, not inference availability or compatibility. +After publication and agent setup, live CUJs must compare discovery/pickers and +complete file tasks on every in-scope compatible model. Claude evidence is the +response-reported model; Codex evidence is the client-selected model. Neither proves +the executed gateway backing destination. Discovery, config, or startup alone is +insufficient; tests and CI remain outside this fixture-only layer. Suggested focused local checks (no workspace or bearer needed): diff --git a/tests/README.md b/tests/README.md index facdd6a51..7a9acc605 100644 --- a/tests/README.md +++ b/tests/README.md @@ -187,7 +187,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | | `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`); explicitly launch `ug claude` above the tier | Bare launches check Claude/Sonnet below the tier and the Codex/Luna recommendation above it; `ug usage` agrees with backend spend, threshold, and percentage. Explicit `ug claude` displays the backend's Codex/Luna recommendation while its generated model setting and native header select Sonnet. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | -| CUJ3 model discovery (`test_case_03_*`, `test_case_04_*`) | Configure both agents in the dedicated workspace using the published `ug_e2e.models` schema pointer | Claude exposes Sonnet/Haiku/Kimi, Codex exposes GPT Luna/Kimi, and Gemini plus all other-schema entries are absent from exact native picker inventories. Bare `ug` and explicit `ug claude` select Claude/Sonnet for separate TUI tasks; Claude print mode, Codex TUI, and Codex exec retain their configured defaults. Every additional compatible model completes a headless task with native model identity. Native picker and task coverage still require a live journey pass; MCP and skills are deferred | +| CUJ3 model discovery (`test_case_03_*`, `test_case_04_*`) | Configure published `ug_e2e.models` policy in the dedicated workspace | Exact Claude Sonnet/Haiku/Kimi and Codex GPT Luna/Kimi catalogs/pickers; Gemini and other-schema decoys excluded. Bare `ug` and `ug claude` complete separate Sonnet TUI tasks; Claude print and Codex TUI/exec tasks use configured defaults without overrides. Every extra compatible model completes a headless task; routing stays off. Model evidence is limited as described below; MCP/skills are deferred | | `test_managed_fixture_claude_rejects_provider_override`, `test_managed_fixture_fresh_claude_rejects_provider_override`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | | `test_managed_fixture_codex_rejects_provider_override_after_configure`, `test_managed_fixture_codex_rejects_provider_override_from_fresh_state`, `test_case_06_*` | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state | @@ -230,7 +230,10 @@ Claude discovery assertions match numbered picker rows, not startup banners or footers. Offline regressions cover that distinction, native Haiku/Opus/Sonnet deduplication, and raw catalog ID/display-name rows for scoped pickers. Managed discovery expectations come from separate read-only, provider-scoped -model-list requests; they do not rely solely on ug's generated catalog. +model-list requests; they do not rely solely on ug's generated catalog. CUJ3 tasks +check Claude's response-reported model and Codex's client-selected model; neither +proves the executed gateway backing destination. Native picker/task coverage +requires a live journey pass, not collection or lint success. The dedicated-workspace CUJ suite adds three independently runnable CUJ2 MPS+MCP cases: one configuration case and separate Codex and Claude TUI task cases. The workspace is diff --git a/tests/cuj3_fixture_data.py b/tests/cuj3_fixture_data.py new file mode 100644 index 000000000..51d59c591 --- /dev/null +++ b/tests/cuj3_fixture_data.py @@ -0,0 +1,35 @@ +"""Shared model-service inputs for CUJ3 component tests.""" + + +def model_readback(*, server_defaults): + destination = { + "name": "system.ai.databricks-gpt-6-luna", + "destination_type": "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", + "pay_per_token_config": {"model": "models/system.ai.databricks-gpt-6-luna"}, + } + routing = {"destinations": [destination]} + if server_defaults: + destination.update(traffic_percentage=100, is_deleted=False) + routing["fallback"] = {"destinations": []} + return { + "name": "model-services/ug_e2e.models.gpt_luna", + "config": {"routing": routing, "usage_tracking": {"enabled": True}}, + } + + +INVALID_DESTINATIONS = [ + {"name": "system.ai.wrong_source"}, + {"name": "other.ai.fixture"}, + {"destination_type": "DESTINATION_TYPE_EXTERNAL_MODEL"}, + {"pay_per_token_config": {"model": "models/system.ai.wrong_source"}}, + {"pay_per_token_config": None}, + {"traffic_percentage": 0}, + {"traffic_percentage": 50}, + {"traffic_percentage": "100"}, + {"traffic_percentage": None}, + {"traffic_percentage": True}, + {"is_deleted": True}, + {"is_deleted": "false"}, + {"is_disabled": True}, + {"disabled": True}, +] diff --git a/tests/integration/README.md b/tests/integration/README.md index cb8744b61..1c2981623 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -562,23 +562,20 @@ cannot still be running when that gate passes. Full coverage on PRs needs no lab ### CUJ3 model-discovery stack -The first CUJ3 layer covers only models, for both Claude and Codex. Provision -`ug_e2e.models` and `ug_e2e.other_models` using `fixtures/cuj/models/provision.py`, -then review and publish `fixtures/cuj/models/cuj3-managed-config.json` in the -dedicated workspace. The model-only fixtures do not require an App, UC connection, -MCP service, or skill bundle. Resource provisioning and config publication are -explicit admin setup, not work performed by the integration tests. - -Each journey calls the real `ug configure` before checking its persisted schema -pointer and defaults. A separate UC inventory check proves that all five in-scope -services exist, including Gemini. It independently requests each agent's scoped catalog, -proves the compatible decoy is accessible in another schema, and compares the -generated catalog and native picker/model-list with that oracle. Native -pickers must contain exactly the expected numbered model rows, with no unmatched, -ambiguous, or duplicate entries; friendly labels cannot hide an excluded service. -The configured default and every additional in-scope compatible model must complete a file task -whose contents are withheld from the prompt, with native model identity evidence. -Discovery or startup alone cannot pass. +CUJ3 first covers Claude/Codex models only. An authorized admin provisions both +schemas with `fixtures/cuj/models/provision.py` and separately reviews/publishes +`fixtures/cuj-3/managed-config.json` in the dedicated workspace. Follow the fixture +README's prerequisites, explicit auth, and preflight-before-write rules. Tests +do not provision resources or policy; Apps, connections, MCP, and skills are not required. + +Each journey runs real `ug configure`, checks persisted schema/defaults and all +seven UC services, and independently fetches scoped and compatible-decoy catalogs. +Generated settings/catalogs, native caches/model lists, and exact numbered picker +rows must agree: no unmatched, ambiguous, duplicate, or excluded entries hidden +by labels. Defaults and every extra compatible model must complete file tasks +with values withheld from prompts. Claude checks the response-reported model; +Codex checks the client-selected model. Neither proves the executed gateway backing +destination; discovery or startup alone cannot pass. The version-agnostic services under `ug_e2e.models` and expected discovery sets are: @@ -590,31 +587,23 @@ The version-agnostic services under `ug_e2e.models` and expected discovery sets | `kimi` | Included | Included | | `gemini_flash` | Excluded | Excluded | -Gemini is inside the managed schema: its exclusion tests agent compatibility, -not schema scoping. The accessible compatible services -`ug_e2e.other_models.claude_decoy` and `ug_e2e.other_models.codex_decoy` separately -test schema exclusion. Exact backing `system.ai` source versions are supplied -explicitly during provisioning. These expected subsets are an intended contract, -not a claim of a live pass: gateway API-type metadata, feature flags, and harness -compatibility must be verified in the dedicated workspace. Unexpected Gemini -visibility fails the tests; no local filtering hides it. - -When the gateway's Claude discovery-alias feature is enabled, non-Claude IDs such -as Kimi are returned as `anthropic-aigw-<8-character SHA-256 prefix>-`. -The journey verifies the checksum and exact service inventory without dropping -entries. Generated settings, the native cache, picker, explicit tasks, and native -transcript identity must retain the actual returned wire ID; Codex keeps raw FQNs. -The dedicated workspace enables this feature: its Claude journey requires the -exact encoded wire inventory as well as service identities and the coding-agent -mode header needed for inference to reverse the alias. - -With the published policy selecting Claude as the default agent, both bare `ug` -and explicit `ug claude` must launch Claude/Sonnet and complete separate real TUI -tasks. `ug claude -p "task"` must -complete a print-mode task on Sonnet without a model override. Codex's default -must complete both a TUI task and `ug codex -- exec --skip-git-repo-check --json "task"` -on `gpt_luna`, also without a model override. Native Codex `-p` selects a profile, -not a headless prompt; `ug codex -p "task"` is not the supported task command. +In-schema Gemini tests compatibility; accessible `ug_e2e.other_models.claude_decoy` +and `ug_e2e.other_models.codex_decoy` test scope exclusion. Provisioning takes explicit +backing `system.ai` versions. These sets require live verification of gateway API-type +metadata, feature flags, and harness compatibility; unexpected Gemini visibility +fails without local filtering. Offline checks are not a live pass. + +The dedicated workspace must enable Claude discovery aliases, including Kimi's +`anthropic-aigw-<8-character SHA-256 prefix>-` form. The journey checks +checksums, exact service/wire inventories, and the coding-agent-mode header needed +to reverse aliases for inference. Settings, cache, picker, tasks, and response-reported +models retain returned wire IDs; Codex keeps raw FQNs. No entries are dropped. + +Bare `ug` and explicit `ug claude` each complete a separate Claude/Sonnet TUI task +and exit normally; `ug claude -p "task"` completes a Sonnet print task. Codex completes +both TUI and `ug codex -- exec --skip-git-repo-check --json "task"` tasks on `gpt_luna`. +All default tasks omit model overrides; routing stays off. Codex `-p` selects a native +profile, not a headless prompt. CI runs `managed and cuj3 and workspace_isolated` in a two-agent matrix using `UG_CUJ3_WORKSPACE` and the shared CUJ SP secrets. The shared managed lanes exclude @@ -628,10 +617,7 @@ python3.12 scripts/run_integration.py \ -- -m 'managed and cuj3 and workspace_isolated' ``` -MCP discovery/execution and skills discovery/invocation remain separate later -layers. The existing full CUJ3 and fixture branches are retained until those -layers are extracted and restacked. The model-only fixture PR is based directly -on main; the discovery-test PR is stacked only on that fixture PR, not CUJ7. +MCP/skills remain separate layers; this model suite does not depend on CUJ7. ### Managed-workspace journeys diff --git a/tests/integration/test_ug_cuj3_model_discovery.py b/tests/integration/test_ug_cuj3_model_discovery.py index 6eadeb458..7e0edd916 100644 --- a/tests/integration/test_ug_cuj3_model_discovery.py +++ b/tests/integration/test_ug_cuj3_model_discovery.py @@ -17,7 +17,7 @@ MODEL_SCHEMA, MODEL_SERVICES, OTHER_MODEL_SCHEMA, - assert_native_model_identity, + assert_completed_task_model, assert_persisted_config, assert_picker_inventory, claude_discovery_model_id, @@ -38,17 +38,15 @@ @pytest.mark.claude @pytest.mark.tui def test_case_03_managed_schema_pointers_claude(live_session, workspace): - """Scenario: configure Claude from the published CUJ3 model schema, then use every model. + """Scenario: configure published CUJ3 policy; launch bare ug, ug claude, and print tasks. - Expected: public configure persists the exact managed schema/defaults. An independent - UC GETs prove all five in-scope services, including Gemini, and both decoys exist. - An independent compatible inventory proves scoped models and the accessible decoy; - generated settings, native gateway cache, and exact numbered picker inventory match the scoped - inventory. Bare ug and explicit ug claude each complete a separate real Claude/Sonnet - TUI file task with native exact default identity and normal exit, without model overrides; - ug claude -p completes a Sonnet print task; every extra model completes a task with native identity. In-scope - Gemini and both out-of-scope decoys are excluded. These intended compatibility assertions - fail if the live backend differs; they are not a claim of live verification. + Expected: persisted schema/defaults and all seven UC services match policy; independent + catalogs, settings, cache, and exact numbered picker agree on Sonnet/Haiku/Kimi, + excluding Gemini and both accessible out-of-scope decoys. Both TUI launches exit + normally after separate Sonnet file tasks; print mode uses Sonnet without an override, + and every extra model completes a headless task, with routing off throughout. + Claude's response-reported model must match; it does not prove the executed gateway + backing destination. These assertions require a live pass, not collection alone. """ session = live_session configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) @@ -127,7 +125,7 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): tui.wait_for_task(default_task, timeout=240) tui.exit_normally() default_task.assert_completed(session, "claude") - assert_native_model_identity(session, "claude", default_task.value, CLAUDE_DEFAULT) + assert_completed_task_model(session, "claude", default_task.value, CLAUDE_DEFAULT) explicit_task = FileTask(session) with AgentTerminal( @@ -138,7 +136,7 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): tui.wait_for_task(explicit_task, timeout=240) tui.exit_normally() explicit_task.assert_completed(session, "claude") - assert_native_model_identity(session, "claude", explicit_task.value, CLAUDE_DEFAULT) + assert_completed_task_model(session, "claude", explicit_task.value, CLAUDE_DEFAULT) gateway_ids = session.claude_gateway_model_ids() assert len(gateway_ids) == len(set(gateway_ids)), gateway_ids @@ -158,7 +156,7 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): ) print_task.assert_headless_answer("claude", result) print_task.assert_completed(session, "claude") - assert_native_model_identity(session, "claude", print_task.value, CLAUDE_DEFAULT) + assert_completed_task_model(session, "claude", print_task.value, CLAUDE_DEFAULT) for model in parent_catalog.model_ids: if model == CLAUDE_DEFAULT: @@ -179,23 +177,22 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): ) task.assert_headless_answer("claude", result) task.assert_completed(session, "claude") - assert_native_model_identity(session, "claude", task.value, model) + assert_completed_task_model(session, "claude", task.value, model) session.assert_not_routed() @pytest.mark.codex @pytest.mark.tui def test_case_04_managed_schema_pointers_codex(live_session, workspace): - """Scenario: configure Codex from the published CUJ3 model schema, then use every model. + """Scenario: configure published CUJ3 policy; launch Codex TUI and ug codex -- exec. - Expected: public configure persists the exact managed schema/defaults. An independent - UC GETs prove all five in-scope services, including Gemini, and both decoys exist. - Independent compatible inventories, generated catalogs, real model/list protocol results, - and the exact numbered native /model picker inventory agree. A default TUI task, a default exec task, - and every extra-model task complete with native identity. Codex -p is a native profile - option, not print mode: headless tasks use ug codex -- exec. In-scope Gemini and both - out-of-scope decoys are excluded. Intended compatibility fails if the backend differs; - these expectations are not a claim of live verification. + Expected: persisted schema/defaults and all seven UC services match policy; independent + catalogs, generated catalogs, model/list, and exact numbered picker agree on GPT Luna/Kimi, + excluding Gemini and both accessible out-of-scope decoys. TUI exits normally after a + GPT Luna file task; exec uses that default without an override, and every extra model + completes a headless task, with routing off throughout. Native -p selects a profile. + Codex's client-selected model must match; it does not prove the executed gateway + backing destination. These assertions require a live pass, not collection alone. """ session = live_session configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) @@ -271,7 +268,7 @@ def test_case_04_managed_schema_pointers_codex(live_session, workspace): tui.wait_for_task(default_task, timeout=240) tui.exit_normally() default_task.assert_completed(session, "codex") - assert_native_model_identity(session, "codex", default_task.value, CODEX_DEFAULT) + assert_completed_task_model(session, "codex", default_task.value, CODEX_DEFAULT) exec_task = FileTask(session) result = session.run( @@ -285,7 +282,7 @@ def test_case_04_managed_schema_pointers_codex(live_session, workspace): ) exec_task.assert_headless_answer("codex", result) exec_task.assert_completed(session, "codex") - assert_native_model_identity(session, "codex", exec_task.value, CODEX_DEFAULT) + assert_completed_task_model(session, "codex", exec_task.value, CODEX_DEFAULT) for model in parent_catalog.model_ids: if model == CODEX_DEFAULT: @@ -304,5 +301,5 @@ def test_case_04_managed_schema_pointers_codex(live_session, workspace): ) task.assert_headless_answer("codex", result) task.assert_completed(session, "codex") - assert_native_model_identity(session, "codex", task.value, model) + assert_completed_task_model(session, "codex", task.value, model) session.assert_not_routed() diff --git a/tests/integration/utils/cuj3.py b/tests/integration/utils/cuj3.py index 63ce7cae7..1ddb1f9a2 100644 --- a/tests/integration/utils/cuj3.py +++ b/tests/integration/utils/cuj3.py @@ -10,7 +10,7 @@ import urllib.request from dataclasses import dataclass -from .evidence import agent_sessions, assistant_answers +from .evidence import agent_sessions, is_child_session from .provider_catalog import parse_anthropic_provider_page, parse_codex_provider_catalog MODEL_SCHEMA = "ug_e2e.models" @@ -50,9 +50,16 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return None -def fetch_model_service_inventory(workspace: str, token: str) -> dict[str, tuple[str, ...]]: - """Prove every fixture exists independently of agent-compatible model lists.""" - origin = urllib.parse.urlsplit(workspace) +def safe_https_json_get( + workspace: str, token: str, path: str, *, headers: dict[str, str] | None = None +) -> object: + """GET JSON from an explicit workspace origin without redirects or secret diagnostics.""" + assert isinstance(workspace, str), "Expected an HTTPS workspace origin" + try: + origin = urllib.parse.urlsplit(workspace) + port = origin.port + except ValueError: + raise AssertionError("Expected a valid HTTPS workspace origin") from None assert ( origin.scheme == "https" and origin.hostname @@ -61,26 +68,38 @@ def fetch_model_service_inventory(workspace: str, token: str) -> dict[str, tuple and origin.path in {"", "/"} and not origin.query and not origin.fragment + and (port is None or 0 < port <= 65535) + and not re.search(r"[\s\\]", workspace) ), "Expected an HTTPS workspace origin without credentials" - assert token, "Expected an explicit workspace bearer" - opener = urllib.request.build_opener(NoRedirect()) + assert isinstance(token, str) and token.strip() and not re.search(r"[\r\n]", token), ( + "Expected an explicit workspace bearer" + ) + assert path.startswith("/") and not path.startswith("//"), "Expected a workspace API path" + request = urllib.request.Request( + f"https://{origin.netloc}{path}", + headers={ + **(headers or {}), + "Authorization": f"Bearer {token}", + "Accept": "application/json", + }, + method="GET", + ) + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + raise AssertionError(f"Workspace JSON GET returned HTTP {error.code}") from None + except (urllib.error.URLError, OSError, ValueError): + raise AssertionError("Workspace JSON GET failed or returned invalid JSON") from None + + +def fetch_model_service_inventory(workspace: str, token: str) -> dict[str, tuple[str, ...]]: + """Prove every fixture exists independently of agent-compatible model lists.""" inventory: dict[str, tuple[str, ...]] = {} for model in sorted(MODEL_SERVICES | {CLAUDE_DECOY, CODEX_DECOY}): - url = f"https://{origin.netloc}/api/2.1/unity-catalog/model-services/{model}" - request = urllib.request.Request( - url, - headers={"Authorization": f"Bearer {token}", "Accept": "application/json"}, - method="GET", + payload = safe_https_json_get( + workspace, token, f"/api/2.1/unity-catalog/model-services/{model}" ) - try: - with opener.open(request, timeout=30) as response: - payload = json.load(response) - except urllib.error.HTTPError as error: - raise AssertionError(f"Model service GET {model} returned HTTP {error.code}") from None - except (urllib.error.URLError, OSError, ValueError): - raise AssertionError( - f"Model service GET {model} failed or returned invalid JSON" - ) from None assert isinstance(payload, dict), f"Model service {model} must be an object" config = payload.get("config") assert isinstance(config, dict), f"Model service {model} must have a config" @@ -171,24 +190,12 @@ class ParentCatalog: def fetch_codex_parent_catalog(workspace: str, token: str, parent_schema: str) -> ParentCatalog: """Keep the raw Codex catalog so hidden entries cannot conceal exclusions.""" - assert workspace.startswith("https://") and token and parent_schema - url = f"{workspace.rstrip('/')}/ai-gateway/codex/v1/models" - request = urllib.request.Request( - url, - headers={ - "Authorization": f"Bearer {token}", - "Accept": "application/json", - MODEL_HEADER: parent_schema, - }, + assert parent_schema, "Expected an explicit parent schema" + payload = safe_https_json_get( + workspace, token, "/ai-gateway/codex/v1/models", headers={MODEL_HEADER: parent_schema} ) - try: - with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response: - payload = json.load(response) - except urllib.error.HTTPError as error: - raise AssertionError(f"Codex catalog GET returned HTTP {error.code}") from None - except (urllib.error.URLError, OSError, ValueError): - raise AssertionError("Codex catalog GET failed or returned invalid JSON") from None model_ids = parse_codex_provider_catalog(payload) + assert isinstance(payload, dict), "Expected a Codex catalog object" display_names = {} for entry in payload["models"]: if entry.get("slug") in model_ids: @@ -200,7 +207,7 @@ def fetch_codex_parent_catalog(workspace: str, token: str, parent_schema: str) - def fetch_claude_parent_catalog(workspace: str, token: str, parent_schema: str) -> ParentCatalog: """Read the real parent-scoped Anthropic catalog with bounded pagination.""" - assert workspace.startswith("https://") and token and parent_schema + assert parent_schema, "Expected an explicit parent schema" model_ids: list[str] = [] display_names: dict[str, str | None] = {} pages: list[dict] = [] @@ -210,27 +217,17 @@ def fetch_claude_parent_catalog(workspace: str, token: str, parent_schema: str) query = {"limit": "1000"} if cursor is not None: query["after_id"] = cursor - url = ( - f"{workspace.rstrip('/')}/ai-gateway/anthropic/v1/models?" - f"{urllib.parse.urlencode(query)}" - ) - request = urllib.request.Request( - url, + payload = safe_https_json_get( + workspace, + token, + f"/ai-gateway/anthropic/v1/models?{urllib.parse.urlencode(query)}", headers={ - "Authorization": f"Bearer {token}", - "Accept": "application/json", "Anthropic-Version": "2023-06-01", MODEL_HEADER: parent_schema, }, ) - try: - with urllib.request.urlopen(request, timeout=30) as response: - payload = json.load(response) - except urllib.error.HTTPError as error: - raise AssertionError(f"GET {url} returned HTTP {error.code}: {error.reason}") from error - except (urllib.error.URLError, OSError) as error: - raise AssertionError(f"GET {url} failed: {error}") from error page = parse_anthropic_provider_page(payload) + assert isinstance(payload, dict), "Expected an Anthropic catalog object" pages.append(payload) for model_id, display_name in page.models: assert model_id not in display_names, f"duplicate Claude model id: {model_id}" @@ -283,41 +280,96 @@ def assert_persisted_config(session, workspace: str) -> dict: return config -def native_model_ids(session, agent: str, answer_value: str) -> set[str]: - """Correlate a completed file answer with its native transcript model identity.""" - assert agent in {"claude", "codex"}, agent +def _evidence_id(value: object, field: str) -> str: + assert isinstance(value, str) and re.fullmatch(r"\S+", value), ( + f"Missing or malformed completed-task {field}" + ) + return value + + +def claude_completed_task_models(records: list[dict], answer_value: str) -> set[str]: + """Parse Claude's response-reported model on the assistant's matching answer.""" + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) found: set[str] = set() - for records in agent_sessions(session, agent).values(): - if agent == "claude": - for record in records: - if not any(answer_value in answer for answer in assistant_answers(agent, [record])): - continue - message = record.get("message") or {} - model = message.get("model") - if isinstance(model, str) and model: - found.add(model.removesuffix("[1m]")) - else: - completed = { - (record.get("payload") or {}).get("turn_id") - for record in records - if record.get("type") == "event_msg" - and (record.get("payload") or {}).get("type") == "task_complete" - and answer_value in ((record.get("payload") or {}).get("last_agent_message") or "") - } - assert all(isinstance(turn_id, str) and turn_id for turn_id in completed), completed - for record in records: - if record.get("type") != "turn_context": - continue - payload = record.get("payload") or {} - if payload.get("turn_id") in completed and isinstance(payload.get("model"), str): - found.add(payload["model"]) + for record in records: + if record.get("type") != "assistant": + continue + message = record.get("message") + assert isinstance(message, dict), "Malformed Claude assistant message" + assert message.get("role") == "assistant", "Missing Claude assistant role" + content = message.get("content") + assert isinstance(content, list), "Malformed Claude assistant content" + for part in content: + assert isinstance(part, dict), "Malformed Claude assistant content block" + if part.get("type") != "text": + continue + text = part.get("text") + assert isinstance(text, str), "Malformed Claude assistant text" + if answer_value in text: + model = _evidence_id(message.get("model"), "model") + found.add(_evidence_id(model.removesuffix("[1m]"), "model")) return found -def assert_native_model_identity(session, agent: str, answer_value: str, expected: str) -> None: - observed = native_model_ids(session, agent, answer_value) +def codex_completed_task_models(records: list[dict], answer_value: str) -> set[str]: + """Join Codex's completed answer turn to its client-selected context model.""" + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) + completed: set[str] = set() + for record in records: + if record.get("type") != "event_msg": + continue + payload = record.get("payload") + assert isinstance(payload, dict), "Malformed Codex event payload" + if payload.get("type") != "task_complete": + continue + answer = payload.get("last_agent_message") + assert isinstance(answer, str), "Missing or malformed Codex completed answer" + if answer_value in answer: + completed.add(_evidence_id(payload.get("turn_id"), "turn ID")) + models: dict[str, set[str]] = {turn_id: set() for turn_id in completed} + for record in records: + if record.get("type") != "turn_context": + continue + payload = record.get("payload") + assert isinstance(payload, dict), "Malformed Codex turn context" + turn_id = _evidence_id(payload.get("turn_id"), "turn ID") + if turn_id in completed: + models[turn_id].add(_evidence_id(payload.get("model"), "model")) + assert all(models.values()), "Missing Codex model context for completed answer turn" + return {model for turn_models in models.values() for model in turn_models} + + +def completed_task_models(session, agent: str, answer_value: str) -> set[str]: + """Read parent task model evidence, not proof of the gateway's destination.""" + assert agent in {"claude", "codex"}, agent + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) + adapter = claude_completed_task_models if agent == "claude" else codex_completed_task_models + return { + model + for path, records in agent_sessions(session, agent).items() + if not is_child_session(agent, path, records) + for model in adapter(records, answer_value) + } + + +def assert_completed_task_model(session, agent: str, answer_value: str, expected: str) -> None: + expected = _evidence_id(expected, "expected model") + observed = completed_task_models(session, agent, answer_value) session.record( - f"native-model-{agent}-{answer_value[:12]}.json", - {"expected": expected, "observed": sorted(observed)}, + f"completed-task-model-{agent}-{answer_value[:12]}.json", + { + "expected": expected, + "observed": sorted(observed), + "evidence_kind": ( + "response-reported model" if agent == "claude" else "client-selected model" + ), + "gateway_destination_proven": False, + }, ) assert observed == {expected}, {"expected": expected, "observed": sorted(observed)} diff --git a/tests/test_cuj3_model_fixtures.py b/tests/test_cuj3_model_fixtures.py index 580980487..29552d8ba 100644 --- a/tests/test_cuj3_model_fixtures.py +++ b/tests/test_cuj3_model_fixtures.py @@ -10,6 +10,8 @@ import pytest +from tests.cuj3_fixture_data import INVALID_DESTINATIONS, model_readback + ROOT = Path(__file__).parent.parent SCRIPT = ROOT / "fixtures/cuj/models/provision.py" @@ -64,40 +66,6 @@ def complete_inventory(provisioner): return inventory -def model_readback(*, server_defaults): - destination = { - "name": "system.ai.databricks-gpt-6-luna", - "destination_type": "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", - "pay_per_token_config": {"model": "models/system.ai.databricks-gpt-6-luna"}, - } - routing = {"destinations": [destination]} - if server_defaults: - destination.update(traffic_percentage=100, is_deleted=False) - routing["fallback"] = {"destinations": []} - return { - "name": "model-services/ug_e2e.models.gpt_luna", - "config": {"routing": routing, "usage_tracking": {"enabled": True}}, - } - - -INVALID_DESTINATIONS = [ - {"name": "system.ai.wrong_source"}, - {"name": "other.ai.fixture"}, - {"destination_type": "DESTINATION_TYPE_EXTERNAL_MODEL"}, - {"pay_per_token_config": {"model": "models/system.ai.wrong_source"}}, - {"pay_per_token_config": None}, - {"traffic_percentage": 0}, - {"traffic_percentage": 50}, - {"traffic_percentage": "100"}, - {"traffic_percentage": None}, - {"traffic_percentage": True}, - {"is_deleted": True}, - {"is_deleted": "false"}, - {"is_disabled": True}, - {"disabled": True}, -] - - def test_model_fixture_create_body_uses_canonical_sdk_fields(provisioner): assert provisioner.model_body("system.ai.databricks-gpt-6-luna") == { "config": { @@ -184,7 +152,7 @@ def test_model_fixture_dry_plan_requires_no_credentials_or_network(): def test_model_fixture_config_is_model_only(): - config = json.loads(SCRIPT.with_name("cuj3-managed-config.json").read_text()) + config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) assert "mcp_servers" not in config assert "skills" not in config agents = {entry["agent"]: entry["config"] for entry in config["enabled_agents"]} diff --git a/tests/test_integration_cuj3.py b/tests/test_integration_cuj3.py index 610c39d78..217ec3b67 100644 --- a/tests/test_integration_cuj3.py +++ b/tests/test_integration_cuj3.py @@ -4,11 +4,13 @@ import json import runpy import urllib.error +import urllib.response from pathlib import Path from types import SimpleNamespace import pytest +from tests.cuj3_fixture_data import INVALID_DESTINATIONS, model_readback from tests.integration.utils import cuj3 ROOT = Path(__file__).parent.parent @@ -20,40 +22,6 @@ def provisioner(): return SimpleNamespace(**runpy.run_path(str(SCRIPT))) -def model_readback(*, server_defaults): - destination = { - "name": "system.ai.databricks-gpt-6-luna", - "destination_type": "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", - "pay_per_token_config": {"model": "models/system.ai.databricks-gpt-6-luna"}, - } - routing = {"destinations": [destination]} - if server_defaults: - destination.update(traffic_percentage=100, is_deleted=False) - routing["fallback"] = {"destinations": []} - return { - "name": "model-services/ug_e2e.models.gpt_luna", - "config": {"routing": routing, "usage_tracking": {"enabled": True}}, - } - - -INVALID_DESTINATIONS = [ - {"name": "system.ai.wrong_source"}, - {"name": "other.ai.fixture"}, - {"destination_type": "DESTINATION_TYPE_EXTERNAL_MODEL"}, - {"pay_per_token_config": {"model": "models/system.ai.wrong_source"}}, - {"pay_per_token_config": None}, - {"traffic_percentage": 0}, - {"traffic_percentage": 50}, - {"traffic_percentage": "100"}, - {"traffic_percentage": None}, - {"traffic_percentage": True}, - {"is_deleted": True}, - {"is_deleted": "false"}, - {"is_disabled": True}, - {"disabled": True}, -] - - @pytest.mark.parametrize( ("wire_id", "expected"), [ @@ -139,7 +107,8 @@ def build_opener(handler): assert set(inventory) == expected assert inventory[cuj3.GEMINI_MODEL] == ("system.ai.databricks-gpt-6-luna",) assert len(requests) == len(expected) - assert len(handlers) == 1 and isinstance(handlers[0], cuj3.NoRedirect) + assert len(handlers) == len(expected) + assert all(isinstance(handler, cuj3.NoRedirect) for handler in handlers) for request, timeout in requests: assert request.get_method() == "GET" assert request.full_url.startswith( @@ -195,7 +164,7 @@ def open(self, request, *, timeout): return io.BytesIO(json.dumps(provisioner.model_body("system.ai.fixture")).encode()) monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: MissingGeminiOpener()) - with pytest.raises(AssertionError, match=r"gemini_flash.*HTTP 404"): + with pytest.raises(AssertionError, match=r"HTTP 404"): cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") @@ -295,7 +264,7 @@ def test_picker_inventory_rejects_model_id_prefix_matches(agent, suffix): def test_model_policy_allows_server_metadata_and_later_stack_fields(): - config = json.loads(SCRIPT.with_name("cuj3-managed-config.json").read_text()) + config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) config["update_time"] = "2026-10-02T00:00:00Z" config["name"] = "coding-agent-configs/cuj3" config["mcp_servers"] = {"unity_catalog_location": "ug_e2e.tools"} @@ -303,7 +272,7 @@ def test_model_policy_allows_server_metadata_and_later_stack_fields(): def test_model_policy_rejects_a_default_outside_the_managed_scope(): - config = json.loads(SCRIPT.with_name("cuj3-managed-config.json").read_text()) + config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) config["enabled_agents"][1]["config"]["default_models"]["default_model"] = cuj3.CODEX_DECOY with pytest.raises(AssertionError): cuj3.assert_cuj3_config(config) @@ -323,7 +292,7 @@ def test_codex_model_identity_uses_only_the_completed_answer_turn(monkeypatch): }, ] monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) - assert cuj3.native_model_ids(None, "codex", "withheld-file-value") == {cuj3.CODEX_DEFAULT} + assert cuj3.completed_task_models(None, "codex", "withheld-file-value") == {cuj3.CODEX_DEFAULT} def test_codex_model_identity_rejects_prompt_only_evidence(monkeypatch): @@ -335,7 +304,7 @@ def test_codex_model_identity_rejects_prompt_only_evidence(monkeypatch): }, ] monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) - assert cuj3.native_model_ids(None, "codex", "withheld-file-value") == set() + assert cuj3.completed_task_models(None, "codex", "withheld-file-value") == set() def test_claude_model_identity_uses_assistant_answer_not_tool_output(monkeypatch): @@ -354,4 +323,373 @@ def test_claude_model_identity_uses_assistant_answer_not_tool_output(monkeypatch }, ] monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) - assert cuj3.native_model_ids(None, "claude", "value") == {cuj3.CLAUDE_DEFAULT} + assert cuj3.completed_task_models(None, "claude", "value") == {cuj3.CLAUDE_DEFAULT} + + +FETCHERS = [ + cuj3.fetch_model_service_inventory, + cuj3.fetch_claude_parent_catalog, + cuj3.fetch_codex_parent_catalog, +] + + +def fetch_inventory(fetcher, workspace="https://workspace.invalid", token="test-bearer"): + if fetcher is cuj3.fetch_model_service_inventory: + return fetcher(workspace, token) + return fetcher(workspace, token, cuj3.MODEL_SCHEMA) + + +@pytest.mark.parametrize("fetcher", FETCHERS) +@pytest.mark.parametrize( + "workspace", + [ + None, + 7, + "", + "http://workspace.invalid", + "https://user:secret@workspace.invalid", + "https://workspace.invalid/path", + "https://workspace.invalid?secret=value", + "https://workspace.invalid#fragment", + "https:///missing-host", + "https://workspace.invalid:bad", + "https://workspace.invalid:0", + "https://workspace.invalid:70000", + "https://workspace.invalid\n", + "https://workspace.invalid\\@other.invalid", + ], +) +def test_all_cuj3_gets_reject_invalid_workspace_origins(monkeypatch, fetcher, workspace): + def unexpected_open(*args, **kwargs): + pytest.fail("Invalid workspace must not reach the HTTP boundary") + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", unexpected_open) + with pytest.raises(AssertionError, match="HTTPS workspace origin") as failure: + fetch_inventory(fetcher, workspace) + assert "secret" not in str(failure.value) + + +@pytest.mark.parametrize("fetcher", FETCHERS) +@pytest.mark.parametrize("token", [None, "", " ", "bearer\nsecret", "bearer\rsecret"]) +def test_all_cuj3_gets_require_explicit_valid_bearers(monkeypatch, fetcher, token): + def unexpected_open(*args, **kwargs): + pytest.fail("Invalid bearer must not reach the HTTP boundary") + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", unexpected_open) + with pytest.raises(AssertionError, match="explicit workspace bearer"): + fetch_inventory(fetcher, token=token) + + +@pytest.mark.parametrize("fetcher", FETCHERS) +@pytest.mark.parametrize("failure_kind", ["http", "url", "os", "json", "encoding"]) +def test_all_cuj3_get_errors_are_sanitized(monkeypatch, fetcher, failure_kind): + secret = "private-bearer-and-response-secret" + + class FailedOpener: + def open(self, request, *, timeout): + assert timeout == 30 + assert request.get_method() == "GET" + assert request.get_header("Authorization") == f"Bearer {secret}" + if failure_kind == "http": + raise urllib.error.HTTPError(request.full_url, 403, secret, {}, None) + if failure_kind == "url": + raise urllib.error.URLError(secret) + if failure_kind == "os": + raise OSError(secret) + return io.BytesIO(secret.encode() if failure_kind == "json" else b"\xff") + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: FailedOpener()) + with pytest.raises(AssertionError, match="Workspace JSON GET") as failure: + fetch_inventory(fetcher, token=secret) + assert secret not in str(failure.value) + assert "workspace.invalid" not in str(failure.value) + assert failure.value.__suppress_context__ + + +@pytest.mark.parametrize("fetcher", FETCHERS) +@pytest.mark.parametrize("status", [301, 302, 303, 307, 308]) +@pytest.mark.parametrize("destination", ["https://other.invalid/secret", "/redirected"]) +def test_all_cuj3_gets_deny_redirects(monkeypatch, fetcher, status, destination): + requests = [] + build_opener = cuj3.urllib.request.build_opener + + class BoundaryHTTPS(cuj3.urllib.request.HTTPSHandler): + def https_open(self, request): + requests.append(request) + response = urllib.response.addinfourl( + io.BytesIO(b"{}"), {"Location": destination}, request.full_url, status + ) + response.msg = "private-redirect-reason" + return response + + monkeypatch.setattr( + cuj3.urllib.request, + "build_opener", + lambda handler: build_opener(handler, BoundaryHTTPS()), + ) + with pytest.raises(AssertionError, match=f"HTTP {status}") as failure: + fetch_inventory(fetcher) + assert len(requests) == 1 + assert requests[0].get_header("Authorization") == "Bearer test-bearer" + assert destination not in str(failure.value) + assert "private-redirect-reason" not in str(failure.value) + + +def test_claude_parent_catalog_retains_pages_labels_and_scoped_requests(monkeypatch): + pages = [ + { + "data": [{"id": cuj3.CLAUDE_DEFAULT, "display_name": "Sonnet"}], + "has_more": True, + "last_id": "cursor with / and ?", + }, + { + "data": [{"id": cuj3.claude_discovery_model_id(f"{cuj3.MODEL_SCHEMA}.kimi")}], + "has_more": False, + }, + ] + requests = [] + + class CatalogOpener: + def open(self, request, *, timeout): + requests.append(request) + assert timeout == 30 + assert request.get_header("Authorization") == "Bearer test-bearer" + assert request.get_header(cuj3.MODEL_HEADER.capitalize()) == cuj3.MODEL_SCHEMA + assert request.get_header("Anthropic-version") == "2023-06-01" + return io.BytesIO(json.dumps(pages[len(requests) - 1]).encode()) + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: CatalogOpener()) + catalog = cuj3.fetch_claude_parent_catalog( + "https://workspace.invalid/", "test-bearer", cuj3.MODEL_SCHEMA + ) + assert catalog.payloads == tuple(pages) + assert catalog.model_ids == (pages[0]["data"][0]["id"], pages[1]["data"][0]["id"]) + assert catalog.display_names == {catalog.model_ids[0]: "Sonnet", catalog.model_ids[1]: None} + assert cuj3.urllib.parse.parse_qs(cuj3.urllib.parse.urlsplit(requests[1].full_url).query) == { + "limit": ["1000"], + "after_id": ["cursor with / and ?"], + } + + +@pytest.mark.parametrize("failure_kind", ["duplicate", "cursor", "limit"]) +def test_claude_parent_catalog_rejects_duplicate_or_unbounded_pagination(monkeypatch, failure_kind): + requests = [] + + class CatalogOpener: + def open(self, request, *, timeout): + requests.append(request) + position = len(requests) + payload = { + "data": [ + {"id": "duplicate" if failure_kind == "duplicate" else f"model-{position}"} + ], + "has_more": True, + "last_id": "repeated" if failure_kind == "cursor" else f"cursor-{position}", + } + return io.BytesIO(json.dumps(payload).encode()) + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: CatalogOpener()) + with pytest.raises(AssertionError): + cuj3.fetch_claude_parent_catalog( + "https://workspace.invalid", "test-bearer", cuj3.MODEL_SCHEMA + ) + assert len(requests) == (20 if failure_kind == "limit" else 2) + + +def test_codex_parent_catalog_retains_unfiltered_payload_and_validates_duplicates(monkeypatch): + payload = { + "models": [ + {"slug": cuj3.CODEX_DEFAULT, "display_name": "Luna", "visibility": "list"}, + {"slug": cuj3.CODEX_DECOY, "visibility": "hidden"}, + ] + } + + class CatalogOpener: + def open(self, request, *, timeout): + assert timeout == 30 + assert request.get_header(cuj3.MODEL_HEADER.capitalize()) == cuj3.MODEL_SCHEMA + return io.BytesIO(json.dumps(payload).encode()) + + monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: CatalogOpener()) + catalog = cuj3.fetch_codex_parent_catalog( + "https://workspace.invalid", "test-bearer", cuj3.MODEL_SCHEMA + ) + assert catalog.payloads == (payload,) + assert catalog.model_ids == (cuj3.CODEX_DEFAULT,) + assert catalog.display_names == {cuj3.CODEX_DEFAULT: "Luna"} + payload["models"].append(payload["models"][0]) + with pytest.raises(AssertionError, match="repeated"): + cuj3.fetch_codex_parent_catalog( + "https://workspace.invalid", "test-bearer", cuj3.MODEL_SCHEMA + ) + + +def completed_records(agent, model, answer="value", turn_id="matching"): + if agent == "claude": + return [ + { + "type": "assistant", + "message": { + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": answer}], + }, + } + ] + return [ + {"type": "turn_context", "payload": {"turn_id": turn_id, "model": model}}, + { + "type": "event_msg", + "payload": {"type": "task_complete", "turn_id": turn_id, "last_agent_message": answer}, + }, + ] + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("parent_answer", [None, "value", "unrelated-answer"]) +def test_completed_task_models_excludes_child_only_and_conflicting_child_evidence( + monkeypatch, agent, parent_answer +): + child = completed_records(agent, "child-model") + if agent == "codex": + child.insert(0, {"type": "session_meta", "payload": {"source": {"subagent": "spawn"}}}) + sessions = {"project/subagents/child.jsonl": child} + if parent_answer is not None: + sessions["project/parent.jsonl"] = completed_records(agent, "parent-model", parent_answer) + monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: sessions) + expected = {"parent-model"} if parent_answer == "value" else set() + assert cuj3.completed_task_models(None, agent, "value") == expected + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("model", [None, "", " ", 5, {}, [], "model with whitespace"]) +def test_completed_task_models_fails_closed_on_malformed_model(monkeypatch, agent, model): + records = completed_records(agent, model) + monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"parent": records}) + with pytest.raises(AssertionError, match="model"): + cuj3.completed_task_models(None, agent, "value") + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_completed_task_models_fails_closed_on_missing_model(monkeypatch, agent): + records = completed_records(agent, "model") + del records[0]["message" if agent == "claude" else "payload"]["model"] + monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"parent": records}) + with pytest.raises(AssertionError, match="model"): + cuj3.completed_task_models(None, agent, "value") + + +@pytest.mark.parametrize("turn_id", [None, "", " ", 5, {}, [], "turn with whitespace"]) +@pytest.mark.parametrize("record_index", [0, 1]) +def test_codex_completed_task_models_fails_closed_on_malformed_turn_ids(turn_id, record_index): + records = completed_records("codex", "model") + records[record_index]["payload"]["turn_id"] = turn_id + with pytest.raises(AssertionError, match="turn ID"): + cuj3.codex_completed_task_models(records, "value") + + +@pytest.mark.parametrize("record_index", [0, 1]) +def test_codex_completed_task_models_fails_closed_on_missing_turn_ids(record_index): + records = completed_records("codex", "model") + del records[record_index]["payload"]["turn_id"] + with pytest.raises(AssertionError, match="turn ID"): + cuj3.codex_completed_task_models(records, "value") + + +def test_codex_completed_task_models_requires_context_in_same_session(monkeypatch): + context, completion = completed_records("codex", "model") + monkeypatch.setattr( + cuj3, + "agent_sessions", + lambda *_: {"context-session": [context], "answer-session": [completion]}, + ) + with pytest.raises(AssertionError, match="Missing Codex model context"): + cuj3.completed_task_models(None, "codex", "value") + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("models", [[], ["unexpected"], ["expected", "conflicting"]]) +def test_completed_task_model_assertion_requires_exact_singleton(monkeypatch, agent, models): + records = [record for model in models for record in completed_records(agent, model)] + monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"parent": records}) + evidence = {} + session = SimpleNamespace(record=lambda name, payload: evidence.update({name: payload})) + with pytest.raises(AssertionError): + cuj3.assert_completed_task_model(session, agent, "value", "expected") + assert next(iter(evidence.values()))["observed"] == sorted(models) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_completed_task_model_records_evidence_limits(monkeypatch, agent): + monkeypatch.setattr( + cuj3, "agent_sessions", lambda *_: {"parent": completed_records(agent, "expected")} + ) + evidence = {} + session = SimpleNamespace(record=lambda name, payload: evidence.update({name: payload})) + cuj3.assert_completed_task_model(session, agent, "value", "expected") + assert evidence == { + f"completed-task-model-{agent}-value.json": { + "expected": "expected", + "observed": ["expected"], + "evidence_kind": ( + "response-reported model" if agent == "claude" else "client-selected model" + ), + "gateway_destination_proven": False, + } + } + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("answer_value", [None, "", " \t", 1, [], {}]) +def test_completed_task_models_rejects_empty_or_malformed_expected_answers( + monkeypatch, agent, answer_value +): + def unexpected_read(*args): + pytest.fail("Invalid answer must not read session evidence") + + monkeypatch.setattr(cuj3, "agent_sessions", unexpected_read) + with pytest.raises(AssertionError, match="nonempty answer value"): + cuj3.completed_task_models(None, agent, answer_value) + adapter = ( + cuj3.claude_completed_task_models if agent == "claude" else cuj3.codex_completed_task_models + ) + with pytest.raises(AssertionError, match="nonempty answer value"): + adapter([], answer_value) + + +@pytest.mark.parametrize( + "message", + [None, [], {}, {"role": "assistant"}, {"role": "assistant", "content": "value"}], +) +def test_claude_completed_task_models_rejects_malformed_message_metadata(message): + with pytest.raises(AssertionError): + cuj3.claude_completed_task_models([{"type": "assistant", "message": message}], "value") + + +@pytest.mark.parametrize("content", [[None], [{"type": "text"}], [{"type": "text", "text": 4}]]) +def test_claude_completed_task_models_rejects_malformed_answer_blocks(content): + records = completed_records("claude", "model") + records[0]["message"]["content"] = content + with pytest.raises(AssertionError): + cuj3.claude_completed_task_models(records, "value") + + +@pytest.mark.parametrize("payload", [None, [], "value"]) +@pytest.mark.parametrize("record_type", ["event_msg", "turn_context"]) +def test_codex_completed_task_models_rejects_malformed_payloads(payload, record_type): + with pytest.raises(AssertionError): + cuj3.codex_completed_task_models([{"type": record_type, "payload": payload}], "value") + + +@pytest.mark.parametrize("answer", [None, [], {}, 7]) +def test_codex_completed_task_models_rejects_malformed_completed_answers(answer): + records = completed_records("codex", "model") + records[1]["payload"]["last_agent_message"] = answer + with pytest.raises(AssertionError, match="completed answer"): + cuj3.codex_completed_task_models(records, "value") + + +def test_claude_completed_task_models_normalizes_only_context_window_suffix(): + assert cuj3.claude_completed_task_models( + completed_records("claude", f"{cuj3.CLAUDE_DEFAULT}[1m]"), "value" + ) == {cuj3.CLAUDE_DEFAULT} From bc8a6476141172b444230ad108b537ebaaa041fe Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 19:22:11 +0000 Subject: [PATCH 04/19] AIGTWY-4878 Use Databricks SDK for model fixture provisioning --- fixtures/cuj/models/README.md | 30 ++++--- fixtures/cuj/models/provision.py | 145 +++++++++++++++++------------- tests/test_cuj3_model_fixtures.py | 109 ++++++++++++++++------ 3 files changed, 187 insertions(+), 97 deletions(-) diff --git a/fixtures/cuj/models/README.md b/fixtures/cuj/models/README.md index 1d94ace1a..7295cadd7 100644 --- a/fixtures/cuj/models/README.md +++ b/fixtures/cuj/models/README.md @@ -18,8 +18,10 @@ explicitly to its schema; inventory ordering does not determine scope. ## Prerequisites and setup order -1. Use Python 3.12+ and select the exact dedicated CUJ3 HTTPS workspace origin. - No SDK, CLI profile, or implicit workspace/authentication fallback is used. +1. Use Python 3.12+, an environment with `databricks-sdk>=0.135.0`, and select the + exact dedicated CUJ3 HTTPS workspace origin. No CLI profile or implicit + workspace/authentication fallback is used. The offline plan does not create a + workspace client or read credentials. 2. An operator must explicitly select seven existing, canonical `system.ai.` registered-model FQNs in that workspace/region, including both decoys, not friendly model-service aliases. Both `--validate` and `--apply` GET every unique source at @@ -52,10 +54,11 @@ explicitly to its schema; inventory ordering does not determine scope. and has no MCP/skill scope. Those omissions do not remove unrelated pre-existing local MCP/skill config. -From the repository root, without needing a bearer for the default offline plan: +From the repository root, without needing a bearer for the default offline plan +(`uv run` supplies the project SDK environment): ```bash -python3 fixtures/cuj/models/provision.py \ +uv run python fixtures/cuj/models/provision.py \ --workspace "$UG_CUJ3_WORKSPACE" \ --bearer-env DATABRICKS_BEARER \ --gpt-luna-source "$GPT_LUNA_SOURCE" \ @@ -69,13 +72,14 @@ python3 fixtures/cuj/models/provision.py \ Each source variable is a concrete `system.ai.` FQN. The default plan checks syntax and prints inventory without network access or reading a bearer; it cannot -check existing resources. `--validate` performs authenticated read-only GETs and -fails on missing resources or invalid services. It is mutually exclusive with +check existing resources. `--validate` performs authenticated, read-only SDK GETs +and fails on missing resources or invalid services. It is mutually exclusive with `--apply`, which completes full source/inventory/routing preflight **before any POST**, creates only missing catalog → schemas → services, and reads back all seven services. -Mismatches abort: there is no overwrite, update, delete, or cleanup path. Concurrent -create conflicts and partial failures stop the non-atomic run; created resources -persist. Inspect with `--validate` before retrying. +Requests use a 30-second SDK timeout and a one-second SDK retry budget. Mismatches +abort: there is no overwrite, update, delete, or cleanup path. Concurrent create +conflicts and partial failures stop the non-atomic run; created resources persist. +Inspect with `--validate` before retrying. ## Compatibility and validation boundary @@ -93,6 +97,12 @@ from the managed catalog. The API uses `/api/2.1/unity-catalog/model-services`, a schema `parent`, and canonical snake_case fields from Universe's Python SDK model-service sample. +Provisioning uses the SDK's public `WorkspaceClient` and `api_client.do` with raw +REST calls. Its per-request bearer callback rejects redirects before requests +follows them and requires every successful GET response to be a JSON object before +the SDK parses it. That preserves unknown routing flags and prevents SDK coercion +of JSON `null` or strings from weakening validation; private sessions, global +environment, and settings are not changed. Creation sets `pay_per_token_config.model = models/system.ai.` and `traffic_percentage = 100`; usage tracking is always on and its removed/reserved configuration field is omitted. Preflight/readback require one destination with @@ -116,5 +126,5 @@ Suggested focused local checks (no workspace or bearer needed): ```bash uv run ruff check fixtures/cuj/models/provision.py uv run ruff format --check fixtures/cuj/models/provision.py -python3 fixtures/cuj/models/provision.py --help +uv run python fixtures/cuj/models/provision.py --help ``` diff --git a/fixtures/cuj/models/provision.py b/fixtures/cuj/models/provision.py index ef03c7cd2..c07535795 100644 --- a/fixtures/cuj/models/provision.py +++ b/fixtures/cuj/models/provision.py @@ -3,13 +3,15 @@ from __future__ import annotations import argparse -import json import os import re import sys -import urllib.error import urllib.parse -import urllib.request + +from databricks.sdk import WorkspaceClient +from databricks.sdk.core import Config +from databricks.sdk.errors import DatabricksError, NotFound +from requests.exceptions import RequestException UC = "/api/2.1/unity-catalog" SCHEMAS = ("models", "other_models") @@ -27,71 +29,88 @@ DESTINATION_TYPE = "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL" -class ApiError(RuntimeError): - def __init__(self, method: str, path: str, status: int): - super().__init__(f"{method} {path.split('?', 1)[0]} returned HTTP {status}") - self.status = status - - -class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - def workspace_origin(workspace: str) -> str: - parsed = urllib.parse.urlsplit(workspace) + try: + parsed = urllib.parse.urlsplit(workspace) + port = parsed.port + except (TypeError, ValueError): + raise ValueError( + "--workspace must be an HTTPS workspace origin without credentials" + ) from None if ( - parsed.scheme != "https" + not isinstance(workspace, str) + or parsed.scheme != "https" or not parsed.hostname or parsed.username is not None or parsed.password is not None or parsed.path not in ("", "/") or parsed.query or parsed.fragment + or (port is not None and not 0 < port <= 65535) + or re.search(r"[\s\\]", workspace) ): raise ValueError("--workspace must be an HTTPS workspace origin without credentials") return f"https://{parsed.netloc}" -class Client: - def __init__(self, workspace: str, bearer: str): - self.origin = workspace_origin(workspace) - self.bearer = bearer - self.opener = urllib.request.build_opener(NoRedirect()) +def require(condition: bool, message: str): + if not condition: + raise RuntimeError(message) - def request(self, method: str, path: str, *, params=None, body=None): - query = f"?{urllib.parse.urlencode(params)}" if params else "" - data = json.dumps(body).encode() if body is not None else None - headers = {"Authorization": f"Bearer {self.bearer}"} - if data is not None: - headers["Content-Type"] = "application/json" - request = urllib.request.Request( - self.origin + path + query, data=data, headers=headers, method=method - ) - try: - with self.opener.open(request, timeout=60) as response: - payload = response.read() - except urllib.error.HTTPError as error: - raise ApiError(method, path, error.code) from None - except (urllib.error.URLError, TimeoutError, OSError): - raise RuntimeError("Workspace request failed; check connectivity and TLS") from None - try: - return json.loads(payload) if payload else None - except (ValueError, UnicodeError): - raise RuntimeError("Workspace returned an invalid JSON response") from None - def get_optional(self, path: str): - try: - return self.request("GET", path) - except ApiError as error: - if error.status == 404: - return None - raise +def bearer_auth(bearer: str): + def reject_redirect(response, **kwargs): + if 300 <= response.status_code < 400: + response.close() + raise AssertionError( + f"Workspace request redirected with HTTP {response.status_code}" + ) from None + if ( + 200 <= response.status_code < 300 + and response.request.method == "GET" + and not isinstance(response.json(), dict) + ): + raise AssertionError("Workspace GET returned a non-object JSON response") from None + return response + def authenticate(request): + request.headers["Authorization"] = f"Bearer {bearer}" + request.register_hook("response", reject_redirect) + return request -def require(condition: bool, message: str): - if not condition: - raise RuntimeError(message) + return authenticate + + +def _request(api_client, auth, method: str, path: str, *, query=None, body=None, missing_ok=False): + try: + return api_client.do(method, path, query=query, body=body, auth=auth) + except NotFound: + if missing_ok: + return None + raise RuntimeError("Workspace API request failed") from None + except DatabricksError: + raise RuntimeError("Workspace API request failed") from None + except (RequestException, OSError, ValueError): + raise RuntimeError("Workspace request failed; check connectivity and TLS") from None + + +def _get_optional(api_client, auth, path: str): + return _request(api_client, auth, "GET", path, missing_ok=True) + + +def workspace_client(origin: str, bearer: str): + try: + return WorkspaceClient( + config=Config( + host=origin, + token=bearer, + auth_type="pat", + http_timeout_seconds=30, + retry_timeout_seconds=1, + ) + ) + except (DatabricksError, RequestException, OSError, ValueError): + raise RuntimeError("Workspace SDK client initialization failed") from None def model_body(source: str): @@ -150,9 +169,10 @@ def print_plan(sources: dict[str, str]): ) -def provision(client: Client, sources: dict[str, str], apply: bool): +def provision(api_client, sources: dict[str, str], apply: bool, bearer: str): + auth = bearer_auth(bearer) for source in sorted(set(sources.values())): - registered_model = client.get_optional(f"{UC}/models/{source}") + registered_model = _get_optional(api_client, auth, f"{UC}/models/{source}") require( isinstance(registered_model, dict) and registered_model.get("full_name") == source, f"Source registered model {source} is missing or has an unexpected full_name; " @@ -161,14 +181,14 @@ def provision(client: Client, sources: dict[str, str], apply: bool): ) print(f"source registered model {source}: validated") missing = [] - if client.get_optional(f"{UC}/catalogs/ug_e2e") is None: + if _get_optional(api_client, auth, f"{UC}/catalogs/ug_e2e") is None: missing.append((f"{UC}/catalogs", None, {"name": "ug_e2e", "comment": "UG CUJ fixtures"})) print("catalog ug_e2e: would create") else: print("catalog ug_e2e: existing") for schema in SCHEMAS: fqn = f"ug_e2e.{schema}" - if client.get_optional(f"{UC}/schemas/{fqn}") is None: + if _get_optional(api_client, auth, f"{UC}/schemas/{fqn}") is None: missing.append( ( f"{UC}/schemas", @@ -183,7 +203,7 @@ def provision(client: Client, sources: dict[str, str], apply: bool): schema = MODEL_SCHEMAS[leaf] fqn = f"ug_e2e.{schema}.{leaf}" source = sources[leaf] - existing = client.get_optional(f"{UC}/model-services/{fqn}") + existing = _get_optional(api_client, auth, f"{UC}/model-services/{fqn}") if existing is None: missing.append( ( @@ -201,11 +221,15 @@ def provision(client: Client, sources: dict[str, str], apply: bool): require(not missing, "Inventory is incomplete; review the offline plan before --apply") return for path, params, body in missing: - client.request("POST", path, params=params, body=body) + _request(api_client, auth, "POST", path, query=params, body=body) for leaf in MODEL_LEAVES: schema = MODEL_SCHEMAS[leaf] fqn = f"ug_e2e.{schema}.{leaf}" - validate_model(client.request("GET", f"{UC}/model-services/{fqn}"), fqn, sources[leaf]) + validate_model( + _request(api_client, auth, "GET", f"{UC}/model-services/{fqn}"), + fqn, + sources[leaf], + ) print("Model-only fixture inventory is ready; managed config was not published") @@ -246,12 +270,13 @@ def main(): require( not any(character.isspace() for character in bearer), "Bearer must not contain whitespace" ) - provision(Client(origin, bearer), sources, args.apply) + client = workspace_client(origin, bearer) + provision(client.api_client, sources, args.apply, bearer=bearer) if __name__ == "__main__": try: main() - except (ApiError, RuntimeError, ValueError) as error: + except (AssertionError, RuntimeError, ValueError) as error: print(f"model fixture provisioning failed: {error}", file=sys.stderr) sys.exit(1) diff --git a/tests/test_cuj3_model_fixtures.py b/tests/test_cuj3_model_fixtures.py index 29552d8ba..61f78a951 100644 --- a/tests/test_cuj3_model_fixtures.py +++ b/tests/test_cuj3_model_fixtures.py @@ -9,6 +9,7 @@ from types import SimpleNamespace import pytest +from databricks.sdk.errors import NotFound from tests.cuj3_fixture_data import INVALID_DESTINATIONS, model_readback @@ -25,25 +26,29 @@ def model_sources(provisioner): return {leaf: f"system.ai.{leaf}" for leaf in provisioner.MODEL_LEAVES} -class RecordingClient: +class RecordingApiClient: def __init__(self, inventory): self.inventory = inventory self.requests = [] + self.auth_requests = [] - def get_optional(self, path): - self.requests.append(("GET", path, None, None)) - return self.inventory.get(path) - - def request(self, method, path, *, params=None, body=None): - self.requests.append((method, path, params, body)) + def do(self, method, path, *, query=None, headers=None, body=None, auth=None): + self.requests.append((method, path, query, body)) + self.auth_requests.append((method, path, auth is not None)) if method == "GET": + if path not in self.inventory: + raise NotFound("not found") return self.inventory[path] if path.endswith("model-services"): - schema = params["parent"].removeprefix("schemas/") - self.inventory[f"{path}/{schema}.{params['model_service_id']}"] = body + schema = query["parent"].removeprefix("schemas/") + self.inventory[f"{path}/{schema}.{query['model_service_id']}"] = body return body +def run_provision(provisioner, client, sources, *, apply): + return provisioner.provision(client, sources, apply=apply, bearer="component-token") + + def source_inventory(provisioner, sources=None): sources = model_sources(provisioner) if sources is None else sources return { @@ -102,9 +107,9 @@ def test_model_fixture_rejects_invalid_readback_before_writes(provisioner, desti sources = model_sources(provisioner) sources["gpt_luna"] = "system.ai.databricks-gpt-6-luna" inventory.update(source_inventory(provisioner, sources)) - client = RecordingClient(inventory) + client = RecordingApiClient(inventory) with pytest.raises(RuntimeError, match="refusing to overwrite"): - provisioner.provision(client, sources, apply=True) + run_provision(provisioner, client, sources, apply=True) assert all(method == "GET" for method, *_ in client.requests) @@ -120,9 +125,9 @@ def test_model_fixture_rejects_multiple_sources_before_writes(provisioner, fallb else: routing["destinations"].extend(extra) inventory[f"{provisioner.UC}/model-services/ug_e2e.models.gpt_luna"] = payload - client = RecordingClient(inventory) + client = RecordingApiClient(inventory) with pytest.raises(RuntimeError, match="refusing to overwrite"): - provisioner.provision(client, model_sources(provisioner), apply=True) + run_provision(provisioner, client, model_sources(provisioner), apply=True) assert all(method == "GET" for method, *_ in client.requests) @@ -151,6 +156,56 @@ def test_model_fixture_dry_plan_requires_no_credentials_or_network(): assert "skills" not in result.stdout +def test_model_fixture_auth_callback_sets_bearer_and_rejects_redirects(provisioner): + class PreparedRequest: + def __init__(self): + self.headers = {} + self.hooks = {} + + def register_hook(self, event, hook): + self.hooks.setdefault(event, []).append(hook) + + callback = provisioner.bearer_auth("component-token") + request = PreparedRequest() + assert callback(request) is request + assert request.headers == {"Authorization": "Bearer component-token"} + redirect_hook = request.hooks["response"][0] + response = SimpleNamespace( + status_code=200, + request=SimpleNamespace(method="GET"), + json=lambda: {"config": {}}, + close=lambda: None, + ) + assert redirect_hook(response) is response + closed = [] + with pytest.raises(AssertionError, match="redirected") as error: + redirect_hook(SimpleNamespace(status_code=307, close=lambda: closed.append(True))) + assert "component-token" not in str(error.value) + assert closed == [True] + with pytest.raises(AssertionError, match="non-object"): + redirect_hook( + SimpleNamespace( + status_code=200, + request=SimpleNamespace(method="GET"), + json=lambda: None, + ) + ) + + +def test_model_fixture_sanitizes_required_not_found_errors(provisioner): + class ErrorApiClient: + def do(self, method, path, *, query=None, headers=None, body=None, auth=None): + raise NotFound("secret response body") + + client = ErrorApiClient() + auth = provisioner.bearer_auth("component-token") + assert provisioner._get_optional(client, auth, "/optional") is None + for method in ("GET", "POST"): + with pytest.raises(RuntimeError, match="^Workspace API request failed$") as error: + provisioner._request(client, auth, method, "/required") + assert "secret response body" not in str(error.value) + + def test_model_fixture_config_is_model_only(): config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) assert "mcp_servers" not in config @@ -172,9 +227,9 @@ def test_model_fixture_config_is_model_only(): def test_model_fixture_validate_missing_inventory_performs_no_writes(provisioner): - client = RecordingClient(source_inventory(provisioner)) + client = RecordingApiClient(source_inventory(provisioner)) with pytest.raises(RuntimeError, match="Inventory is incomplete"): - provisioner.provision(client, model_sources(provisioner), apply=False) + run_provision(provisioner, client, model_sources(provisioner), apply=False) assert len(client.requests) == 17 assert all(method == "GET" for method, *_ in client.requests) @@ -188,9 +243,9 @@ def test_model_fixture_source_preflight_failure_performs_no_writes( inventory = source_inventory(provisioner) source = sorted(sources.values())[-1] inventory[f"{provisioner.UC}/models/{source}"] = registered_model - client = RecordingClient(inventory) + client = RecordingApiClient(inventory) with pytest.raises(RuntimeError, match="canonical registered-model FQN.*model-service alias"): - provisioner.provision(client, sources, apply=apply) + run_provision(provisioner, client, sources, apply=apply) assert len(client.requests) == len(set(sources.values())) assert all(method == "GET" and "/models/" in path for method, path, *_ in client.requests) @@ -205,17 +260,17 @@ def test_model_fixture_source_preflight_rejects_friendly_alias_without_resolving inventory[f"{provisioner.UC}/model-services/system.ai.gpt-6-luna"] = model_readback( server_defaults=True ) - client = RecordingClient(inventory) + client = RecordingApiClient(inventory) with pytest.raises(RuntimeError, match="system.ai.gpt-6-luna.*model-service alias"): - provisioner.provision(client, sources, apply=True) + run_provision(provisioner, client, sources, apply=True) assert all(method == "GET" and "/models/" in path for method, path, *_ in client.requests) assert not any("databricks-gpt-6-luna" in path for _, path, *_ in client.requests) def test_model_fixture_source_preflight_checks_each_unique_source_once(provisioner): sources = dict.fromkeys(provisioner.MODEL_LEAVES, "system.ai.databricks-gpt-6-luna") - client = RecordingClient(source_inventory(provisioner, sources)) - provisioner.provision(client, sources, apply=True) + client = RecordingApiClient(source_inventory(provisioner, sources)) + run_provision(provisioner, client, sources, apply=True) source_reads = [request for request in client.requests if "/models/" in request[1]] assert source_reads == [ ("GET", f"{provisioner.UC}/models/system.ai.databricks-gpt-6-luna", None, None) @@ -229,8 +284,8 @@ def test_model_fixture_existing_inventory_is_not_rewritten(provisioner, apply): for path, payload in inventory.items(): if "/model-services/" in path: payload["config"]["routing"]["destinations"][0]["is_deleted"] = False - client = RecordingClient(inventory) - provisioner.provision(client, model_sources(provisioner), apply=apply) + client = RecordingApiClient(inventory) + run_provision(provisioner, client, model_sources(provisioner), apply=apply) assert all(method == "GET" for method, *_ in client.requests) @@ -240,15 +295,15 @@ def test_model_fixture_mismatch_aborts_before_any_creation(provisioner): inventory[f"{provisioner.UC}/model-services/ug_e2e.other_models.codex_decoy"] = ( provisioner.model_body("system.ai.wrong_model") ) - client = RecordingClient(inventory) + client = RecordingApiClient(inventory) with pytest.raises(RuntimeError, match="refusing to overwrite"): - provisioner.provision(client, model_sources(provisioner), apply=True) + run_provision(provisioner, client, model_sources(provisioner), apply=True) assert all(method == "GET" for method, *_ in client.requests) def test_model_fixture_apply_only_creates_models_in_dependency_order(provisioner): - client = RecordingClient(source_inventory(provisioner)) - provisioner.provision(client, model_sources(provisioner), apply=True) + client = RecordingApiClient(source_inventory(provisioner)) + run_provision(provisioner, client, model_sources(provisioner), apply=True) writes = [request for request in client.requests if request[0] != "GET"] assert [request[1] for request in writes] == [ f"{provisioner.UC}/catalogs", From 42a4165abfbc0237f3b8588a20ff298ac954aee3 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 19:22:28 +0000 Subject: [PATCH 05/19] AIGTWY-4878 Simplify catalog discovery journeys and share agent helpers --- tests/README.md | 12 +- tests/integration/README.md | 43 +- .../catalog_discovery_expectations.py | 38 + tests/integration/pytest.ini | 2 +- ...covery.py => test_ug_catalog_discovery.py} | 101 +-- .../test_ug_claude_managed_model_discovery.py | 4 +- .../test_ug_codex_managed_model_discovery.py | 4 +- tests/integration/utils/agents/__init__.py | 11 +- tests/integration/utils/agents/claude.py | 43 +- tests/integration/utils/agents/codex.py | 45 +- tests/integration/utils/cuj3.py | 375 ---------- tests/integration/utils/evidence.py | 35 +- tests/integration/utils/http.py | 63 ++ tests/integration/utils/managed.py | 10 + tests/integration/utils/model_discovery.py | 60 ++ tests/integration/utils/provider_catalog.py | 74 +- tests/test_integration_cuj3.py | 695 ------------------ tests/test_integration_evidence.py | 239 ++++++ tests/test_integration_http.py | 116 +++ tests/test_integration_managed_config.py | 96 +++ tests/test_integration_model_discovery.py | 131 +++- tests/test_integration_model_policy.py | 50 ++ tests/test_integration_provider_catalog.py | 227 +++++- 23 files changed, 1286 insertions(+), 1188 deletions(-) create mode 100644 tests/integration/catalog_discovery_expectations.py rename tests/integration/{test_ug_cuj3_model_discovery.py => test_ug_catalog_discovery.py} (77%) delete mode 100644 tests/integration/utils/cuj3.py create mode 100644 tests/integration/utils/http.py delete mode 100644 tests/test_integration_cuj3.py create mode 100644 tests/test_integration_http.py create mode 100644 tests/test_integration_managed_config.py create mode 100644 tests/test_integration_model_policy.py diff --git a/tests/README.md b/tests/README.md index 7a9acc605..5c13608f5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -187,10 +187,10 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | | `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`); explicitly launch `ug claude` above the tier | Bare launches check Claude/Sonnet below the tier and the Codex/Luna recommendation above it; `ug usage` agrees with backend spend, threshold, and percentage. Explicit `ug claude` displays the backend's Codex/Luna recommendation while its generated model setting and native header select Sonnet. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | -| CUJ3 model discovery (`test_case_03_*`, `test_case_04_*`) | Configure published `ug_e2e.models` policy in the dedicated workspace | Exact Claude Sonnet/Haiku/Kimi and Codex GPT Luna/Kimi catalogs/pickers; Gemini and other-schema decoys excluded. Bare `ug` and `ug claude` complete separate Sonnet TUI tasks; Claude print and Codex TUI/exec tasks use configured defaults without overrides. Every extra compatible model completes a headless task; routing stays off. Model evidence is limited as described below; MCP/skills are deferred | -| `test_managed_fixture_claude_rejects_provider_override`, `test_managed_fixture_fresh_claude_rejects_provider_override`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | +| Catalog discovery (`test_catalog_discovery_claude`, `test_catalog_discovery_codex`) | Agent-specific journeys using shared discovery/evidence helpers; configure published `ug_e2e.models` policy in the dedicated workspace | Exact Claude Sonnet/Haiku/Kimi and Codex GPT Luna/Kimi catalogs/pickers; Gemini and other-schema decoys excluded. Bare `ug` and `ug claude` complete separate Sonnet TUI tasks; Claude print and Codex TUI/exec tasks use configured defaults without overrides. Every extra compatible model completes a headless task; routing stays off. Model evidence is limited as described below; MCP/skills are deferred | +| `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | -| `test_managed_fixture_codex_rejects_provider_override_after_configure`, `test_managed_fixture_codex_rejects_provider_override_from_fresh_state`, `test_case_06_*` | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state | +| `test_case_04_*`, `test_case_06_*` | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state | | `test_ug_configure_managed_codex_catalog_fallback` | Configure from an injected managed response containing a GPT model absent from Codex's bundled catalog | Actionable metadata warning; conservative catalog entry for the unknown model; real Codex prompt on the valid default model | | `test_managed_fixture_codex_http_headers_in_managed_file` | Interactive PTY configure with injected managed `http_headers` for Codex | The specified header (`x-databricks-workspace`) lands in `model_providers.Databricks.http_headers` in `/etc/codex/managed_config.toml` with the exact admin value | | `test_managed_claude_mps_defaults_accompany_discovery`, `test_managed_claude_parent_schema_defaults_accompany_discovery` | Configure from the published admin config and launch Claude with MPS on `eng-ml-inference-batch-inference-us-west-2` and Unity Catalog discovery on `eng-ml-inference-ap-northeast-2`, respectively | Both generated settings files retain every admin-authored default alongside the source header and every independently fetched catalog model with its label; MPS pickers keep family shortcut rows separate from catalog entries; only UC Opus/Sonnet family ids gain `[1m]` | @@ -203,11 +203,11 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_and_ucode_web_search_helpers_preserve_mcp_stdio` | Initialize and list tools through both web-search helper commands | Exactly the MCP JSON-RPC responses; no text/ANSI contamination; existing server/tool identities preserved; no model request | With Claude and Codex selected there are **62 live cases** (12 marked TUI cases), -**8 managed-workspace cases** (marker `managed`, including two dedicated CUJ3 model journeys, run against workspaces that +**8 managed-workspace cases** (marker `managed`, including two dedicated catalog discovery journeys, run against workspaces that publish a CodingAgentConfig), **1 two-workspace case** (marker `workspace_switch`), **25 managed-fixture cases** (marker `managed_fixture`, with only the CodingAgentConfig input injected), and **7 installation checks**. The 14 retained numbered scenarios -comprise **22 explicit journeys**: 8 managed-fixture configured/fresh executions, 2 CUJ3 model journeys, and 12 unmanaged +comprise **22 explicit journeys**: 8 managed-fixture configured/fresh executions, 2 catalog discovery journeys, and 12 unmanaged executions. Four provider-override journeys retain their assertions under descriptive names. Thirteen additional managed-fixture cases cover focused model, MCP, skills, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases @@ -230,7 +230,7 @@ Claude discovery assertions match numbered picker rows, not startup banners or footers. Offline regressions cover that distinction, native Haiku/Opus/Sonnet deduplication, and raw catalog ID/display-name rows for scoped pickers. Managed discovery expectations come from separate read-only, provider-scoped -model-list requests; they do not rely solely on ug's generated catalog. CUJ3 tasks +model-list requests; they do not rely solely on ug's generated catalog. Catalog discovery tasks check Claude's response-reported model and Codex's client-selected model; neither proves the executed gateway backing destination. Native picker/task coverage requires a live journey pass, not collection or lint success. diff --git a/tests/integration/README.md b/tests/integration/README.md index 1c2981623..d3dbfe3cc 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -10,7 +10,7 @@ CUJ2 adds three separately collected cases for exact MPS/MCP configuration, Code and Claude inference. This suite runs the **installed product** through subprocesses, against the same -`UCODE_TEST_WORKSPACE` used by the existing e2e tests; CUJ3 uses a separate managed +`UCODE_TEST_WORKSPACE` used by the existing e2e tests; catalog discovery uses a separate managed workspace selected explicitly for its model-only journeys. It does not import `ucode`, patch application functions, substitute agent executables, run a fake gateway, or construct ug state files. The normal test suite checks these boundaries. @@ -230,6 +230,7 @@ test_ug_claude_managed_model_discovery.py # fetched/reused Claude MPS policy cas test_ug_codex_managed_model_discovery.py # fetched/reused Codex MPS policy cases test_ug_claude_model_discovery.py # unmanaged scenarios 7, 9, 11, 13 test_ug_codex_model_discovery.py # unmanaged scenarios 8, 10, 12, 14 +test_ug_catalog_discovery.py # dedicated managed Claude/Codex catalogs and tasks test_ug_configure_managed.py # managed workspace: static model list/catalog pointer, no agent selector test_ug_configure_managed_models.py # injected model sources, smart-routing banner, Codex fallback metadata test_ug_configure_managed_mcp.py # injected managed MCP list @@ -242,6 +243,9 @@ utils/ # process/terminal/evidence helpers and Do `conftest.py`, `pytest.ini`, and this README stay at the suite root for pytest discovery and run instructions. +Agent catalog requests use the explicitly supplied workspace and bearer, reject +redirects, and retain schema/provider headers and raw pagination evidence. + Each test has a `Scenario:` / `Expected:` docstring and shows its own public configure command, launch, user action, and assertions. Shared code only handles process/terminal mechanics, evidence, and cleanup. Fixtures supply an isolated @@ -361,7 +365,7 @@ real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **8 managed-workspace cases** (one per agent, an idempotent -re-configure, a cache-TTL journey, two Claude defaults cases, and two CUJ3 model journeys; marker `managed`) run against +re-configure, a cache-TTL journey, two Claude defaults cases, and two catalog discovery journeys; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** uses two real workspaces and checks skills MCP cleanup and a completed Claude task. A further **25 `managed_fixture` @@ -385,7 +389,7 @@ constants in the runner; CI only needs `UG_MPS_DEFAULTS_CLIENT_SECRET` for west- mints short-lived tokens and passes bearers to pytest; each test selects its target bearer for `ug configure` and Claude. The client secrets do not enter the pytest process. The 14 retained numbered scenarios comprise 22 explicit journeys: 8 managed-fixture, -2 CUJ3 model, and 12 unmanaged executions. Four provider-override journeys retain their +2 catalog discovery, and 12 unmanaged executions. Four provider-override journeys retain their assertions under descriptive names; the complete integration suite collects 103 executions. See the named coverage and gaps matrix in [../README.md](../README.md). @@ -552,7 +556,7 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, -both full lanes, both CUJ3 model-discovery lanes, and both **Managed config** lanes to pass for full/live runs. Each tracing +both full lanes, both Catalog discovery lanes, and both **Managed config** lanes to pass for full/live runs. Each tracing journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke, TUI, and installation subsets do not select managed tests and do not require them. @@ -560,16 +564,28 @@ The advisory Windows installation and headless lanes are not yet included in tha The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. -### CUJ3 model-discovery stack +### Catalog discovery + +Agent selection follows the rest of the suite: explicit `test_catalog_discovery_` +journeys carry agent markers; `UG_INTEGRATION_AGENTS` filters collection, and CI runs an +agent matrix. Launch commands, TUI tasks, defaults, and expected model sets stay visible +in each journey. Shared per-agent discovery and transcript helpers live in `utils/agents/`; +process, terminal, and task orchestration stay in the common harness. + +For another agent, add its helper module and evidence adapter, explicit journey and +fixture expectations. Register its package/version and marker in the runner/collection +setup, implement any native terminal support, and add it to the CI matrix. Existing +journeys need no renaming or generic agent branches. -CUJ3 first covers Claude/Codex models only. An authorized admin provisions both +Catalog discovery covers Claude/Codex models only. An authorized admin provisions both schemas with `fixtures/cuj/models/provision.py` and separately reviews/publishes `fixtures/cuj-3/managed-config.json` in the dedicated workspace. Follow the fixture README's prerequisites, explicit auth, and preflight-before-write rules. Tests do not provision resources or policy; Apps, connections, MCP, and skills are not required. -Each journey runs real `ug configure`, checks persisted schema/defaults and all -seven UC services, and independently fetches scoped and compatible-decoy catalogs. +Model-service fixtures are trusted prerequisites; journeys do not revalidate their +backing destinations. Each journey runs real `ug configure`, checks persisted +schema/defaults, and independently fetches scoped and compatible-decoy catalogs. Generated settings/catalogs, native caches/model lists, and exact numbered picker rows must agree: no unmatched, ambiguous, duplicate, or excluded entries hidden by labels. Defaults and every extra compatible model must complete file tasks @@ -602,19 +618,18 @@ models retain returned wire IDs; Codex keeps raw FQNs. No entries are dropped. Bare `ug` and explicit `ug claude` each complete a separate Claude/Sonnet TUI task and exit normally; `ug claude -p "task"` completes a Sonnet print task. Codex completes both TUI and `ug codex -- exec --skip-git-repo-check --json "task"` tasks on `gpt_luna`. -All default tasks omit model overrides; routing stays off. Codex `-p` selects a native -profile, not a headless prompt. +All default tasks omit model overrides; routing stays off. -CI runs `managed and cuj3 and workspace_isolated` in a two-agent matrix using +CI runs `managed and catalog_discovery and workspace_isolated` in a two-agent matrix using `UG_CUJ3_WORKSPACE` and the shared CUJ SP secrets. The shared managed lanes exclude -`workspace_isolated`; both CUJ3 legs are required for full/live runs. Local runs +`workspace_isolated`; both catalog discovery legs are required for full/live runs. Local runs must provide an explicit workspace and authenticated profile or bearer: ```bash python3.12 scripts/run_integration.py \ --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ - --workspace "$CUJ3_WORKSPACE" --profile YOUR_CUJ3_PROFILE \ - -- -m 'managed and cuj3 and workspace_isolated' + --workspace "$CATALOG_DISCOVERY_WORKSPACE" --profile YOUR_PROFILE \ + -- -m 'managed and catalog_discovery and workspace_isolated' ``` MCP/skills remain separate layers; this model suite does not depend on CUJ7. diff --git a/tests/integration/catalog_discovery_expectations.py b/tests/integration/catalog_discovery_expectations.py new file mode 100644 index 000000000..528d760fd --- /dev/null +++ b/tests/integration/catalog_discovery_expectations.py @@ -0,0 +1,38 @@ +"""Fixture expectations owned by the catalog discovery journey, not shared utilities.""" + +MODEL_SCHEMA = "ug_e2e.models" +OTHER_MODEL_SCHEMA = "ug_e2e.other_models" +CLAUDE_DEFAULT = f"{MODEL_SCHEMA}.claude_sonnet" +CODEX_DEFAULT = f"{MODEL_SCHEMA}.gpt_luna" +GEMINI_MODEL = f"{MODEL_SCHEMA}.gemini_flash" +CLAUDE_MODELS = frozenset({CLAUDE_DEFAULT, f"{MODEL_SCHEMA}.claude_haiku", f"{MODEL_SCHEMA}.kimi"}) +CODEX_MODELS = frozenset({CODEX_DEFAULT, f"{MODEL_SCHEMA}.kimi"}) +MODEL_SERVICES = CLAUDE_MODELS | CODEX_MODELS | {GEMINI_MODEL} +CLAUDE_DECOY = f"{OTHER_MODEL_SCHEMA}.claude_decoy" +CODEX_DECOY = f"{OTHER_MODEL_SCHEMA}.codex_decoy" + + +def assert_model_policy(config: dict) -> None: + """Require exact model policy while allowing unrelated resource fields.""" + assert config["spec_version"] == 1, config + assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE", config + entries = config["enabled_agents"] + assert isinstance(entries, list) and len(entries) == 2, config + assert all(isinstance(entry, dict) for entry in entries), entries + by_agent = {entry.get("agent"): entry.get("config") for entry in entries} + expected_defaults = { + "CODING_AGENT_CLAUDE_CODE": { + "default_model": CLAUDE_DEFAULT, + "default_sonnet_model": CLAUDE_DEFAULT, + }, + "CODING_AGENT_CODEX": {"default_model": CODEX_DEFAULT}, + } + assert set(by_agent) == set(expected_defaults), by_agent + for agent, defaults in expected_defaults.items(): + agent_config = by_agent[agent] + assert isinstance(agent_config, dict), agent_config + assert agent_config.get("models") == {"unity_catalog_location": MODEL_SCHEMA}, agent_config + assert agent_config.get("default_models") == defaults, agent_config + for key in ("smart_routing", "tracing"): + settings = agent_config.get(key) + assert isinstance(settings, dict) and settings.get("enabled") is False, agent_config diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 7cdb26711..5b1671408 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -10,7 +10,7 @@ markers = smoke: Databricks Hosted, custom OAuth CLI TUI, and headless prompt for each agent tui: real interactive terminal boot, keyboard input, exit and reopen workspace_isolated: requires a dedicated workspace instead of the shared integration workspace - cuj3: managed schema-scoped model discovery and inference in the dedicated CUJ3 workspace + catalog_discovery: managed schema-scoped model discovery and inference in the dedicated workspace claude: only runs when Claude Code is explicitly selected codex: only runs when Codex is explicitly selected opencode: only runs when OpenCode is explicitly selected diff --git a/tests/integration/test_ug_cuj3_model_discovery.py b/tests/integration/test_ug_catalog_discovery.py similarity index 77% rename from tests/integration/test_ug_cuj3_model_discovery.py rename to tests/integration/test_ug_catalog_discovery.py index 7e0edd916..fa7dc7aa3 100644 --- a/tests/integration/test_ug_cuj3_model_discovery.py +++ b/tests/integration/test_ug_catalog_discovery.py @@ -1,11 +1,11 @@ -"""CUJ3: complete managed, schema-scoped model discovery journeys.""" +"""Catalog discovery: complete managed, schema-scoped model discovery journeys.""" import json import os import tomllib import pytest -from utils.cuj3 import ( +from catalog_discovery_expectations import ( CLAUDE_DECOY, CLAUDE_DEFAULT, CLAUDE_MODELS, @@ -13,34 +13,31 @@ CODEX_DEFAULT, CODEX_MODELS, GEMINI_MODEL, - MODEL_HEADER, MODEL_SCHEMA, - MODEL_SERVICES, OTHER_MODEL_SCHEMA, - assert_completed_task_model, - assert_persisted_config, + assert_model_policy, +) +from utils.agents import claude, codex +from utils.evidence import FileTask, assert_completed_task_model +from utils.managed import read_persisted_managed_config +from utils.model_discovery import ( assert_picker_inventory, - claude_discovery_model_id, - claude_model_service_id, - codex_model_in_picker, - fetch_claude_parent_catalog, - fetch_codex_parent_catalog, - fetch_model_service_inventory, ) -from utils.evidence import FileTask -from utils.model_discovery import claude_model_in_picker -from utils.provider_catalog import parse_codex_provider_catalog +from utils.provider_catalog import ( + MODEL_SERVICE_PARENT_SCHEMA_HEADER, + parse_codex_provider_catalog, +) from utils.terminal import AgentTerminal -pytestmark = [pytest.mark.managed, pytest.mark.cuj3, pytest.mark.workspace_isolated] +pytestmark = [pytest.mark.managed, pytest.mark.catalog_discovery, pytest.mark.workspace_isolated] @pytest.mark.claude @pytest.mark.tui -def test_case_03_managed_schema_pointers_claude(live_session, workspace): - """Scenario: configure published CUJ3 policy; launch bare ug, ug claude, and print tasks. +def test_catalog_discovery_claude(live_session, workspace): + """Scenario: configure published Catalog discovery policy; launch bare ug, ug claude, and print tasks. - Expected: persisted schema/defaults and all seven UC services match policy; independent + Expected: persisted schema/defaults match policy; independent catalogs, settings, cache, and exact numbered picker agree on Sonnet/Haiku/Kimi, excluding Gemini and both accessible out-of-scope decoys. Both TUI launches exit normally after separate Sonnet file tasks; print mode uses Sonnet without an override, @@ -51,16 +48,15 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): session = live_session configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) assert "Select coding agents to configure:" not in configured.stdout, configured.stdout - assert_persisted_config(session, workspace) + persisted = read_persisted_managed_config(session, workspace) + assert_model_policy(persisted["config"]) + session.record("catalog-discovery-managed-config.json", persisted) bearer = os.environ["DATABRICKS_BEARER"] - inventory = fetch_model_service_inventory(workspace, bearer) - session.record("cuj3-model-service-sources.json", inventory) - assert set(inventory) == MODEL_SERVICES | {CLAUDE_DECOY, CODEX_DECOY}, inventory - parent_catalog = fetch_claude_parent_catalog(workspace, bearer, MODEL_SCHEMA) - decoy_catalog = fetch_claude_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) + parent_catalog = claude.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) + decoy_catalog = claude.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) session.record( - "cuj3-parent-inventory.json", + "catalog-discovery-parent-inventory.json", { "scoped": parent_catalog.model_ids, "decoy": decoy_catalog.model_ids, @@ -69,20 +65,23 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): "decoy_pages": decoy_catalog.payloads, }, ) - service_ids = [claude_model_service_id(model) for model in parent_catalog.model_ids] + service_ids = [claude.model_service_id(model) for model in parent_catalog.model_ids] assert len(service_ids) == len(set(service_ids)), parent_catalog assert set(service_ids) == CLAUDE_MODELS, parent_catalog assert set(parent_catalog.model_ids) == { - claude_discovery_model_id(model) for model in CLAUDE_MODELS + claude.discovery_model_id(model) for model in CLAUDE_MODELS }, parent_catalog - decoy_ids = [claude_model_service_id(model) for model in decoy_catalog.model_ids] + decoy_ids = [claude.model_service_id(model) for model in decoy_catalog.model_ids] assert len(decoy_ids) == len(set(decoy_ids)), decoy_catalog assert set(decoy_ids) == {CLAUDE_DECOY}, decoy_catalog - assert set(decoy_catalog.model_ids) == {claude_discovery_model_id(CLAUDE_DECOY)}, decoy_catalog + assert set(decoy_catalog.model_ids) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog default_task = FileTask(session) with AgentTerminal( - session, "claude", [str(session.binary)], "cuj3-bare-ug-claude-default-and-picker" + session, + "claude", + [str(session.binary)], + "catalog-discovery-bare-ug-claude-default-and-picker", ) as tui: tui.boot() settings = json.loads((session.home / ".claude/ucode-settings.json").read_text()) @@ -96,7 +95,7 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): assert ( env.get("ANTHROPIC_CUSTOM_HEADERS", "") .splitlines() - .count(f"{MODEL_HEADER}: {MODEL_SCHEMA}") + .count(f"{MODEL_SERVICE_PARENT_SCHEMA_HEADER}: {MODEL_SCHEMA}") == 1 ), settings assert not {"availableModels", "enforceAvailableModels"} & settings.keys(), settings @@ -110,10 +109,10 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): for option in options: if display_name := parent_catalog.display_names[option["model"]]: assert option["label"] == display_name, option - session.record("cuj3-claude-settings.json", settings) + session.record("catalog-discovery-claude-settings.json", settings) picker_screen = tui.open_model_picker( model_visible=lambda screen: all( - claude_model_in_picker(screen, model, parent_catalog.display_names[model]) + claude.model_in_picker(screen, model, parent_catalog.display_names[model]) for model in parent_catalog.model_ids ) ) @@ -129,7 +128,10 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): explicit_task = FileTask(session) with AgentTerminal( - session, "claude", [str(session.binary), "claude"], "cuj3-explicit-ug-claude-default" + session, + "claude", + [str(session.binary), "claude"], + "catalog-discovery-explicit-ug-claude-default", ) as tui: tui.boot() tui.submit(explicit_task.prompt) @@ -183,30 +185,29 @@ def test_case_03_managed_schema_pointers_claude(live_session, workspace): @pytest.mark.codex @pytest.mark.tui -def test_case_04_managed_schema_pointers_codex(live_session, workspace): - """Scenario: configure published CUJ3 policy; launch Codex TUI and ug codex -- exec. +def test_catalog_discovery_codex(live_session, workspace): + """Scenario: configure published Catalog discovery policy; launch Codex TUI and ug codex -- exec. - Expected: persisted schema/defaults and all seven UC services match policy; independent + Expected: persisted schema/defaults match policy; independent catalogs, generated catalogs, model/list, and exact numbered picker agree on GPT Luna/Kimi, excluding Gemini and both accessible out-of-scope decoys. TUI exits normally after a GPT Luna file task; exec uses that default without an override, and every extra model - completes a headless task, with routing off throughout. Native -p selects a profile. + completes a headless task, with routing off throughout. Codex's client-selected model must match; it does not prove the executed gateway backing destination. These assertions require a live pass, not collection alone. """ session = live_session configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) assert "Select coding agents to configure:" not in configured.stdout, configured.stdout - assert_persisted_config(session, workspace) + persisted = read_persisted_managed_config(session, workspace) + assert_model_policy(persisted["config"]) + session.record("catalog-discovery-managed-config.json", persisted) bearer = os.environ["DATABRICKS_BEARER"] - inventory = fetch_model_service_inventory(workspace, bearer) - session.record("cuj3-model-service-sources.json", inventory) - assert set(inventory) == MODEL_SERVICES | {CLAUDE_DECOY, CODEX_DECOY}, inventory - parent_catalog = fetch_codex_parent_catalog(workspace, bearer, MODEL_SCHEMA) - decoy_catalog = fetch_codex_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) + parent_catalog = codex.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) + decoy_catalog = codex.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) session.record( - "cuj3-parent-inventory.json", + "catalog-discovery-parent-inventory.json", { "scoped": parent_catalog.model_ids, "decoy": decoy_catalog.model_ids, @@ -244,19 +245,19 @@ def test_case_04_managed_schema_pointers_codex(live_session, workspace): label = entry.get("display_name") assert isinstance(label, str) and label.strip(), entry display_names[entry["slug"]] = label - session.record("cuj3-codex-catalog.json", catalog) + session.record("catalog-discovery-codex-catalog.json", catalog) default_task = FileTask(session) with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "cuj3-codex-default" + session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-default" ) as tui: tui.boot() config = tomllib.loads((session.home / ".codex/ucode.config.toml").read_text()) assert config.get("model") == CODEX_DEFAULT, config - session.record("cuj3-codex-config.json", config) + session.record("catalog-discovery-codex-config.json", config) picker_screen = tui.open_codex_model_picker( model_visible=lambda screen: all( - codex_model_in_picker(screen, model, display_names[model]) + codex.model_in_picker(screen, model, display_names[model]) for model in parent_catalog.model_ids ) ) diff --git a/tests/integration/test_ug_claude_managed_model_discovery.py b/tests/integration/test_ug_claude_managed_model_discovery.py index e759823e2..f674d225e 100644 --- a/tests/integration/test_ug_claude_managed_model_discovery.py +++ b/tests/integration/test_ug_claude_managed_model_discovery.py @@ -186,7 +186,7 @@ def test_case_01_fresh_managed_claude_uses_admin_discovery( _assert_managed_provider_in_picker(session, workspace, screen, _managed_claude_provider_catalog) -def test_managed_fixture_claude_rejects_provider_override(live_session, workspace, claude_provider): +def test_case_03_managed_claude_rejects_provider_override(live_session, workspace, claude_provider): """Scenario: configure managed Claude, then pass --provider. Expected: ug rejects the override without changing agent-owned state/files. @@ -215,7 +215,7 @@ def test_managed_fixture_claude_rejects_provider_override(live_session, workspac _assert_rejected_before_claude_started(session, result, before) -def test_managed_fixture_fresh_claude_rejects_provider_override( +def test_case_03_fresh_managed_claude_rejects_provider_override( live_session, workspace, claude_provider ): """Scenario: pass --provider while launching managed Claude from fresh state. diff --git a/tests/integration/test_ug_codex_managed_model_discovery.py b/tests/integration/test_ug_codex_managed_model_discovery.py index f7484759b..980f69085 100644 --- a/tests/integration/test_ug_codex_managed_model_discovery.py +++ b/tests/integration/test_ug_codex_managed_model_discovery.py @@ -193,7 +193,7 @@ def test_case_02_managed_codex_uses_admin_discovery_from_fresh_state( assert app_models == models, (app_models, models) -def test_managed_fixture_codex_rejects_provider_override_after_configure( +def test_case_04_managed_codex_rejects_provider_override_after_configure( live_session, workspace, codex_provider ): """Scenario: configure managed Codex, then pass a --provider override. @@ -225,7 +225,7 @@ def test_managed_fixture_codex_rejects_provider_override_after_configure( _assert_rejected_before_codex_started(session, result, before) -def test_managed_fixture_codex_rejects_provider_override_from_fresh_state( +def test_case_04_managed_codex_rejects_provider_override_from_fresh_state( live_session, workspace, codex_provider ): """Scenario: pass --workspace and a --provider override from fresh state. diff --git a/tests/integration/utils/agents/__init__.py b/tests/integration/utils/agents/__init__.py index 1a952fc10..ae5f60cc1 100644 --- a/tests/integration/utils/agents/__init__.py +++ b/tests/integration/utils/agents/__init__.py @@ -1 +1,10 @@ -"""Native agent transcript helpers used by the shared evidence layer.""" +"""Shared validation for agent-specific integration evidence.""" + +import re + + +def _evidence_id(value: object, field: str) -> str: + assert isinstance(value, str) and re.fullmatch(r"\S+", value), ( + f"Missing or malformed completed-task {field}" + ) + return value diff --git a/tests/integration/utils/agents/claude.py b/tests/integration/utils/agents/claude.py index 2c79f661d..4846ceaaa 100644 --- a/tests/integration/utils/agents/claude.py +++ b/tests/integration/utils/agents/claude.py @@ -1,6 +1,21 @@ -"""Claude transcript helpers for integration journeys.""" +"""Claude-specific integration evidence and model discovery helpers.""" + +from ..model_discovery import claude_discovery_model_id as discovery_model_id +from ..model_discovery import claude_model_in_picker as model_in_picker +from ..provider_catalog import fetch_anthropic_parent_catalog as fetch_parent_catalog +from . import _evidence_id + +__all__ = [ + "EVIDENCE_KIND", + "assistant_answers", + "completed_task_models", + "discovery_model_id", + "fetch_parent_catalog", + "model_in_picker", +] SESSION_DIRECTORY = ".claude/projects" +EVIDENCE_KIND = "response-reported model" def is_child_session(path: str, records: list[dict]) -> bool: @@ -20,3 +35,29 @@ def assistant_answers(records: list[dict]) -> list[str]: if part.get("type") == "text" and isinstance(part.get("text"), str) ) return answers + + +def completed_task_models(records: list[dict], answer_value: str) -> set[str]: + """Parse Claude's response-reported model on the assistant's matching answer.""" + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) + found: set[str] = set() + for record in records: + if record.get("type") != "assistant": + continue + message = record.get("message") + assert isinstance(message, dict), "Malformed Claude assistant message" + assert message.get("role") == "assistant", "Missing Claude assistant role" + content = message.get("content") + assert isinstance(content, list), "Malformed Claude assistant content" + for part in content: + assert isinstance(part, dict), "Malformed Claude assistant content block" + if part.get("type") != "text": + continue + text = part.get("text") + assert isinstance(text, str), "Malformed Claude assistant text" + if answer_value in text: + model = _evidence_id(message.get("model"), "model") + found.add(_evidence_id(model.removesuffix("[1m]"), "model")) + return found diff --git a/tests/integration/utils/agents/codex.py b/tests/integration/utils/agents/codex.py index fea968ce8..99e505420 100644 --- a/tests/integration/utils/agents/codex.py +++ b/tests/integration/utils/agents/codex.py @@ -1,6 +1,19 @@ -"""Codex transcript helpers for integration journeys.""" +"""Codex-specific integration evidence and model discovery helpers.""" + +from ..model_discovery import codex_model_in_picker as model_in_picker +from ..provider_catalog import fetch_codex_parent_catalog as fetch_parent_catalog +from . import _evidence_id + +__all__ = [ + "EVIDENCE_KIND", + "assistant_answers", + "completed_task_models", + "fetch_parent_catalog", + "model_in_picker", +] SESSION_DIRECTORY = ".codex/sessions" +EVIDENCE_KIND = "client-selected model" def is_child_session(path: str, records: list[dict]) -> bool: @@ -21,3 +34,33 @@ def assistant_answers(records: list[dict]) -> list[str]: if payload.get("type") == "task_complete" and payload.get("last_agent_message"): answers.append(payload["last_agent_message"]) return answers + + +def completed_task_models(records: list[dict], answer_value: str) -> set[str]: + """Join Codex's completed answer turn to its client-selected context model.""" + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) + completed: set[str] = set() + for record in records: + if record.get("type") != "event_msg": + continue + payload = record.get("payload") + assert isinstance(payload, dict), "Malformed Codex event payload" + if payload.get("type") != "task_complete": + continue + answer = payload.get("last_agent_message") + assert isinstance(answer, str), "Missing or malformed Codex completed answer" + if answer_value in answer: + completed.add(_evidence_id(payload.get("turn_id"), "turn ID")) + models: dict[str, set[str]] = {turn_id: set() for turn_id in completed} + for record in records: + if record.get("type") != "turn_context": + continue + payload = record.get("payload") + assert isinstance(payload, dict), "Malformed Codex turn context" + turn_id = _evidence_id(payload.get("turn_id"), "turn ID") + if turn_id in completed: + models[turn_id].add(_evidence_id(payload.get("model"), "model")) + assert all(models.values()), "Missing Codex model context for completed answer turn" + return {model for turn_models in models.values() for model in turn_models} diff --git a/tests/integration/utils/cuj3.py b/tests/integration/utils/cuj3.py deleted file mode 100644 index 1ddb1f9a2..000000000 --- a/tests/integration/utils/cuj3.py +++ /dev/null @@ -1,375 +0,0 @@ -"""Black-box model-discovery evidence for the dedicated CUJ3 workspace.""" - -from __future__ import annotations - -import hashlib -import json -import re -import urllib.error -import urllib.parse -import urllib.request -from dataclasses import dataclass - -from .evidence import agent_sessions, is_child_session -from .provider_catalog import parse_anthropic_provider_page, parse_codex_provider_catalog - -MODEL_SCHEMA = "ug_e2e.models" -OTHER_MODEL_SCHEMA = "ug_e2e.other_models" -MODEL_HEADER = "Databricks-Model-Service-Parent-Schema" -CLAUDE_DEFAULT = f"{MODEL_SCHEMA}.claude_sonnet" -CODEX_DEFAULT = f"{MODEL_SCHEMA}.gpt_luna" -GEMINI_MODEL = f"{MODEL_SCHEMA}.gemini_flash" -CLAUDE_MODELS = frozenset({CLAUDE_DEFAULT, f"{MODEL_SCHEMA}.claude_haiku", f"{MODEL_SCHEMA}.kimi"}) -CODEX_MODELS = frozenset({CODEX_DEFAULT, f"{MODEL_SCHEMA}.kimi"}) -MODEL_SERVICES = CLAUDE_MODELS | CODEX_MODELS | {GEMINI_MODEL} -CLAUDE_DECOY = f"{OTHER_MODEL_SCHEMA}.claude_decoy" -CODEX_DECOY = f"{OTHER_MODEL_SCHEMA}.codex_decoy" - - -def claude_discovery_model_id(model_id: str) -> str: - """Encode the exact wire ID expected by this workspace's Claude discovery feature.""" - if "claude" in model_id.lower() or "anthropic" in model_id.lower(): - return model_id - checksum = hashlib.sha256(model_id.encode("utf-8")).hexdigest()[:8] - return f"anthropic-aigw-{checksum}-{model_id}" - - -def claude_model_service_id(model_id: str) -> str: - """Recover service identity only from the gateway's checksum-valid discovery alias.""" - alias = re.fullmatch(r"anthropic-aigw-([0-9a-f]{8})-(.+)", model_id) - if alias is None: - return model_id - checksum, original = alias.groups() - if hashlib.sha256(original.encode("utf-8")).hexdigest()[:8] != checksum: - return model_id - return original - - -class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - -def safe_https_json_get( - workspace: str, token: str, path: str, *, headers: dict[str, str] | None = None -) -> object: - """GET JSON from an explicit workspace origin without redirects or secret diagnostics.""" - assert isinstance(workspace, str), "Expected an HTTPS workspace origin" - try: - origin = urllib.parse.urlsplit(workspace) - port = origin.port - except ValueError: - raise AssertionError("Expected a valid HTTPS workspace origin") from None - assert ( - origin.scheme == "https" - and origin.hostname - and origin.username is None - and origin.password is None - and origin.path in {"", "/"} - and not origin.query - and not origin.fragment - and (port is None or 0 < port <= 65535) - and not re.search(r"[\s\\]", workspace) - ), "Expected an HTTPS workspace origin without credentials" - assert isinstance(token, str) and token.strip() and not re.search(r"[\r\n]", token), ( - "Expected an explicit workspace bearer" - ) - assert path.startswith("/") and not path.startswith("//"), "Expected a workspace API path" - request = urllib.request.Request( - f"https://{origin.netloc}{path}", - headers={ - **(headers or {}), - "Authorization": f"Bearer {token}", - "Accept": "application/json", - }, - method="GET", - ) - try: - with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response: - return json.load(response) - except urllib.error.HTTPError as error: - raise AssertionError(f"Workspace JSON GET returned HTTP {error.code}") from None - except (urllib.error.URLError, OSError, ValueError): - raise AssertionError("Workspace JSON GET failed or returned invalid JSON") from None - - -def fetch_model_service_inventory(workspace: str, token: str) -> dict[str, tuple[str, ...]]: - """Prove every fixture exists independently of agent-compatible model lists.""" - inventory: dict[str, tuple[str, ...]] = {} - for model in sorted(MODEL_SERVICES | {CLAUDE_DECOY, CODEX_DECOY}): - payload = safe_https_json_get( - workspace, token, f"/api/2.1/unity-catalog/model-services/{model}" - ) - assert isinstance(payload, dict), f"Model service {model} must be an object" - config = payload.get("config") - assert isinstance(config, dict), f"Model service {model} must have a config" - routing = config.get("routing") - assert isinstance(routing, dict), f"Model service {model} must have routing" - assert routing.get("fallback") in (None, {}, {"destinations": []}), ( - f"Model service {model} must not have fallback destinations" - ) - destinations = routing.get("destinations") - assert isinstance(destinations, list) and len(destinations) == 1, ( - f"Model service {model} must have exactly one routing destination" - ) - sources: list[str] = [] - for destination in destinations: - assert isinstance(destination, dict), f"Invalid destination for {model}" - source = destination.get("name") - assert isinstance(source, str) and re.fullmatch( - r"system\.ai\.[A-Za-z0-9_-]+", source - ), f"Model service {model} must target an existing system.ai source" - assert destination.get("destination_type") == ( - "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL" - ), f"Unexpected destination type for {model}" - target = destination.get("pay_per_token_config") - assert isinstance(target, dict) and target.get("model") == f"models/{source}", ( - f"Model service {model} has inconsistent source routing" - ) - traffic = destination.get("traffic_percentage", 100) - assert type(traffic) is int and traffic == 100, ( - f"Model service {model} must route 100 percent of traffic to its source" - ) - assert all( - destination.get(field, False) is False - for field in ("is_deleted", "is_disabled", "disabled") - ), f"Model service {model} has a disabled or deleted destination" - sources.append(source) - inventory[model] = tuple(sources) - return inventory - - -def codex_model_in_picker(screen: str, model_id: str, display_name: str) -> bool: - """Match native numbered picker rows, not banners, prompt text, or footers.""" - rows = re.findall(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)][ \t]+([^\n]+)$", screen) - return any(_picker_row_matches(row, model_id, display_name) for row in rows) - - -def _picker_row_matches(label: str, model_id: str, display_name: str | None) -> bool: - return any( - re.search(rf"(? None: - """Require exactly one unambiguous numbered native row per expected model.""" - assert agent in {"claude", "codex"}, agent - assert display_names, "Expected a nonempty provider model inventory" - rows = re.findall(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)](?:[ \t]+([^\n]*))?$", screen) - assert rows, f"No numbered {agent} picker rows:\n{screen}" - observed: list[str] = [] - for label in rows: - matched = { - model - for model, display_name in display_names.items() - if _picker_row_matches(label, model, display_name) - } - assert len(matched) == 1, ( - f"Unmatched or ambiguous {agent} picker row {label!r}: {sorted(matched)}" - ) - model = matched.pop() - assert model not in observed, f"Duplicate {agent} picker model row: {model}" - observed.append(model) - assert set(observed) == set(display_names), { - "agent": agent, - "expected": sorted(display_names), - "observed": observed, - } - - -@dataclass(frozen=True) -class ParentCatalog: - """Independently observed provider inventory, labels, and unfiltered payloads.""" - - model_ids: tuple[str, ...] - display_names: dict[str, str | None] - payloads: tuple[dict, ...] - - -def fetch_codex_parent_catalog(workspace: str, token: str, parent_schema: str) -> ParentCatalog: - """Keep the raw Codex catalog so hidden entries cannot conceal exclusions.""" - assert parent_schema, "Expected an explicit parent schema" - payload = safe_https_json_get( - workspace, token, "/ai-gateway/codex/v1/models", headers={MODEL_HEADER: parent_schema} - ) - model_ids = parse_codex_provider_catalog(payload) - assert isinstance(payload, dict), "Expected a Codex catalog object" - display_names = {} - for entry in payload["models"]: - if entry.get("slug") in model_ids: - label = entry.get("display_name") - assert isinstance(label, str) and label.strip(), "Invalid Codex catalog display name" - display_names[entry["slug"]] = label - return ParentCatalog(model_ids, display_names, (payload,)) - - -def fetch_claude_parent_catalog(workspace: str, token: str, parent_schema: str) -> ParentCatalog: - """Read the real parent-scoped Anthropic catalog with bounded pagination.""" - assert parent_schema, "Expected an explicit parent schema" - model_ids: list[str] = [] - display_names: dict[str, str | None] = {} - pages: list[dict] = [] - cursor: str | None = None - seen_cursors: set[str] = set() - for _page_number in range(20): - query = {"limit": "1000"} - if cursor is not None: - query["after_id"] = cursor - payload = safe_https_json_get( - workspace, - token, - f"/ai-gateway/anthropic/v1/models?{urllib.parse.urlencode(query)}", - headers={ - "Anthropic-Version": "2023-06-01", - MODEL_HEADER: parent_schema, - }, - ) - page = parse_anthropic_provider_page(payload) - assert isinstance(payload, dict), "Expected an Anthropic catalog object" - pages.append(payload) - for model_id, display_name in page.models: - assert model_id not in display_names, f"duplicate Claude model id: {model_id}" - model_ids.append(model_id) - display_names[model_id] = display_name - if not page.has_more: - return ParentCatalog(tuple(model_ids), display_names, tuple(pages)) - assert page.last_id is not None and page.last_id not in seen_cursors, payload - seen_cursors.add(page.last_id) - cursor = page.last_id - raise AssertionError("Claude parent catalog exceeded 20 pages") - - -def assert_cuj3_config(config: dict) -> None: - """Require exact model policy while allowing unrelated resource fields.""" - assert config["spec_version"] == 1, config - assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE", config - entries = config["enabled_agents"] - assert isinstance(entries, list) and len(entries) == 2, config - assert all(isinstance(entry, dict) for entry in entries), entries - by_agent = {entry.get("agent"): entry.get("config") for entry in entries} - expected_defaults = { - "CODING_AGENT_CLAUDE_CODE": { - "default_model": CLAUDE_DEFAULT, - "default_sonnet_model": CLAUDE_DEFAULT, - }, - "CODING_AGENT_CODEX": {"default_model": CODEX_DEFAULT}, - } - assert set(by_agent) == set(expected_defaults), by_agent - for agent, defaults in expected_defaults.items(): - agent_config = by_agent[agent] - assert isinstance(agent_config, dict), agent_config - assert agent_config.get("models") == {"unity_catalog_location": MODEL_SCHEMA}, agent_config - assert agent_config.get("default_models") == defaults, agent_config - for key in ("smart_routing", "tracing"): - settings = agent_config.get(key) - assert isinstance(settings, dict) and settings.get("enabled") is False, agent_config - - -def assert_persisted_config(session, workspace: str) -> dict: - """Validate the policy persisted by public configure, never a setup precheck.""" - path = session.home / ".ucode" / "managed-config.json" - assert path.is_file(), path - persisted = json.loads(path.read_text()) - assert isinstance(persisted, dict) and persisted.get("workspace") == workspace, persisted - config = persisted.get("config") - assert isinstance(config, dict), persisted - assert_cuj3_config(config) - session.record("cuj3-managed-config.json", persisted) - return config - - -def _evidence_id(value: object, field: str) -> str: - assert isinstance(value, str) and re.fullmatch(r"\S+", value), ( - f"Missing or malformed completed-task {field}" - ) - return value - - -def claude_completed_task_models(records: list[dict], answer_value: str) -> set[str]: - """Parse Claude's response-reported model on the assistant's matching answer.""" - assert isinstance(answer_value, str) and answer_value.strip(), ( - "Expected a nonempty answer value" - ) - found: set[str] = set() - for record in records: - if record.get("type") != "assistant": - continue - message = record.get("message") - assert isinstance(message, dict), "Malformed Claude assistant message" - assert message.get("role") == "assistant", "Missing Claude assistant role" - content = message.get("content") - assert isinstance(content, list), "Malformed Claude assistant content" - for part in content: - assert isinstance(part, dict), "Malformed Claude assistant content block" - if part.get("type") != "text": - continue - text = part.get("text") - assert isinstance(text, str), "Malformed Claude assistant text" - if answer_value in text: - model = _evidence_id(message.get("model"), "model") - found.add(_evidence_id(model.removesuffix("[1m]"), "model")) - return found - - -def codex_completed_task_models(records: list[dict], answer_value: str) -> set[str]: - """Join Codex's completed answer turn to its client-selected context model.""" - assert isinstance(answer_value, str) and answer_value.strip(), ( - "Expected a nonempty answer value" - ) - completed: set[str] = set() - for record in records: - if record.get("type") != "event_msg": - continue - payload = record.get("payload") - assert isinstance(payload, dict), "Malformed Codex event payload" - if payload.get("type") != "task_complete": - continue - answer = payload.get("last_agent_message") - assert isinstance(answer, str), "Missing or malformed Codex completed answer" - if answer_value in answer: - completed.add(_evidence_id(payload.get("turn_id"), "turn ID")) - models: dict[str, set[str]] = {turn_id: set() for turn_id in completed} - for record in records: - if record.get("type") != "turn_context": - continue - payload = record.get("payload") - assert isinstance(payload, dict), "Malformed Codex turn context" - turn_id = _evidence_id(payload.get("turn_id"), "turn ID") - if turn_id in completed: - models[turn_id].add(_evidence_id(payload.get("model"), "model")) - assert all(models.values()), "Missing Codex model context for completed answer turn" - return {model for turn_models in models.values() for model in turn_models} - - -def completed_task_models(session, agent: str, answer_value: str) -> set[str]: - """Read parent task model evidence, not proof of the gateway's destination.""" - assert agent in {"claude", "codex"}, agent - assert isinstance(answer_value, str) and answer_value.strip(), ( - "Expected a nonempty answer value" - ) - adapter = claude_completed_task_models if agent == "claude" else codex_completed_task_models - return { - model - for path, records in agent_sessions(session, agent).items() - if not is_child_session(agent, path, records) - for model in adapter(records, answer_value) - } - - -def assert_completed_task_model(session, agent: str, answer_value: str, expected: str) -> None: - expected = _evidence_id(expected, "expected model") - observed = completed_task_models(session, agent, answer_value) - session.record( - f"completed-task-model-{agent}-{answer_value[:12]}.json", - { - "expected": expected, - "observed": sorted(observed), - "evidence_kind": ( - "response-reported model" if agent == "claude" else "client-selected model" - ), - "gateway_destination_proven": False, - }, - ) - assert observed == {expected}, {"expected": expected, "observed": sorted(observed)} diff --git a/tests/integration/utils/evidence.py b/tests/integration/utils/evidence.py index b5dff142a..938fb7eba 100644 --- a/tests/integration/utils/evidence.py +++ b/tests/integration/utils/evidence.py @@ -5,7 +5,7 @@ import uuid from pathlib import Path -from .agents import claude, codex +from .agents import _evidence_id, claude, codex _AGENT_HELPERS = {"claude": claude, "codex": codex} @@ -56,6 +56,39 @@ def is_child_session(agent: str, path: str, records: list[dict]) -> bool: return _AGENT_HELPERS.get(agent, codex).is_child_session(path, records) +def completed_task_models(session, agent: str, answer_value: str) -> set[str]: + """Read parent task model evidence, not proof of the gateway's destination.""" + assert agent in _AGENT_HELPERS, f"Unsupported evidence agent: {agent}" + adapter = _AGENT_HELPERS[agent] + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) + sessions = agent_sessions(session, agent) + session.record("agent-sessions.json", sessions) + return { + model + for path, records in sessions.items() + if not is_child_session(agent, path, records) + for model in adapter.completed_task_models(records, answer_value) + } + + +def assert_completed_task_model(session, agent: str, answer_value: str, expected: str) -> None: + expected = _evidence_id(expected, "expected model") + observed = completed_task_models(session, agent, answer_value) + adapter = _AGENT_HELPERS[agent] + session.record( + f"completed-task-model-{agent}-{answer_value[:12]}.json", + { + "expected": expected, + "observed": sorted(observed), + "evidence_kind": adapter.EVIDENCE_KIND, + "gateway_destination_proven": False, + }, + ) + assert observed == {expected}, {"expected": expected, "observed": sorted(observed)} + + def assistant_answer_contains(session, agent: str, value: str, *, child: bool = False) -> bool: """Whether a native parent or child assistant answer contains ``value``.""" return any( diff --git a/tests/integration/utils/http.py b/tests/integration/utils/http.py new file mode 100644 index 000000000..1c35cff5e --- /dev/null +++ b/tests/integration/utils/http.py @@ -0,0 +1,63 @@ +"""Read-only workspace JSON transport with strict origins and no redirects.""" + +from __future__ import annotations + +import json +import re +import urllib.error +import urllib.parse +import urllib.request + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def workspace_origin(workspace: str, token: str) -> str: + """Validate an explicit HTTPS origin and bearer before constructing a client.""" + assert isinstance(workspace, str), "Expected an HTTPS workspace origin" + try: + origin = urllib.parse.urlsplit(workspace) + port = origin.port + except ValueError: + raise AssertionError("Expected a valid HTTPS workspace origin") from None + assert ( + origin.scheme == "https" + and origin.hostname + and origin.username is None + and origin.password is None + and origin.path in {"", "/"} + and not origin.query + and not origin.fragment + and (port is None or 0 < port <= 65535) + and not re.search(r"[\s\\]", workspace) + ), "Expected an HTTPS workspace origin without credentials" + assert isinstance(token, str) and token.strip() and not re.search(r"[\r\n]", token), ( + "Expected an explicit workspace bearer" + ) + return f"https://{origin.netloc}" + + +def safe_https_json_get( + workspace: str, token: str, path: str, *, headers: dict[str, str] | None = None +) -> object: + """GET JSON from an explicit workspace origin without redirects or secret diagnostics.""" + origin = workspace_origin(workspace, token) + assert path.startswith("/") and not path.startswith("//"), "Expected a workspace API path" + request = urllib.request.Request( + f"{origin}{path}", + headers={ + **(headers or {}), + "Authorization": f"Bearer {token}", + "Accept": "application/json", + }, + method="GET", + ) + try: + with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + raise AssertionError(f"Workspace JSON GET returned HTTP {error.code}") from None + except (urllib.error.URLError, OSError, ValueError): + raise AssertionError("Workspace JSON GET failed or returned invalid JSON") from None diff --git a/tests/integration/utils/managed.py b/tests/integration/utils/managed.py index 3599d38f9..997d96948 100644 --- a/tests/integration/utils/managed.py +++ b/tests/integration/utils/managed.py @@ -13,6 +13,16 @@ MANAGED_CONFIGS_PATH = "/api/ai-gateway/v2/coding-agent-configs" +def read_persisted_managed_config(session, workspace: str) -> dict: + """Read the configure result without imposing a journey-specific policy.""" + path = session.home / ".ucode" / "managed-config.json" + assert path.is_file(), path + persisted = json.loads(path.read_text()) + assert isinstance(persisted, dict) and persisted.get("workspace") == workspace, persisted + assert isinstance(persisted.get("config"), dict), persisted + return persisted + + def assert_no_managed_config(payload: object) -> None: """Validate the real List response before claiming unmanaged-workspace coverage.""" configs = payload.get("coding_agent_configs", []) if isinstance(payload, dict) else payload diff --git a/tests/integration/utils/model_discovery.py b/tests/integration/utils/model_discovery.py index 36171de96..bfb338e94 100644 --- a/tests/integration/utils/model_discovery.py +++ b/tests/integration/utils/model_discovery.py @@ -1,5 +1,6 @@ """Validation helpers for model-discovery evidence.""" +import hashlib import re _CLAUDE_GATEWAY_ALIAS = re.compile(r"^anthropic-aigw-[0-9a-fA-F]{8}-(?P.*)$") @@ -12,6 +13,25 @@ _SYSTEM_AI_PREFIX = "system.ai." +def claude_discovery_model_id(model_id: str) -> str: + """Encode the exact wire ID expected by Claude's gateway discovery feature.""" + if "claude" in model_id.lower() or "anthropic" in model_id.lower(): + return model_id + checksum = hashlib.sha256(model_id.encode("utf-8")).hexdigest()[:8] + return f"anthropic-aigw-{checksum}-{model_id}" + + +def claude_model_service_id(model_id: str) -> str: + """Recover service identity only from the gateway's checksum-valid discovery alias.""" + alias = re.fullmatch(r"anthropic-aigw-([0-9a-f]{8})-(.+)", model_id) + if alias is None: + return model_id + checksum, original = alias.groups() + if hashlib.sha256(original.encode("utf-8")).hexdigest()[:8] != checksum: + return model_id + return original + + def claude_model_in_picker(screen: str, model_id: str, display_name: str | None) -> bool: """Return whether a Claude model appears in a numbered picker row. @@ -70,3 +90,43 @@ def claude_system_model_ids(models: list[dict]) -> list[str]: ), f"Expected a system.ai model id: {model_id!r}" canonical_ids.append(canonical_id) return canonical_ids + + +def codex_model_in_picker(screen: str, model_id: str, display_name: str) -> bool: + """Match native numbered picker rows, not banners, prompt text, or footers.""" + rows = re.findall(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)][ \t]+([^\n]+)$", screen) + return any(_picker_row_matches(row, model_id, display_name) for row in rows) + + +def _picker_row_matches(label: str, model_id: str, display_name: str | None) -> bool: + return any( + re.search(rf"(? None: + """Require exactly one unambiguous numbered native row per expected model.""" + assert agent in {"claude", "codex"}, agent + assert display_names, "Expected a nonempty provider model inventory" + rows = re.findall(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)](?:[ \t]+([^\n]*))?$", screen) + assert rows, f"No numbered {agent} picker rows:\n{screen}" + observed: list[str] = [] + for label in rows: + matched = { + model + for model, display_name in display_names.items() + if _picker_row_matches(label, model, display_name) + } + assert len(matched) == 1, ( + f"Unmatched or ambiguous {agent} picker row {label!r}: {sorted(matched)}" + ) + model = matched.pop() + assert model not in observed, f"Duplicate {agent} picker model row: {model}" + observed.append(model) + assert set(observed) == set(display_names), { + "agent": agent, + "expected": sorted(display_names), + "observed": observed, + } diff --git a/tests/integration/utils/provider_catalog.py b/tests/integration/utils/provider_catalog.py index d0a5c00fc..d3f1632df 100644 --- a/tests/integration/utils/provider_catalog.py +++ b/tests/integration/utils/provider_catalog.py @@ -12,7 +12,7 @@ _ANTHROPIC_MODELS_PATH = "/ai-gateway/anthropic/v1/models" _CODEX_MODELS_PATH = "/ai-gateway/codex/v1/models" _PROVIDER_HEADER = "Databricks-Model-Provider-Service" -_PARENT_SCHEMA_HEADER = "Databricks-Model-Service-Parent-Schema" +MODEL_SERVICE_PARENT_SCHEMA_HEADER = "Databricks-Model-Service-Parent-Schema" _ANTHROPIC_VERSION = "2023-06-01" _ANTHROPIC_PAGE_SIZE = 1000 _ANTHROPIC_MAX_PAGES = 20 @@ -33,6 +33,7 @@ class AnthropicProviderCatalog: model_ids: tuple[str, ...] display_names: dict[str, str | None] + payloads: tuple[dict, ...] = () @dataclass(frozen=True) @@ -40,6 +41,7 @@ class CodexProviderCatalog: """The list-visible model slugs from a Codex provider catalog.""" model_ids: tuple[str, ...] + payloads: tuple[dict, ...] = () def _fail(message: str) -> NoReturn: @@ -110,8 +112,6 @@ def parse_codex_provider_catalog(payload: object) -> tuple[str, ...]: if not isinstance(visibility, str): _fail(f"Codex provider model {index} had an invalid visibility") if visibility != "list": - # Hidden/native aliases are part of the provider response but are not exposed by - # Codex's model/list request used as the observable catalog below. continue slug = raw_model.get("slug") if not isinstance(slug, str) or not slug.strip(): @@ -153,51 +153,62 @@ def _get_json(url: str, headers: dict[str, str]) -> object: raise AssertionError(f"GET {url} returned invalid JSON: {error.msg}") from error -def _validate_request_inputs(workspace: str, token: str, provider_service: str) -> str: +def _validate_request_inputs(workspace: str, token: str) -> str: if not isinstance(workspace, str) or not workspace.rstrip("/"): _fail("provider catalog request requires a workspace URL") if not isinstance(token, str) or not token: _fail("provider catalog request requires an explicit bearer token") - if not isinstance(provider_service, str) or not provider_service.strip(): - _fail("provider catalog request requires a provider service") return workspace.rstrip("/") +def _validate_scope(scope: str) -> None: + if not isinstance(scope, str) or not scope.strip(): + _fail("provider catalog request requires an explicit catalog scope") + + def fetch_anthropic_provider_catalog( workspace: str, token: str, provider_service: str ) -> AnthropicProviderCatalog: """Fetch and validate the complete Anthropic provider catalog with bounded pagination.""" - return _fetch_anthropic_catalog(workspace, token, provider_service, _PROVIDER_HEADER) + _validate_scope(provider_service) + return _fetch_anthropic_catalog(workspace, token, {_PROVIDER_HEADER: provider_service}) def fetch_anthropic_parent_catalog( workspace: str, token: str, parent_schema: str ) -> AnthropicProviderCatalog: - """Fetch the Anthropic-compatible models advertised for a Unity Catalog parent schema.""" - return _fetch_anthropic_catalog(workspace, token, parent_schema, _PARENT_SCHEMA_HEADER) + """Fetch the complete Anthropic catalog scoped to a Unity Catalog parent schema.""" + _validate_scope(parent_schema) + return _fetch_anthropic_catalog( + workspace, token, {MODEL_SERVICE_PARENT_SCHEMA_HEADER: parent_schema} + ) def _fetch_anthropic_catalog( - workspace: str, token: str, source: str, source_header: str + workspace: str, token: str, scope_headers: dict[str, str] ) -> AnthropicProviderCatalog: - base_url = _validate_request_inputs(workspace, token, source) + base_url = _validate_request_inputs(workspace, token) headers = { "Authorization": f"Bearer {token}", "Accept": "application/json", "Anthropic-Version": _ANTHROPIC_VERSION, - source_header: source, + **scope_headers, } model_ids: list[str] = [] display_names: dict[str, str | None] = {} + payloads: list[dict] = [] cursor: str | None = None cursors: set[str] = set() for _page_number in range(_ANTHROPIC_MAX_PAGES): query = {"limit": str(_ANTHROPIC_PAGE_SIZE)} if cursor is not None: query["after_id"] = cursor - url = f"{base_url}{_ANTHROPIC_MODELS_PATH}?{urllib.parse.urlencode(query)}" - page = parse_anthropic_provider_page(_get_json(url, headers)) + path = f"{_ANTHROPIC_MODELS_PATH}?{urllib.parse.urlencode(query)}" + payload = _get_json(f"{base_url}{path}", headers) + page = parse_anthropic_provider_page(payload) + assert isinstance(payload, dict), "Expected an Anthropic catalog object" + payloads.append(payload) for model_id, display_name in page.models: if model_id in display_names: _fail(f"Anthropic provider catalog repeated model id {model_id!r}") @@ -205,8 +216,8 @@ def _fetch_anthropic_catalog( display_names[model_id] = display_name if not page.has_more: - return AnthropicProviderCatalog(tuple(model_ids), display_names) - assert page.last_id is not None # checked by parse_anthropic_provider_page + return AnthropicProviderCatalog(tuple(model_ids), display_names, tuple(payloads)) + assert page.last_id is not None if page.last_id in cursors: _fail(f"Anthropic provider catalog repeated pagination cursor {page.last_id!r}") cursors.add(page.last_id) @@ -219,29 +230,28 @@ def fetch_codex_provider_catalog( workspace: str, token: str, provider_service: str ) -> CodexProviderCatalog: """Fetch and validate the list-visible Codex provider catalog.""" - base_url = _validate_request_inputs(workspace, token, provider_service) + _validate_scope(provider_service) + return _fetch_codex_catalog(workspace, token, {_PROVIDER_HEADER: provider_service}) + + +def _fetch_codex_catalog( + workspace: str, token: str, scope_headers: dict[str, str] +) -> CodexProviderCatalog: + base_url = _validate_request_inputs(workspace, token) payload = _get_json( f"{base_url}{_CODEX_MODELS_PATH}", - { - "Authorization": f"Bearer {token}", - "Accept": "application/json", - _PROVIDER_HEADER: provider_service, - }, + {"Authorization": f"Bearer {token}", "Accept": "application/json", **scope_headers}, ) - return CodexProviderCatalog(parse_codex_provider_catalog(payload)) + model_ids = parse_codex_provider_catalog(payload) + assert isinstance(payload, dict), "Expected a Codex catalog object" + return CodexProviderCatalog(model_ids, (payload,)) def fetch_codex_parent_catalog( workspace: str, token: str, parent_schema: str ) -> CodexProviderCatalog: """Fetch the API-compatible models advertised for a Unity Catalog parent schema.""" - base_url = _validate_request_inputs(workspace, token, parent_schema) - payload = _get_json( - f"{base_url}{_CODEX_MODELS_PATH}", - { - "Authorization": f"Bearer {token}", - "Accept": "application/json", - _PARENT_SCHEMA_HEADER: parent_schema, - }, + _validate_scope(parent_schema) + return _fetch_codex_catalog( + workspace, token, {MODEL_SERVICE_PARENT_SCHEMA_HEADER: parent_schema} ) - return CodexProviderCatalog(parse_codex_provider_catalog(payload)) diff --git a/tests/test_integration_cuj3.py b/tests/test_integration_cuj3.py deleted file mode 100644 index 217ec3b67..000000000 --- a/tests/test_integration_cuj3.py +++ /dev/null @@ -1,695 +0,0 @@ -"""Component checks for CUJ3 discovery helpers and native model evidence.""" - -import io -import json -import runpy -import urllib.error -import urllib.response -from pathlib import Path -from types import SimpleNamespace - -import pytest - -from tests.cuj3_fixture_data import INVALID_DESTINATIONS, model_readback -from tests.integration.utils import cuj3 - -ROOT = Path(__file__).parent.parent -SCRIPT = ROOT / "fixtures/cuj/models/provision.py" - - -@pytest.fixture -def provisioner(): - return SimpleNamespace(**runpy.run_path(str(SCRIPT))) - - -@pytest.mark.parametrize( - ("wire_id", "expected"), - [ - ("ug_e2e.models.kimi", "ug_e2e.models.kimi"), - ("anthropic-aigw-7bf7897e-ug_e2e.models.kimi", "ug_e2e.models.kimi"), - ("ug_e2e.models.claude_sonnet", "ug_e2e.models.claude_sonnet"), - ( - "anthropic-aigw-00000000-ug_e2e.models.kimi", - "anthropic-aigw-00000000-ug_e2e.models.kimi", - ), - ( - "anthropic-aigw-7bf7897e-ug_e2e.models.kimi_v2", - "anthropic-aigw-7bf7897e-ug_e2e.models.kimi_v2", - ), - ( - "anthropic-aigw-7bf7897e-ug_e2e.other_models.kimi", - "anthropic-aigw-7bf7897e-ug_e2e.other_models.kimi", - ), - ( - "anthropic-aigw-7BF7897E-ug_e2e.models.kimi", - "anthropic-aigw-7BF7897E-ug_e2e.models.kimi", - ), - ], -) -def test_claude_service_identity_requires_exact_gateway_alias_checksum(wire_id, expected): - assert cuj3.claude_model_service_id(wire_id) == expected - - -def test_claude_discovery_requires_exact_gateway_wire_ids(): - assert {cuj3.claude_discovery_model_id(model) for model in cuj3.CLAUDE_MODELS} == { - "ug_e2e.models.claude_haiku", - "ug_e2e.models.claude_sonnet", - "anthropic-aigw-7bf7897e-ug_e2e.models.kimi", - } - assert cuj3.claude_discovery_model_id(cuj3.CLAUDE_DECOY) == cuj3.CLAUDE_DECOY - - -def test_model_discovery_matrix_filters_families_without_changing_schema_scope(provisioner): - expected_models = { - "ug_e2e.models.gpt_luna", - "ug_e2e.models.claude_haiku", - "ug_e2e.models.claude_sonnet", - "ug_e2e.models.kimi", - "ug_e2e.models.gemini_flash", - } - assert cuj3.MODEL_SERVICES == expected_models - assert cuj3.CLAUDE_MODELS == { - "ug_e2e.models.claude_haiku", - "ug_e2e.models.claude_sonnet", - "ug_e2e.models.kimi", - } - assert cuj3.CODEX_MODELS == {"ug_e2e.models.gpt_luna", "ug_e2e.models.kimi"} - assert cuj3.GEMINI_MODEL in cuj3.MODEL_SERVICES - assert cuj3.GEMINI_MODEL not in cuj3.CLAUDE_MODELS | cuj3.CODEX_MODELS - assert cuj3.CLAUDE_DEFAULT in cuj3.CLAUDE_MODELS - assert cuj3.CODEX_DEFAULT in cuj3.CODEX_MODELS - assert cuj3.CLAUDE_DECOY not in cuj3.MODEL_SERVICES - assert cuj3.CODEX_DECOY not in cuj3.MODEL_SERVICES - assert { - f"ug_e2e.models.{leaf}" for leaf in provisioner.MODEL_LEAVES if not leaf.endswith("decoy") - } == expected_models - - -@pytest.mark.parametrize("server_defaults", [False, True]) -def test_live_inventory_proves_gemini_exists_without_using_agent_discovery( - monkeypatch, server_defaults -): - requests = [] - handlers = [] - - class InventoryOpener: - def open(self, request, *, timeout): - requests.append((request, timeout)) - return io.BytesIO(json.dumps(model_readback(server_defaults=server_defaults)).encode()) - - def build_opener(handler): - handlers.append(handler) - return InventoryOpener() - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", build_opener) - inventory = cuj3.fetch_model_service_inventory("https://workspace.invalid/", "test-bearer") - expected = cuj3.MODEL_SERVICES | {cuj3.CLAUDE_DECOY, cuj3.CODEX_DECOY} - assert set(inventory) == expected - assert inventory[cuj3.GEMINI_MODEL] == ("system.ai.databricks-gpt-6-luna",) - assert len(requests) == len(expected) - assert len(handlers) == len(expected) - assert all(isinstance(handler, cuj3.NoRedirect) for handler in handlers) - for request, timeout in requests: - assert request.get_method() == "GET" - assert request.full_url.startswith( - "https://workspace.invalid/api/2.1/unity-catalog/model-services/" - ) - assert request.data is None - assert request.get_header("Authorization") == "Bearer test-bearer" - assert timeout == 30 - - -@pytest.mark.parametrize("destination_updates", INVALID_DESTINATIONS) -def test_live_inventory_rejects_invalid_source_routing(monkeypatch, destination_updates): - payload = model_readback(server_defaults=True) - payload["config"]["routing"]["destinations"][0].update(destination_updates) - - class InvalidOpener: - def open(self, request, *, timeout): - return io.BytesIO(json.dumps(payload).encode()) - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: InvalidOpener()) - with pytest.raises(AssertionError): - cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") - - -@pytest.mark.parametrize( - "payload", - [ - None, - {}, - {"config": {}}, - {"config": {"routing": {"destinations": []}}}, - {"config": {"routing": {"destinations": [None]}}}, - {"config": {"routing": {"destinations": [{"name": "system.ai.fixture"}]}}}, - ], -) -def test_live_inventory_rejects_missing_or_malformed_services(monkeypatch, payload): - class InvalidOpener: - def open(self, request, *, timeout): - return io.BytesIO(json.dumps(payload).encode()) - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: InvalidOpener()) - with pytest.raises(AssertionError): - cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") - - -def test_live_inventory_fails_on_missing_gemini_even_when_other_models_exist( - provisioner, monkeypatch -): - class MissingGeminiOpener: - def open(self, request, *, timeout): - if request.full_url.endswith(cuj3.GEMINI_MODEL): - raise urllib.error.HTTPError(request.full_url, 404, "Not found", {}, None) - return io.BytesIO(json.dumps(provisioner.model_body("system.ai.fixture")).encode()) - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: MissingGeminiOpener()) - with pytest.raises(AssertionError, match=r"HTTP 404"): - cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") - - -@pytest.mark.parametrize("fallback", [False, True]) -def test_live_inventory_rejects_multiple_backing_models(provisioner, monkeypatch, fallback): - payload = provisioner.model_body("system.ai.fixture") - routing = payload["config"]["routing"] - if fallback: - routing["fallback"] = {"destinations": routing["destinations"]} - else: - routing["destinations"] *= 2 - - class MultipleSourceOpener: - def open(self, request, *, timeout): - return io.BytesIO(json.dumps(payload).encode()) - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: MultipleSourceOpener()) - with pytest.raises(AssertionError): - cuj3.fetch_model_service_inventory("https://workspace.invalid", "test-bearer") - - -@pytest.mark.parametrize("cursor", ["", "❯ ", "› ", "> "]) -@pytest.mark.parametrize("label", ["ug_e2e.models.gpt_luna", "GPT Luna"]) -def test_codex_picker_matches_model_only_in_numbered_rows(cursor, label): - screen = f"Select Model and Effort\n {cursor}1. {label} (current)\nEnter to confirm" - assert cuj3.codex_model_in_picker(screen, cuj3.CODEX_DEFAULT, "GPT Luna") - - -@pytest.mark.parametrize( - "screen", - [ - "GPT Luna\nSelect Model and Effort\n 1. Other model", - "Select Model and Effort\n 1. Other model\nCurrent model: GPT Luna", - " 1. ug_e2e.models.gpt_luna_v2", - " 1.\nGPT Luna", - "", - ], -) -def test_codex_picker_rejects_nonrows_and_different_model_ids(screen): - assert not cuj3.codex_model_in_picker(screen, cuj3.CODEX_DEFAULT, "GPT Luna") - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_picker_inventory_accepts_exact_rows_and_ignores_banner_text(agent): - models = cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS - labels = {model: model.rsplit(".", 1)[-1].replace("_", " ").title() for model in models} - rows = "\n".join( - f" {position}. {label}" for position, label in enumerate(labels.values(), start=1) - ) - screen = f"Banner: Gemini Flash\nSelect model\n{rows}\nEnter to confirm" - cuj3.assert_picker_inventory(screen, agent, labels) - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("excluded_label", ["Gemini Flash", "Friendly out-of-scope model"]) -def test_picker_inventory_rejects_extra_friendly_label_rows(agent, excluded_label): - models = cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS - labels = dict.fromkeys(models, None) - rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) - screen = f"Select model\n{rows}\n {len(models) + 1}. {excluded_label}" - with pytest.raises(AssertionError): - cuj3.assert_picker_inventory(screen, agent, labels) - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_picker_inventory_rejects_duplicate_and_missing_rows(agent): - models = sorted(cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS) - labels = dict.fromkeys(models, None) - rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) - with pytest.raises(AssertionError): - cuj3.assert_picker_inventory(f"{rows}\n {len(models) + 1}. {models[0]}", agent, labels) - with pytest.raises(AssertionError): - cuj3.assert_picker_inventory(f" 1. {models[0]}", agent, labels) - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_picker_inventory_rejects_ambiguous_labels_and_split_line_evidence(agent): - models = sorted(cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS) - with pytest.raises(AssertionError): - cuj3.assert_picker_inventory( - " 1. Shared label", agent, dict.fromkeys(models, "Shared label") - ) - with pytest.raises(AssertionError): - cuj3.assert_picker_inventory(f" 1.\n{models[0]}", agent, dict.fromkeys(models, None)) - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("suffix", ["_v2", "-decoy", ".other"]) -def test_picker_inventory_rejects_model_id_prefix_matches(agent, suffix): - models = sorted(cuj3.CLAUDE_MODELS if agent == "claude" else cuj3.CODEX_MODELS) - rows = "\n".join( - f" {position}. {model}{suffix if position == 1 else ''}" - for position, model in enumerate(models, start=1) - ) - with pytest.raises(AssertionError): - cuj3.assert_picker_inventory(rows, agent, dict.fromkeys(models, None)) - - -def test_model_policy_allows_server_metadata_and_later_stack_fields(): - config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) - config["update_time"] = "2026-10-02T00:00:00Z" - config["name"] = "coding-agent-configs/cuj3" - config["mcp_servers"] = {"unity_catalog_location": "ug_e2e.tools"} - cuj3.assert_cuj3_config(config) - - -def test_model_policy_rejects_a_default_outside_the_managed_scope(): - config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) - config["enabled_agents"][1]["config"]["default_models"]["default_model"] = cuj3.CODEX_DECOY - with pytest.raises(AssertionError): - cuj3.assert_cuj3_config(config) - - -def test_codex_model_identity_uses_only_the_completed_answer_turn(monkeypatch): - records = [ - {"type": "turn_context", "payload": {"turn_id": "other", "model": cuj3.CODEX_DECOY}}, - {"type": "turn_context", "payload": {"turn_id": "matching", "model": cuj3.CODEX_DEFAULT}}, - { - "type": "event_msg", - "payload": { - "type": "task_complete", - "turn_id": "matching", - "last_agent_message": "withheld-file-value", - }, - }, - ] - monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) - assert cuj3.completed_task_models(None, "codex", "withheld-file-value") == {cuj3.CODEX_DEFAULT} - - -def test_codex_model_identity_rejects_prompt_only_evidence(monkeypatch): - records = [ - {"type": "turn_context", "payload": {"turn_id": "matching", "model": cuj3.CODEX_DEFAULT}}, - { - "type": "response_item", - "payload": {"type": "message", "role": "user", "content": "withheld-file-value"}, - }, - ] - monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) - assert cuj3.completed_task_models(None, "codex", "withheld-file-value") == set() - - -def test_claude_model_identity_uses_assistant_answer_not_tool_output(monkeypatch): - records = [ - { - "type": "user", - "message": {"model": cuj3.CLAUDE_DECOY, "content": [{"type": "text", "text": "value"}]}, - }, - { - "type": "assistant", - "message": { - "role": "assistant", - "model": cuj3.CLAUDE_DEFAULT, - "content": [{"type": "text", "text": "value"}], - }, - }, - ] - monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"transcript": records}) - assert cuj3.completed_task_models(None, "claude", "value") == {cuj3.CLAUDE_DEFAULT} - - -FETCHERS = [ - cuj3.fetch_model_service_inventory, - cuj3.fetch_claude_parent_catalog, - cuj3.fetch_codex_parent_catalog, -] - - -def fetch_inventory(fetcher, workspace="https://workspace.invalid", token="test-bearer"): - if fetcher is cuj3.fetch_model_service_inventory: - return fetcher(workspace, token) - return fetcher(workspace, token, cuj3.MODEL_SCHEMA) - - -@pytest.mark.parametrize("fetcher", FETCHERS) -@pytest.mark.parametrize( - "workspace", - [ - None, - 7, - "", - "http://workspace.invalid", - "https://user:secret@workspace.invalid", - "https://workspace.invalid/path", - "https://workspace.invalid?secret=value", - "https://workspace.invalid#fragment", - "https:///missing-host", - "https://workspace.invalid:bad", - "https://workspace.invalid:0", - "https://workspace.invalid:70000", - "https://workspace.invalid\n", - "https://workspace.invalid\\@other.invalid", - ], -) -def test_all_cuj3_gets_reject_invalid_workspace_origins(monkeypatch, fetcher, workspace): - def unexpected_open(*args, **kwargs): - pytest.fail("Invalid workspace must not reach the HTTP boundary") - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", unexpected_open) - with pytest.raises(AssertionError, match="HTTPS workspace origin") as failure: - fetch_inventory(fetcher, workspace) - assert "secret" not in str(failure.value) - - -@pytest.mark.parametrize("fetcher", FETCHERS) -@pytest.mark.parametrize("token", [None, "", " ", "bearer\nsecret", "bearer\rsecret"]) -def test_all_cuj3_gets_require_explicit_valid_bearers(monkeypatch, fetcher, token): - def unexpected_open(*args, **kwargs): - pytest.fail("Invalid bearer must not reach the HTTP boundary") - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", unexpected_open) - with pytest.raises(AssertionError, match="explicit workspace bearer"): - fetch_inventory(fetcher, token=token) - - -@pytest.mark.parametrize("fetcher", FETCHERS) -@pytest.mark.parametrize("failure_kind", ["http", "url", "os", "json", "encoding"]) -def test_all_cuj3_get_errors_are_sanitized(monkeypatch, fetcher, failure_kind): - secret = "private-bearer-and-response-secret" - - class FailedOpener: - def open(self, request, *, timeout): - assert timeout == 30 - assert request.get_method() == "GET" - assert request.get_header("Authorization") == f"Bearer {secret}" - if failure_kind == "http": - raise urllib.error.HTTPError(request.full_url, 403, secret, {}, None) - if failure_kind == "url": - raise urllib.error.URLError(secret) - if failure_kind == "os": - raise OSError(secret) - return io.BytesIO(secret.encode() if failure_kind == "json" else b"\xff") - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: FailedOpener()) - with pytest.raises(AssertionError, match="Workspace JSON GET") as failure: - fetch_inventory(fetcher, token=secret) - assert secret not in str(failure.value) - assert "workspace.invalid" not in str(failure.value) - assert failure.value.__suppress_context__ - - -@pytest.mark.parametrize("fetcher", FETCHERS) -@pytest.mark.parametrize("status", [301, 302, 303, 307, 308]) -@pytest.mark.parametrize("destination", ["https://other.invalid/secret", "/redirected"]) -def test_all_cuj3_gets_deny_redirects(monkeypatch, fetcher, status, destination): - requests = [] - build_opener = cuj3.urllib.request.build_opener - - class BoundaryHTTPS(cuj3.urllib.request.HTTPSHandler): - def https_open(self, request): - requests.append(request) - response = urllib.response.addinfourl( - io.BytesIO(b"{}"), {"Location": destination}, request.full_url, status - ) - response.msg = "private-redirect-reason" - return response - - monkeypatch.setattr( - cuj3.urllib.request, - "build_opener", - lambda handler: build_opener(handler, BoundaryHTTPS()), - ) - with pytest.raises(AssertionError, match=f"HTTP {status}") as failure: - fetch_inventory(fetcher) - assert len(requests) == 1 - assert requests[0].get_header("Authorization") == "Bearer test-bearer" - assert destination not in str(failure.value) - assert "private-redirect-reason" not in str(failure.value) - - -def test_claude_parent_catalog_retains_pages_labels_and_scoped_requests(monkeypatch): - pages = [ - { - "data": [{"id": cuj3.CLAUDE_DEFAULT, "display_name": "Sonnet"}], - "has_more": True, - "last_id": "cursor with / and ?", - }, - { - "data": [{"id": cuj3.claude_discovery_model_id(f"{cuj3.MODEL_SCHEMA}.kimi")}], - "has_more": False, - }, - ] - requests = [] - - class CatalogOpener: - def open(self, request, *, timeout): - requests.append(request) - assert timeout == 30 - assert request.get_header("Authorization") == "Bearer test-bearer" - assert request.get_header(cuj3.MODEL_HEADER.capitalize()) == cuj3.MODEL_SCHEMA - assert request.get_header("Anthropic-version") == "2023-06-01" - return io.BytesIO(json.dumps(pages[len(requests) - 1]).encode()) - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: CatalogOpener()) - catalog = cuj3.fetch_claude_parent_catalog( - "https://workspace.invalid/", "test-bearer", cuj3.MODEL_SCHEMA - ) - assert catalog.payloads == tuple(pages) - assert catalog.model_ids == (pages[0]["data"][0]["id"], pages[1]["data"][0]["id"]) - assert catalog.display_names == {catalog.model_ids[0]: "Sonnet", catalog.model_ids[1]: None} - assert cuj3.urllib.parse.parse_qs(cuj3.urllib.parse.urlsplit(requests[1].full_url).query) == { - "limit": ["1000"], - "after_id": ["cursor with / and ?"], - } - - -@pytest.mark.parametrize("failure_kind", ["duplicate", "cursor", "limit"]) -def test_claude_parent_catalog_rejects_duplicate_or_unbounded_pagination(monkeypatch, failure_kind): - requests = [] - - class CatalogOpener: - def open(self, request, *, timeout): - requests.append(request) - position = len(requests) - payload = { - "data": [ - {"id": "duplicate" if failure_kind == "duplicate" else f"model-{position}"} - ], - "has_more": True, - "last_id": "repeated" if failure_kind == "cursor" else f"cursor-{position}", - } - return io.BytesIO(json.dumps(payload).encode()) - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: CatalogOpener()) - with pytest.raises(AssertionError): - cuj3.fetch_claude_parent_catalog( - "https://workspace.invalid", "test-bearer", cuj3.MODEL_SCHEMA - ) - assert len(requests) == (20 if failure_kind == "limit" else 2) - - -def test_codex_parent_catalog_retains_unfiltered_payload_and_validates_duplicates(monkeypatch): - payload = { - "models": [ - {"slug": cuj3.CODEX_DEFAULT, "display_name": "Luna", "visibility": "list"}, - {"slug": cuj3.CODEX_DECOY, "visibility": "hidden"}, - ] - } - - class CatalogOpener: - def open(self, request, *, timeout): - assert timeout == 30 - assert request.get_header(cuj3.MODEL_HEADER.capitalize()) == cuj3.MODEL_SCHEMA - return io.BytesIO(json.dumps(payload).encode()) - - monkeypatch.setattr(cuj3.urllib.request, "build_opener", lambda *_: CatalogOpener()) - catalog = cuj3.fetch_codex_parent_catalog( - "https://workspace.invalid", "test-bearer", cuj3.MODEL_SCHEMA - ) - assert catalog.payloads == (payload,) - assert catalog.model_ids == (cuj3.CODEX_DEFAULT,) - assert catalog.display_names == {cuj3.CODEX_DEFAULT: "Luna"} - payload["models"].append(payload["models"][0]) - with pytest.raises(AssertionError, match="repeated"): - cuj3.fetch_codex_parent_catalog( - "https://workspace.invalid", "test-bearer", cuj3.MODEL_SCHEMA - ) - - -def completed_records(agent, model, answer="value", turn_id="matching"): - if agent == "claude": - return [ - { - "type": "assistant", - "message": { - "role": "assistant", - "model": model, - "content": [{"type": "text", "text": answer}], - }, - } - ] - return [ - {"type": "turn_context", "payload": {"turn_id": turn_id, "model": model}}, - { - "type": "event_msg", - "payload": {"type": "task_complete", "turn_id": turn_id, "last_agent_message": answer}, - }, - ] - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("parent_answer", [None, "value", "unrelated-answer"]) -def test_completed_task_models_excludes_child_only_and_conflicting_child_evidence( - monkeypatch, agent, parent_answer -): - child = completed_records(agent, "child-model") - if agent == "codex": - child.insert(0, {"type": "session_meta", "payload": {"source": {"subagent": "spawn"}}}) - sessions = {"project/subagents/child.jsonl": child} - if parent_answer is not None: - sessions["project/parent.jsonl"] = completed_records(agent, "parent-model", parent_answer) - monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: sessions) - expected = {"parent-model"} if parent_answer == "value" else set() - assert cuj3.completed_task_models(None, agent, "value") == expected - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("model", [None, "", " ", 5, {}, [], "model with whitespace"]) -def test_completed_task_models_fails_closed_on_malformed_model(monkeypatch, agent, model): - records = completed_records(agent, model) - monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"parent": records}) - with pytest.raises(AssertionError, match="model"): - cuj3.completed_task_models(None, agent, "value") - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_completed_task_models_fails_closed_on_missing_model(monkeypatch, agent): - records = completed_records(agent, "model") - del records[0]["message" if agent == "claude" else "payload"]["model"] - monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"parent": records}) - with pytest.raises(AssertionError, match="model"): - cuj3.completed_task_models(None, agent, "value") - - -@pytest.mark.parametrize("turn_id", [None, "", " ", 5, {}, [], "turn with whitespace"]) -@pytest.mark.parametrize("record_index", [0, 1]) -def test_codex_completed_task_models_fails_closed_on_malformed_turn_ids(turn_id, record_index): - records = completed_records("codex", "model") - records[record_index]["payload"]["turn_id"] = turn_id - with pytest.raises(AssertionError, match="turn ID"): - cuj3.codex_completed_task_models(records, "value") - - -@pytest.mark.parametrize("record_index", [0, 1]) -def test_codex_completed_task_models_fails_closed_on_missing_turn_ids(record_index): - records = completed_records("codex", "model") - del records[record_index]["payload"]["turn_id"] - with pytest.raises(AssertionError, match="turn ID"): - cuj3.codex_completed_task_models(records, "value") - - -def test_codex_completed_task_models_requires_context_in_same_session(monkeypatch): - context, completion = completed_records("codex", "model") - monkeypatch.setattr( - cuj3, - "agent_sessions", - lambda *_: {"context-session": [context], "answer-session": [completion]}, - ) - with pytest.raises(AssertionError, match="Missing Codex model context"): - cuj3.completed_task_models(None, "codex", "value") - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("models", [[], ["unexpected"], ["expected", "conflicting"]]) -def test_completed_task_model_assertion_requires_exact_singleton(monkeypatch, agent, models): - records = [record for model in models for record in completed_records(agent, model)] - monkeypatch.setattr(cuj3, "agent_sessions", lambda *_: {"parent": records}) - evidence = {} - session = SimpleNamespace(record=lambda name, payload: evidence.update({name: payload})) - with pytest.raises(AssertionError): - cuj3.assert_completed_task_model(session, agent, "value", "expected") - assert next(iter(evidence.values()))["observed"] == sorted(models) - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_completed_task_model_records_evidence_limits(monkeypatch, agent): - monkeypatch.setattr( - cuj3, "agent_sessions", lambda *_: {"parent": completed_records(agent, "expected")} - ) - evidence = {} - session = SimpleNamespace(record=lambda name, payload: evidence.update({name: payload})) - cuj3.assert_completed_task_model(session, agent, "value", "expected") - assert evidence == { - f"completed-task-model-{agent}-value.json": { - "expected": "expected", - "observed": ["expected"], - "evidence_kind": ( - "response-reported model" if agent == "claude" else "client-selected model" - ), - "gateway_destination_proven": False, - } - } - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("answer_value", [None, "", " \t", 1, [], {}]) -def test_completed_task_models_rejects_empty_or_malformed_expected_answers( - monkeypatch, agent, answer_value -): - def unexpected_read(*args): - pytest.fail("Invalid answer must not read session evidence") - - monkeypatch.setattr(cuj3, "agent_sessions", unexpected_read) - with pytest.raises(AssertionError, match="nonempty answer value"): - cuj3.completed_task_models(None, agent, answer_value) - adapter = ( - cuj3.claude_completed_task_models if agent == "claude" else cuj3.codex_completed_task_models - ) - with pytest.raises(AssertionError, match="nonempty answer value"): - adapter([], answer_value) - - -@pytest.mark.parametrize( - "message", - [None, [], {}, {"role": "assistant"}, {"role": "assistant", "content": "value"}], -) -def test_claude_completed_task_models_rejects_malformed_message_metadata(message): - with pytest.raises(AssertionError): - cuj3.claude_completed_task_models([{"type": "assistant", "message": message}], "value") - - -@pytest.mark.parametrize("content", [[None], [{"type": "text"}], [{"type": "text", "text": 4}]]) -def test_claude_completed_task_models_rejects_malformed_answer_blocks(content): - records = completed_records("claude", "model") - records[0]["message"]["content"] = content - with pytest.raises(AssertionError): - cuj3.claude_completed_task_models(records, "value") - - -@pytest.mark.parametrize("payload", [None, [], "value"]) -@pytest.mark.parametrize("record_type", ["event_msg", "turn_context"]) -def test_codex_completed_task_models_rejects_malformed_payloads(payload, record_type): - with pytest.raises(AssertionError): - cuj3.codex_completed_task_models([{"type": record_type, "payload": payload}], "value") - - -@pytest.mark.parametrize("answer", [None, [], {}, 7]) -def test_codex_completed_task_models_rejects_malformed_completed_answers(answer): - records = completed_records("codex", "model") - records[1]["payload"]["last_agent_message"] = answer - with pytest.raises(AssertionError, match="completed answer"): - cuj3.codex_completed_task_models(records, "value") - - -def test_claude_completed_task_models_normalizes_only_context_window_suffix(): - assert cuj3.claude_completed_task_models( - completed_records("claude", f"{cuj3.CLAUDE_DEFAULT}[1m]"), "value" - ) == {cuj3.CLAUDE_DEFAULT} diff --git a/tests/test_integration_evidence.py b/tests/test_integration_evidence.py index f618304d1..4b6b31e1f 100644 --- a/tests/test_integration_evidence.py +++ b/tests/test_integration_evidence.py @@ -1,9 +1,11 @@ """Unit checks for interpreting terminal evidence, not live agent substitutes.""" import json +from types import SimpleNamespace import pytest +from tests.integration.utils import evidence from tests.integration.utils.evidence import ( SubagentCalculation, assert_no_terminal_api_error, @@ -89,3 +91,240 @@ def test_tagged_calculation_requires_the_native_child_answer(tmp_path, agent): assert task.marker in task.prompt assert f'task name "{task.marker}"' in task.prompt assert "1+1" in task.prompt + + +def test_codex_model_identity_uses_only_the_completed_answer_turn(monkeypatch): + records = [ + { + "type": "turn_context", + "payload": {"turn_id": "other", "model": "catalog.other_models.codex_decoy"}, + }, + { + "type": "turn_context", + "payload": {"turn_id": "matching", "model": "catalog.models.gpt_luna"}, + }, + { + "type": "event_msg", + "payload": { + "type": "task_complete", + "turn_id": "matching", + "last_agent_message": "withheld-file-value", + }, + }, + ] + monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"transcript": records}) + assert evidence.completed_task_models(None, "codex", "withheld-file-value") == { + "catalog.models.gpt_luna" + } + + +def test_codex_model_identity_rejects_prompt_only_evidence(monkeypatch): + records = [ + { + "type": "turn_context", + "payload": {"turn_id": "matching", "model": "catalog.models.gpt_luna"}, + }, + { + "type": "response_item", + "payload": {"type": "message", "role": "user", "content": "withheld-file-value"}, + }, + ] + monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"transcript": records}) + assert evidence.completed_task_models(None, "codex", "withheld-file-value") == set() + + +def test_claude_model_identity_uses_assistant_answer_not_tool_output(monkeypatch): + records = [ + { + "type": "user", + "message": { + "model": "catalog.other_models.claude_decoy", + "content": [{"type": "text", "text": "value"}], + }, + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "model": "catalog.models.claude_sonnet", + "content": [{"type": "text", "text": "value"}], + }, + }, + ] + monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"transcript": records}) + assert evidence.completed_task_models(None, "claude", "value") == { + "catalog.models.claude_sonnet" + } + + +def completed_records(agent, model, answer="value", turn_id="matching"): + if agent == "claude": + return [ + { + "type": "assistant", + "message": { + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": answer}], + }, + } + ] + return [ + {"type": "turn_context", "payload": {"turn_id": turn_id, "model": model}}, + { + "type": "event_msg", + "payload": {"type": "task_complete", "turn_id": turn_id, "last_agent_message": answer}, + }, + ] + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("parent_answer", [None, "value", "unrelated-answer"]) +def test_completed_task_models_excludes_child_only_and_conflicting_child_evidence( + monkeypatch, agent, parent_answer +): + child = completed_records(agent, "child-model") + if agent == "codex": + child.insert(0, {"type": "session_meta", "payload": {"source": {"subagent": "spawn"}}}) + sessions = {"project/subagents/child.jsonl": child} + if parent_answer is not None: + sessions["project/parent.jsonl"] = completed_records(agent, "parent-model", parent_answer) + monkeypatch.setattr(evidence, "agent_sessions", lambda *_: sessions) + expected = {"parent-model"} if parent_answer == "value" else set() + assert evidence.completed_task_models(None, agent, "value") == expected + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("model", [None, "", " ", 5, {}, [], "model with whitespace"]) +def test_completed_task_models_fails_closed_on_malformed_model(monkeypatch, agent, model): + records = completed_records(agent, model) + monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"parent": records}) + with pytest.raises(AssertionError, match="model"): + evidence.completed_task_models(None, agent, "value") + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_completed_task_models_fails_closed_on_missing_model(monkeypatch, agent): + records = completed_records(agent, "model") + del records[0]["message" if agent == "claude" else "payload"]["model"] + monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"parent": records}) + with pytest.raises(AssertionError, match="model"): + evidence.completed_task_models(None, agent, "value") + + +@pytest.mark.parametrize("turn_id", [None, "", " ", 5, {}, [], "turn with whitespace"]) +@pytest.mark.parametrize("record_index", [0, 1]) +def test_codex_completed_task_models_fails_closed_on_malformed_turn_ids(turn_id, record_index): + records = completed_records("codex", "model") + records[record_index]["payload"]["turn_id"] = turn_id + with pytest.raises(AssertionError, match="turn ID"): + evidence.codex_completed_task_models(records, "value") + + +@pytest.mark.parametrize("record_index", [0, 1]) +def test_codex_completed_task_models_fails_closed_on_missing_turn_ids(record_index): + records = completed_records("codex", "model") + del records[record_index]["payload"]["turn_id"] + with pytest.raises(AssertionError, match="turn ID"): + evidence.codex_completed_task_models(records, "value") + + +def test_codex_completed_task_models_requires_context_in_same_session(monkeypatch): + context, completion = completed_records("codex", "model") + monkeypatch.setattr( + evidence, + "agent_sessions", + lambda *_: {"context-session": [context], "answer-session": [completion]}, + ) + with pytest.raises(AssertionError, match="Missing Codex model context"): + evidence.completed_task_models(None, "codex", "value") + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("models", [[], ["unexpected"], ["expected", "conflicting"]]) +def test_completed_task_model_assertion_requires_exact_singleton(monkeypatch, agent, models): + records = [record for model in models for record in completed_records(agent, model)] + monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"parent": records}) + artifacts = {} + session = SimpleNamespace(record=lambda name, payload: artifacts.update({name: payload})) + with pytest.raises(AssertionError): + evidence.assert_completed_task_model(session, agent, "value", "expected") + assert next(iter(artifacts.values()))["observed"] == sorted(models) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_completed_task_model_records_evidence_limits(monkeypatch, agent): + monkeypatch.setattr( + evidence, "agent_sessions", lambda *_: {"parent": completed_records(agent, "expected")} + ) + artifacts = {} + session = SimpleNamespace(record=lambda name, payload: artifacts.update({name: payload})) + evidence.assert_completed_task_model(session, agent, "value", "expected") + assert artifacts == { + f"completed-task-model-{agent}-value.json": { + "expected": "expected", + "observed": ["expected"], + "evidence_kind": ( + "response-reported model" if agent == "claude" else "client-selected model" + ), + "gateway_destination_proven": False, + } + } + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("answer_value", [None, "", " \t", 1, [], {}]) +def test_completed_task_models_rejects_empty_or_malformed_expected_answers( + monkeypatch, agent, answer_value +): + def unexpected_read(*args): + pytest.fail("Invalid answer must not read session evidence") + + monkeypatch.setattr(evidence, "agent_sessions", unexpected_read) + with pytest.raises(AssertionError, match="nonempty answer value"): + evidence.completed_task_models(None, agent, answer_value) + adapter = ( + evidence.claude_completed_task_models + if agent == "claude" + else evidence.codex_completed_task_models + ) + with pytest.raises(AssertionError, match="nonempty answer value"): + adapter([], answer_value) + + +@pytest.mark.parametrize( + "message", + [None, [], {}, {"role": "assistant"}, {"role": "assistant", "content": "value"}], +) +def test_claude_completed_task_models_rejects_malformed_message_metadata(message): + with pytest.raises(AssertionError): + evidence.claude_completed_task_models([{"type": "assistant", "message": message}], "value") + + +@pytest.mark.parametrize("content", [[None], [{"type": "text"}], [{"type": "text", "text": 4}]]) +def test_claude_completed_task_models_rejects_malformed_answer_blocks(content): + records = completed_records("claude", "model") + records[0]["message"]["content"] = content + with pytest.raises(AssertionError): + evidence.claude_completed_task_models(records, "value") + + +@pytest.mark.parametrize("payload", [None, [], "value"]) +@pytest.mark.parametrize("record_type", ["event_msg", "turn_context"]) +def test_codex_completed_task_models_rejects_malformed_payloads(payload, record_type): + with pytest.raises(AssertionError): + evidence.codex_completed_task_models([{"type": record_type, "payload": payload}], "value") + + +@pytest.mark.parametrize("answer", [None, [], {}, 7]) +def test_codex_completed_task_models_rejects_malformed_completed_answers(answer): + records = completed_records("codex", "model") + records[1]["payload"]["last_agent_message"] = answer + with pytest.raises(AssertionError, match="completed answer"): + evidence.codex_completed_task_models(records, "value") + + +def test_claude_completed_task_models_normalizes_only_context_window_suffix(): + assert evidence.claude_completed_task_models( + completed_records("claude", "catalog.models.claude_sonnet[1m]"), "value" + ) == {"catalog.models.claude_sonnet"} diff --git a/tests/test_integration_http.py b/tests/test_integration_http.py new file mode 100644 index 000000000..ef6df7c8e --- /dev/null +++ b/tests/test_integration_http.py @@ -0,0 +1,116 @@ +"""Offline component checks for read-only workspace HTTP boundaries.""" + +import io +import urllib.error +import urllib.response + +import pytest + +from tests.integration.utils import http, provider_catalog + +FETCHERS = [ + provider_catalog.fetch_anthropic_parent_catalog, + provider_catalog.fetch_codex_parent_catalog, + provider_catalog.fetch_anthropic_provider_catalog, + provider_catalog.fetch_codex_provider_catalog, +] + + +def fetch_inventory(fetcher, workspace="https://workspace.invalid", token="test-bearer"): + return fetcher(workspace, token, "catalog.models") + + +@pytest.mark.parametrize("fetcher", FETCHERS) +@pytest.mark.parametrize( + "workspace", + [ + None, + 7, + "", + "http://workspace.invalid", + "https://user:secret@workspace.invalid", + "https://workspace.invalid/path", + "https://workspace.invalid?secret=value", + "https://workspace.invalid#fragment", + "https:///missing-host", + "https://workspace.invalid:bad", + "https://workspace.invalid:0", + "https://workspace.invalid:70000", + "https://workspace.invalid\n", + "https://workspace.invalid\\@other.invalid", + ], +) +def test_catalog_gets_reject_invalid_workspace_origins(monkeypatch, fetcher, workspace): + def unexpected_open(*args, **kwargs): + pytest.fail("Invalid workspace must not reach the HTTP boundary") + + monkeypatch.setattr(http.urllib.request, "build_opener", unexpected_open) + with pytest.raises(AssertionError, match="HTTPS workspace origin") as failure: + fetch_inventory(fetcher, workspace) + assert "secret" not in str(failure.value) + + +@pytest.mark.parametrize("fetcher", FETCHERS) +@pytest.mark.parametrize("token", [None, "", " ", "bearer\nsecret", "bearer\rsecret"]) +def test_catalog_gets_require_explicit_valid_bearers(monkeypatch, fetcher, token): + def unexpected_open(*args, **kwargs): + pytest.fail("Invalid bearer must not reach the HTTP boundary") + + monkeypatch.setattr(http.urllib.request, "build_opener", unexpected_open) + with pytest.raises(AssertionError, match="explicit workspace bearer"): + fetch_inventory(fetcher, token=token) + + +@pytest.mark.parametrize("fetcher", FETCHERS) +@pytest.mark.parametrize("failure_kind", ["http", "url", "os", "json", "encoding"]) +def test_catalog_get_errors_are_sanitized(monkeypatch, fetcher, failure_kind): + secret = "private-bearer-and-response-secret" + + class FailedOpener: + def open(self, request, *, timeout): + assert timeout == 30 + assert request.get_method() == "GET" + assert request.get_header("Authorization") == f"Bearer {secret}" + if failure_kind == "http": + raise urllib.error.HTTPError(request.full_url, 403, secret, {}, None) + if failure_kind == "url": + raise urllib.error.URLError(secret) + if failure_kind == "os": + raise OSError(secret) + return io.BytesIO(secret.encode() if failure_kind == "json" else b"\xff") + + monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: FailedOpener()) + with pytest.raises(AssertionError, match="Workspace JSON GET") as failure: + fetch_inventory(fetcher, token=secret) + assert secret not in str(failure.value) + assert "workspace.invalid" not in str(failure.value) + assert failure.value.__suppress_context__ + + +@pytest.mark.parametrize("fetcher", FETCHERS) +@pytest.mark.parametrize("status", [301, 302, 303, 307, 308]) +@pytest.mark.parametrize("destination", ["https://other.invalid/secret", "/redirected"]) +def test_catalog_gets_deny_redirects(monkeypatch, fetcher, status, destination): + requests = [] + build_opener = http.urllib.request.build_opener + + class BoundaryHTTPS(http.urllib.request.HTTPSHandler): + def https_open(self, request): + requests.append(request) + response = urllib.response.addinfourl( + io.BytesIO(b"{}"), {"Location": destination}, request.full_url, status + ) + response.msg = "private-redirect-reason" + return response + + monkeypatch.setattr( + http.urllib.request, + "build_opener", + lambda handler: build_opener(handler, BoundaryHTTPS()), + ) + with pytest.raises(AssertionError, match=f"HTTP {status}") as failure: + fetch_inventory(fetcher) + assert len(requests) == 1 + assert requests[0].get_header("Authorization") == "Bearer test-bearer" + assert destination not in str(failure.value) + assert "private-redirect-reason" not in str(failure.value) diff --git a/tests/test_integration_managed_config.py b/tests/test_integration_managed_config.py new file mode 100644 index 000000000..111fd1e82 --- /dev/null +++ b/tests/test_integration_managed_config.py @@ -0,0 +1,96 @@ +"""Component checks for reading persisted managed-config evidence.""" + +import json +from types import SimpleNamespace + +import pytest + +from tests.integration.utils.managed import read_persisted_managed_config + +WORKSPACE = "https://workspace.invalid" + + +@pytest.fixture +def session(tmp_path): + return SimpleNamespace(home=tmp_path / "session-home") + + +def _write_persisted_config(session, envelope): + path = session.home / ".ucode" / "managed-config.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps(envelope), encoding="utf-8") + return path + + +@pytest.mark.parametrize( + "config", + [ + {}, + { + "default_agent": "unrelated-agent", + "custom_policy": {"labels": ["independent", "policy"], "enabled": True}, + }, + ], +) +def test_read_persisted_managed_config_returns_envelope_without_enforcing_policy(session, config): + envelope = {"workspace": WORKSPACE, "config": config, "metadata": {"revision": 7}} + path = _write_persisted_config(session, envelope) + original = path.read_text(encoding="utf-8") + + assert read_persisted_managed_config(session, WORKSPACE) == envelope + assert path.read_text(encoding="utf-8") == original + + +@pytest.mark.parametrize("directory_instead", [False, True]) +def test_read_persisted_managed_config_requires_a_file(session, directory_instead): + if directory_instead: + (session.home / ".ucode" / "managed-config.json").mkdir(parents=True) + + with pytest.raises(AssertionError): + read_persisted_managed_config(session, WORKSPACE) + + +@pytest.mark.parametrize("contents", ["", "{", '{"workspace":', "{} trailing-data"]) +def test_read_persisted_managed_config_rejects_invalid_json(session, contents): + path = _write_persisted_config(session, {}) + path.write_text(contents, encoding="utf-8") + + with pytest.raises(json.JSONDecodeError): + read_persisted_managed_config(session, WORKSPACE) + + +@pytest.mark.parametrize("envelope", [None, [], ["config"], "config", False, 7]) +def test_read_persisted_managed_config_requires_a_dict_envelope(session, envelope): + _write_persisted_config(session, envelope) + + with pytest.raises(AssertionError): + read_persisted_managed_config(session, WORKSPACE) + + +@pytest.mark.parametrize( + "workspace", + [None, "", "https://other-workspace.invalid", WORKSPACE + "/", WORKSPACE.upper(), 7, {}, []], +) +def test_read_persisted_managed_config_requires_an_exact_workspace_match(session, workspace): + _write_persisted_config(session, {"workspace": workspace, "config": {}}) + + with pytest.raises(AssertionError): + read_persisted_managed_config(session, WORKSPACE) + + +@pytest.mark.parametrize("config", [None, [], ["policy"], "policy", False, 7]) +def test_read_persisted_managed_config_requires_a_dict_config(session, config): + _write_persisted_config(session, {"workspace": WORKSPACE, "config": config}) + + with pytest.raises(AssertionError): + read_persisted_managed_config(session, WORKSPACE) + + +@pytest.mark.parametrize("field", ["workspace", "config"]) +def test_read_persisted_managed_config_rejects_missing_fields(session, field): + envelope = {"workspace": WORKSPACE, "config": {}} + del envelope[field] + _write_persisted_config(session, envelope) + + with pytest.raises(AssertionError): + read_persisted_managed_config(session, WORKSPACE) diff --git a/tests/test_integration_model_discovery.py b/tests/test_integration_model_discovery.py index fd8860e7f..27bf981fb 100644 --- a/tests/test_integration_model_discovery.py +++ b/tests/test_integration_model_discovery.py @@ -2,7 +2,21 @@ import pytest -from tests.integration.utils.model_discovery import claude_model_in_picker, claude_system_model_ids +from tests.integration.utils.model_discovery import ( + assert_picker_inventory, + claude_discovery_model_id, + claude_model_in_picker, + claude_model_service_id, + claude_system_model_ids, + codex_model_in_picker, +) + +PICKER_MODELS = { + "claude": frozenset( + {"catalog.models.claude_sonnet", "catalog.models.claude_haiku", "catalog.models.kimi"} + ), + "codex": frozenset({"catalog.models.gpt_luna", "catalog.models.kimi"}), +} @pytest.mark.parametrize("cursor", ["", "❯ ", "› ", "> "]) @@ -131,3 +145,118 @@ def test_claude_system_model_ids_rejects_invalid_or_out_of_scope_ids(model_id): def test_claude_system_model_ids_rejects_empty_missing_and_duplicate_ids(models): with pytest.raises(AssertionError): claude_system_model_ids(models) + + +@pytest.mark.parametrize( + ("wire_id", "expected"), + [ + ("catalog.models.kimi", "catalog.models.kimi"), + ("anthropic-aigw-69e2a9a0-catalog.models.kimi", "catalog.models.kimi"), + ("catalog.models.claude_sonnet", "catalog.models.claude_sonnet"), + ( + "anthropic-aigw-00000000-catalog.models.kimi", + "anthropic-aigw-00000000-catalog.models.kimi", + ), + ( + "anthropic-aigw-69e2a9a0-catalog.models.kimi_v2", + "anthropic-aigw-69e2a9a0-catalog.models.kimi_v2", + ), + ( + "anthropic-aigw-69e2a9a0-catalog.other_models.kimi", + "anthropic-aigw-69e2a9a0-catalog.other_models.kimi", + ), + ( + "anthropic-aigw-69E2A9A0-catalog.models.kimi", + "anthropic-aigw-69E2A9A0-catalog.models.kimi", + ), + ], +) +def test_claude_service_identity_requires_exact_gateway_alias_checksum(wire_id, expected): + assert claude_model_service_id(wire_id) == expected + + +def test_claude_discovery_requires_exact_gateway_wire_ids(): + assert {claude_discovery_model_id(model) for model in PICKER_MODELS["claude"]} == { + "catalog.models.claude_haiku", + "catalog.models.claude_sonnet", + "anthropic-aigw-69e2a9a0-catalog.models.kimi", + } + assert ( + claude_discovery_model_id("catalog.other_models.claude_decoy") + == "catalog.other_models.claude_decoy" + ) + + +@pytest.mark.parametrize("cursor", ["", "❯ ", "› ", "> "]) +@pytest.mark.parametrize("label", ["catalog.models.gpt_luna", "GPT Luna"]) +def test_codex_picker_matches_model_only_in_numbered_rows(cursor, label): + screen = f"Select Model and Effort\n {cursor}1. {label} (current)\nEnter to confirm" + assert codex_model_in_picker(screen, "catalog.models.gpt_luna", "GPT Luna") + + +@pytest.mark.parametrize( + "screen", + [ + "GPT Luna\nSelect Model and Effort\n 1. Other model", + "Select Model and Effort\n 1. Other model\nCurrent model: GPT Luna", + " 1. catalog.models.gpt_luna_v2", + " 1.\nGPT Luna", + "", + ], +) +def test_codex_picker_rejects_nonrows_and_different_model_ids(screen): + assert not codex_model_in_picker(screen, "catalog.models.gpt_luna", "GPT Luna") + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_picker_inventory_accepts_exact_rows_and_ignores_banner_text(agent): + models = PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"] + labels = {model: model.rsplit(".", 1)[-1].replace("_", " ").title() for model in models} + rows = "\n".join( + f" {position}. {label}" for position, label in enumerate(labels.values(), start=1) + ) + screen = f"Banner: Gemini Flash\nSelect model\n{rows}\nEnter to confirm" + assert_picker_inventory(screen, agent, labels) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("excluded_label", ["Gemini Flash", "Friendly out-of-scope model"]) +def test_picker_inventory_rejects_extra_friendly_label_rows(agent, excluded_label): + models = PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"] + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"Select model\n{rows}\n {len(models) + 1}. {excluded_label}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, agent, labels) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_picker_inventory_rejects_duplicate_and_missing_rows(agent): + models = sorted(PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + with pytest.raises(AssertionError): + assert_picker_inventory(f"{rows}\n {len(models) + 1}. {models[0]}", agent, labels) + with pytest.raises(AssertionError): + assert_picker_inventory(f" 1. {models[0]}", agent, labels) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_picker_inventory_rejects_ambiguous_labels_and_split_line_evidence(agent): + models = sorted(PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"]) + with pytest.raises(AssertionError): + assert_picker_inventory(" 1. Shared label", agent, dict.fromkeys(models, "Shared label")) + with pytest.raises(AssertionError): + assert_picker_inventory(f" 1.\n{models[0]}", agent, dict.fromkeys(models, None)) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("suffix", ["_v2", "-decoy", ".other"]) +def test_picker_inventory_rejects_model_id_prefix_matches(agent, suffix): + models = sorted(PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"]) + rows = "\n".join( + f" {position}. {model}{suffix if position == 1 else ''}" + for position, model in enumerate(models, start=1) + ) + with pytest.raises(AssertionError): + assert_picker_inventory(rows, agent, dict.fromkeys(models, None)) diff --git a/tests/test_integration_model_policy.py b/tests/test_integration_model_policy.py new file mode 100644 index 000000000..fdb4fe5d3 --- /dev/null +++ b/tests/test_integration_model_policy.py @@ -0,0 +1,50 @@ +"""Component checks for catalog discovery fixture expectations.""" + +import json +from pathlib import Path + +import pytest + +from tests.integration import catalog_discovery_expectations as expectations + +ROOT = Path(__file__).parent.parent + + +def test_model_discovery_matrix_filters_families_without_changing_schema_scope(): + expected_models = { + "ug_e2e.models.gpt_luna", + "ug_e2e.models.claude_haiku", + "ug_e2e.models.claude_sonnet", + "ug_e2e.models.kimi", + "ug_e2e.models.gemini_flash", + } + assert expectations.MODEL_SERVICES == expected_models + assert expectations.CLAUDE_MODELS == { + "ug_e2e.models.claude_haiku", + "ug_e2e.models.claude_sonnet", + "ug_e2e.models.kimi", + } + assert expectations.CODEX_MODELS == {"ug_e2e.models.gpt_luna", "ug_e2e.models.kimi"} + assert expectations.GEMINI_MODEL in expectations.MODEL_SERVICES + assert expectations.GEMINI_MODEL not in expectations.CLAUDE_MODELS | expectations.CODEX_MODELS + assert expectations.CLAUDE_DEFAULT in expectations.CLAUDE_MODELS + assert expectations.CODEX_DEFAULT in expectations.CODEX_MODELS + assert expectations.CLAUDE_DECOY not in expectations.MODEL_SERVICES + assert expectations.CODEX_DECOY not in expectations.MODEL_SERVICES + + +def test_model_policy_allows_server_metadata_and_later_stack_fields(): + config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) + config["update_time"] = "2026-10-02T00:00:00Z" + config["name"] = "coding-agent-configs/cuj3" + config["mcp_servers"] = {"unity_catalog_location": "ug_e2e.tools"} + expectations.assert_model_policy(config) + + +def test_model_policy_rejects_a_default_outside_the_managed_scope(): + config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) + config["enabled_agents"][1]["config"]["default_models"]["default_model"] = ( + expectations.CODEX_DECOY + ) + with pytest.raises(AssertionError): + expectations.assert_model_policy(config) diff --git a/tests/test_integration_provider_catalog.py b/tests/test_integration_provider_catalog.py index 41aa1f66d..39e2e0ad8 100644 --- a/tests/test_integration_provider_catalog.py +++ b/tests/test_integration_provider_catalog.py @@ -7,9 +7,32 @@ import pytest +from tests.integration.utils import http from tests.integration.utils import provider_catalog as catalog +def _mock_json_boundary(monkeypatch, get_json): + class CatalogOpener: + def open(self, request, *, timeout): + assert request.get_method() == "GET" + assert timeout == 30 + names = ( + "Authorization", + "Accept", + "Anthropic-Version", + "Databricks-Model-Provider-Service", + catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER, + ) + headers = { + name: request.get_header(name.capitalize()) + for name in names + if request.has_header(name.capitalize()) + } + return io.BytesIO(json.dumps(get_json(request.full_url, headers)).encode()) + + monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: CatalogOpener()) + + def _page(*ids, has_more=False, last_id=None): return { "data": [{"id": model_id, "display_name": model_id} for model_id in ids], @@ -83,7 +106,7 @@ def get_json(url, headers): requests.append((url, headers)) return next(pages) - monkeypatch.setattr(catalog, "_get_json", get_json) + _mock_json_boundary(monkeypatch, get_json) result = catalog.fetch_anthropic_provider_catalog("https://workspace/", "token", "c.s.mps") assert result.model_ids == ("a", "b") assert result.display_names == {"a": "a", "b": "b"} @@ -93,6 +116,8 @@ def get_json(url, headers): for _, headers in requests: assert headers["Authorization"] == "Bearer token" assert headers["Databricks-Model-Provider-Service"] == "c.s.mps" + assert headers["Anthropic-Version"] == "2023-06-01" + assert catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER not in headers pages = iter([_page("a", has_more=True, last_id="a"), _page("a")]) with pytest.raises(AssertionError, match="repeated model id"): @@ -103,14 +128,14 @@ def test_anthropic_provider_fetch_rejects_repeated_cursor(monkeypatch): pages = iter( [_page("a", has_more=True, last_id="cursor"), _page("b", has_more=True, last_id="cursor")] ) - monkeypatch.setattr(catalog, "_get_json", lambda *args: next(pages)) + _mock_json_boundary(monkeypatch, lambda *args: next(pages)) with pytest.raises(AssertionError, match="repeated pagination cursor"): catalog.fetch_anthropic_provider_catalog("https://workspace", "token", "c.s.mps") def test_anthropic_provider_fetch_has_a_page_bound(monkeypatch): pages = iter(_page(str(i), has_more=True, last_id=str(i)) for i in range(20)) - monkeypatch.setattr(catalog, "_get_json", lambda *args: next(pages)) + _mock_json_boundary(monkeypatch, lambda *args: next(pages)) with pytest.raises(AssertionError, match="exceeded 20 pages"): catalog.fetch_anthropic_provider_catalog("https://workspace", "token", "c.s.mps") @@ -127,10 +152,15 @@ def urlopen(request, timeout): assert request.full_url == "https://workspace/ai-gateway/codex/v1/models" assert request.get_header("Authorization") == "Bearer token" assert request.get_header("Databricks-model-provider-service") == "c.s.mps" + assert not request.has_header(catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER.capitalize()) assert timeout == 30 return Response(json.dumps(payload).encode()) - monkeypatch.setattr(catalog.urllib.request, "urlopen", urlopen) + monkeypatch.setattr( + http.urllib.request, + "build_opener", + lambda *_: type("CatalogOpener", (), {"open": staticmethod(urlopen)})(), + ) result = catalog.fetch_codex_provider_catalog("https://workspace/", "token", "c.s.mps") assert result.model_ids == ("gpt-a",) @@ -144,7 +174,7 @@ def get_json(url, headers): assert "Databricks-Model-Provider-Service" not in headers return {"models": [{"slug": model, "visibility": "list"} for model in ids]} - monkeypatch.setattr(catalog, "_get_json", get_json) + _mock_json_boundary(monkeypatch, get_json) result = catalog.fetch_codex_parent_catalog("https://workspace/", "token", "c.s") assert result.model_ids == tuple(ids) @@ -154,6 +184,191 @@ def test_provider_fetch_does_not_hide_http_failures(monkeypatch, status): def urlopen(request, timeout): raise urllib.error.HTTPError(request.full_url, status, "failure", {}, None) - monkeypatch.setattr(catalog.urllib.request, "urlopen", urlopen) + monkeypatch.setattr( + http.urllib.request, + "build_opener", + lambda *_: type("CatalogOpener", (), {"open": staticmethod(urlopen)})(), + ) with pytest.raises(AssertionError, match=f"HTTP {status}"): catalog.fetch_codex_provider_catalog("https://workspace", "token", "c.s.mps") + + +def test_anthropic_parent_catalog_retains_pages_labels_and_scoped_requests(monkeypatch): + pages = [ + { + "data": [{"id": "catalog.models.claude_sonnet", "display_name": "Sonnet"}], + "has_more": True, + "last_id": "cursor with / and ?", + }, + { + "data": [{"id": "anthropic-aigw-12345678-catalog.models.kimi"}], + "has_more": False, + }, + ] + requests = [] + + class CatalogOpener: + def open(self, request, *, timeout): + requests.append(request) + assert timeout == 30 + assert request.get_header("Authorization") == "Bearer test-bearer" + assert ( + request.get_header(catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER.capitalize()) + == "catalog.models" + ) + assert request.get_header("Anthropic-version") == "2023-06-01" + assert not request.has_header("Databricks-model-provider-service") + return io.BytesIO(json.dumps(pages[len(requests) - 1]).encode()) + + monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: CatalogOpener()) + result = catalog.fetch_anthropic_parent_catalog( + "https://workspace.invalid/", "test-bearer", "catalog.models" + ) + assert result.payloads == tuple(pages) + assert result.model_ids == (pages[0]["data"][0]["id"], pages[1]["data"][0]["id"]) + assert result.display_names == {result.model_ids[0]: "Sonnet", result.model_ids[1]: None} + assert http.urllib.parse.parse_qs(http.urllib.parse.urlsplit(requests[1].full_url).query) == { + "limit": ["1000"], + "after_id": ["cursor with / and ?"], + } + + +@pytest.mark.parametrize("failure_kind", ["duplicate", "cursor", "limit"]) +def test_anthropic_parent_catalog_rejects_duplicate_or_unbounded_pagination( + monkeypatch, failure_kind +): + requests = [] + + class CatalogOpener: + def open(self, request, *, timeout): + requests.append(request) + position = len(requests) + payload = { + "data": [ + {"id": "duplicate" if failure_kind == "duplicate" else f"model-{position}"} + ], + "has_more": True, + "last_id": "repeated" if failure_kind == "cursor" else f"cursor-{position}", + } + return io.BytesIO(json.dumps(payload).encode()) + + monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: CatalogOpener()) + with pytest.raises(AssertionError): + catalog.fetch_anthropic_parent_catalog( + "https://workspace.invalid", "test-bearer", "catalog.models" + ) + assert len(requests) == (20 if failure_kind == "limit" else 2) + + +def test_codex_parent_catalog_retains_unfiltered_payload_and_validates_duplicates(monkeypatch): + payload = { + "models": [ + {"slug": "catalog.models.gpt_luna", "display_name": "Luna", "visibility": "list"}, + {"slug": "catalog.other_models.codex_decoy", "visibility": "hidden"}, + ] + } + + class CatalogOpener: + def open(self, request, *, timeout): + assert timeout == 30 + assert ( + request.get_header(catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER.capitalize()) + == "catalog.models" + ) + assert not request.has_header("Databricks-model-provider-service") + return io.BytesIO(json.dumps(payload).encode()) + + monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: CatalogOpener()) + result = catalog.fetch_codex_parent_catalog( + "https://workspace.invalid", "test-bearer", "catalog.models" + ) + assert result.payloads == (payload,) + assert result.model_ids == ("catalog.models.gpt_luna",) + assert result.display_names == {"catalog.models.gpt_luna": "Luna"} + payload["models"].append(payload["models"][0]) + with pytest.raises(AssertionError, match="repeated"): + catalog.fetch_codex_parent_catalog( + "https://workspace.invalid", "test-bearer", "catalog.models" + ) + + +def test_provider_catalog_constructors_preserve_existing_arguments_and_isolate_defaults(): + anthropic = catalog.AnthropicProviderCatalog(("claude-a",), {"claude-a": None}) + codex = catalog.CodexProviderCatalog(("gpt-a",)) + other_codex = catalog.CodexProviderCatalog(("gpt-b",)) + assert anthropic.payloads == () + assert codex.payloads == () + assert codex.display_names == {} + codex.display_names["gpt-a"] = "Model A" + assert other_codex.display_names == {} + + +@pytest.mark.parametrize( + "fetcher", [catalog.fetch_anthropic_provider_catalog, catalog.fetch_anthropic_parent_catalog] +) +def test_anthropic_catalog_preserves_raw_pages_and_optional_labels(monkeypatch, fetcher): + payloads = [ + { + "data": [{"id": "b", "visibility": "hidden", "extra": [1, 2]}], + "has_more": True, + "last_id": "cursor", + "metadata": {"raw": True}, + }, + {"data": [{"id": "a", "display_name": "Model A"}], "has_more": False}, + ] + pages = iter(payloads) + _mock_json_boundary(monkeypatch, lambda *args: next(pages)) + result = fetcher("https://workspace", "token", "catalog.models") + assert result.payloads == tuple(payloads) + assert result.model_ids == ("b", "a") + assert result.display_names == {"b": None, "a": "Model A"} + + +@pytest.mark.parametrize( + "fetcher", [catalog.fetch_codex_provider_catalog, catalog.fetch_codex_parent_catalog] +) +def test_codex_catalog_preserves_hidden_entries_order_and_optional_labels(monkeypatch, fetcher): + payload = { + "models": [ + {"slug": "b", "visibility": "list"}, + {"slug": "b", "visibility": "hidden", "display_name": 7}, + {"visibility": "hidden", "extra": {"raw": True}}, + {"slug": "a", "visibility": "list", "display_name": "Model A"}, + ], + "metadata": {"raw": True}, + } + _mock_json_boundary(monkeypatch, lambda *args: payload) + result = fetcher("https://workspace", "token", "catalog.models") + assert result.payloads == (payload,) + assert result.model_ids == ("b", "a") + assert result.display_names == {"b": None, "a": "Model A"} + + +@pytest.mark.parametrize( + "fetcher", [catalog.fetch_codex_provider_catalog, catalog.fetch_codex_parent_catalog] +) +@pytest.mark.parametrize("display_name", ["", " ", 7, [], {}]) +def test_codex_catalog_rejects_invalid_optional_display_names(monkeypatch, fetcher, display_name): + payload = {"models": [{"slug": "a", "visibility": "list", "display_name": display_name}]} + _mock_json_boundary(monkeypatch, lambda *args: payload) + with pytest.raises(AssertionError, match="display name"): + fetcher("https://workspace", "token", "catalog.models") + + +@pytest.mark.parametrize( + "fetcher", + [ + catalog.fetch_anthropic_provider_catalog, + catalog.fetch_anthropic_parent_catalog, + catalog.fetch_codex_provider_catalog, + catalog.fetch_codex_parent_catalog, + ], +) +@pytest.mark.parametrize("scope", [None, "", " ", 7]) +def test_provider_catalog_requests_require_an_explicit_scope(monkeypatch, fetcher, scope): + def unexpected_open(*args, **kwargs): + pytest.fail("Invalid catalog scope must not reach the HTTP boundary") + + monkeypatch.setattr(http.urllib.request, "build_opener", unexpected_open) + with pytest.raises(AssertionError, match="explicit catalog scope"): + fetcher("https://workspace", "token", scope) From 8a030844a02e6bfaf2b0291932b0d78be5994972 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 19:28:30 +0000 Subject: [PATCH 06/19] AIGTWY-4878 Make catalog discovery tests independent of fixture provisioning --- fixtures/cuj-3/managed-config.json | 27 -- fixtures/cuj/models/README.md | 130 ---------- fixtures/cuj/models/provision.py | 282 -------------------- tests/README.md | 5 +- tests/cuj3_fixture_data.py | 35 --- tests/integration/README.md | 16 +- tests/test_cuj3_model_fixtures.py | 345 ------------------------- tests/test_integration_model_policy.py | 54 ++-- 8 files changed, 50 insertions(+), 844 deletions(-) delete mode 100644 fixtures/cuj-3/managed-config.json delete mode 100644 fixtures/cuj/models/README.md delete mode 100644 fixtures/cuj/models/provision.py delete mode 100644 tests/cuj3_fixture_data.py delete mode 100644 tests/test_cuj3_model_fixtures.py diff --git a/fixtures/cuj-3/managed-config.json b/fixtures/cuj-3/managed-config.json deleted file mode 100644 index 1c1beb448..000000000 --- a/fixtures/cuj-3/managed-config.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "spec_version": 1, - "default_agent": "CODING_AGENT_CLAUDE_CODE", - "enabled_agents": [ - { - "agent": "CODING_AGENT_CLAUDE_CODE", - "config": { - "models": { "unity_catalog_location": "ug_e2e.models" }, - "default_models": { - "default_model": "ug_e2e.models.claude_sonnet", - "default_sonnet_model": "ug_e2e.models.claude_sonnet" - }, - "smart_routing": { "enabled": false }, - "tracing": { "enabled": false } - } - }, - { - "agent": "CODING_AGENT_CODEX", - "config": { - "models": { "unity_catalog_location": "ug_e2e.models" }, - "default_models": { "default_model": "ug_e2e.models.gpt_luna" }, - "smart_routing": { "enabled": false }, - "tracing": { "enabled": false } - } - } - ] -} diff --git a/fixtures/cuj/models/README.md b/fixtures/cuj/models/README.md deleted file mode 100644 index 7295cadd7..000000000 --- a/fixtures/cuj/models/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# CUJ3 model-only fixtures - -This layer provisions seven UC model services and, when missing, the `ug_e2e` -catalog and its `models` and `other_models` schemas. Apps, connections, MCP, and -skills are not prerequisites. Agent installation, credentials, grants, and -managed-config publication are separate operator setup. - -| Schema | Services | Purpose | -| --- | --- | --- | -| `ug_e2e.models` | `gpt_luna` | Codex default; GPT family | -| `ug_e2e.models` | `claude_haiku`, `claude_sonnet` | Claude family; Sonnet is the Claude default | -| `ug_e2e.models` | `kimi` | Shared Claude/Codex compatible model | -| `ug_e2e.models` | `gemini_flash` | Gemini family; intended exclusion from both agents | -| `ug_e2e.other_models` | `claude_decoy`, `codex_decoy` | Compatible, accessible, out-of-scope decoys | - -Service identities are stable aliases, not source model names. Each leaf maps -explicitly to its schema; inventory ordering does not determine scope. - -## Prerequisites and setup order - -1. Use Python 3.12+, an environment with `databricks-sdk>=0.135.0`, and select the - exact dedicated CUJ3 HTTPS workspace origin. No CLI profile or implicit - workspace/authentication fallback is used. The offline plan does not create a - workspace client or read credentials. -2. An operator must explicitly select seven existing, canonical `system.ai.` - registered-model FQNs in that workspace/region, including both decoys, not - friendly model-service aliases. Both `--validate` and `--apply` GET every unique source at - `/api/2.1/unity-catalog/models/{source}` and require its `full_name` to exactly - match the explicit source before any writes. Missing or mismatched sources fail; - the script never resolves aliases, discovers/replaces sources, or guesses versions/defaults. - Choose GPT, Claude Haiku, Claude Sonnet, Kimi, and Gemini Flash sources for their aliases. - Haiku, Sonnet, Kimi, and the Claude decoy must support Claude's wire protocol; - GPT Luna, Kimi, and the Codex decoy must support Codex's wire protocol. - Decoys may reuse compatible in-scope sources; their UC identities remain distinct. -3. For `--validate` or `--apply` only, supply a workspace bearer through - `DATABRICKS_BEARER` (or an explicitly chosen - `--bearer-env` variable) using your approved secret mechanism. Never put tokens - in arguments, source, shell history, logs, or captured output; disable shell - tracing. The script prints only resource names, source names, and sanitized - errors, never tokens or server response bodies. Redirects are refused. -4. The provisioning principal needs permission to read the existing inventory, - create missing catalog/schemas/services, and use the chosen source models. - If it cannot create the catalog, have an authorized operator create it first. - Separately grant the CUJ execution principal `USE_CATALOG`, `USE_SCHEMA`, and - the required model-service invocation/discovery permissions for **both** schemas. - Decoys must be accessible so exclusion proves scope filtering, not missing grants. -5. Review the offline plan's workspace, seven mappings, permissions, and request - format. Run `--validate`, then explicitly use `--apply` for missing resources. - Rerun `--validate` and require zero missing resources and a successful exit. -6. An authorized administrator separately reviews and publishes - `fixtures/cuj-3/managed-config.json`; this script never changes workspace policy. - It points both agents to `ug_e2e.models`, selects Claude as the default agent, - Sonnet for Claude and GPT Luna for Codex, disables smart routing/tracing, - and has no MCP/skill scope. - Those omissions do not remove unrelated pre-existing local MCP/skill config. - -From the repository root, without needing a bearer for the default offline plan -(`uv run` supplies the project SDK environment): - -```bash -uv run python fixtures/cuj/models/provision.py \ - --workspace "$UG_CUJ3_WORKSPACE" \ - --bearer-env DATABRICKS_BEARER \ - --gpt-luna-source "$GPT_LUNA_SOURCE" \ - --claude-haiku-source "$CLAUDE_HAIKU_SOURCE" \ - --claude-sonnet-source "$CLAUDE_SONNET_SOURCE" \ - --kimi-source "$KIMI_SOURCE" \ - --gemini-flash-source "$GEMINI_FLASH_SOURCE" \ - --claude-decoy-source "$CLAUDE_DECOY_SOURCE" \ - --codex-decoy-source "$CODEX_DECOY_SOURCE" -``` - -Each source variable is a concrete `system.ai.` FQN. The default plan checks -syntax and prints inventory without network access or reading a bearer; it cannot -check existing resources. `--validate` performs authenticated, read-only SDK GETs -and fails on missing resources or invalid services. It is mutually exclusive with -`--apply`, which completes full source/inventory/routing preflight **before any POST**, -creates only missing catalog → schemas → services, and reads back all seven services. -Requests use a 30-second SDK timeout and a one-second SDK retry budget. Mismatches -abort: there is no overwrite, update, delete, or cleanup path. Concurrent create -conflicts and partial failures stop the non-atomic run; created resources persist. -Inspect with `--validate` before retrying. - -## Compatibility and validation boundary - -The intended agent-specific catalog contract is: - -| Agent | Exact `ug_e2e.models` leaves | Default | -| --- | --- | --- | -| Claude | `claude_haiku`, `claude_sonnet`, `kimi` | `claude_sonnet` | -| Codex | `gpt_luna`, `kimi` | `gpt_luna` | - -`gemini_flash` is excluded from both catalogs. API-specific filtering of the shared -schema pointer must produce these exact sets, without a fixture-side allowlist. -Each compatible decoy must be discoverable under `ug_e2e.other_models` but absent -from the managed catalog. - -The API uses `/api/2.1/unity-catalog/model-services`, a schema `parent`, and -canonical snake_case fields from Universe's Python SDK model-service sample. -Provisioning uses the SDK's public `WorkspaceClient` and `api_client.do` with raw -REST calls. Its per-request bearer callback rejects redirects before requests -follows them and requires every successful GET response to be a JSON object before -the SDK parses it. That preserves unknown routing flags and prevents SDK coercion -of JSON `null` or strings from weakening validation; private sessions, global -environment, and settings are not changed. -Creation sets `pay_per_token_config.model = models/system.ai.` and -`traffic_percentage = 100`; usage tracking is always on and its removed/reserved -configuration field is omitted. Preflight/readback require one destination with -the selected source name, pay-per-token type, and matching model target. Server -metadata/defaults (`is_deleted = false`, empty fallback routing, omitted traffic -percentage for a single primary) are accepted. Deleted/disabled destinations, -explicit traffic other than 100, fallbacks, or source/type/target mismatches fail. -Usage-tracking metadata is not a routing check. - -Offline checks do not verify workspace API availability, source existence, or wire -compatibility; confirm these in the intended workspace before apply. Provisioning -checks source metadata and routing, not inference availability or compatibility. -After publication and agent setup, live CUJs must compare discovery/pickers and -complete file tasks on every in-scope compatible model. Claude evidence is the -response-reported model; Codex evidence is the client-selected model. Neither proves -the executed gateway backing destination. Discovery, config, or startup alone is -insufficient; tests and CI remain outside this fixture-only layer. - -Suggested focused local checks (no workspace or bearer needed): - -```bash -uv run ruff check fixtures/cuj/models/provision.py -uv run ruff format --check fixtures/cuj/models/provision.py -uv run python fixtures/cuj/models/provision.py --help -``` diff --git a/fixtures/cuj/models/provision.py b/fixtures/cuj/models/provision.py deleted file mode 100644 index c07535795..000000000 --- a/fixtures/cuj/models/provision.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Plan seven CUJ3 models offline; authenticate only for --validate or --apply.""" - -from __future__ import annotations - -import argparse -import os -import re -import sys -import urllib.parse - -from databricks.sdk import WorkspaceClient -from databricks.sdk.core import Config -from databricks.sdk.errors import DatabricksError, NotFound -from requests.exceptions import RequestException - -UC = "/api/2.1/unity-catalog" -SCHEMAS = ("models", "other_models") -MODEL_SCHEMAS = { - "gpt_luna": "models", - "claude_haiku": "models", - "claude_sonnet": "models", - "kimi": "models", - "gemini_flash": "models", - "claude_decoy": "other_models", - "codex_decoy": "other_models", -} -MODEL_LEAVES = tuple(MODEL_SCHEMAS) -SOURCE = re.compile(r"system\.ai\.[A-Za-z0-9_-]+\Z") -DESTINATION_TYPE = "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL" - - -def workspace_origin(workspace: str) -> str: - try: - parsed = urllib.parse.urlsplit(workspace) - port = parsed.port - except (TypeError, ValueError): - raise ValueError( - "--workspace must be an HTTPS workspace origin without credentials" - ) from None - if ( - not isinstance(workspace, str) - or parsed.scheme != "https" - or not parsed.hostname - or parsed.username is not None - or parsed.password is not None - or parsed.path not in ("", "/") - or parsed.query - or parsed.fragment - or (port is not None and not 0 < port <= 65535) - or re.search(r"[\s\\]", workspace) - ): - raise ValueError("--workspace must be an HTTPS workspace origin without credentials") - return f"https://{parsed.netloc}" - - -def require(condition: bool, message: str): - if not condition: - raise RuntimeError(message) - - -def bearer_auth(bearer: str): - def reject_redirect(response, **kwargs): - if 300 <= response.status_code < 400: - response.close() - raise AssertionError( - f"Workspace request redirected with HTTP {response.status_code}" - ) from None - if ( - 200 <= response.status_code < 300 - and response.request.method == "GET" - and not isinstance(response.json(), dict) - ): - raise AssertionError("Workspace GET returned a non-object JSON response") from None - return response - - def authenticate(request): - request.headers["Authorization"] = f"Bearer {bearer}" - request.register_hook("response", reject_redirect) - return request - - return authenticate - - -def _request(api_client, auth, method: str, path: str, *, query=None, body=None, missing_ok=False): - try: - return api_client.do(method, path, query=query, body=body, auth=auth) - except NotFound: - if missing_ok: - return None - raise RuntimeError("Workspace API request failed") from None - except DatabricksError: - raise RuntimeError("Workspace API request failed") from None - except (RequestException, OSError, ValueError): - raise RuntimeError("Workspace request failed; check connectivity and TLS") from None - - -def _get_optional(api_client, auth, path: str): - return _request(api_client, auth, "GET", path, missing_ok=True) - - -def workspace_client(origin: str, bearer: str): - try: - return WorkspaceClient( - config=Config( - host=origin, - token=bearer, - auth_type="pat", - http_timeout_seconds=30, - retry_timeout_seconds=1, - ) - ) - except (DatabricksError, RequestException, OSError, ValueError): - raise RuntimeError("Workspace SDK client initialization failed") from None - - -def model_body(source: str): - return { - "config": { - "routing": { - "destinations": [ - { - "name": source, - "destination_type": DESTINATION_TYPE, - "pay_per_token_config": {"model": f"models/{source}"}, - "traffic_percentage": 100, - } - ] - }, - } - } - - -def validate_model(existing: dict, fqn: str, source: str): - message = f"Model service {fqn} has unexpected routing; refusing to overwrite it" - require(isinstance(existing, dict) and isinstance(existing.get("config"), dict), message) - routing = existing["config"].get("routing") - require(isinstance(routing, dict), message) - require(routing.get("fallback") in (None, {}, {"destinations": []}), message) - destinations = routing.get("destinations") - require(isinstance(destinations, list) and len(destinations) == 1, message) - destination = destinations[0] - require(isinstance(destination, dict), message) - target = destination.get("pay_per_token_config") - traffic = destination.get("traffic_percentage", 100) - require( - destination.get("name") == source - and destination.get("destination_type") == DESTINATION_TYPE - and isinstance(target, dict) - and target.get("model") == f"models/{source}" - and type(traffic) is int - and traffic == 100 - and all( - destination.get(field, False) is False - for field in ("is_deleted", "is_disabled", "disabled") - ), - message, - ) - - -def print_plan(sources: dict[str, str]): - print("catalog ug_e2e: ensure only if missing") - for schema in SCHEMAS: - print(f"schema ug_e2e.{schema}: ensure only if missing") - for leaf in MODEL_LEAVES: - schema = MODEL_SCHEMAS[leaf] - print(f"model service ug_e2e.{schema}.{leaf}: planned -> {sources[leaf]}") - print( - "Offline dry plan: no network or writes; no bearer required; existing inventory not checked" - ) - - -def provision(api_client, sources: dict[str, str], apply: bool, bearer: str): - auth = bearer_auth(bearer) - for source in sorted(set(sources.values())): - registered_model = _get_optional(api_client, auth, f"{UC}/models/{source}") - require( - isinstance(registered_model, dict) and registered_model.get("full_name") == source, - f"Source registered model {source} is missing or has an unexpected full_name; " - "supply an exact canonical registered-model FQN, not a model-service alias; " - "no writes performed", - ) - print(f"source registered model {source}: validated") - missing = [] - if _get_optional(api_client, auth, f"{UC}/catalogs/ug_e2e") is None: - missing.append((f"{UC}/catalogs", None, {"name": "ug_e2e", "comment": "UG CUJ fixtures"})) - print("catalog ug_e2e: would create") - else: - print("catalog ug_e2e: existing") - for schema in SCHEMAS: - fqn = f"ug_e2e.{schema}" - if _get_optional(api_client, auth, f"{UC}/schemas/{fqn}") is None: - missing.append( - ( - f"{UC}/schemas", - None, - {"name": schema, "catalog_name": "ug_e2e", "comment": "UG model fixture scope"}, - ) - ) - print(f"schema {fqn}: would create") - else: - print(f"schema {fqn}: existing") - for leaf in MODEL_LEAVES: - schema = MODEL_SCHEMAS[leaf] - fqn = f"ug_e2e.{schema}.{leaf}" - source = sources[leaf] - existing = _get_optional(api_client, auth, f"{UC}/model-services/{fqn}") - if existing is None: - missing.append( - ( - f"{UC}/model-services", - {"parent": f"schemas/ug_e2e.{schema}", "model_service_id": leaf}, - model_body(source), - ) - ) - print(f"model service {fqn}: would create -> {source}") - else: - validate_model(existing, fqn, source) - print(f"model service {fqn}: validated -> {source}") - if not apply: - print(f"Validation: {len(missing)} missing resources; no writes performed") - require(not missing, "Inventory is incomplete; review the offline plan before --apply") - return - for path, params, body in missing: - _request(api_client, auth, "POST", path, query=params, body=body) - for leaf in MODEL_LEAVES: - schema = MODEL_SCHEMAS[leaf] - fqn = f"ug_e2e.{schema}.{leaf}" - validate_model( - _request(api_client, auth, "GET", f"{UC}/model-services/{fqn}"), - fqn, - sources[leaf], - ) - print("Model-only fixture inventory is ready; managed config was not published") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--workspace", required=True, help="Exact HTTPS workspace origin") - parser.add_argument( - "--bearer-env", - default="DATABRICKS_BEARER", - help="Environment variable containing an explicit workspace bearer token", - ) - for leaf in MODEL_LEAVES: - parser.add_argument( - f"--{leaf.replace('_', '-')}-source", - required=True, - help="Canonical existing system.ai registered-model FQN, not a model-service alias", - ) - modes = parser.add_mutually_exclusive_group() - modes.add_argument( - "--validate", - action="store_true", - help="Authenticate and validate existing resources; GET only", - ) - modes.add_argument("--apply", action="store_true", help="Create missing resources after review") - args = parser.parse_args() - origin = workspace_origin(args.workspace) - sources = {leaf: getattr(args, f"{leaf}_source") for leaf in MODEL_LEAVES} - require( - all(SOURCE.fullmatch(source) for source in sources.values()), - "Every model source must be an existing system.ai model FQN", - ) - print(f"workspace: {origin}") - if not args.validate and not args.apply: - print_plan(sources) - return - bearer = os.environ.get(args.bearer_env, "") - require(bool(bearer.strip()), "Bearer environment variable must contain a workspace token") - require( - not any(character.isspace() for character in bearer), "Bearer must not contain whitespace" - ) - client = workspace_client(origin, bearer) - provision(client.api_client, sources, args.apply, bearer=bearer) - - -if __name__ == "__main__": - try: - main() - except (AssertionError, RuntimeError, ValueError) as error: - print(f"model fixture provisioning failed: {error}", file=sys.stderr) - sys.exit(1) diff --git a/tests/README.md b/tests/README.md index 5c13608f5..2a45465f2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -207,8 +207,9 @@ With Claude and Codex selected there are **62 live cases** (12 marked TUI cases) publish a CodingAgentConfig), **1 two-workspace case** (marker `workspace_switch`), **25 managed-fixture cases** (marker `managed_fixture`, with only the CodingAgentConfig input injected), and **7 installation checks**. The 14 retained numbered scenarios -comprise **22 explicit journeys**: 8 managed-fixture configured/fresh executions, 2 catalog discovery journeys, and 12 unmanaged -executions. Four provider-override journeys retain their assertions under descriptive names. Thirteen additional managed-fixture cases cover focused model, MCP, skills, +comprise **24 explicit journeys**: 12 managed-fixture configured/fresh executions and 12 unmanaged +executions. Two additional catalog discovery journeys cover the preconfigured dedicated workspace. +Thirteen additional managed-fixture cases cover focused model, MCP, skills, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. diff --git a/tests/cuj3_fixture_data.py b/tests/cuj3_fixture_data.py deleted file mode 100644 index 51d59c591..000000000 --- a/tests/cuj3_fixture_data.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Shared model-service inputs for CUJ3 component tests.""" - - -def model_readback(*, server_defaults): - destination = { - "name": "system.ai.databricks-gpt-6-luna", - "destination_type": "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", - "pay_per_token_config": {"model": "models/system.ai.databricks-gpt-6-luna"}, - } - routing = {"destinations": [destination]} - if server_defaults: - destination.update(traffic_percentage=100, is_deleted=False) - routing["fallback"] = {"destinations": []} - return { - "name": "model-services/ug_e2e.models.gpt_luna", - "config": {"routing": routing, "usage_tracking": {"enabled": True}}, - } - - -INVALID_DESTINATIONS = [ - {"name": "system.ai.wrong_source"}, - {"name": "other.ai.fixture"}, - {"destination_type": "DESTINATION_TYPE_EXTERNAL_MODEL"}, - {"pay_per_token_config": {"model": "models/system.ai.wrong_source"}}, - {"pay_per_token_config": None}, - {"traffic_percentage": 0}, - {"traffic_percentage": 50}, - {"traffic_percentage": "100"}, - {"traffic_percentage": None}, - {"traffic_percentage": True}, - {"is_deleted": True}, - {"is_deleted": "false"}, - {"is_disabled": True}, - {"disabled": True}, -] diff --git a/tests/integration/README.md b/tests/integration/README.md index d3dbfe3cc..2b4a7049a 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -388,9 +388,9 @@ constants in the runner; CI only needs `UG_MPS_DEFAULTS_CLIENT_SECRET` for west- `UG_PARENT_SCHEMA_DEFAULTS_CLIENT_SECRET` for northeast-2. As with the base workspace, the runner mints short-lived tokens and passes bearers to pytest; each test selects its target bearer for `ug configure` and Claude. The client secrets do not enter the pytest process. -The 14 retained numbered scenarios comprise 22 explicit journeys: 8 managed-fixture, -2 catalog discovery, and 12 unmanaged executions. Four provider-override journeys retain their -assertions under descriptive names; the complete integration suite collects 103 executions. See the named coverage and gaps matrix in +The 14 retained numbered scenarios comprise 24 explicit journeys: 12 managed-fixture +and 12 unmanaged executions. Two additional catalog discovery journeys bring the +complete integration suite to 103 executions. See the named coverage and gaps matrix in [../README.md](../README.md). ```bash @@ -577,11 +577,11 @@ fixture expectations. Register its package/version and marker in the runner/coll setup, implement any native terminal support, and add it to the CI matrix. Existing journeys need no renaming or generic agent branches. -Catalog discovery covers Claude/Codex models only. An authorized admin provisions both -schemas with `fixtures/cuj/models/provision.py` and separately reviews/publishes -`fixtures/cuj-3/managed-config.json` in the dedicated workspace. Follow the fixture -README's prerequisites, explicit auth, and preflight-before-write rules. Tests -do not provision resources or policy; Apps, connections, MCP, and skills are not required. +The dedicated workspace must already contain the services listed below and publish +a CodingAgentConfig enabling both agents with `ug_e2e.models` as their model source. +Its default agent is Claude; defaults are Claude Sonnet (including the Sonnet family +default) and Codex GPT Luna, with smart routing and tracing disabled. Tests do not +provision resources or policy; Apps, connections, MCP, and skills are not required. Model-service fixtures are trusted prerequisites; journeys do not revalidate their backing destinations. Each journey runs real `ug configure`, checks persisted diff --git a/tests/test_cuj3_model_fixtures.py b/tests/test_cuj3_model_fixtures.py deleted file mode 100644 index 61f78a951..000000000 --- a/tests/test_cuj3_model_fixtures.py +++ /dev/null @@ -1,345 +0,0 @@ -"""Component safety checks for the model-only workspace provisioning script.""" - -import json -import os -import runpy -import subprocess -import sys -from pathlib import Path -from types import SimpleNamespace - -import pytest -from databricks.sdk.errors import NotFound - -from tests.cuj3_fixture_data import INVALID_DESTINATIONS, model_readback - -ROOT = Path(__file__).parent.parent -SCRIPT = ROOT / "fixtures/cuj/models/provision.py" - - -@pytest.fixture -def provisioner(): - return SimpleNamespace(**runpy.run_path(str(SCRIPT))) - - -def model_sources(provisioner): - return {leaf: f"system.ai.{leaf}" for leaf in provisioner.MODEL_LEAVES} - - -class RecordingApiClient: - def __init__(self, inventory): - self.inventory = inventory - self.requests = [] - self.auth_requests = [] - - def do(self, method, path, *, query=None, headers=None, body=None, auth=None): - self.requests.append((method, path, query, body)) - self.auth_requests.append((method, path, auth is not None)) - if method == "GET": - if path not in self.inventory: - raise NotFound("not found") - return self.inventory[path] - if path.endswith("model-services"): - schema = query["parent"].removeprefix("schemas/") - self.inventory[f"{path}/{schema}.{query['model_service_id']}"] = body - return body - - -def run_provision(provisioner, client, sources, *, apply): - return provisioner.provision(client, sources, apply=apply, bearer="component-token") - - -def source_inventory(provisioner, sources=None): - sources = model_sources(provisioner) if sources is None else sources - return { - f"{provisioner.UC}/models/{source}": {"full_name": source} - for source in set(sources.values()) - } - - -def complete_inventory(provisioner): - inventory = source_inventory(provisioner) - inventory[f"{provisioner.UC}/catalogs/ug_e2e"] = {} - inventory.update( - {f"{provisioner.UC}/schemas/ug_e2e.{schema}": {} for schema in provisioner.SCHEMAS} - ) - for leaf, source in model_sources(provisioner).items(): - schema = "other_models" if leaf.endswith("decoy") else "models" - inventory[f"{provisioner.UC}/model-services/ug_e2e.{schema}.{leaf}"] = ( - provisioner.model_body(source) - ) - return inventory - - -def test_model_fixture_create_body_uses_canonical_sdk_fields(provisioner): - assert provisioner.model_body("system.ai.databricks-gpt-6-luna") == { - "config": { - "routing": { - "destinations": [ - { - "name": "system.ai.databricks-gpt-6-luna", - "destination_type": "DESTINATION_TYPE_PAY_PER_TOKEN_FOUNDATION_MODEL", - "pay_per_token_config": {"model": "models/system.ai.databricks-gpt-6-luna"}, - "traffic_percentage": 100, - } - ] - }, - } - } - - -@pytest.mark.parametrize("server_defaults", [False, True]) -def test_model_fixture_accepts_canonical_readback(provisioner, server_defaults): - provisioner.validate_model( - model_readback(server_defaults=server_defaults), - "ug_e2e.models.gpt_luna", - "system.ai.databricks-gpt-6-luna", - ) - - -@pytest.mark.parametrize("destination_updates", INVALID_DESTINATIONS) -def test_model_fixture_rejects_invalid_readback_before_writes(provisioner, destination_updates): - inventory = complete_inventory(provisioner) - inventory.pop(f"{provisioner.UC}/catalogs/ug_e2e") - payload = model_readback(server_defaults=True) - payload["config"]["routing"]["destinations"][0].update(destination_updates) - inventory[f"{provisioner.UC}/model-services/ug_e2e.models.gpt_luna"] = payload - sources = model_sources(provisioner) - sources["gpt_luna"] = "system.ai.databricks-gpt-6-luna" - inventory.update(source_inventory(provisioner, sources)) - client = RecordingApiClient(inventory) - with pytest.raises(RuntimeError, match="refusing to overwrite"): - run_provision(provisioner, client, sources, apply=True) - assert all(method == "GET" for method, *_ in client.requests) - - -@pytest.mark.parametrize("fallback", [False, True]) -def test_model_fixture_rejects_multiple_sources_before_writes(provisioner, fallback): - inventory = complete_inventory(provisioner) - inventory.pop(f"{provisioner.UC}/catalogs/ug_e2e") - payload = provisioner.model_body("system.ai.gpt_luna") - routing = payload["config"]["routing"] - extra = provisioner.model_body("system.ai.other")["config"]["routing"]["destinations"] - if fallback: - routing["fallback"] = {"destinations": extra} - else: - routing["destinations"].extend(extra) - inventory[f"{provisioner.UC}/model-services/ug_e2e.models.gpt_luna"] = payload - client = RecordingApiClient(inventory) - with pytest.raises(RuntimeError, match="refusing to overwrite"): - run_provision(provisioner, client, model_sources(provisioner), apply=True) - assert all(method == "GET" for method, *_ in client.requests) - - -def test_model_fixture_dry_plan_requires_no_credentials_or_network(): - arguments = [sys.executable, str(SCRIPT), "--workspace", "https://workspace.invalid"] - for leaf in ( - "gpt-luna", - "claude-haiku", - "claude-sonnet", - "kimi", - "gemini-flash", - "claude-decoy", - "codex-decoy", - ): - arguments.extend([f"--{leaf}-source", f"system.ai.{leaf}"]) - environment = os.environ.copy() - environment.pop("DATABRICKS_BEARER", None) - result = subprocess.run(arguments, env=environment, capture_output=True, text=True, timeout=10) - assert result.returncode == 0, result.stderr - assert "no network or writes; no bearer required" in result.stdout - assert result.stdout.count("model service ug_e2e.") == 7 - assert "model service ug_e2e.models.gemini_flash" in result.stdout - assert "model service ug_e2e.other_models.claude_decoy" in result.stdout - assert "model service ug_e2e.other_models.codex_decoy" in result.stdout - assert "mcp-services" not in result.stdout - assert "skills" not in result.stdout - - -def test_model_fixture_auth_callback_sets_bearer_and_rejects_redirects(provisioner): - class PreparedRequest: - def __init__(self): - self.headers = {} - self.hooks = {} - - def register_hook(self, event, hook): - self.hooks.setdefault(event, []).append(hook) - - callback = provisioner.bearer_auth("component-token") - request = PreparedRequest() - assert callback(request) is request - assert request.headers == {"Authorization": "Bearer component-token"} - redirect_hook = request.hooks["response"][0] - response = SimpleNamespace( - status_code=200, - request=SimpleNamespace(method="GET"), - json=lambda: {"config": {}}, - close=lambda: None, - ) - assert redirect_hook(response) is response - closed = [] - with pytest.raises(AssertionError, match="redirected") as error: - redirect_hook(SimpleNamespace(status_code=307, close=lambda: closed.append(True))) - assert "component-token" not in str(error.value) - assert closed == [True] - with pytest.raises(AssertionError, match="non-object"): - redirect_hook( - SimpleNamespace( - status_code=200, - request=SimpleNamespace(method="GET"), - json=lambda: None, - ) - ) - - -def test_model_fixture_sanitizes_required_not_found_errors(provisioner): - class ErrorApiClient: - def do(self, method, path, *, query=None, headers=None, body=None, auth=None): - raise NotFound("secret response body") - - client = ErrorApiClient() - auth = provisioner.bearer_auth("component-token") - assert provisioner._get_optional(client, auth, "/optional") is None - for method in ("GET", "POST"): - with pytest.raises(RuntimeError, match="^Workspace API request failed$") as error: - provisioner._request(client, auth, method, "/required") - assert "secret response body" not in str(error.value) - - -def test_model_fixture_config_is_model_only(): - config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) - assert "mcp_servers" not in config - assert "skills" not in config - agents = {entry["agent"]: entry["config"] for entry in config["enabled_agents"]} - assert set(agents) == {"CODING_AGENT_CLAUDE_CODE", "CODING_AGENT_CODEX"} - assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE" - assert agents["CODING_AGENT_CLAUDE_CODE"]["default_models"] == { - "default_model": "ug_e2e.models.claude_sonnet", - "default_sonnet_model": "ug_e2e.models.claude_sonnet", - } - assert agents["CODING_AGENT_CODEX"]["default_models"] == { - "default_model": "ug_e2e.models.gpt_luna" - } - for agent_config in agents.values(): - assert agent_config["models"] == {"unity_catalog_location": "ug_e2e.models"} - assert agent_config["smart_routing"] == {"enabled": False} - assert agent_config["tracing"] == {"enabled": False} - - -def test_model_fixture_validate_missing_inventory_performs_no_writes(provisioner): - client = RecordingApiClient(source_inventory(provisioner)) - with pytest.raises(RuntimeError, match="Inventory is incomplete"): - run_provision(provisioner, client, model_sources(provisioner), apply=False) - assert len(client.requests) == 17 - assert all(method == "GET" for method, *_ in client.requests) - - -@pytest.mark.parametrize("apply", [False, True]) -@pytest.mark.parametrize("registered_model", [None, {}, {"full_name": "system.ai.other"}, []]) -def test_model_fixture_source_preflight_failure_performs_no_writes( - provisioner, apply, registered_model -): - sources = model_sources(provisioner) - inventory = source_inventory(provisioner) - source = sorted(sources.values())[-1] - inventory[f"{provisioner.UC}/models/{source}"] = registered_model - client = RecordingApiClient(inventory) - with pytest.raises(RuntimeError, match="canonical registered-model FQN.*model-service alias"): - run_provision(provisioner, client, sources, apply=apply) - assert len(client.requests) == len(set(sources.values())) - assert all(method == "GET" and "/models/" in path for method, path, *_ in client.requests) - - -def test_model_fixture_source_preflight_rejects_friendly_alias_without_resolving(provisioner): - sources = model_sources(provisioner) - inventory = source_inventory(provisioner) - sources["gpt_luna"] = "system.ai.gpt-6-luna" - inventory[f"{provisioner.UC}/models/system.ai.databricks-gpt-6-luna"] = { - "full_name": "system.ai.databricks-gpt-6-luna" - } - inventory[f"{provisioner.UC}/model-services/system.ai.gpt-6-luna"] = model_readback( - server_defaults=True - ) - client = RecordingApiClient(inventory) - with pytest.raises(RuntimeError, match="system.ai.gpt-6-luna.*model-service alias"): - run_provision(provisioner, client, sources, apply=True) - assert all(method == "GET" and "/models/" in path for method, path, *_ in client.requests) - assert not any("databricks-gpt-6-luna" in path for _, path, *_ in client.requests) - - -def test_model_fixture_source_preflight_checks_each_unique_source_once(provisioner): - sources = dict.fromkeys(provisioner.MODEL_LEAVES, "system.ai.databricks-gpt-6-luna") - client = RecordingApiClient(source_inventory(provisioner, sources)) - run_provision(provisioner, client, sources, apply=True) - source_reads = [request for request in client.requests if "/models/" in request[1]] - assert source_reads == [ - ("GET", f"{provisioner.UC}/models/system.ai.databricks-gpt-6-luna", None, None) - ] - assert client.requests[0] == source_reads[0] - - -@pytest.mark.parametrize("apply", [False, True]) -def test_model_fixture_existing_inventory_is_not_rewritten(provisioner, apply): - inventory = complete_inventory(provisioner) - for path, payload in inventory.items(): - if "/model-services/" in path: - payload["config"]["routing"]["destinations"][0]["is_deleted"] = False - client = RecordingApiClient(inventory) - run_provision(provisioner, client, model_sources(provisioner), apply=apply) - assert all(method == "GET" for method, *_ in client.requests) - - -def test_model_fixture_mismatch_aborts_before_any_creation(provisioner): - inventory = complete_inventory(provisioner) - inventory.pop(f"{provisioner.UC}/catalogs/ug_e2e") - inventory[f"{provisioner.UC}/model-services/ug_e2e.other_models.codex_decoy"] = ( - provisioner.model_body("system.ai.wrong_model") - ) - client = RecordingApiClient(inventory) - with pytest.raises(RuntimeError, match="refusing to overwrite"): - run_provision(provisioner, client, model_sources(provisioner), apply=True) - assert all(method == "GET" for method, *_ in client.requests) - - -def test_model_fixture_apply_only_creates_models_in_dependency_order(provisioner): - client = RecordingApiClient(source_inventory(provisioner)) - run_provision(provisioner, client, model_sources(provisioner), apply=True) - writes = [request for request in client.requests if request[0] != "GET"] - assert [request[1] for request in writes] == [ - f"{provisioner.UC}/catalogs", - *[f"{provisioner.UC}/schemas"] * 2, - *[f"{provisioner.UC}/model-services"] * 7, - ] - assert all(request[0] == "POST" for request in writes) - assert all(request[0] == "GET" for request in client.requests[:17]) - assert {request[1] for request in client.requests[:7]} == set(source_inventory(provisioner)) - model_writes = [request for request in writes if request[1].endswith("model-services")] - expected_parents = { - "gpt_luna": "schemas/ug_e2e.models", - "claude_haiku": "schemas/ug_e2e.models", - "claude_sonnet": "schemas/ug_e2e.models", - "kimi": "schemas/ug_e2e.models", - "gemini_flash": "schemas/ug_e2e.models", - "claude_decoy": "schemas/ug_e2e.other_models", - "codex_decoy": "schemas/ug_e2e.other_models", - } - assert { - request[2]["model_service_id"]: request[2]["parent"] for request in model_writes - } == expected_parents - for request in model_writes: - leaf = request[2]["model_service_id"] - assert request[3] == provisioner.model_body(f"system.ai.{leaf}") - - -@pytest.mark.parametrize( - "workspace", - [ - "http://workspace.invalid", - "https://user:secret@workspace.invalid", - "https://workspace.invalid/path", - "https://workspace.invalid/?token=secret", - ], -) -def test_model_fixture_rejects_non_origin_workspaces(provisioner, workspace): - with pytest.raises(ValueError, match="HTTPS workspace origin"): - provisioner.workspace_origin(workspace) diff --git a/tests/test_integration_model_policy.py b/tests/test_integration_model_policy.py index fdb4fe5d3..7bfc6b030 100644 --- a/tests/test_integration_model_policy.py +++ b/tests/test_integration_model_policy.py @@ -1,13 +1,39 @@ -"""Component checks for catalog discovery fixture expectations.""" - -import json -from pathlib import Path +"""Component checks for catalog discovery model-policy expectations.""" import pytest from tests.integration import catalog_discovery_expectations as expectations -ROOT = Path(__file__).parent.parent + +@pytest.fixture +def managed_policy(): + return { + "spec_version": 1, + "default_agent": "CODING_AGENT_CLAUDE_CODE", + "enabled_agents": [ + { + "agent": "CODING_AGENT_CLAUDE_CODE", + "config": { + "models": {"unity_catalog_location": "ug_e2e.models"}, + "default_models": { + "default_model": "ug_e2e.models.claude_sonnet", + "default_sonnet_model": "ug_e2e.models.claude_sonnet", + }, + "smart_routing": {"enabled": False}, + "tracing": {"enabled": False}, + }, + }, + { + "agent": "CODING_AGENT_CODEX", + "config": { + "models": {"unity_catalog_location": "ug_e2e.models"}, + "default_models": {"default_model": "ug_e2e.models.gpt_luna"}, + "smart_routing": {"enabled": False}, + "tracing": {"enabled": False}, + }, + }, + ], + } def test_model_discovery_matrix_filters_families_without_changing_schema_scope(): @@ -33,18 +59,16 @@ def test_model_discovery_matrix_filters_families_without_changing_schema_scope() assert expectations.CODEX_DECOY not in expectations.MODEL_SERVICES -def test_model_policy_allows_server_metadata_and_later_stack_fields(): - config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) - config["update_time"] = "2026-10-02T00:00:00Z" - config["name"] = "coding-agent-configs/cuj3" - config["mcp_servers"] = {"unity_catalog_location": "ug_e2e.tools"} - expectations.assert_model_policy(config) +def test_model_policy_allows_server_metadata_and_later_stack_fields(managed_policy): + managed_policy["update_time"] = "2026-10-02T00:00:00Z" + managed_policy["name"] = "coding-agent-configs/cuj3" + managed_policy["mcp_servers"] = {"unity_catalog_location": "ug_e2e.tools"} + expectations.assert_model_policy(managed_policy) -def test_model_policy_rejects_a_default_outside_the_managed_scope(): - config = json.loads((ROOT / "fixtures/cuj-3/managed-config.json").read_text()) - config["enabled_agents"][1]["config"]["default_models"]["default_model"] = ( +def test_model_policy_rejects_a_default_outside_the_managed_scope(managed_policy): + managed_policy["enabled_agents"][1]["config"]["default_models"]["default_model"] = ( expectations.CODEX_DECOY ) with pytest.raises(AssertionError): - expectations.assert_model_policy(config) + expectations.assert_model_policy(managed_policy) From c373f723dd16eb482295a71472d90be303496a2f Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 19:36:31 +0000 Subject: [PATCH 07/19] AIGTWY-4878 Extract reusable native agent integration helpers --- tests/README.md | 13 ++---- tests/integration/README.md | 88 +++---------------------------------- 2 files changed, 10 insertions(+), 91 deletions(-) diff --git a/tests/README.md b/tests/README.md index 2a45465f2..66d24fa4f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -187,7 +187,6 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_configure_managed_claude`, `test_ug_configure_managed_codex` | Configure against a workspace that publishes a managed CodingAgentConfig | No agent selector; each agent's generated config exposes exactly the admin's static model_services; real gateway prompt on launch. The Codex case also checks the shared catalog pointer, restart guidance, and a fresh bare app-server's visible model list | | `e2e_cuj/test_ug_budget_defaults.py` | Launch bare `ug` with separate low-spend and above-tier principals against the fixed 1% tier (`spending_percentage=0.01`); explicitly launch `ug claude` above the tier | Bare launches check Claude/Sonnet below the tier and the Codex/Luna recommendation above it; `ug usage` agrees with backend spend, threshold, and percentage. Explicit `ug claude` displays the backend's Codex/Luna recommendation while its generated model setting and native header select Sonnet. No budget writes or inference tasks. Run in the shared `E2E CUJs` job with both credential pairs documented in `integration/README.md`. | | `test_case_01_*` | Launch managed Claude without defaults after configure and from fresh state | Claude receives the admin MPS header; its gateway cache and replacement picker match the independently fetched provider model IDs; catalog labels are preserved and a model appears in a numbered picker row | -| Catalog discovery (`test_catalog_discovery_claude`, `test_catalog_discovery_codex`) | Agent-specific journeys using shared discovery/evidence helpers; configure published `ug_e2e.models` policy in the dedicated workspace | Exact Claude Sonnet/Haiku/Kimi and Codex GPT Luna/Kimi catalogs/pickers; Gemini and other-schema decoys excluded. Bare `ug` and `ug claude` complete separate Sonnet TUI tasks; Claude print and Codex TUI/exec tasks use configured defaults without overrides. Every extra compatible model completes a headless task; routing stays off. Model evidence is limited as described below; MCP/skills are deferred | | `test_case_03_*`, `test_case_05_*` | Pass a provider or model-location override to managed Claude after configure and from fresh state | ug rejects the override before Claude starts and preserves agent-owned state | | `test_case_02_*` | Launch managed Codex after configure and from fresh state | The scoped and stable catalogs, ug-launched app server, and fresh bare app server match the independently fetched admin MPS model IDs. The configured case uses real `ug revert` to remove ug's shared pointer and stable file while preserving a user setting | | `test_case_04_*`, `test_case_06_*` | Pass a provider or model-location override to managed Codex after configure and from fresh state | ug rejects the override before Codex starts and preserves agent-owned state | @@ -203,13 +202,12 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_and_ucode_web_search_helpers_preserve_mcp_stdio` | Initialize and list tools through both web-search helper commands | Exactly the MCP JSON-RPC responses; no text/ANSI contamination; existing server/tool identities preserved; no model request | With Claude and Codex selected there are **62 live cases** (12 marked TUI cases), -**8 managed-workspace cases** (marker `managed`, including two dedicated catalog discovery journeys, run against workspaces that +**6 managed-workspace cases** (marker `managed`, run against workspaces that publish a CodingAgentConfig), **1 two-workspace case** (marker `workspace_switch`), **25 managed-fixture cases** (marker `managed_fixture`, with only the CodingAgentConfig input injected), and **7 installation checks**. The 14 retained numbered scenarios -comprise **24 explicit journeys**: 12 managed-fixture configured/fresh executions and 12 unmanaged -executions. Two additional catalog discovery journeys cover the preconfigured dedicated workspace. -Thirteen additional managed-fixture cases cover focused model, MCP, skills, +comprise **24 explicit journeys**: 12 managed configured/fresh executions and 12 unmanaged +executions. Thirteen additional managed-fixture cases cover focused model, MCP, skills, and lifecycle shapes; two published-config cases cover Claude defaults. Parametrization varies argument spelling or routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks configuration TUI journeys. @@ -231,10 +229,7 @@ Claude discovery assertions match numbered picker rows, not startup banners or footers. Offline regressions cover that distinction, native Haiku/Opus/Sonnet deduplication, and raw catalog ID/display-name rows for scoped pickers. Managed discovery expectations come from separate read-only, provider-scoped -model-list requests; they do not rely solely on ug's generated catalog. Catalog discovery tasks -check Claude's response-reported model and Codex's client-selected model; neither -proves the executed gateway backing destination. Native picker/task coverage -requires a live journey pass, not collection or lint success. +model-list requests; they do not rely solely on ug's generated catalog. The dedicated-workspace CUJ suite adds three independently runnable CUJ2 MPS+MCP cases: one configuration case and separate Codex and Claude TUI task cases. The workspace is diff --git a/tests/integration/README.md b/tests/integration/README.md index 2b4a7049a..8cdd60186 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -10,8 +10,7 @@ CUJ2 adds three separately collected cases for exact MPS/MCP configuration, Code and Claude inference. This suite runs the **installed product** through subprocesses, against the same -`UCODE_TEST_WORKSPACE` used by the existing e2e tests; catalog discovery uses a separate managed -workspace selected explicitly for its model-only journeys. It does not import `ucode`, +`UCODE_TEST_WORKSPACE` used by the existing e2e tests. It does not import `ucode`, patch application functions, substitute agent executables, run a fake gateway, or construct ug state files. The normal test suite checks these boundaries. @@ -230,7 +229,6 @@ test_ug_claude_managed_model_discovery.py # fetched/reused Claude MPS policy cas test_ug_codex_managed_model_discovery.py # fetched/reused Codex MPS policy cases test_ug_claude_model_discovery.py # unmanaged scenarios 7, 9, 11, 13 test_ug_codex_model_discovery.py # unmanaged scenarios 8, 10, 12, 14 -test_ug_catalog_discovery.py # dedicated managed Claude/Codex catalogs and tasks test_ug_configure_managed.py # managed workspace: static model list/catalog pointer, no agent selector test_ug_configure_managed_models.py # injected model sources, smart-routing banner, Codex fallback metadata test_ug_configure_managed_mcp.py # injected managed MCP list @@ -243,9 +241,6 @@ utils/ # process/terminal/evidence helpers and Do `conftest.py`, `pytest.ini`, and this README stay at the suite root for pytest discovery and run instructions. -Agent catalog requests use the explicitly supplied workspace and bearer, reject -redirects, and retain schema/provider headers and raw pagination evidence. - Each test has a `Scenario:` / `Expected:` docstring and shows its own public configure command, launch, user action, and assertions. Shared code only handles process/terminal mechanics, evidence, and cleanup. Fixtures supply an isolated @@ -364,8 +359,8 @@ they only configure, list models, and open/close the picker. Other live CUJs per real model tasks. There are **62 live cases** (including 12 marked TUI journeys) and **7 installation -checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **8 managed-workspace cases** (one per agent, an idempotent -re-configure, a cache-TTL journey, two Claude defaults cases, and two catalog discovery journeys; marker `managed`) run against +checks** with Claude and Codex; selecting OpenCode adds one live headless case. A separate **6 managed-workspace cases** (one per agent, an idempotent +re-configure, a cache-TTL journey, and two Claude defaults cases; marker `managed`) run against workspaces that publish CodingAgentConfigs; see "Managed-workspace journeys" below. One **`workspace_switch` case** uses two real workspaces and checks skills MCP cleanup and a completed Claude task. A further **25 `managed_fixture` @@ -388,9 +383,8 @@ constants in the runner; CI only needs `UG_MPS_DEFAULTS_CLIENT_SECRET` for west- `UG_PARENT_SCHEMA_DEFAULTS_CLIENT_SECRET` for northeast-2. As with the base workspace, the runner mints short-lived tokens and passes bearers to pytest; each test selects its target bearer for `ug configure` and Claude. The client secrets do not enter the pytest process. -The 14 retained numbered scenarios comprise 24 explicit journeys: 12 managed-fixture -and 12 unmanaged executions. Two additional catalog discovery journeys bring the -complete integration suite to 103 executions. See the named coverage and gaps matrix in +The 14 retained numbered scenarios comprise 24 explicit journeys: 12 managed and 12 unmanaged +executions; the complete integration suite collects 101 executions. See the named coverage and gaps matrix in [../README.md](../README.md). ```bash @@ -556,7 +550,7 @@ No test retries or assertion changes compensate for capacity failures. Both matrices use `fail-fast: false` and upload uniquely named evidence even when the other agent fails. The **All integration tests** check requires installation, workspace validation, smoke, -both full lanes, both Catalog discovery lanes, and both **Managed config** lanes to pass for full/live runs. Each tracing +both full lanes, and both **Managed config** lanes to pass for full/live runs. Each tracing journey is included in its agent's Full lane. The managed lanes do not use `continue-on-error`: a failure, cancellation, or unexpected skip fails the aggregate check. Manual smoke, TUI, and installation subsets do not select managed tests and do not require them. @@ -564,76 +558,6 @@ The advisory Windows installation and headless lanes are not yet included in tha The existing required `e2e` context also waits for the complete integration workflow, so integration cannot still be running when that gate passes. Full coverage on PRs needs no label or opt-in. -### Catalog discovery - -Agent selection follows the rest of the suite: explicit `test_catalog_discovery_` -journeys carry agent markers; `UG_INTEGRATION_AGENTS` filters collection, and CI runs an -agent matrix. Launch commands, TUI tasks, defaults, and expected model sets stay visible -in each journey. Shared per-agent discovery and transcript helpers live in `utils/agents/`; -process, terminal, and task orchestration stay in the common harness. - -For another agent, add its helper module and evidence adapter, explicit journey and -fixture expectations. Register its package/version and marker in the runner/collection -setup, implement any native terminal support, and add it to the CI matrix. Existing -journeys need no renaming or generic agent branches. - -The dedicated workspace must already contain the services listed below and publish -a CodingAgentConfig enabling both agents with `ug_e2e.models` as their model source. -Its default agent is Claude; defaults are Claude Sonnet (including the Sonnet family -default) and Codex GPT Luna, with smart routing and tracing disabled. Tests do not -provision resources or policy; Apps, connections, MCP, and skills are not required. - -Model-service fixtures are trusted prerequisites; journeys do not revalidate their -backing destinations. Each journey runs real `ug configure`, checks persisted -schema/defaults, and independently fetches scoped and compatible-decoy catalogs. -Generated settings/catalogs, native caches/model lists, and exact numbered picker -rows must agree: no unmatched, ambiguous, duplicate, or excluded entries hidden -by labels. Defaults and every extra compatible model must complete file tasks -with values withheld from prompts. Claude checks the response-reported model; -Codex checks the client-selected model. Neither proves the executed gateway backing -destination; discovery or startup alone cannot pass. - -The version-agnostic services under `ug_e2e.models` and expected discovery sets are: - -| Service | Claude Code | Codex | -| --- | --- | --- | -| `gpt_luna` | Excluded | Included; default | -| `claude_haiku` | Included | Excluded | -| `claude_sonnet` | Included; default | Excluded | -| `kimi` | Included | Included | -| `gemini_flash` | Excluded | Excluded | - -In-schema Gemini tests compatibility; accessible `ug_e2e.other_models.claude_decoy` -and `ug_e2e.other_models.codex_decoy` test scope exclusion. Provisioning takes explicit -backing `system.ai` versions. These sets require live verification of gateway API-type -metadata, feature flags, and harness compatibility; unexpected Gemini visibility -fails without local filtering. Offline checks are not a live pass. - -The dedicated workspace must enable Claude discovery aliases, including Kimi's -`anthropic-aigw-<8-character SHA-256 prefix>-` form. The journey checks -checksums, exact service/wire inventories, and the coding-agent-mode header needed -to reverse aliases for inference. Settings, cache, picker, tasks, and response-reported -models retain returned wire IDs; Codex keeps raw FQNs. No entries are dropped. - -Bare `ug` and explicit `ug claude` each complete a separate Claude/Sonnet TUI task -and exit normally; `ug claude -p "task"` completes a Sonnet print task. Codex completes -both TUI and `ug codex -- exec --skip-git-repo-check --json "task"` tasks on `gpt_luna`. -All default tasks omit model overrides; routing stays off. - -CI runs `managed and catalog_discovery and workspace_isolated` in a two-agent matrix using -`UG_CUJ3_WORKSPACE` and the shared CUJ SP secrets. The shared managed lanes exclude -`workspace_isolated`; both catalog discovery legs are required for full/live runs. Local runs -must provide an explicit workspace and authenticated profile or bearer: - -```bash -python3.12 scripts/run_integration.py \ - --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ - --workspace "$CATALOG_DISCOVERY_WORKSPACE" --profile YOUR_PROFILE \ - -- -m 'managed and catalog_discovery and workspace_isolated' -``` - -MCP/skills remain separate layers; this model suite does not depend on CUJ7. - ### Managed-workspace journeys `test_ug_configure_managed.py` (marker `managed`, not `live`) runs in its own per-agent From d1b4bb61a97677d4c349f6f718519cb70307a6a8 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 20:23:34 +0000 Subject: [PATCH 08/19] AIGTWY-4878 Simplify catalog discovery coverage and supporting helpers --- .../catalog_discovery_expectations.py | 27 ---- .../integration/test_ug_catalog_discovery.py | 42 +----- tests/integration/utils/model_discovery.py | 11 -- tests/test_integration_evidence.py | 142 +++++++++--------- tests/test_integration_http.py | 33 ++-- tests/test_integration_model_discovery.py | 29 ---- tests/test_integration_model_policy.py | 74 --------- tests/test_integration_provider_catalog.py | 26 +--- 8 files changed, 88 insertions(+), 296 deletions(-) delete mode 100644 tests/test_integration_model_policy.py diff --git a/tests/integration/catalog_discovery_expectations.py b/tests/integration/catalog_discovery_expectations.py index 528d760fd..2a17409c3 100644 --- a/tests/integration/catalog_discovery_expectations.py +++ b/tests/integration/catalog_discovery_expectations.py @@ -7,32 +7,5 @@ GEMINI_MODEL = f"{MODEL_SCHEMA}.gemini_flash" CLAUDE_MODELS = frozenset({CLAUDE_DEFAULT, f"{MODEL_SCHEMA}.claude_haiku", f"{MODEL_SCHEMA}.kimi"}) CODEX_MODELS = frozenset({CODEX_DEFAULT, f"{MODEL_SCHEMA}.kimi"}) -MODEL_SERVICES = CLAUDE_MODELS | CODEX_MODELS | {GEMINI_MODEL} CLAUDE_DECOY = f"{OTHER_MODEL_SCHEMA}.claude_decoy" CODEX_DECOY = f"{OTHER_MODEL_SCHEMA}.codex_decoy" - - -def assert_model_policy(config: dict) -> None: - """Require exact model policy while allowing unrelated resource fields.""" - assert config["spec_version"] == 1, config - assert config["default_agent"] == "CODING_AGENT_CLAUDE_CODE", config - entries = config["enabled_agents"] - assert isinstance(entries, list) and len(entries) == 2, config - assert all(isinstance(entry, dict) for entry in entries), entries - by_agent = {entry.get("agent"): entry.get("config") for entry in entries} - expected_defaults = { - "CODING_AGENT_CLAUDE_CODE": { - "default_model": CLAUDE_DEFAULT, - "default_sonnet_model": CLAUDE_DEFAULT, - }, - "CODING_AGENT_CODEX": {"default_model": CODEX_DEFAULT}, - } - assert set(by_agent) == set(expected_defaults), by_agent - for agent, defaults in expected_defaults.items(): - agent_config = by_agent[agent] - assert isinstance(agent_config, dict), agent_config - assert agent_config.get("models") == {"unity_catalog_location": MODEL_SCHEMA}, agent_config - assert agent_config.get("default_models") == defaults, agent_config - for key in ("smart_routing", "tracing"): - settings = agent_config.get(key) - assert isinstance(settings, dict) and settings.get("enabled") is False, agent_config diff --git a/tests/integration/test_ug_catalog_discovery.py b/tests/integration/test_ug_catalog_discovery.py index fa7dc7aa3..6ce129f0c 100644 --- a/tests/integration/test_ug_catalog_discovery.py +++ b/tests/integration/test_ug_catalog_discovery.py @@ -15,7 +15,6 @@ GEMINI_MODEL, MODEL_SCHEMA, OTHER_MODEL_SCHEMA, - assert_model_policy, ) from utils.agents import claude, codex from utils.evidence import FileTask, assert_completed_task_model @@ -35,21 +34,16 @@ @pytest.mark.claude @pytest.mark.tui def test_catalog_discovery_claude(live_session, workspace): - """Scenario: configure published Catalog discovery policy; launch bare ug, ug claude, and print tasks. + """Scenario: configure the managed workspace; launch bare ug, ug claude, and print tasks. - Expected: persisted schema/defaults match policy; independent - catalogs, settings, cache, and exact numbered picker agree on Sonnet/Haiku/Kimi, - excluding Gemini and both accessible out-of-scope decoys. Both TUI launches exit - normally after separate Sonnet file tasks; print mode uses Sonnet without an override, - and every extra model completes a headless task, with routing off throughout. - Claude's response-reported model must match; it does not prove the executed gateway - backing destination. These assertions require a live pass, not collection alone. + Expected: Sonnet/Haiku/Kimi appear in discovery and the picker, but Gemini and decoys + do not. TUI and print tasks use the Sonnet default without overrides; other compatible + models complete print tasks. Answers report the expected model, with routing off. """ session = live_session configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) assert "Select coding agents to configure:" not in configured.stdout, configured.stdout persisted = read_persisted_managed_config(session, workspace) - assert_model_policy(persisted["config"]) session.record("catalog-discovery-managed-config.json", persisted) bearer = os.environ["DATABRICKS_BEARER"] @@ -65,15 +59,9 @@ def test_catalog_discovery_claude(live_session, workspace): "decoy_pages": decoy_catalog.payloads, }, ) - service_ids = [claude.model_service_id(model) for model in parent_catalog.model_ids] - assert len(service_ids) == len(set(service_ids)), parent_catalog - assert set(service_ids) == CLAUDE_MODELS, parent_catalog assert set(parent_catalog.model_ids) == { claude.discovery_model_id(model) for model in CLAUDE_MODELS }, parent_catalog - decoy_ids = [claude.model_service_id(model) for model in decoy_catalog.model_ids] - assert len(decoy_ids) == len(set(decoy_ids)), decoy_catalog - assert set(decoy_ids) == {CLAUDE_DECOY}, decoy_catalog assert set(decoy_catalog.model_ids) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog default_task = FileTask(session) @@ -123,7 +111,6 @@ def test_catalog_discovery_claude(live_session, workspace): tui.submit(default_task.prompt) tui.wait_for_task(default_task, timeout=240) tui.exit_normally() - default_task.assert_completed(session, "claude") assert_completed_task_model(session, "claude", default_task.value, CLAUDE_DEFAULT) explicit_task = FileTask(session) @@ -137,13 +124,11 @@ def test_catalog_discovery_claude(live_session, workspace): tui.submit(explicit_task.prompt) tui.wait_for_task(explicit_task, timeout=240) tui.exit_normally() - explicit_task.assert_completed(session, "claude") assert_completed_task_model(session, "claude", explicit_task.value, CLAUDE_DEFAULT) gateway_ids = session.claude_gateway_model_ids() assert len(gateway_ids) == len(set(gateway_ids)), gateway_ids assert set(gateway_ids) == set(parent_catalog.model_ids), gateway_ids - assert not {GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY} & set(gateway_ids), gateway_ids print_task = FileTask(session) result = session.run( @@ -157,7 +142,6 @@ def test_catalog_discovery_claude(live_session, workspace): timeout=240, ) print_task.assert_headless_answer("claude", result) - print_task.assert_completed(session, "claude") assert_completed_task_model(session, "claude", print_task.value, CLAUDE_DEFAULT) for model in parent_catalog.model_ids: @@ -178,7 +162,6 @@ def test_catalog_discovery_claude(live_session, workspace): timeout=240, ) task.assert_headless_answer("claude", result) - task.assert_completed(session, "claude") assert_completed_task_model(session, "claude", task.value, model) session.assert_not_routed() @@ -186,21 +169,16 @@ def test_catalog_discovery_claude(live_session, workspace): @pytest.mark.codex @pytest.mark.tui def test_catalog_discovery_codex(live_session, workspace): - """Scenario: configure published Catalog discovery policy; launch Codex TUI and ug codex -- exec. + """Scenario: configure the managed workspace; launch Codex TUI and ug codex -- exec. - Expected: persisted schema/defaults match policy; independent - catalogs, generated catalogs, model/list, and exact numbered picker agree on GPT Luna/Kimi, - excluding Gemini and both accessible out-of-scope decoys. TUI exits normally after a - GPT Luna file task; exec uses that default without an override, and every extra model - completes a headless task, with routing off throughout. - Codex's client-selected model must match; it does not prove the executed gateway - backing destination. These assertions require a live pass, not collection alone. + Expected: GPT Luna/Kimi appear in model/list and the picker, but Gemini and decoys + do not. TUI and exec tasks use the GPT Luna default without overrides; Kimi completes + an exec task. Completed turns select the expected model, with routing off. """ session = live_session configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) assert "Select coding agents to configure:" not in configured.stdout, configured.stdout persisted = read_persisted_managed_config(session, workspace) - assert_model_policy(persisted["config"]) session.record("catalog-discovery-managed-config.json", persisted) bearer = os.environ["DATABRICKS_BEARER"] @@ -229,7 +207,6 @@ def test_catalog_discovery_codex(live_session, workspace): models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) assert len(models) == len(set(models)), models assert set(models) == set(parent_catalog.model_ids), models - assert not {GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY} & set(models), models catalog_paths = list((session.home / ".ucode").glob("codex-model-catalog-*.json")) assert len(catalog_paths) == 1, catalog_paths catalog = json.loads(catalog_paths[0].read_text()) @@ -268,7 +245,6 @@ def test_catalog_discovery_codex(live_session, workspace): tui.submit(default_task.prompt) tui.wait_for_task(default_task, timeout=240) tui.exit_normally() - default_task.assert_completed(session, "codex") assert_completed_task_model(session, "codex", default_task.value, CODEX_DEFAULT) exec_task = FileTask(session) @@ -282,7 +258,6 @@ def test_catalog_discovery_codex(live_session, workspace): timeout=240, ) exec_task.assert_headless_answer("codex", result) - exec_task.assert_completed(session, "codex") assert_completed_task_model(session, "codex", exec_task.value, CODEX_DEFAULT) for model in parent_catalog.model_ids: @@ -301,6 +276,5 @@ def test_catalog_discovery_codex(live_session, workspace): timeout=240, ) task.assert_headless_answer("codex", result) - task.assert_completed(session, "codex") assert_completed_task_model(session, "codex", task.value, model) session.assert_not_routed() diff --git a/tests/integration/utils/model_discovery.py b/tests/integration/utils/model_discovery.py index bfb338e94..941337627 100644 --- a/tests/integration/utils/model_discovery.py +++ b/tests/integration/utils/model_discovery.py @@ -21,17 +21,6 @@ def claude_discovery_model_id(model_id: str) -> str: return f"anthropic-aigw-{checksum}-{model_id}" -def claude_model_service_id(model_id: str) -> str: - """Recover service identity only from the gateway's checksum-valid discovery alias.""" - alias = re.fullmatch(r"anthropic-aigw-([0-9a-f]{8})-(.+)", model_id) - if alias is None: - return model_id - checksum, original = alias.groups() - if hashlib.sha256(original.encode("utf-8")).hexdigest()[:8] != checksum: - return model_id - return original - - def claude_model_in_picker(screen: str, model_id: str, display_name: str | None) -> bool: """Return whether a Claude model appears in a numbered picker row. diff --git a/tests/test_integration_evidence.py b/tests/test_integration_evidence.py index 4b6b31e1f..bc724beaf 100644 --- a/tests/test_integration_evidence.py +++ b/tests/test_integration_evidence.py @@ -1,11 +1,11 @@ """Unit checks for interpreting terminal evidence, not live agent substitutes.""" import json -from types import SimpleNamespace import pytest from tests.integration.utils import evidence +from tests.integration.utils.agents import _evidence_id, claude, codex from tests.integration.utils.evidence import ( SubagentCalculation, assert_no_terminal_api_error, @@ -16,9 +16,19 @@ class _Session: def __init__(self, home): self.home = home + self.artifacts = {} - def record(self, _name, _value): - pass + def record(self, name, value): + self.artifacts[name] = value + + +def _transcript_session(home, agent, transcripts): + directory = home / {"claude": ".claude/projects", "codex": ".codex/sessions"}[agent] + for name, records in transcripts.items(): + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(record) + "\n" for record in records)) + return _Session(home) def _write_answer(home, agent, *, child, value): @@ -93,7 +103,7 @@ def test_tagged_calculation_requires_the_native_child_answer(tmp_path, agent): assert "1+1" in task.prompt -def test_codex_model_identity_uses_only_the_completed_answer_turn(monkeypatch): +def test_codex_model_identity_uses_only_the_completed_answer_turn(): records = [ { "type": "turn_context", @@ -112,13 +122,12 @@ def test_codex_model_identity_uses_only_the_completed_answer_turn(monkeypatch): }, }, ] - monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"transcript": records}) - assert evidence.completed_task_models(None, "codex", "withheld-file-value") == { + assert codex.completed_task_models(records, "withheld-file-value") == { "catalog.models.gpt_luna" } -def test_codex_model_identity_rejects_prompt_only_evidence(monkeypatch): +def test_codex_model_identity_rejects_prompt_only_evidence(): records = [ { "type": "turn_context", @@ -129,11 +138,10 @@ def test_codex_model_identity_rejects_prompt_only_evidence(monkeypatch): "payload": {"type": "message", "role": "user", "content": "withheld-file-value"}, }, ] - monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"transcript": records}) - assert evidence.completed_task_models(None, "codex", "withheld-file-value") == set() + assert codex.completed_task_models(records, "withheld-file-value") == set() -def test_claude_model_identity_uses_assistant_answer_not_tool_output(monkeypatch): +def test_claude_model_identity_uses_assistant_answer_not_tool_output(): records = [ { "type": "user", @@ -151,10 +159,7 @@ def test_claude_model_identity_uses_assistant_answer_not_tool_output(monkeypatch }, }, ] - monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"transcript": records}) - assert evidence.completed_task_models(None, "claude", "value") == { - "catalog.models.claude_sonnet" - } + assert claude.completed_task_models(records, "value") == {"catalog.models.claude_sonnet"} def completed_records(agent, model, answer="value", turn_id="matching"): @@ -181,7 +186,7 @@ def completed_records(agent, model, answer="value", turn_id="matching"): @pytest.mark.parametrize("agent", ["claude", "codex"]) @pytest.mark.parametrize("parent_answer", [None, "value", "unrelated-answer"]) def test_completed_task_models_excludes_child_only_and_conflicting_child_evidence( - monkeypatch, agent, parent_answer + tmp_path, agent, parent_answer ): child = completed_records(agent, "child-model") if agent == "codex": @@ -189,36 +194,33 @@ def test_completed_task_models_excludes_child_only_and_conflicting_child_evidenc sessions = {"project/subagents/child.jsonl": child} if parent_answer is not None: sessions["project/parent.jsonl"] = completed_records(agent, "parent-model", parent_answer) - monkeypatch.setattr(evidence, "agent_sessions", lambda *_: sessions) + session = _transcript_session(tmp_path, agent, sessions) expected = {"parent-model"} if parent_answer == "value" else set() - assert evidence.completed_task_models(None, agent, "value") == expected + assert evidence.completed_task_models(session, agent, "value") == expected -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("model", [None, "", " ", 5, {}, [], "model with whitespace"]) -def test_completed_task_models_fails_closed_on_malformed_model(monkeypatch, agent, model): - records = completed_records(agent, model) - monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"parent": records}) - with pytest.raises(AssertionError, match="model"): - evidence.completed_task_models(None, agent, "value") +@pytest.mark.parametrize("value", [None, "", " ", 5, {}, [], "model with whitespace"]) +def test_completed_task_ids_require_nonempty_strings_without_whitespace(value): + with pytest.raises(AssertionError, match="Missing or malformed completed-task model"): + _evidence_id(value, "model") @pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_completed_task_models_fails_closed_on_missing_model(monkeypatch, agent): +def test_completed_task_models_fails_closed_on_missing_model(tmp_path, agent): records = completed_records(agent, "model") del records[0]["message" if agent == "claude" else "payload"]["model"] - monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"parent": records}) + session = _transcript_session(tmp_path, agent, {"parent.jsonl": records}) with pytest.raises(AssertionError, match="model"): - evidence.completed_task_models(None, agent, "value") + evidence.completed_task_models(session, agent, "value") -@pytest.mark.parametrize("turn_id", [None, "", " ", 5, {}, [], "turn with whitespace"]) +@pytest.mark.parametrize("turn_id", ["", "turn with whitespace"]) @pytest.mark.parametrize("record_index", [0, 1]) def test_codex_completed_task_models_fails_closed_on_malformed_turn_ids(turn_id, record_index): records = completed_records("codex", "model") records[record_index]["payload"]["turn_id"] = turn_id with pytest.raises(AssertionError, match="turn ID"): - evidence.codex_completed_task_models(records, "value") + codex.completed_task_models(records, "value") @pytest.mark.parametrize("record_index", [0, 1]) @@ -226,68 +228,57 @@ def test_codex_completed_task_models_fails_closed_on_missing_turn_ids(record_ind records = completed_records("codex", "model") del records[record_index]["payload"]["turn_id"] with pytest.raises(AssertionError, match="turn ID"): - evidence.codex_completed_task_models(records, "value") + codex.completed_task_models(records, "value") -def test_codex_completed_task_models_requires_context_in_same_session(monkeypatch): +def test_codex_completed_task_models_requires_context_in_same_session(tmp_path): context, completion = completed_records("codex", "model") - monkeypatch.setattr( - evidence, - "agent_sessions", - lambda *_: {"context-session": [context], "answer-session": [completion]}, + session = _transcript_session( + tmp_path, + "codex", + {"context-session.jsonl": [context], "answer-session.jsonl": [completion]}, ) with pytest.raises(AssertionError, match="Missing Codex model context"): - evidence.completed_task_models(None, "codex", "value") + evidence.completed_task_models(session, "codex", "value") @pytest.mark.parametrize("agent", ["claude", "codex"]) @pytest.mark.parametrize("models", [[], ["unexpected"], ["expected", "conflicting"]]) -def test_completed_task_model_assertion_requires_exact_singleton(monkeypatch, agent, models): +def test_completed_task_model_assertion_requires_exact_singleton(tmp_path, agent, models): records = [record for model in models for record in completed_records(agent, model)] - monkeypatch.setattr(evidence, "agent_sessions", lambda *_: {"parent": records}) - artifacts = {} - session = SimpleNamespace(record=lambda name, payload: artifacts.update({name: payload})) + session = _transcript_session(tmp_path, agent, {"parent.jsonl": records}) with pytest.raises(AssertionError): evidence.assert_completed_task_model(session, agent, "value", "expected") - assert next(iter(artifacts.values()))["observed"] == sorted(models) + assert session.artifacts[f"completed-task-model-{agent}-value.json"]["observed"] == sorted( + models + ) @pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_completed_task_model_records_evidence_limits(monkeypatch, agent): - monkeypatch.setattr( - evidence, "agent_sessions", lambda *_: {"parent": completed_records(agent, "expected")} +def test_completed_task_model_records_evidence_limits(tmp_path, agent): + session = _transcript_session( + tmp_path, agent, {"parent.jsonl": completed_records(agent, "expected")} ) - artifacts = {} - session = SimpleNamespace(record=lambda name, payload: artifacts.update({name: payload})) evidence.assert_completed_task_model(session, agent, "value", "expected") - assert artifacts == { - f"completed-task-model-{agent}-value.json": { - "expected": "expected", - "observed": ["expected"], - "evidence_kind": ( - "response-reported model" if agent == "claude" else "client-selected model" - ), - "gateway_destination_proven": False, - } + assert session.artifacts["agent-sessions.json"] == { + "parent.jsonl": completed_records(agent, "expected") + } + assert session.artifacts[f"completed-task-model-{agent}-value.json"] == { + "expected": "expected", + "observed": ["expected"], + "evidence_kind": ( + "response-reported model" if agent == "claude" else "client-selected model" + ), + "gateway_destination_proven": False, } @pytest.mark.parametrize("agent", ["claude", "codex"]) @pytest.mark.parametrize("answer_value", [None, "", " \t", 1, [], {}]) -def test_completed_task_models_rejects_empty_or_malformed_expected_answers( - monkeypatch, agent, answer_value -): - def unexpected_read(*args): - pytest.fail("Invalid answer must not read session evidence") - - monkeypatch.setattr(evidence, "agent_sessions", unexpected_read) +def test_completed_task_models_rejects_empty_or_malformed_expected_answers(agent, answer_value): with pytest.raises(AssertionError, match="nonempty answer value"): evidence.completed_task_models(None, agent, answer_value) - adapter = ( - evidence.claude_completed_task_models - if agent == "claude" - else evidence.codex_completed_task_models - ) + adapter = claude.completed_task_models if agent == "claude" else codex.completed_task_models with pytest.raises(AssertionError, match="nonempty answer value"): adapter([], answer_value) @@ -298,7 +289,7 @@ def unexpected_read(*args): ) def test_claude_completed_task_models_rejects_malformed_message_metadata(message): with pytest.raises(AssertionError): - evidence.claude_completed_task_models([{"type": "assistant", "message": message}], "value") + claude.completed_task_models([{"type": "assistant", "message": message}], "value") @pytest.mark.parametrize("content", [[None], [{"type": "text"}], [{"type": "text", "text": 4}]]) @@ -306,14 +297,14 @@ def test_claude_completed_task_models_rejects_malformed_answer_blocks(content): records = completed_records("claude", "model") records[0]["message"]["content"] = content with pytest.raises(AssertionError): - evidence.claude_completed_task_models(records, "value") + claude.completed_task_models(records, "value") @pytest.mark.parametrize("payload", [None, [], "value"]) @pytest.mark.parametrize("record_type", ["event_msg", "turn_context"]) def test_codex_completed_task_models_rejects_malformed_payloads(payload, record_type): with pytest.raises(AssertionError): - evidence.codex_completed_task_models([{"type": record_type, "payload": payload}], "value") + codex.completed_task_models([{"type": record_type, "payload": payload}], "value") @pytest.mark.parametrize("answer", [None, [], {}, 7]) @@ -321,10 +312,17 @@ def test_codex_completed_task_models_rejects_malformed_completed_answers(answer) records = completed_records("codex", "model") records[1]["payload"]["last_agent_message"] = answer with pytest.raises(AssertionError, match="completed answer"): - evidence.codex_completed_task_models(records, "value") + codex.completed_task_models(records, "value") def test_claude_completed_task_models_normalizes_only_context_window_suffix(): - assert evidence.claude_completed_task_models( + assert claude.completed_task_models( completed_records("claude", "catalog.models.claude_sonnet[1m]"), "value" ) == {"catalog.models.claude_sonnet"} + + +@pytest.mark.parametrize("agent", ["opencode", "unknown"]) +def test_completed_task_model_rejects_unsupported_agents(agent): + session = _Session(None) + with pytest.raises(AssertionError, match="Unsupported evidence agent"): + evidence.assert_completed_task_model(session, agent, "value", "model") diff --git a/tests/test_integration_http.py b/tests/test_integration_http.py index ef6df7c8e..3167c5cf3 100644 --- a/tests/test_integration_http.py +++ b/tests/test_integration_http.py @@ -6,21 +6,9 @@ import pytest -from tests.integration.utils import http, provider_catalog +from tests.integration.utils import http -FETCHERS = [ - provider_catalog.fetch_anthropic_parent_catalog, - provider_catalog.fetch_codex_parent_catalog, - provider_catalog.fetch_anthropic_provider_catalog, - provider_catalog.fetch_codex_provider_catalog, -] - -def fetch_inventory(fetcher, workspace="https://workspace.invalid", token="test-bearer"): - return fetcher(workspace, token, "catalog.models") - - -@pytest.mark.parametrize("fetcher", FETCHERS) @pytest.mark.parametrize( "workspace", [ @@ -40,30 +28,28 @@ def fetch_inventory(fetcher, workspace="https://workspace.invalid", token="test- "https://workspace.invalid\\@other.invalid", ], ) -def test_catalog_gets_reject_invalid_workspace_origins(monkeypatch, fetcher, workspace): +def test_workspace_json_get_rejects_invalid_origins(monkeypatch, workspace): def unexpected_open(*args, **kwargs): pytest.fail("Invalid workspace must not reach the HTTP boundary") monkeypatch.setattr(http.urllib.request, "build_opener", unexpected_open) with pytest.raises(AssertionError, match="HTTPS workspace origin") as failure: - fetch_inventory(fetcher, workspace) + http.safe_https_json_get(workspace, "test-bearer", "/api/models") assert "secret" not in str(failure.value) -@pytest.mark.parametrize("fetcher", FETCHERS) @pytest.mark.parametrize("token", [None, "", " ", "bearer\nsecret", "bearer\rsecret"]) -def test_catalog_gets_require_explicit_valid_bearers(monkeypatch, fetcher, token): +def test_workspace_json_get_requires_an_explicit_valid_bearer(monkeypatch, token): def unexpected_open(*args, **kwargs): pytest.fail("Invalid bearer must not reach the HTTP boundary") monkeypatch.setattr(http.urllib.request, "build_opener", unexpected_open) with pytest.raises(AssertionError, match="explicit workspace bearer"): - fetch_inventory(fetcher, token=token) + http.safe_https_json_get("https://workspace.invalid", token, "/api/models") -@pytest.mark.parametrize("fetcher", FETCHERS) @pytest.mark.parametrize("failure_kind", ["http", "url", "os", "json", "encoding"]) -def test_catalog_get_errors_are_sanitized(monkeypatch, fetcher, failure_kind): +def test_workspace_json_get_errors_are_sanitized(monkeypatch, failure_kind): secret = "private-bearer-and-response-secret" class FailedOpener: @@ -81,16 +67,15 @@ def open(self, request, *, timeout): monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: FailedOpener()) with pytest.raises(AssertionError, match="Workspace JSON GET") as failure: - fetch_inventory(fetcher, token=secret) + http.safe_https_json_get("https://workspace.invalid", secret, "/api/models") assert secret not in str(failure.value) assert "workspace.invalid" not in str(failure.value) assert failure.value.__suppress_context__ -@pytest.mark.parametrize("fetcher", FETCHERS) @pytest.mark.parametrize("status", [301, 302, 303, 307, 308]) @pytest.mark.parametrize("destination", ["https://other.invalid/secret", "/redirected"]) -def test_catalog_gets_deny_redirects(monkeypatch, fetcher, status, destination): +def test_workspace_json_get_denies_redirects(monkeypatch, status, destination): requests = [] build_opener = http.urllib.request.build_opener @@ -109,7 +94,7 @@ def https_open(self, request): lambda handler: build_opener(handler, BoundaryHTTPS()), ) with pytest.raises(AssertionError, match=f"HTTP {status}") as failure: - fetch_inventory(fetcher) + http.safe_https_json_get("https://workspace.invalid", "test-bearer", "/api/models") assert len(requests) == 1 assert requests[0].get_header("Authorization") == "Bearer test-bearer" assert destination not in str(failure.value) diff --git a/tests/test_integration_model_discovery.py b/tests/test_integration_model_discovery.py index 27bf981fb..350016ac3 100644 --- a/tests/test_integration_model_discovery.py +++ b/tests/test_integration_model_discovery.py @@ -6,7 +6,6 @@ assert_picker_inventory, claude_discovery_model_id, claude_model_in_picker, - claude_model_service_id, claude_system_model_ids, codex_model_in_picker, ) @@ -147,34 +146,6 @@ def test_claude_system_model_ids_rejects_empty_missing_and_duplicate_ids(models) claude_system_model_ids(models) -@pytest.mark.parametrize( - ("wire_id", "expected"), - [ - ("catalog.models.kimi", "catalog.models.kimi"), - ("anthropic-aigw-69e2a9a0-catalog.models.kimi", "catalog.models.kimi"), - ("catalog.models.claude_sonnet", "catalog.models.claude_sonnet"), - ( - "anthropic-aigw-00000000-catalog.models.kimi", - "anthropic-aigw-00000000-catalog.models.kimi", - ), - ( - "anthropic-aigw-69e2a9a0-catalog.models.kimi_v2", - "anthropic-aigw-69e2a9a0-catalog.models.kimi_v2", - ), - ( - "anthropic-aigw-69e2a9a0-catalog.other_models.kimi", - "anthropic-aigw-69e2a9a0-catalog.other_models.kimi", - ), - ( - "anthropic-aigw-69E2A9A0-catalog.models.kimi", - "anthropic-aigw-69E2A9A0-catalog.models.kimi", - ), - ], -) -def test_claude_service_identity_requires_exact_gateway_alias_checksum(wire_id, expected): - assert claude_model_service_id(wire_id) == expected - - def test_claude_discovery_requires_exact_gateway_wire_ids(): assert {claude_discovery_model_id(model) for model in PICKER_MODELS["claude"]} == { "catalog.models.claude_haiku", diff --git a/tests/test_integration_model_policy.py b/tests/test_integration_model_policy.py deleted file mode 100644 index 7bfc6b030..000000000 --- a/tests/test_integration_model_policy.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Component checks for catalog discovery model-policy expectations.""" - -import pytest - -from tests.integration import catalog_discovery_expectations as expectations - - -@pytest.fixture -def managed_policy(): - return { - "spec_version": 1, - "default_agent": "CODING_AGENT_CLAUDE_CODE", - "enabled_agents": [ - { - "agent": "CODING_AGENT_CLAUDE_CODE", - "config": { - "models": {"unity_catalog_location": "ug_e2e.models"}, - "default_models": { - "default_model": "ug_e2e.models.claude_sonnet", - "default_sonnet_model": "ug_e2e.models.claude_sonnet", - }, - "smart_routing": {"enabled": False}, - "tracing": {"enabled": False}, - }, - }, - { - "agent": "CODING_AGENT_CODEX", - "config": { - "models": {"unity_catalog_location": "ug_e2e.models"}, - "default_models": {"default_model": "ug_e2e.models.gpt_luna"}, - "smart_routing": {"enabled": False}, - "tracing": {"enabled": False}, - }, - }, - ], - } - - -def test_model_discovery_matrix_filters_families_without_changing_schema_scope(): - expected_models = { - "ug_e2e.models.gpt_luna", - "ug_e2e.models.claude_haiku", - "ug_e2e.models.claude_sonnet", - "ug_e2e.models.kimi", - "ug_e2e.models.gemini_flash", - } - assert expectations.MODEL_SERVICES == expected_models - assert expectations.CLAUDE_MODELS == { - "ug_e2e.models.claude_haiku", - "ug_e2e.models.claude_sonnet", - "ug_e2e.models.kimi", - } - assert expectations.CODEX_MODELS == {"ug_e2e.models.gpt_luna", "ug_e2e.models.kimi"} - assert expectations.GEMINI_MODEL in expectations.MODEL_SERVICES - assert expectations.GEMINI_MODEL not in expectations.CLAUDE_MODELS | expectations.CODEX_MODELS - assert expectations.CLAUDE_DEFAULT in expectations.CLAUDE_MODELS - assert expectations.CODEX_DEFAULT in expectations.CODEX_MODELS - assert expectations.CLAUDE_DECOY not in expectations.MODEL_SERVICES - assert expectations.CODEX_DECOY not in expectations.MODEL_SERVICES - - -def test_model_policy_allows_server_metadata_and_later_stack_fields(managed_policy): - managed_policy["update_time"] = "2026-10-02T00:00:00Z" - managed_policy["name"] = "coding-agent-configs/cuj3" - managed_policy["mcp_servers"] = {"unity_catalog_location": "ug_e2e.tools"} - expectations.assert_model_policy(managed_policy) - - -def test_model_policy_rejects_a_default_outside_the_managed_scope(managed_policy): - managed_policy["enabled_agents"][1]["config"]["default_models"]["default_model"] = ( - expectations.CODEX_DECOY - ) - with pytest.raises(AssertionError): - expectations.assert_model_policy(managed_policy) diff --git a/tests/test_integration_provider_catalog.py b/tests/test_integration_provider_catalog.py index 39e2e0ad8..afe12b4f8 100644 --- a/tests/test_integration_provider_catalog.py +++ b/tests/test_integration_provider_catalog.py @@ -284,7 +284,6 @@ def open(self, request, *, timeout): ) assert result.payloads == (payload,) assert result.model_ids == ("catalog.models.gpt_luna",) - assert result.display_names == {"catalog.models.gpt_luna": "Luna"} payload["models"].append(payload["models"][0]) with pytest.raises(AssertionError, match="repeated"): catalog.fetch_codex_parent_catalog( @@ -292,17 +291,6 @@ def open(self, request, *, timeout): ) -def test_provider_catalog_constructors_preserve_existing_arguments_and_isolate_defaults(): - anthropic = catalog.AnthropicProviderCatalog(("claude-a",), {"claude-a": None}) - codex = catalog.CodexProviderCatalog(("gpt-a",)) - other_codex = catalog.CodexProviderCatalog(("gpt-b",)) - assert anthropic.payloads == () - assert codex.payloads == () - assert codex.display_names == {} - codex.display_names["gpt-a"] = "Model A" - assert other_codex.display_names == {} - - @pytest.mark.parametrize( "fetcher", [catalog.fetch_anthropic_provider_catalog, catalog.fetch_anthropic_parent_catalog] ) @@ -327,7 +315,7 @@ def test_anthropic_catalog_preserves_raw_pages_and_optional_labels(monkeypatch, @pytest.mark.parametrize( "fetcher", [catalog.fetch_codex_provider_catalog, catalog.fetch_codex_parent_catalog] ) -def test_codex_catalog_preserves_hidden_entries_order_and_optional_labels(monkeypatch, fetcher): +def test_codex_catalog_preserves_hidden_entries_and_order(monkeypatch, fetcher): payload = { "models": [ {"slug": "b", "visibility": "list"}, @@ -341,18 +329,6 @@ def test_codex_catalog_preserves_hidden_entries_order_and_optional_labels(monkey result = fetcher("https://workspace", "token", "catalog.models") assert result.payloads == (payload,) assert result.model_ids == ("b", "a") - assert result.display_names == {"b": None, "a": "Model A"} - - -@pytest.mark.parametrize( - "fetcher", [catalog.fetch_codex_provider_catalog, catalog.fetch_codex_parent_catalog] -) -@pytest.mark.parametrize("display_name", ["", " ", 7, [], {}]) -def test_codex_catalog_rejects_invalid_optional_display_names(monkeypatch, fetcher, display_name): - payload = {"models": [{"slug": "a", "visibility": "list", "display_name": display_name}]} - _mock_json_boundary(monkeypatch, lambda *args: payload) - with pytest.raises(AssertionError, match="display name"): - fetcher("https://workspace", "token", "catalog.models") @pytest.mark.parametrize( From fcba0c18bbbf9d0343b7048830334d499aef173d Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 20:37:15 +0000 Subject: [PATCH 09/19] AIGTWY-4878 Remove unrelated catalog HTTP hardening --- tests/integration/utils/http.py | 63 ------------- tests/test_integration_http.py | 101 --------------------- tests/test_integration_provider_catalog.py | 25 ++--- 3 files changed, 8 insertions(+), 181 deletions(-) delete mode 100644 tests/integration/utils/http.py delete mode 100644 tests/test_integration_http.py diff --git a/tests/integration/utils/http.py b/tests/integration/utils/http.py deleted file mode 100644 index 1c35cff5e..000000000 --- a/tests/integration/utils/http.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Read-only workspace JSON transport with strict origins and no redirects.""" - -from __future__ import annotations - -import json -import re -import urllib.error -import urllib.parse -import urllib.request - - -class NoRedirect(urllib.request.HTTPRedirectHandler): - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - -def workspace_origin(workspace: str, token: str) -> str: - """Validate an explicit HTTPS origin and bearer before constructing a client.""" - assert isinstance(workspace, str), "Expected an HTTPS workspace origin" - try: - origin = urllib.parse.urlsplit(workspace) - port = origin.port - except ValueError: - raise AssertionError("Expected a valid HTTPS workspace origin") from None - assert ( - origin.scheme == "https" - and origin.hostname - and origin.username is None - and origin.password is None - and origin.path in {"", "/"} - and not origin.query - and not origin.fragment - and (port is None or 0 < port <= 65535) - and not re.search(r"[\s\\]", workspace) - ), "Expected an HTTPS workspace origin without credentials" - assert isinstance(token, str) and token.strip() and not re.search(r"[\r\n]", token), ( - "Expected an explicit workspace bearer" - ) - return f"https://{origin.netloc}" - - -def safe_https_json_get( - workspace: str, token: str, path: str, *, headers: dict[str, str] | None = None -) -> object: - """GET JSON from an explicit workspace origin without redirects or secret diagnostics.""" - origin = workspace_origin(workspace, token) - assert path.startswith("/") and not path.startswith("//"), "Expected a workspace API path" - request = urllib.request.Request( - f"{origin}{path}", - headers={ - **(headers or {}), - "Authorization": f"Bearer {token}", - "Accept": "application/json", - }, - method="GET", - ) - try: - with urllib.request.build_opener(NoRedirect()).open(request, timeout=30) as response: - return json.load(response) - except urllib.error.HTTPError as error: - raise AssertionError(f"Workspace JSON GET returned HTTP {error.code}") from None - except (urllib.error.URLError, OSError, ValueError): - raise AssertionError("Workspace JSON GET failed or returned invalid JSON") from None diff --git a/tests/test_integration_http.py b/tests/test_integration_http.py deleted file mode 100644 index 3167c5cf3..000000000 --- a/tests/test_integration_http.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Offline component checks for read-only workspace HTTP boundaries.""" - -import io -import urllib.error -import urllib.response - -import pytest - -from tests.integration.utils import http - - -@pytest.mark.parametrize( - "workspace", - [ - None, - 7, - "", - "http://workspace.invalid", - "https://user:secret@workspace.invalid", - "https://workspace.invalid/path", - "https://workspace.invalid?secret=value", - "https://workspace.invalid#fragment", - "https:///missing-host", - "https://workspace.invalid:bad", - "https://workspace.invalid:0", - "https://workspace.invalid:70000", - "https://workspace.invalid\n", - "https://workspace.invalid\\@other.invalid", - ], -) -def test_workspace_json_get_rejects_invalid_origins(monkeypatch, workspace): - def unexpected_open(*args, **kwargs): - pytest.fail("Invalid workspace must not reach the HTTP boundary") - - monkeypatch.setattr(http.urllib.request, "build_opener", unexpected_open) - with pytest.raises(AssertionError, match="HTTPS workspace origin") as failure: - http.safe_https_json_get(workspace, "test-bearer", "/api/models") - assert "secret" not in str(failure.value) - - -@pytest.mark.parametrize("token", [None, "", " ", "bearer\nsecret", "bearer\rsecret"]) -def test_workspace_json_get_requires_an_explicit_valid_bearer(monkeypatch, token): - def unexpected_open(*args, **kwargs): - pytest.fail("Invalid bearer must not reach the HTTP boundary") - - monkeypatch.setattr(http.urllib.request, "build_opener", unexpected_open) - with pytest.raises(AssertionError, match="explicit workspace bearer"): - http.safe_https_json_get("https://workspace.invalid", token, "/api/models") - - -@pytest.mark.parametrize("failure_kind", ["http", "url", "os", "json", "encoding"]) -def test_workspace_json_get_errors_are_sanitized(monkeypatch, failure_kind): - secret = "private-bearer-and-response-secret" - - class FailedOpener: - def open(self, request, *, timeout): - assert timeout == 30 - assert request.get_method() == "GET" - assert request.get_header("Authorization") == f"Bearer {secret}" - if failure_kind == "http": - raise urllib.error.HTTPError(request.full_url, 403, secret, {}, None) - if failure_kind == "url": - raise urllib.error.URLError(secret) - if failure_kind == "os": - raise OSError(secret) - return io.BytesIO(secret.encode() if failure_kind == "json" else b"\xff") - - monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: FailedOpener()) - with pytest.raises(AssertionError, match="Workspace JSON GET") as failure: - http.safe_https_json_get("https://workspace.invalid", secret, "/api/models") - assert secret not in str(failure.value) - assert "workspace.invalid" not in str(failure.value) - assert failure.value.__suppress_context__ - - -@pytest.mark.parametrize("status", [301, 302, 303, 307, 308]) -@pytest.mark.parametrize("destination", ["https://other.invalid/secret", "/redirected"]) -def test_workspace_json_get_denies_redirects(monkeypatch, status, destination): - requests = [] - build_opener = http.urllib.request.build_opener - - class BoundaryHTTPS(http.urllib.request.HTTPSHandler): - def https_open(self, request): - requests.append(request) - response = urllib.response.addinfourl( - io.BytesIO(b"{}"), {"Location": destination}, request.full_url, status - ) - response.msg = "private-redirect-reason" - return response - - monkeypatch.setattr( - http.urllib.request, - "build_opener", - lambda handler: build_opener(handler, BoundaryHTTPS()), - ) - with pytest.raises(AssertionError, match=f"HTTP {status}") as failure: - http.safe_https_json_get("https://workspace.invalid", "test-bearer", "/api/models") - assert len(requests) == 1 - assert requests[0].get_header("Authorization") == "Bearer test-bearer" - assert destination not in str(failure.value) - assert "private-redirect-reason" not in str(failure.value) diff --git a/tests/test_integration_provider_catalog.py b/tests/test_integration_provider_catalog.py index afe12b4f8..12ad91f1c 100644 --- a/tests/test_integration_provider_catalog.py +++ b/tests/test_integration_provider_catalog.py @@ -7,7 +7,6 @@ import pytest -from tests.integration.utils import http from tests.integration.utils import provider_catalog as catalog @@ -30,7 +29,7 @@ def open(self, request, *, timeout): } return io.BytesIO(json.dumps(get_json(request.full_url, headers)).encode()) - monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: CatalogOpener()) + monkeypatch.setattr(catalog.urllib.request, "urlopen", CatalogOpener().open) def _page(*ids, has_more=False, last_id=None): @@ -156,11 +155,7 @@ def urlopen(request, timeout): assert timeout == 30 return Response(json.dumps(payload).encode()) - monkeypatch.setattr( - http.urllib.request, - "build_opener", - lambda *_: type("CatalogOpener", (), {"open": staticmethod(urlopen)})(), - ) + monkeypatch.setattr(catalog.urllib.request, "urlopen", urlopen) result = catalog.fetch_codex_provider_catalog("https://workspace/", "token", "c.s.mps") assert result.model_ids == ("gpt-a",) @@ -184,11 +179,7 @@ def test_provider_fetch_does_not_hide_http_failures(monkeypatch, status): def urlopen(request, timeout): raise urllib.error.HTTPError(request.full_url, status, "failure", {}, None) - monkeypatch.setattr( - http.urllib.request, - "build_opener", - lambda *_: type("CatalogOpener", (), {"open": staticmethod(urlopen)})(), - ) + monkeypatch.setattr(catalog.urllib.request, "urlopen", urlopen) with pytest.raises(AssertionError, match=f"HTTP {status}"): catalog.fetch_codex_provider_catalog("https://workspace", "token", "c.s.mps") @@ -220,14 +211,14 @@ def open(self, request, *, timeout): assert not request.has_header("Databricks-model-provider-service") return io.BytesIO(json.dumps(pages[len(requests) - 1]).encode()) - monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: CatalogOpener()) + monkeypatch.setattr(catalog.urllib.request, "urlopen", CatalogOpener().open) result = catalog.fetch_anthropic_parent_catalog( "https://workspace.invalid/", "test-bearer", "catalog.models" ) assert result.payloads == tuple(pages) assert result.model_ids == (pages[0]["data"][0]["id"], pages[1]["data"][0]["id"]) assert result.display_names == {result.model_ids[0]: "Sonnet", result.model_ids[1]: None} - assert http.urllib.parse.parse_qs(http.urllib.parse.urlsplit(requests[1].full_url).query) == { + assert parse_qs(urlparse(requests[1].full_url).query) == { "limit": ["1000"], "after_id": ["cursor with / and ?"], } @@ -252,7 +243,7 @@ def open(self, request, *, timeout): } return io.BytesIO(json.dumps(payload).encode()) - monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: CatalogOpener()) + monkeypatch.setattr(catalog.urllib.request, "urlopen", CatalogOpener().open) with pytest.raises(AssertionError): catalog.fetch_anthropic_parent_catalog( "https://workspace.invalid", "test-bearer", "catalog.models" @@ -278,7 +269,7 @@ def open(self, request, *, timeout): assert not request.has_header("Databricks-model-provider-service") return io.BytesIO(json.dumps(payload).encode()) - monkeypatch.setattr(http.urllib.request, "build_opener", lambda *_: CatalogOpener()) + monkeypatch.setattr(catalog.urllib.request, "urlopen", CatalogOpener().open) result = catalog.fetch_codex_parent_catalog( "https://workspace.invalid", "test-bearer", "catalog.models" ) @@ -345,6 +336,6 @@ def test_provider_catalog_requests_require_an_explicit_scope(monkeypatch, fetche def unexpected_open(*args, **kwargs): pytest.fail("Invalid catalog scope must not reach the HTTP boundary") - monkeypatch.setattr(http.urllib.request, "build_opener", unexpected_open) + monkeypatch.setattr(catalog.urllib.request, "urlopen", unexpected_open) with pytest.raises(AssertionError, match="explicit catalog scope"): fetcher("https://workspace", "token", scope) From 4a9319cef169cffb77274d22ac6e21889314eb05 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 21:14:44 +0000 Subject: [PATCH 10/19] AIGTWY-4878 Split catalog discovery into focused user journeys --- .../integration/test_ug_catalog_discovery.py | 370 +++++++++--------- tests/integration/utils/managed.py | 10 - tests/test_integration_managed_config.py | 96 ----- 3 files changed, 183 insertions(+), 293 deletions(-) delete mode 100644 tests/test_integration_managed_config.py diff --git a/tests/integration/test_ug_catalog_discovery.py b/tests/integration/test_ug_catalog_discovery.py index 6ce129f0c..4048fc927 100644 --- a/tests/integration/test_ug_catalog_discovery.py +++ b/tests/integration/test_ug_catalog_discovery.py @@ -1,8 +1,6 @@ -"""Catalog discovery: complete managed, schema-scoped model discovery journeys.""" +"""Managed catalog discovery, default launches, and explicit model selection.""" -import json import os -import tomllib import pytest from catalog_discovery_expectations import ( @@ -18,14 +16,7 @@ ) from utils.agents import claude, codex from utils.evidence import FileTask, assert_completed_task_model -from utils.managed import read_persisted_managed_config -from utils.model_discovery import ( - assert_picker_inventory, -) -from utils.provider_catalog import ( - MODEL_SERVICE_PARENT_SCHEMA_HEADER, - parse_codex_provider_catalog, -) +from utils.model_discovery import assert_picker_inventory from utils.terminal import AgentTerminal pytestmark = [pytest.mark.managed, pytest.mark.catalog_discovery, pytest.mark.workspace_isolated] @@ -33,71 +24,31 @@ @pytest.mark.claude @pytest.mark.tui -def test_catalog_discovery_claude(live_session, workspace): - """Scenario: configure the managed workspace; launch bare ug, ug claude, and print tasks. +def test_catalog_discovery_claude_picker_preserves_default(live_session, workspace): + """Scenario: configure, launch ug claude, inspect its picker, then submit a task. - Expected: Sonnet/Haiku/Kimi appear in discovery and the picker, but Gemini and decoys - do not. TUI and print tasks use the Sonnet default without overrides; other compatible - models complete print tasks. Answers report the expected model, with routing off. + Expected: only Sonnet/Haiku/Kimi appear; Gemini and decoys are absent. + Dismissing the picker without a selection preserves Sonnet for the completed task. """ session = live_session - configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - assert "Select coding agents to configure:" not in configured.stdout, configured.stdout - persisted = read_persisted_managed_config(session, workspace) - session.record("catalog-discovery-managed-config.json", persisted) + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) bearer = os.environ["DATABRICKS_BEARER"] parent_catalog = claude.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) decoy_catalog = claude.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) - session.record( - "catalog-discovery-parent-inventory.json", - { - "scoped": parent_catalog.model_ids, - "decoy": decoy_catalog.model_ids, - "display_names": parent_catalog.display_names, - "pages": parent_catalog.payloads, - "decoy_pages": decoy_catalog.payloads, - }, - ) assert set(parent_catalog.model_ids) == { claude.discovery_model_id(model) for model in CLAUDE_MODELS }, parent_catalog assert set(decoy_catalog.model_ids) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog - default_task = FileTask(session) + task = FileTask(session) with AgentTerminal( session, "claude", - [str(session.binary)], - "catalog-discovery-bare-ug-claude-default-and-picker", + [str(session.binary), "claude"], + "catalog-discovery-claude-picker", ) as tui: tui.boot() - settings = json.loads((session.home / ".claude/ucode-settings.json").read_text()) - env = settings["env"] - assert env.get("ANTHROPIC_MODEL") == CLAUDE_DEFAULT, settings - assert env.get("ANTHROPIC_DEFAULT_SONNET_MODEL") == CLAUDE_DEFAULT, settings - assert ( - "x-databricks-use-coding-agent-mode: true" - in env.get("ANTHROPIC_CUSTOM_HEADERS", "").splitlines() - ), settings - assert ( - env.get("ANTHROPIC_CUSTOM_HEADERS", "") - .splitlines() - .count(f"{MODEL_SERVICE_PARENT_SCHEMA_HEADER}: {MODEL_SCHEMA}") - == 1 - ), settings - assert not {"availableModels", "enforceAvailableModels"} & settings.keys(), settings - picker = settings["modelPicker"] - assert picker.get("replaceBuiltInOptions") is True, picker - options = picker["options"] - assert isinstance(options, list) and len(options) == len(parent_catalog.model_ids), picker - picker_ids = [option["model"] for option in options] - assert len(picker_ids) == len(set(picker_ids)), picker - assert set(picker_ids) == set(parent_catalog.model_ids), picker - for option in options: - if display_name := parent_catalog.display_names[option["model"]]: - assert option["label"] == display_name, option - session.record("catalog-discovery-claude-settings.json", settings) picker_screen = tui.open_model_picker( model_visible=lambda screen: all( claude.model_in_picker(screen, model, parent_catalog.display_names[model]) @@ -108,130 +59,45 @@ def test_catalog_discovery_claude(live_session, workspace): for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): assert excluded not in picker_screen, picker_screen assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen - tui.submit(default_task.prompt) - tui.wait_for_task(default_task, timeout=240) + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) tui.exit_normally() - assert_completed_task_model(session, "claude", default_task.value, CLAUDE_DEFAULT) - - explicit_task = FileTask(session) - with AgentTerminal( - session, - "claude", - [str(session.binary), "claude"], - "catalog-discovery-explicit-ug-claude-default", - ) as tui: - tui.boot() - tui.submit(explicit_task.prompt) - tui.wait_for_task(explicit_task, timeout=240) - tui.exit_normally() - assert_completed_task_model(session, "claude", explicit_task.value, CLAUDE_DEFAULT) - - gateway_ids = session.claude_gateway_model_ids() - assert len(gateway_ids) == len(set(gateway_ids)), gateway_ids - assert set(gateway_ids) == set(parent_catalog.model_ids), gateway_ids - - print_task = FileTask(session) - result = session.run( - "claude", - "-p", - print_task.prompt, - "--output-format", - "json", - "--allowedTools", - "Read", - timeout=240, - ) - print_task.assert_headless_answer("claude", result) - assert_completed_task_model(session, "claude", print_task.value, CLAUDE_DEFAULT) - - for model in parent_catalog.model_ids: - if model == CLAUDE_DEFAULT: - continue - task = FileTask(session) - result = session.run( - "claude", - "--", - "-p", - task.prompt, - "--output-format", - "json", - "--allowedTools", - "Read", - "--model", - model, - timeout=240, - ) - task.assert_headless_answer("claude", result) - assert_completed_task_model(session, "claude", task.value, model) + assert_completed_task_model(session, "claude", task.value, CLAUDE_DEFAULT) session.assert_not_routed() @pytest.mark.codex @pytest.mark.tui -def test_catalog_discovery_codex(live_session, workspace): - """Scenario: configure the managed workspace; launch Codex TUI and ug codex -- exec. +def test_catalog_discovery_codex_picker_preserves_default(live_session, workspace): + """Scenario: configure, query Codex model/list, inspect its TUI picker, then submit a task. - Expected: GPT Luna/Kimi appear in model/list and the picker, but Gemini and decoys - do not. TUI and exec tasks use the GPT Luna default without overrides; Kimi completes - an exec task. Completed turns select the expected model, with routing off. + Expected: model/list and the picker expose only GPT Luna/Kimi, not Gemini or decoys. + Dismissing the picker preserves GPT Luna for the completed task's selected model. """ session = live_session - configured = session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - assert "Select coding agents to configure:" not in configured.stdout, configured.stdout - persisted = read_persisted_managed_config(session, workspace) - session.record("catalog-discovery-managed-config.json", persisted) + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) bearer = os.environ["DATABRICKS_BEARER"] parent_catalog = codex.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) decoy_catalog = codex.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) - session.record( - "catalog-discovery-parent-inventory.json", - { - "scoped": parent_catalog.model_ids, - "decoy": decoy_catalog.model_ids, - "pages": parent_catalog.payloads, - "decoy_pages": decoy_catalog.payloads, - }, - ) assert set(parent_catalog.model_ids) == CODEX_MODELS, parent_catalog assert set(decoy_catalog.model_ids) == {CODEX_DECOY}, decoy_catalog - provider_ids = { - entry.get("slug") for page in parent_catalog.payloads for entry in page["models"] - } - assert not {GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY} & provider_ids, parent_catalog - decoy_provider_ids = { - entry.get("slug") for page in decoy_catalog.payloads for entry in page["models"] - } - assert not {GEMINI_MODEL, CLAUDE_DECOY} & decoy_provider_ids, decoy_catalog models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) assert len(models) == len(set(models)), models - assert set(models) == set(parent_catalog.model_ids), models - catalog_paths = list((session.home / ".ucode").glob("codex-model-catalog-*.json")) - assert len(catalog_paths) == 1, catalog_paths - catalog = json.loads(catalog_paths[0].read_text()) - assert json.loads((session.home / ".ucode/codex-model-catalog.json").read_text()) == catalog - catalog_ids = parse_codex_provider_catalog(catalog) - assert set(catalog_ids) == set(parent_catalog.model_ids), catalog - assert models == list(catalog_ids), (models, catalog_ids) - all_catalog_ids = {entry["slug"] for entry in catalog["models"]} - assert not {GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY} & all_catalog_ids, catalog - display_names = {} - for entry in catalog["models"]: - if entry["slug"] in CODEX_MODELS: - label = entry.get("display_name") - assert isinstance(label, str) and label.strip(), entry - display_names[entry["slug"]] = label - session.record("catalog-discovery-codex-catalog.json", catalog) - - default_task = FileTask(session) + assert set(models) == CODEX_MODELS, models + display_names = { + entry["slug"]: entry.get("display_name") or entry["slug"] + for page in parent_catalog.payloads + for entry in page["models"] + if entry["slug"] in CODEX_MODELS + } + + task = FileTask(session) with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-default" + session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-picker" ) as tui: tui.boot() - config = tomllib.loads((session.home / ".codex/ucode.config.toml").read_text()) - assert config.get("model") == CODEX_DEFAULT, config - session.record("catalog-discovery-codex-config.json", config) picker_screen = tui.open_codex_model_picker( model_visible=lambda screen: all( codex.model_in_picker(screen, model, display_names[model]) @@ -242,39 +108,169 @@ def test_catalog_discovery_codex(live_session, workspace): for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): assert excluded not in picker_screen, picker_screen assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen - tui.submit(default_task.prompt) - tui.wait_for_task(default_task, timeout=240) + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) tui.exit_normally() - assert_completed_task_model(session, "codex", default_task.value, CODEX_DEFAULT) + assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + session.assert_not_routed() - exec_task = FileTask(session) + +@pytest.mark.claude +@pytest.mark.tui +def test_catalog_discovery_bare_ug_uses_claude_default(live_session, workspace): + """Scenario: configure the managed workspace, launch bare ug, and submit its first task. + + Expected: Claude completes the task on Sonnet without an agent or model override. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + with AgentTerminal( + session, "claude", [str(session.binary)], "catalog-discovery-bare-ug-default" + ) as tui: + tui.boot() + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + tui.exit_normally() + assert_completed_task_model(session, "claude", task.value, CLAUDE_DEFAULT) + session.assert_not_routed() + + +@pytest.mark.claude +@pytest.mark.tui +def test_catalog_discovery_claude_tui_uses_default(live_session, workspace): + """Scenario: configure, launch ug claude, and submit its first task without opening a picker. + + Expected: Claude completes the task on the configured Sonnet default without overrides. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + with AgentTerminal( + session, + "claude", + [str(session.binary), "claude"], + "catalog-discovery-claude-tui-default", + ) as tui: + tui.boot() + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + tui.exit_normally() + assert_completed_task_model(session, "claude", task.value, CLAUDE_DEFAULT) + session.assert_not_routed() + + +@pytest.mark.codex +@pytest.mark.tui +def test_catalog_discovery_codex_tui_uses_default(live_session, workspace): + """Scenario: configure, launch ug codex, and submit its first task without opening a picker. + + Expected: Codex completes the task with GPT Luna selected, without a model override. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + with AgentTerminal( + session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-tui-default" + ) as tui: + tui.boot() + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + tui.exit_normally() + assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + session.assert_not_routed() + + +@pytest.mark.claude +def test_catalog_discovery_claude_headless_uses_default(live_session, workspace): + """Scenario: configure, then run ug claude -p with a file task and no model override. + + Expected: the print task completes with a Sonnet response. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + result = session.run( + "claude", + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + timeout=240, + ) + task.assert_headless_answer("claude", result) + assert_completed_task_model(session, "claude", task.value, CLAUDE_DEFAULT) + session.assert_not_routed() + + +@pytest.mark.codex +def test_catalog_discovery_codex_headless_uses_default(live_session, workspace): + """Scenario: configure, then run ug codex -- exec with a file task and no model override. + + Expected: the exec task completes with GPT Luna selected. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + result = session.run( + "codex", "--", "exec", "--skip-git-repo-check", "--json", task.prompt, timeout=240 + ) + task.assert_headless_answer("codex", result) + assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + session.assert_not_routed() + + +@pytest.mark.claude +@pytest.mark.parametrize("model", sorted(CLAUDE_MODELS - {CLAUDE_DEFAULT})) +def test_catalog_discovery_claude_explicit_model_completes_task(live_session, workspace, model): + """Scenario: configure, then run a Claude print task with another compatible scoped model. + + Expected: each Haiku/Kimi task completes with the explicitly requested model. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + result = session.run( + "claude", + "--", + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + "--model", + claude.discovery_model_id(model), + timeout=240, + ) + task.assert_headless_answer("claude", result) + assert_completed_task_model(session, "claude", task.value, claude.discovery_model_id(model)) + session.assert_not_routed() + + +@pytest.mark.codex +@pytest.mark.parametrize("model", sorted(CODEX_MODELS - {CODEX_DEFAULT})) +def test_catalog_discovery_codex_explicit_model_completes_task(live_session, workspace, model): + """Scenario: configure, then run a Codex exec task with another compatible scoped model. + + Expected: the Kimi task completes with the explicitly requested model selected. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) result = session.run( "codex", "--", "exec", "--skip-git-repo-check", "--json", - exec_task.prompt, + task.prompt, + "--model", + model, timeout=240, ) - exec_task.assert_headless_answer("codex", result) - assert_completed_task_model(session, "codex", exec_task.value, CODEX_DEFAULT) - - for model in parent_catalog.model_ids: - if model == CODEX_DEFAULT: - continue - task = FileTask(session) - result = session.run( - "codex", - "--", - "exec", - "--skip-git-repo-check", - "--json", - "--model", - model, - task.prompt, - timeout=240, - ) - task.assert_headless_answer("codex", result) - assert_completed_task_model(session, "codex", task.value, model) + task.assert_headless_answer("codex", result) + assert_completed_task_model(session, "codex", task.value, model) session.assert_not_routed() diff --git a/tests/integration/utils/managed.py b/tests/integration/utils/managed.py index 997d96948..3599d38f9 100644 --- a/tests/integration/utils/managed.py +++ b/tests/integration/utils/managed.py @@ -13,16 +13,6 @@ MANAGED_CONFIGS_PATH = "/api/ai-gateway/v2/coding-agent-configs" -def read_persisted_managed_config(session, workspace: str) -> dict: - """Read the configure result without imposing a journey-specific policy.""" - path = session.home / ".ucode" / "managed-config.json" - assert path.is_file(), path - persisted = json.loads(path.read_text()) - assert isinstance(persisted, dict) and persisted.get("workspace") == workspace, persisted - assert isinstance(persisted.get("config"), dict), persisted - return persisted - - def assert_no_managed_config(payload: object) -> None: """Validate the real List response before claiming unmanaged-workspace coverage.""" configs = payload.get("coding_agent_configs", []) if isinstance(payload, dict) else payload diff --git a/tests/test_integration_managed_config.py b/tests/test_integration_managed_config.py deleted file mode 100644 index 111fd1e82..000000000 --- a/tests/test_integration_managed_config.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Component checks for reading persisted managed-config evidence.""" - -import json -from types import SimpleNamespace - -import pytest - -from tests.integration.utils.managed import read_persisted_managed_config - -WORKSPACE = "https://workspace.invalid" - - -@pytest.fixture -def session(tmp_path): - return SimpleNamespace(home=tmp_path / "session-home") - - -def _write_persisted_config(session, envelope): - path = session.home / ".ucode" / "managed-config.json" - path.parent.mkdir(parents=True) - path.write_text(json.dumps(envelope), encoding="utf-8") - return path - - -@pytest.mark.parametrize( - "config", - [ - {}, - { - "default_agent": "unrelated-agent", - "custom_policy": {"labels": ["independent", "policy"], "enabled": True}, - }, - ], -) -def test_read_persisted_managed_config_returns_envelope_without_enforcing_policy(session, config): - envelope = {"workspace": WORKSPACE, "config": config, "metadata": {"revision": 7}} - path = _write_persisted_config(session, envelope) - original = path.read_text(encoding="utf-8") - - assert read_persisted_managed_config(session, WORKSPACE) == envelope - assert path.read_text(encoding="utf-8") == original - - -@pytest.mark.parametrize("directory_instead", [False, True]) -def test_read_persisted_managed_config_requires_a_file(session, directory_instead): - if directory_instead: - (session.home / ".ucode" / "managed-config.json").mkdir(parents=True) - - with pytest.raises(AssertionError): - read_persisted_managed_config(session, WORKSPACE) - - -@pytest.mark.parametrize("contents", ["", "{", '{"workspace":', "{} trailing-data"]) -def test_read_persisted_managed_config_rejects_invalid_json(session, contents): - path = _write_persisted_config(session, {}) - path.write_text(contents, encoding="utf-8") - - with pytest.raises(json.JSONDecodeError): - read_persisted_managed_config(session, WORKSPACE) - - -@pytest.mark.parametrize("envelope", [None, [], ["config"], "config", False, 7]) -def test_read_persisted_managed_config_requires_a_dict_envelope(session, envelope): - _write_persisted_config(session, envelope) - - with pytest.raises(AssertionError): - read_persisted_managed_config(session, WORKSPACE) - - -@pytest.mark.parametrize( - "workspace", - [None, "", "https://other-workspace.invalid", WORKSPACE + "/", WORKSPACE.upper(), 7, {}, []], -) -def test_read_persisted_managed_config_requires_an_exact_workspace_match(session, workspace): - _write_persisted_config(session, {"workspace": workspace, "config": {}}) - - with pytest.raises(AssertionError): - read_persisted_managed_config(session, WORKSPACE) - - -@pytest.mark.parametrize("config", [None, [], ["policy"], "policy", False, 7]) -def test_read_persisted_managed_config_requires_a_dict_config(session, config): - _write_persisted_config(session, {"workspace": WORKSPACE, "config": config}) - - with pytest.raises(AssertionError): - read_persisted_managed_config(session, WORKSPACE) - - -@pytest.mark.parametrize("field", ["workspace", "config"]) -def test_read_persisted_managed_config_rejects_missing_fields(session, field): - envelope = {"workspace": WORKSPACE, "config": {}} - del envelope[field] - _write_persisted_config(session, envelope) - - with pytest.raises(AssertionError): - read_persisted_managed_config(session, WORKSPACE) From 880ec3ad3f9438023fc77f8cd922c7da35ed13bd Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 21:19:32 +0000 Subject: [PATCH 11/19] AIGTWY-4878 Prune generic component validation from catalog journey --- tests/test_integration_evidence.py | 116 +------------------- tests/test_integration_model_discovery.py | 21 ---- tests/test_integration_provider_catalog.py | 117 --------------------- 3 files changed, 3 insertions(+), 251 deletions(-) diff --git a/tests/test_integration_evidence.py b/tests/test_integration_evidence.py index bc724beaf..910984916 100644 --- a/tests/test_integration_evidence.py +++ b/tests/test_integration_evidence.py @@ -5,7 +5,7 @@ import pytest from tests.integration.utils import evidence -from tests.integration.utils.agents import _evidence_id, claude, codex +from tests.integration.utils.agents import claude, codex from tests.integration.utils.evidence import ( SubagentCalculation, assert_no_terminal_api_error, @@ -16,10 +16,9 @@ class _Session: def __init__(self, home): self.home = home - self.artifacts = {} - def record(self, name, value): - self.artifacts[name] = value + def record(self, _name, _value): + pass def _transcript_session(home, agent, transcripts): @@ -199,38 +198,6 @@ def test_completed_task_models_excludes_child_only_and_conflicting_child_evidenc assert evidence.completed_task_models(session, agent, "value") == expected -@pytest.mark.parametrize("value", [None, "", " ", 5, {}, [], "model with whitespace"]) -def test_completed_task_ids_require_nonempty_strings_without_whitespace(value): - with pytest.raises(AssertionError, match="Missing or malformed completed-task model"): - _evidence_id(value, "model") - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_completed_task_models_fails_closed_on_missing_model(tmp_path, agent): - records = completed_records(agent, "model") - del records[0]["message" if agent == "claude" else "payload"]["model"] - session = _transcript_session(tmp_path, agent, {"parent.jsonl": records}) - with pytest.raises(AssertionError, match="model"): - evidence.completed_task_models(session, agent, "value") - - -@pytest.mark.parametrize("turn_id", ["", "turn with whitespace"]) -@pytest.mark.parametrize("record_index", [0, 1]) -def test_codex_completed_task_models_fails_closed_on_malformed_turn_ids(turn_id, record_index): - records = completed_records("codex", "model") - records[record_index]["payload"]["turn_id"] = turn_id - with pytest.raises(AssertionError, match="turn ID"): - codex.completed_task_models(records, "value") - - -@pytest.mark.parametrize("record_index", [0, 1]) -def test_codex_completed_task_models_fails_closed_on_missing_turn_ids(record_index): - records = completed_records("codex", "model") - del records[record_index]["payload"]["turn_id"] - with pytest.raises(AssertionError, match="turn ID"): - codex.completed_task_models(records, "value") - - def test_codex_completed_task_models_requires_context_in_same_session(tmp_path): context, completion = completed_records("codex", "model") session = _transcript_session( @@ -249,80 +216,3 @@ def test_completed_task_model_assertion_requires_exact_singleton(tmp_path, agent session = _transcript_session(tmp_path, agent, {"parent.jsonl": records}) with pytest.raises(AssertionError): evidence.assert_completed_task_model(session, agent, "value", "expected") - assert session.artifacts[f"completed-task-model-{agent}-value.json"]["observed"] == sorted( - models - ) - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_completed_task_model_records_evidence_limits(tmp_path, agent): - session = _transcript_session( - tmp_path, agent, {"parent.jsonl": completed_records(agent, "expected")} - ) - evidence.assert_completed_task_model(session, agent, "value", "expected") - assert session.artifacts["agent-sessions.json"] == { - "parent.jsonl": completed_records(agent, "expected") - } - assert session.artifacts[f"completed-task-model-{agent}-value.json"] == { - "expected": "expected", - "observed": ["expected"], - "evidence_kind": ( - "response-reported model" if agent == "claude" else "client-selected model" - ), - "gateway_destination_proven": False, - } - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("answer_value", [None, "", " \t", 1, [], {}]) -def test_completed_task_models_rejects_empty_or_malformed_expected_answers(agent, answer_value): - with pytest.raises(AssertionError, match="nonempty answer value"): - evidence.completed_task_models(None, agent, answer_value) - adapter = claude.completed_task_models if agent == "claude" else codex.completed_task_models - with pytest.raises(AssertionError, match="nonempty answer value"): - adapter([], answer_value) - - -@pytest.mark.parametrize( - "message", - [None, [], {}, {"role": "assistant"}, {"role": "assistant", "content": "value"}], -) -def test_claude_completed_task_models_rejects_malformed_message_metadata(message): - with pytest.raises(AssertionError): - claude.completed_task_models([{"type": "assistant", "message": message}], "value") - - -@pytest.mark.parametrize("content", [[None], [{"type": "text"}], [{"type": "text", "text": 4}]]) -def test_claude_completed_task_models_rejects_malformed_answer_blocks(content): - records = completed_records("claude", "model") - records[0]["message"]["content"] = content - with pytest.raises(AssertionError): - claude.completed_task_models(records, "value") - - -@pytest.mark.parametrize("payload", [None, [], "value"]) -@pytest.mark.parametrize("record_type", ["event_msg", "turn_context"]) -def test_codex_completed_task_models_rejects_malformed_payloads(payload, record_type): - with pytest.raises(AssertionError): - codex.completed_task_models([{"type": record_type, "payload": payload}], "value") - - -@pytest.mark.parametrize("answer", [None, [], {}, 7]) -def test_codex_completed_task_models_rejects_malformed_completed_answers(answer): - records = completed_records("codex", "model") - records[1]["payload"]["last_agent_message"] = answer - with pytest.raises(AssertionError, match="completed answer"): - codex.completed_task_models(records, "value") - - -def test_claude_completed_task_models_normalizes_only_context_window_suffix(): - assert claude.completed_task_models( - completed_records("claude", "catalog.models.claude_sonnet[1m]"), "value" - ) == {"catalog.models.claude_sonnet"} - - -@pytest.mark.parametrize("agent", ["opencode", "unknown"]) -def test_completed_task_model_rejects_unsupported_agents(agent): - session = _Session(None) - with pytest.raises(AssertionError, match="Unsupported evidence agent"): - evidence.assert_completed_task_model(session, agent, "value", "model") diff --git a/tests/test_integration_model_discovery.py b/tests/test_integration_model_discovery.py index 350016ac3..05d91e083 100644 --- a/tests/test_integration_model_discovery.py +++ b/tests/test_integration_model_discovery.py @@ -210,24 +210,3 @@ def test_picker_inventory_rejects_duplicate_and_missing_rows(agent): assert_picker_inventory(f"{rows}\n {len(models) + 1}. {models[0]}", agent, labels) with pytest.raises(AssertionError): assert_picker_inventory(f" 1. {models[0]}", agent, labels) - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -def test_picker_inventory_rejects_ambiguous_labels_and_split_line_evidence(agent): - models = sorted(PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"]) - with pytest.raises(AssertionError): - assert_picker_inventory(" 1. Shared label", agent, dict.fromkeys(models, "Shared label")) - with pytest.raises(AssertionError): - assert_picker_inventory(f" 1.\n{models[0]}", agent, dict.fromkeys(models, None)) - - -@pytest.mark.parametrize("agent", ["claude", "codex"]) -@pytest.mark.parametrize("suffix", ["_v2", "-decoy", ".other"]) -def test_picker_inventory_rejects_model_id_prefix_matches(agent, suffix): - models = sorted(PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"]) - rows = "\n".join( - f" {position}. {model}{suffix if position == 1 else ''}" - for position, model in enumerate(models, start=1) - ) - with pytest.raises(AssertionError): - assert_picker_inventory(rows, agent, dict.fromkeys(models, None)) diff --git a/tests/test_integration_provider_catalog.py b/tests/test_integration_provider_catalog.py index 12ad91f1c..d5827c175 100644 --- a/tests/test_integration_provider_catalog.py +++ b/tests/test_integration_provider_catalog.py @@ -222,120 +222,3 @@ def open(self, request, *, timeout): "limit": ["1000"], "after_id": ["cursor with / and ?"], } - - -@pytest.mark.parametrize("failure_kind", ["duplicate", "cursor", "limit"]) -def test_anthropic_parent_catalog_rejects_duplicate_or_unbounded_pagination( - monkeypatch, failure_kind -): - requests = [] - - class CatalogOpener: - def open(self, request, *, timeout): - requests.append(request) - position = len(requests) - payload = { - "data": [ - {"id": "duplicate" if failure_kind == "duplicate" else f"model-{position}"} - ], - "has_more": True, - "last_id": "repeated" if failure_kind == "cursor" else f"cursor-{position}", - } - return io.BytesIO(json.dumps(payload).encode()) - - monkeypatch.setattr(catalog.urllib.request, "urlopen", CatalogOpener().open) - with pytest.raises(AssertionError): - catalog.fetch_anthropic_parent_catalog( - "https://workspace.invalid", "test-bearer", "catalog.models" - ) - assert len(requests) == (20 if failure_kind == "limit" else 2) - - -def test_codex_parent_catalog_retains_unfiltered_payload_and_validates_duplicates(monkeypatch): - payload = { - "models": [ - {"slug": "catalog.models.gpt_luna", "display_name": "Luna", "visibility": "list"}, - {"slug": "catalog.other_models.codex_decoy", "visibility": "hidden"}, - ] - } - - class CatalogOpener: - def open(self, request, *, timeout): - assert timeout == 30 - assert ( - request.get_header(catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER.capitalize()) - == "catalog.models" - ) - assert not request.has_header("Databricks-model-provider-service") - return io.BytesIO(json.dumps(payload).encode()) - - monkeypatch.setattr(catalog.urllib.request, "urlopen", CatalogOpener().open) - result = catalog.fetch_codex_parent_catalog( - "https://workspace.invalid", "test-bearer", "catalog.models" - ) - assert result.payloads == (payload,) - assert result.model_ids == ("catalog.models.gpt_luna",) - payload["models"].append(payload["models"][0]) - with pytest.raises(AssertionError, match="repeated"): - catalog.fetch_codex_parent_catalog( - "https://workspace.invalid", "test-bearer", "catalog.models" - ) - - -@pytest.mark.parametrize( - "fetcher", [catalog.fetch_anthropic_provider_catalog, catalog.fetch_anthropic_parent_catalog] -) -def test_anthropic_catalog_preserves_raw_pages_and_optional_labels(monkeypatch, fetcher): - payloads = [ - { - "data": [{"id": "b", "visibility": "hidden", "extra": [1, 2]}], - "has_more": True, - "last_id": "cursor", - "metadata": {"raw": True}, - }, - {"data": [{"id": "a", "display_name": "Model A"}], "has_more": False}, - ] - pages = iter(payloads) - _mock_json_boundary(monkeypatch, lambda *args: next(pages)) - result = fetcher("https://workspace", "token", "catalog.models") - assert result.payloads == tuple(payloads) - assert result.model_ids == ("b", "a") - assert result.display_names == {"b": None, "a": "Model A"} - - -@pytest.mark.parametrize( - "fetcher", [catalog.fetch_codex_provider_catalog, catalog.fetch_codex_parent_catalog] -) -def test_codex_catalog_preserves_hidden_entries_and_order(monkeypatch, fetcher): - payload = { - "models": [ - {"slug": "b", "visibility": "list"}, - {"slug": "b", "visibility": "hidden", "display_name": 7}, - {"visibility": "hidden", "extra": {"raw": True}}, - {"slug": "a", "visibility": "list", "display_name": "Model A"}, - ], - "metadata": {"raw": True}, - } - _mock_json_boundary(monkeypatch, lambda *args: payload) - result = fetcher("https://workspace", "token", "catalog.models") - assert result.payloads == (payload,) - assert result.model_ids == ("b", "a") - - -@pytest.mark.parametrize( - "fetcher", - [ - catalog.fetch_anthropic_provider_catalog, - catalog.fetch_anthropic_parent_catalog, - catalog.fetch_codex_provider_catalog, - catalog.fetch_codex_parent_catalog, - ], -) -@pytest.mark.parametrize("scope", [None, "", " ", 7]) -def test_provider_catalog_requests_require_an_explicit_scope(monkeypatch, fetcher, scope): - def unexpected_open(*args, **kwargs): - pytest.fail("Invalid catalog scope must not reach the HTTP boundary") - - monkeypatch.setattr(catalog.urllib.request, "urlopen", unexpected_open) - with pytest.raises(AssertionError, match="explicit catalog scope"): - fetcher("https://workspace", "token", scope) From e9664e2107f15fcaba20a218a506c01be9196155 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Fri, 2 Oct 2026 23:31:11 +0000 Subject: [PATCH 12/19] Fix Claude catalog journey model evidence and file prompts --- .../integration/test_ug_catalog_discovery.py | 47 +++++++++++++++---- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_ug_catalog_discovery.py b/tests/integration/test_ug_catalog_discovery.py index 4048fc927..8ea95d630 100644 --- a/tests/integration/test_ug_catalog_discovery.py +++ b/tests/integration/test_ug_catalog_discovery.py @@ -1,5 +1,6 @@ """Managed catalog discovery, default launches, and explicit model selection.""" +import json import os import pytest @@ -22,6 +23,29 @@ pytestmark = [pytest.mark.managed, pytest.mark.catalog_discovery, pytest.mark.workspace_isolated] +def _claude_file_task(session): + task = FileTask(session) + task.prompt = ( + f"Use the Read tool to read {session.cwd / task.filename}. Reply with only its contents." + ) + return task + + +def _assert_claude_headless_model(result, expected): + final = None + for line in result.stdout.splitlines(): + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict) and payload.get("type") == "result": + final = payload + assert final is not None and not final.get("is_error"), result.stdout + usage = final["modelUsage"] + assert set(usage) == {expected}, {"expected": expected, "observed": sorted(usage)} + assert usage[expected]["outputTokens"] > 0, usage + + @pytest.mark.claude @pytest.mark.tui def test_catalog_discovery_claude_picker_preserves_default(live_session, workspace): @@ -41,7 +65,7 @@ def test_catalog_discovery_claude_picker_preserves_default(live_session, workspa }, parent_catalog assert set(decoy_catalog.model_ids) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog - task = FileTask(session) + task = _claude_file_task(session) with AgentTerminal( session, "claude", @@ -59,10 +83,11 @@ def test_catalog_discovery_claude_picker_preserves_default(live_session, workspa for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): assert excluded not in picker_screen, picker_screen assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen + assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible tui.submit(task.prompt) tui.wait_for_task(task, timeout=240) tui.exit_normally() - assert_completed_task_model(session, "claude", task.value, CLAUDE_DEFAULT) + task.assert_completed(session, "claude") session.assert_not_routed() @@ -124,15 +149,16 @@ def test_catalog_discovery_bare_ug_uses_claude_default(live_session, workspace): """ session = live_session session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = FileTask(session) + task = _claude_file_task(session) with AgentTerminal( session, "claude", [str(session.binary)], "catalog-discovery-bare-ug-default" ) as tui: tui.boot() + assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible tui.submit(task.prompt) tui.wait_for_task(task, timeout=240) tui.exit_normally() - assert_completed_task_model(session, "claude", task.value, CLAUDE_DEFAULT) + task.assert_completed(session, "claude") session.assert_not_routed() @@ -145,7 +171,7 @@ def test_catalog_discovery_claude_tui_uses_default(live_session, workspace): """ session = live_session session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = FileTask(session) + task = _claude_file_task(session) with AgentTerminal( session, "claude", @@ -153,10 +179,11 @@ def test_catalog_discovery_claude_tui_uses_default(live_session, workspace): "catalog-discovery-claude-tui-default", ) as tui: tui.boot() + assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible tui.submit(task.prompt) tui.wait_for_task(task, timeout=240) tui.exit_normally() - assert_completed_task_model(session, "claude", task.value, CLAUDE_DEFAULT) + task.assert_completed(session, "claude") session.assert_not_routed() @@ -189,7 +216,7 @@ def test_catalog_discovery_claude_headless_uses_default(live_session, workspace) """ session = live_session session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = FileTask(session) + task = _claude_file_task(session) result = session.run( "claude", "-p", @@ -201,7 +228,7 @@ def test_catalog_discovery_claude_headless_uses_default(live_session, workspace) timeout=240, ) task.assert_headless_answer("claude", result) - assert_completed_task_model(session, "claude", task.value, CLAUDE_DEFAULT) + _assert_claude_headless_model(result, CLAUDE_DEFAULT) session.assert_not_routed() @@ -231,7 +258,7 @@ def test_catalog_discovery_claude_explicit_model_completes_task(live_session, wo """ session = live_session session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = FileTask(session) + task = _claude_file_task(session) result = session.run( "claude", "--", @@ -246,7 +273,7 @@ def test_catalog_discovery_claude_explicit_model_completes_task(live_session, wo timeout=240, ) task.assert_headless_answer("claude", result) - assert_completed_task_model(session, "claude", task.value, claude.discovery_model_id(model)) + _assert_claude_headless_model(result, claude.discovery_model_id(model)) session.assert_not_routed() From 292fac44e6e1c201355d84395e0459c6966f61fe Mon Sep 17 00:00:00 2001 From: Tien Le Date: Mon, 5 Oct 2026 14:45:30 +0000 Subject: [PATCH 13/19] Migrate catalog discovery to dedicated-workspace CUJ scaffolding --- scripts/run_integration.py | 39 ++- tests/e2e_cuj/README.md | 57 ++++ .../catalog_discovery_expectations.py | 2 +- tests/e2e_cuj/pytest.ini | 11 + tests/e2e_cuj/test_catalog_discovery.py | 306 ++++++++++++++++++ tests/integration/pytest.ini | 2 - .../integration/test_ug_catalog_discovery.py | 303 ----------------- tests/test_integration_runner.py | 49 +++ 8 files changed, 457 insertions(+), 312 deletions(-) create mode 100644 tests/e2e_cuj/README.md rename tests/{integration => e2e_cuj}/catalog_discovery_expectations.py (84%) create mode 100644 tests/e2e_cuj/pytest.ini create mode 100644 tests/e2e_cuj/test_catalog_discovery.py delete mode 100644 tests/integration/test_ug_catalog_discovery.py diff --git a/scripts/run_integration.py b/scripts/run_integration.py index bc81650d3..6d2a6cbca 100644 --- a/scripts/run_integration.py +++ b/scripts/run_integration.py @@ -279,6 +279,7 @@ def arguments( platform_name = os.name if platform_name is None else platform_name environment = os.environ if environment is None else environment parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--suite", choices=["integration", "e2e-cuj"], default="integration") source = parser.add_mutually_exclusive_group() source.add_argument("--ug-version", default="checkout", help="Exact ug release, or checkout.") source.add_argument( @@ -373,6 +374,14 @@ def arguments( "pytest_args", nargs=argparse.REMAINDER, help="After --, pass pytest filters." ) args = parser.parse_args(argv) + if args.suite == "e2e-cuj": + if args.installation_only or args.headless_only or args.profile: + parser.error("--suite e2e-cuj uses CUJ service-principal auth and journey filters.") + if not all( + environment.get(key, "").strip() + for key in ("UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET") + ): + parser.error("Set UG_CUJ_SP_CLIENT_ID and UG_CUJ_SP_CLIENT_SECRET for e2e-cuj.") if platform_name != "posix" and not (args.installation_only or args.headless_only): parser.error( "Live agent/TUI integration requires POSIX PTY, managed-settings, and signal " @@ -387,7 +396,7 @@ def arguments( filters.add_argument("--maxfail", type=int) extra = args.pytest_args[1:] if args.pytest_args[:1] == ["--"] else args.pytest_args selected = filters.parse_args(extra) - marker = selected.m or "live" + marker = selected.m or ("live" if args.suite == "integration" else None) if args.installation_only: marker = f"installation and ({selected.m})" if selected.m else "installation" args.pytest_args = [] @@ -418,7 +427,7 @@ def arguments( environment.get("DATABRICKS_CLIENT_ID", "").strip() and environment.get("DATABRICKS_CLIENT_SECRET", "").strip() ) - if not ( + if args.suite == "integration" and not ( args.profile or environment.get("DATABRICKS_BEARER", "").strip() or has_client_creds ): parser.error( @@ -523,6 +532,7 @@ def terminate(signum, frame): target_bearers: dict[str, str] = {} client_secrets = ( os.environ.get("DATABRICKS_CLIENT_SECRET", ""), + os.environ.get("UG_CUJ_SP_CLIENT_SECRET", ""), os.environ.get("UG_MPS_DEFAULTS_CLIENT_SECRET", ""), os.environ.get("UG_PARENT_SCHEMA_DEFAULTS_CLIENT_SECRET", ""), ) @@ -825,13 +835,18 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: if not bearer: raise RuntimeError("Selected profile returned no access token.") - if not bearer and not args.profile and not args.installation_only: + if ( + not bearer + and not args.profile + and not args.installation_only + and args.suite == "integration" + ): client_id = os.environ.get("DATABRICKS_CLIENT_ID", "").strip() client_secret = os.environ.get("DATABRICKS_CLIENT_SECRET", "").strip() if client_id and client_secret: bearer = mint_m2m_token(args.workspace, client_id, client_secret) - if not args.installation_only: + if not args.installation_only and args.suite == "integration": for bearer_env, target_workspace, client_id, secret_env in MANAGED_DEFAULTS_TARGETS: secret = os.environ.get(secret_env, "").strip() if args.workspace.rstrip("/") == target_workspace: @@ -840,6 +855,10 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: target_bearers[bearer_env] = mint_m2m_token(target_workspace, client_id, secret) test_dependencies = ["pytest==9.0.3"] + if args.suite == "e2e-cuj": + test_dependencies.append( + next(line for line in freeze.splitlines() if line.startswith("databricks-sdk==")) + ) if os.name == "posix": test_dependencies.extend(["pexpect==4.9.0", "pyte==0.8.2"]) run( @@ -889,9 +908,17 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: runtime_env[f"UG_INTEGRATION_{agent.upper()}_MODEL"] = ( getattr(args, f"{agent}_model") or "" ) - suite = ROOT / "tests/integration" + if args.suite == "e2e-cuj": + for key in ("UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET"): + runtime_env[key] = os.environ[key] + suite = ROOT / "tests" / ("e2e_cuj" if args.suite == "e2e-cuj" else "integration") + report["suite"] = args.suite suite_hash = hashlib.sha256() - for path in [Path(__file__), *sorted(suite.rglob("*.py")), suite / "pytest.ini"]: + suite_files = set(suite.rglob("*.py")) + if args.suite == "e2e-cuj": + suite_files.update((ROOT / "tests/integration/utils").rglob("*.py")) + suite_files.add(ROOT / "tests/integration/conftest.py") + for path in [Path(__file__), *sorted(suite_files), suite / "pytest.ini"]: suite_hash.update(str(path.relative_to(ROOT)).encode() + b"\0" + path.read_bytes()) report["suite_sha256"] = suite_hash.hexdigest() extra = args.pytest_args diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md new file mode 100644 index 000000000..00dd2e904 --- /dev/null +++ b/tests/e2e_cuj/README.md @@ -0,0 +1,57 @@ +# Catalog discovery CUJ + +The dedicated workspace must already publish a CodingAgentConfig enabling both agents +with `ug_e2e.models` as their model source. Claude is the default agent; its default and +Sonnet-family model are `claude_sonnet`, and Codex defaults to `gpt_luna`. Smart routing +and tracing are disabled. Tests neither provision nor validate fixture definitions. + +| Service under `ug_e2e.models` | Claude Code | Codex | +| --- | --- | --- | +| `gpt_luna` | Excluded | Included; default | +| `claude_haiku` | Included | Excluded | +| `claude_sonnet` | Included; default | Excluded | +| `kimi` | Included | Included | +| `gemini_flash` | Excluded | Excluded | + +Independent gateway catalogs, Codex's native model/list, and exact TUI picker inventories must agree. +Compatible `ug_e2e.other_models.claude_decoy` and `codex_decoy` must be discoverable in their +own schema but absent from the scoped picker. Claude discovery aliases must be enabled; +Kimi uses `anthropic-aigw-<8-character SHA-256 prefix>-`. + +Ten independently configured cases cover picker discovery, default launches, and explicit models. +Picker cases dismiss the menu without changing selection, then complete a task on the default. +Separate cases cover bare `ug`, `ug claude`, and `ug codex` TUI first tasks and Claude print/Codex +exec defaults. Each additional compatible model gets its own headless task case. +Success depends on native behavior, not generated settings, managed-config JSON, or catalog caches. +Defaults omit model overrides; expected answers are withheld from prompts. Claude headless results +require the requested service/alias in `modelUsage` with nonzero output tokens; TUI cases check the +selected default in the native banner and a completed assistant answer. Claude transcript model IDs +name the backing model, not the service. Codex evidence joins the completed answer to its +client-selected turn model. None proves the gateway's backing destination. Only live passes +establish coverage. + +The tests in `test_catalog_discovery.py` subclass Lilly's `BaseCujTest`. +`WORKSPACE_URL` comes from the explicit runner workspace; the SDK authenticates with +`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET`. Existing integration helpers +provide fresh installed-agent sessions, real TUI driving, and transcript evidence. + +CI runs a serialized Claude/Codex matrix using `UG_CUJ3_WORKSPACE` and the CUJ SP +secrets. Job concurrency prevents simultaneous runs against that workspace. Both +lanes block full/live runs. Run on a clean host/container with the two SP environment +variables set and an explicit dedicated workspace: + +```bash +python3.12 scripts/run_integration.py --suite e2e-cuj \ + --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ + --workspace "$CATALOG_DISCOVERY_WORKSPACE" \ + -- -m 'managed and catalog_discovery and workspace_isolated' +``` + +MCP/skills discovery remains separate. + +Collection only (no authentication or inference): + +```bash +uv run pytest -c tests/e2e_cuj/pytest.ini --confcutdir=tests/e2e_cuj \ + --collect-only tests/e2e_cuj +``` diff --git a/tests/integration/catalog_discovery_expectations.py b/tests/e2e_cuj/catalog_discovery_expectations.py similarity index 84% rename from tests/integration/catalog_discovery_expectations.py rename to tests/e2e_cuj/catalog_discovery_expectations.py index 2a17409c3..59e54c532 100644 --- a/tests/integration/catalog_discovery_expectations.py +++ b/tests/e2e_cuj/catalog_discovery_expectations.py @@ -1,4 +1,4 @@ -"""Fixture expectations owned by the catalog discovery journey, not shared utilities.""" +"""Model fixtures owned by the catalog discovery journey.""" MODEL_SCHEMA = "ug_e2e.models" OTHER_MODEL_SCHEMA = "ug_e2e.other_models" diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini new file mode 100644 index 000000000..98b8bea60 --- /dev/null +++ b/tests/e2e_cuj/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +testpaths = . +pythonpath = ../.. ../integration +addopts = --strict-markers --tb=short +markers = + managed: requires an admin CodingAgentConfig published on the selected workspace + catalog_discovery: managed schema-scoped model discovery and inference + workspace_isolated: owns a dedicated workspace with no concurrent runs + tui: real interactive terminal and completed task + claude: only runs when Claude Code is explicitly selected + codex: only runs when Codex is explicitly selected diff --git a/tests/e2e_cuj/test_catalog_discovery.py b/tests/e2e_cuj/test_catalog_discovery.py new file mode 100644 index 000000000..d683edbe0 --- /dev/null +++ b/tests/e2e_cuj/test_catalog_discovery.py @@ -0,0 +1,306 @@ +"""Managed catalog discovery, default launches, and explicit model selection.""" + +import json +import os + +import pytest +from utils.agents import claude, codex +from utils.evidence import FileTask, assert_completed_task_model +from utils.model_discovery import assert_picker_inventory +from utils.terminal import AgentTerminal + +from .base import BaseCujTest +from .catalog_discovery_expectations import ( + CLAUDE_DECOY, + CLAUDE_DEFAULT, + CLAUDE_MODELS, + CODEX_DECOY, + CODEX_DEFAULT, + CODEX_MODELS, + GEMINI_MODEL, + MODEL_SCHEMA, + OTHER_MODEL_SCHEMA, +) + +pytestmark = [pytest.mark.managed, pytest.mark.catalog_discovery, pytest.mark.workspace_isolated] + + +def _claude_file_task(session): + task = FileTask(session) + task.prompt = ( + f"Use the Read tool to read {session.cwd / task.filename}. Reply with only its contents." + ) + return task + + +def _assert_claude_headless_model(result, expected): + final = None + for line in result.stdout.splitlines(): + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict) and payload.get("type") == "result": + final = payload + assert final is not None and not final.get("is_error"), result.stdout + usage = final["modelUsage"] + assert set(usage) == {expected}, {"expected": expected, "observed": sorted(usage)} + assert usage[expected]["outputTokens"] > 0, usage + + +class TestCatalogDiscovery(BaseCujTest): + WORKSPACE_URL = os.environ.get("UCODE_TEST_WORKSPACE", "") + + @pytest.mark.claude + @pytest.mark.tui + def test_catalog_discovery_claude_picker_preserves_default(self, live_session, workspace): + """Scenario: configure, launch ug claude, inspect its picker, then submit a task. + + Expected: only Sonnet/Haiku/Kimi appear; Gemini and decoys are absent. + Dismissing the picker without a selection preserves Sonnet for the completed task. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + + bearer = self.workspace.config.authenticate()["Authorization"].removeprefix("Bearer ") + parent_catalog = claude.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) + decoy_catalog = claude.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) + assert set(parent_catalog.model_ids) == { + claude.discovery_model_id(model) for model in CLAUDE_MODELS + }, parent_catalog + assert set(decoy_catalog.model_ids) == {claude.discovery_model_id(CLAUDE_DECOY)}, ( + decoy_catalog + ) + + task = _claude_file_task(session) + with AgentTerminal( + session, + "claude", + [str(session.binary), "claude"], + "catalog-discovery-claude-picker", + ) as tui: + tui.boot() + picker_screen = tui.open_model_picker( + model_visible=lambda screen: all( + claude.model_in_picker(screen, model, parent_catalog.display_names[model]) + for model in parent_catalog.model_ids + ) + ) + assert_picker_inventory(picker_screen, "claude", parent_catalog.display_names) + for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): + assert excluded not in picker_screen, picker_screen + assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen + assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + tui.exit_normally() + task.assert_completed(session, "claude") + session.assert_not_routed() + + @pytest.mark.codex + @pytest.mark.tui + def test_catalog_discovery_codex_picker_preserves_default(self, live_session, workspace): + """Scenario: configure, query Codex model/list, inspect its TUI picker, then submit a task. + + Expected: model/list and the picker expose only GPT Luna/Kimi, not Gemini or decoys. + Dismissing the picker preserves GPT Luna for the completed task's selected model. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + + bearer = self.workspace.config.authenticate()["Authorization"].removeprefix("Bearer ") + parent_catalog = codex.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) + decoy_catalog = codex.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) + assert set(parent_catalog.model_ids) == CODEX_MODELS, parent_catalog + assert set(decoy_catalog.model_ids) == {CODEX_DECOY}, decoy_catalog + + models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) + assert len(models) == len(set(models)), models + assert set(models) == CODEX_MODELS, models + display_names = { + entry["slug"]: entry.get("display_name") or entry["slug"] + for page in parent_catalog.payloads + for entry in page["models"] + if entry["slug"] in CODEX_MODELS + } + + task = FileTask(session) + with AgentTerminal( + session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-picker" + ) as tui: + tui.boot() + picker_screen = tui.open_codex_model_picker( + model_visible=lambda screen: all( + codex.model_in_picker(screen, model, display_names[model]) + for model in parent_catalog.model_ids + ) + ) + assert_picker_inventory(picker_screen, "codex", display_names) + for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): + assert excluded not in picker_screen, picker_screen + assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + tui.exit_normally() + assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + session.assert_not_routed() + + @pytest.mark.claude + @pytest.mark.tui + def test_catalog_discovery_bare_ug_uses_claude_default(self, live_session, workspace): + """Scenario: configure the managed workspace, launch bare ug, and submit its first task. + + Expected: Claude completes the task on Sonnet without an agent or model override. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = _claude_file_task(session) + with AgentTerminal( + session, "claude", [str(session.binary)], "catalog-discovery-bare-ug-default" + ) as tui: + tui.boot() + assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + tui.exit_normally() + task.assert_completed(session, "claude") + session.assert_not_routed() + + @pytest.mark.claude + @pytest.mark.tui + def test_catalog_discovery_claude_tui_uses_default(self, live_session, workspace): + """Scenario: configure, launch ug claude, and submit its first task without opening a picker. + + Expected: Claude completes the task on the configured Sonnet default without overrides. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = _claude_file_task(session) + with AgentTerminal( + session, + "claude", + [str(session.binary), "claude"], + "catalog-discovery-claude-tui-default", + ) as tui: + tui.boot() + assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + tui.exit_normally() + task.assert_completed(session, "claude") + session.assert_not_routed() + + @pytest.mark.codex + @pytest.mark.tui + def test_catalog_discovery_codex_tui_uses_default(self, live_session, workspace): + """Scenario: configure, launch ug codex, and submit its first task without opening a picker. + + Expected: Codex completes the task with GPT Luna selected, without a model override. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + with AgentTerminal( + session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-tui-default" + ) as tui: + tui.boot() + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=240) + tui.exit_normally() + assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + session.assert_not_routed() + + @pytest.mark.claude + def test_catalog_discovery_claude_headless_uses_default(self, live_session, workspace): + """Scenario: configure, then run ug claude -p with a file task and no model override. + + Expected: the print task completes with a Sonnet response. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = _claude_file_task(session) + result = session.run( + "claude", + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + timeout=240, + ) + task.assert_headless_answer("claude", result) + _assert_claude_headless_model(result, CLAUDE_DEFAULT) + session.assert_not_routed() + + @pytest.mark.codex + def test_catalog_discovery_codex_headless_uses_default(self, live_session, workspace): + """Scenario: configure, then run ug codex -- exec with a file task and no model override. + + Expected: the exec task completes with GPT Luna selected. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + result = session.run( + "codex", "--", "exec", "--skip-git-repo-check", "--json", task.prompt, timeout=240 + ) + task.assert_headless_answer("codex", result) + assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + session.assert_not_routed() + + @pytest.mark.claude + @pytest.mark.parametrize("model", sorted(CLAUDE_MODELS - {CLAUDE_DEFAULT})) + def test_catalog_discovery_claude_explicit_model_completes_task( + self, live_session, workspace, model + ): + """Scenario: configure, then run a Claude print task with another compatible scoped model. + + Expected: each Haiku/Kimi task completes with the explicitly requested model. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = _claude_file_task(session) + result = session.run( + "claude", + "--", + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + "--model", + claude.discovery_model_id(model), + timeout=240, + ) + task.assert_headless_answer("claude", result) + _assert_claude_headless_model(result, claude.discovery_model_id(model)) + session.assert_not_routed() + + @pytest.mark.codex + @pytest.mark.parametrize("model", sorted(CODEX_MODELS - {CODEX_DEFAULT})) + def test_catalog_discovery_codex_explicit_model_completes_task( + self, live_session, workspace, model + ): + """Scenario: configure, then run a Codex exec task with another compatible scoped model. + + Expected: the Kimi task completes with the explicitly requested model selected. + """ + session = live_session + session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + task = FileTask(session) + result = session.run( + "codex", + "--", + "exec", + "--skip-git-repo-check", + "--json", + task.prompt, + "--model", + model, + timeout=240, + ) + task.assert_headless_answer("codex", result) + assert_completed_task_model(session, "codex", task.value, model) + session.assert_not_routed() diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 5b1671408..cc0d9b0ca 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -9,8 +9,6 @@ markers = workspace_switch: requires two real workspace hosts and their respective credentials smoke: Databricks Hosted, custom OAuth CLI TUI, and headless prompt for each agent tui: real interactive terminal boot, keyboard input, exit and reopen - workspace_isolated: requires a dedicated workspace instead of the shared integration workspace - catalog_discovery: managed schema-scoped model discovery and inference in the dedicated workspace claude: only runs when Claude Code is explicitly selected codex: only runs when Codex is explicitly selected opencode: only runs when OpenCode is explicitly selected diff --git a/tests/integration/test_ug_catalog_discovery.py b/tests/integration/test_ug_catalog_discovery.py deleted file mode 100644 index 8ea95d630..000000000 --- a/tests/integration/test_ug_catalog_discovery.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Managed catalog discovery, default launches, and explicit model selection.""" - -import json -import os - -import pytest -from catalog_discovery_expectations import ( - CLAUDE_DECOY, - CLAUDE_DEFAULT, - CLAUDE_MODELS, - CODEX_DECOY, - CODEX_DEFAULT, - CODEX_MODELS, - GEMINI_MODEL, - MODEL_SCHEMA, - OTHER_MODEL_SCHEMA, -) -from utils.agents import claude, codex -from utils.evidence import FileTask, assert_completed_task_model -from utils.model_discovery import assert_picker_inventory -from utils.terminal import AgentTerminal - -pytestmark = [pytest.mark.managed, pytest.mark.catalog_discovery, pytest.mark.workspace_isolated] - - -def _claude_file_task(session): - task = FileTask(session) - task.prompt = ( - f"Use the Read tool to read {session.cwd / task.filename}. Reply with only its contents." - ) - return task - - -def _assert_claude_headless_model(result, expected): - final = None - for line in result.stdout.splitlines(): - try: - payload = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(payload, dict) and payload.get("type") == "result": - final = payload - assert final is not None and not final.get("is_error"), result.stdout - usage = final["modelUsage"] - assert set(usage) == {expected}, {"expected": expected, "observed": sorted(usage)} - assert usage[expected]["outputTokens"] > 0, usage - - -@pytest.mark.claude -@pytest.mark.tui -def test_catalog_discovery_claude_picker_preserves_default(live_session, workspace): - """Scenario: configure, launch ug claude, inspect its picker, then submit a task. - - Expected: only Sonnet/Haiku/Kimi appear; Gemini and decoys are absent. - Dismissing the picker without a selection preserves Sonnet for the completed task. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - - bearer = os.environ["DATABRICKS_BEARER"] - parent_catalog = claude.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) - decoy_catalog = claude.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) - assert set(parent_catalog.model_ids) == { - claude.discovery_model_id(model) for model in CLAUDE_MODELS - }, parent_catalog - assert set(decoy_catalog.model_ids) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog - - task = _claude_file_task(session) - with AgentTerminal( - session, - "claude", - [str(session.binary), "claude"], - "catalog-discovery-claude-picker", - ) as tui: - tui.boot() - picker_screen = tui.open_model_picker( - model_visible=lambda screen: all( - claude.model_in_picker(screen, model, parent_catalog.display_names[model]) - for model in parent_catalog.model_ids - ) - ) - assert_picker_inventory(picker_screen, "claude", parent_catalog.display_names) - for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): - assert excluded not in picker_screen, picker_screen - assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen - assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - tui.exit_normally() - task.assert_completed(session, "claude") - session.assert_not_routed() - - -@pytest.mark.codex -@pytest.mark.tui -def test_catalog_discovery_codex_picker_preserves_default(live_session, workspace): - """Scenario: configure, query Codex model/list, inspect its TUI picker, then submit a task. - - Expected: model/list and the picker expose only GPT Luna/Kimi, not Gemini or decoys. - Dismissing the picker preserves GPT Luna for the completed task's selected model. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - - bearer = os.environ["DATABRICKS_BEARER"] - parent_catalog = codex.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) - decoy_catalog = codex.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) - assert set(parent_catalog.model_ids) == CODEX_MODELS, parent_catalog - assert set(decoy_catalog.model_ids) == {CODEX_DECOY}, decoy_catalog - - models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) - assert len(models) == len(set(models)), models - assert set(models) == CODEX_MODELS, models - display_names = { - entry["slug"]: entry.get("display_name") or entry["slug"] - for page in parent_catalog.payloads - for entry in page["models"] - if entry["slug"] in CODEX_MODELS - } - - task = FileTask(session) - with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-picker" - ) as tui: - tui.boot() - picker_screen = tui.open_codex_model_picker( - model_visible=lambda screen: all( - codex.model_in_picker(screen, model, display_names[model]) - for model in parent_catalog.model_ids - ) - ) - assert_picker_inventory(picker_screen, "codex", display_names) - for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): - assert excluded not in picker_screen, picker_screen - assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - tui.exit_normally() - assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) - session.assert_not_routed() - - -@pytest.mark.claude -@pytest.mark.tui -def test_catalog_discovery_bare_ug_uses_claude_default(live_session, workspace): - """Scenario: configure the managed workspace, launch bare ug, and submit its first task. - - Expected: Claude completes the task on Sonnet without an agent or model override. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = _claude_file_task(session) - with AgentTerminal( - session, "claude", [str(session.binary)], "catalog-discovery-bare-ug-default" - ) as tui: - tui.boot() - assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - tui.exit_normally() - task.assert_completed(session, "claude") - session.assert_not_routed() - - -@pytest.mark.claude -@pytest.mark.tui -def test_catalog_discovery_claude_tui_uses_default(live_session, workspace): - """Scenario: configure, launch ug claude, and submit its first task without opening a picker. - - Expected: Claude completes the task on the configured Sonnet default without overrides. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = _claude_file_task(session) - with AgentTerminal( - session, - "claude", - [str(session.binary), "claude"], - "catalog-discovery-claude-tui-default", - ) as tui: - tui.boot() - assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - tui.exit_normally() - task.assert_completed(session, "claude") - session.assert_not_routed() - - -@pytest.mark.codex -@pytest.mark.tui -def test_catalog_discovery_codex_tui_uses_default(live_session, workspace): - """Scenario: configure, launch ug codex, and submit its first task without opening a picker. - - Expected: Codex completes the task with GPT Luna selected, without a model override. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = FileTask(session) - with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-tui-default" - ) as tui: - tui.boot() - tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) - tui.exit_normally() - assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) - session.assert_not_routed() - - -@pytest.mark.claude -def test_catalog_discovery_claude_headless_uses_default(live_session, workspace): - """Scenario: configure, then run ug claude -p with a file task and no model override. - - Expected: the print task completes with a Sonnet response. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = _claude_file_task(session) - result = session.run( - "claude", - "-p", - task.prompt, - "--output-format", - "json", - "--allowedTools", - "Read", - timeout=240, - ) - task.assert_headless_answer("claude", result) - _assert_claude_headless_model(result, CLAUDE_DEFAULT) - session.assert_not_routed() - - -@pytest.mark.codex -def test_catalog_discovery_codex_headless_uses_default(live_session, workspace): - """Scenario: configure, then run ug codex -- exec with a file task and no model override. - - Expected: the exec task completes with GPT Luna selected. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = FileTask(session) - result = session.run( - "codex", "--", "exec", "--skip-git-repo-check", "--json", task.prompt, timeout=240 - ) - task.assert_headless_answer("codex", result) - assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) - session.assert_not_routed() - - -@pytest.mark.claude -@pytest.mark.parametrize("model", sorted(CLAUDE_MODELS - {CLAUDE_DEFAULT})) -def test_catalog_discovery_claude_explicit_model_completes_task(live_session, workspace, model): - """Scenario: configure, then run a Claude print task with another compatible scoped model. - - Expected: each Haiku/Kimi task completes with the explicitly requested model. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = _claude_file_task(session) - result = session.run( - "claude", - "--", - "-p", - task.prompt, - "--output-format", - "json", - "--allowedTools", - "Read", - "--model", - claude.discovery_model_id(model), - timeout=240, - ) - task.assert_headless_answer("claude", result) - _assert_claude_headless_model(result, claude.discovery_model_id(model)) - session.assert_not_routed() - - -@pytest.mark.codex -@pytest.mark.parametrize("model", sorted(CODEX_MODELS - {CODEX_DEFAULT})) -def test_catalog_discovery_codex_explicit_model_completes_task(live_session, workspace, model): - """Scenario: configure, then run a Codex exec task with another compatible scoped model. - - Expected: the Kimi task completes with the explicitly requested model selected. - """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) - task = FileTask(session) - result = session.run( - "codex", - "--", - "exec", - "--skip-git-repo-check", - "--json", - task.prompt, - "--model", - model, - timeout=240, - ) - task.assert_headless_answer("codex", result) - assert_completed_task_model(session, "codex", task.value, model) - session.assert_not_routed() diff --git a/tests/test_integration_runner.py b/tests/test_integration_runner.py index 49f749c95..69a4c9b0d 100644 --- a/tests/test_integration_runner.py +++ b/tests/test_integration_runner.py @@ -143,3 +143,52 @@ def test_windows_policy_paths_match_pinned_agent_locations(): def test_windows_policy_preflight_fails_closed_without_required_roots(): with pytest.raises(RuntimeError, match="PROGRAMDATA"): runner.managed_policy_paths(["codex"], {}, platform_name="nt", system_platform="win32") + + +def test_e2e_cuj_runner_uses_explicit_sp_auth_without_legacy_live_filter(): + args = runner.arguments( + ["--suite", "e2e-cuj", "--claude-version", "2.1.280"], + platform_name="posix", + environment={ + "UCODE_TEST_WORKSPACE": "https://dedicated.example.test", + "UG_CUJ_SP_CLIENT_ID": "client-id", + "UG_CUJ_SP_CLIENT_SECRET": "client-secret", + }, + ) + + assert args.suite == "e2e-cuj" + assert args.workspace == "https://dedicated.example.test" + assert args.pytest_args == [] + + +@pytest.mark.parametrize("missing", ["UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET"]) +def test_e2e_cuj_runner_requires_scaffold_credentials(missing): + environment = { + "UCODE_TEST_WORKSPACE": "https://dedicated.example.test", + "UG_CUJ_SP_CLIENT_ID": "client-id", + "UG_CUJ_SP_CLIENT_SECRET": "client-secret", + "DATABRICKS_BEARER": "not-a-substitute", + } + del environment[missing] + with pytest.raises(SystemExit): + runner.arguments( + ["--suite", "e2e-cuj", "--claude-version", "2.1.280"], + platform_name="posix", + environment=environment, + ) + + +@pytest.mark.parametrize( + "mode", [["--installation-only"], ["--headless-only"], ["--profile", "dev"]] +) +def test_e2e_cuj_runner_rejects_incompatible_modes(mode): + with pytest.raises(SystemExit): + runner.arguments( + ["--suite", "e2e-cuj", "--claude-version", "2.1.280", *mode], + platform_name="posix", + environment={ + "UCODE_TEST_WORKSPACE": "https://dedicated.example.test", + "UG_CUJ_SP_CLIENT_ID": "client-id", + "UG_CUJ_SP_CLIENT_SECRET": "client-secret", + }, + ) From 02745db9a423f9b4607fbfcc3b4d50c8c44251ca Mon Sep 17 00:00:00 2001 From: Tien Le Date: Tue, 6 Oct 2026 13:42:58 +0000 Subject: [PATCH 14/19] Use shared dedicated CUJ framework for catalog discovery --- scripts/run_integration.py | 39 +---- tests/e2e_cuj/README.md | 21 ++- tests/e2e_cuj/pytest.ini | 4 +- tests/e2e_cuj/test_catalog_discovery.py | 192 ++++++++++++++---------- tests/integration/README.md | 4 + tests/test_integration_runner.py | 49 ------ 6 files changed, 131 insertions(+), 178 deletions(-) diff --git a/scripts/run_integration.py b/scripts/run_integration.py index 6d2a6cbca..bc81650d3 100644 --- a/scripts/run_integration.py +++ b/scripts/run_integration.py @@ -279,7 +279,6 @@ def arguments( platform_name = os.name if platform_name is None else platform_name environment = os.environ if environment is None else environment parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--suite", choices=["integration", "e2e-cuj"], default="integration") source = parser.add_mutually_exclusive_group() source.add_argument("--ug-version", default="checkout", help="Exact ug release, or checkout.") source.add_argument( @@ -374,14 +373,6 @@ def arguments( "pytest_args", nargs=argparse.REMAINDER, help="After --, pass pytest filters." ) args = parser.parse_args(argv) - if args.suite == "e2e-cuj": - if args.installation_only or args.headless_only or args.profile: - parser.error("--suite e2e-cuj uses CUJ service-principal auth and journey filters.") - if not all( - environment.get(key, "").strip() - for key in ("UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET") - ): - parser.error("Set UG_CUJ_SP_CLIENT_ID and UG_CUJ_SP_CLIENT_SECRET for e2e-cuj.") if platform_name != "posix" and not (args.installation_only or args.headless_only): parser.error( "Live agent/TUI integration requires POSIX PTY, managed-settings, and signal " @@ -396,7 +387,7 @@ def arguments( filters.add_argument("--maxfail", type=int) extra = args.pytest_args[1:] if args.pytest_args[:1] == ["--"] else args.pytest_args selected = filters.parse_args(extra) - marker = selected.m or ("live" if args.suite == "integration" else None) + marker = selected.m or "live" if args.installation_only: marker = f"installation and ({selected.m})" if selected.m else "installation" args.pytest_args = [] @@ -427,7 +418,7 @@ def arguments( environment.get("DATABRICKS_CLIENT_ID", "").strip() and environment.get("DATABRICKS_CLIENT_SECRET", "").strip() ) - if args.suite == "integration" and not ( + if not ( args.profile or environment.get("DATABRICKS_BEARER", "").strip() or has_client_creds ): parser.error( @@ -532,7 +523,6 @@ def terminate(signum, frame): target_bearers: dict[str, str] = {} client_secrets = ( os.environ.get("DATABRICKS_CLIENT_SECRET", ""), - os.environ.get("UG_CUJ_SP_CLIENT_SECRET", ""), os.environ.get("UG_MPS_DEFAULTS_CLIENT_SECRET", ""), os.environ.get("UG_PARENT_SCHEMA_DEFAULTS_CLIENT_SECRET", ""), ) @@ -835,18 +825,13 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: if not bearer: raise RuntimeError("Selected profile returned no access token.") - if ( - not bearer - and not args.profile - and not args.installation_only - and args.suite == "integration" - ): + if not bearer and not args.profile and not args.installation_only: client_id = os.environ.get("DATABRICKS_CLIENT_ID", "").strip() client_secret = os.environ.get("DATABRICKS_CLIENT_SECRET", "").strip() if client_id and client_secret: bearer = mint_m2m_token(args.workspace, client_id, client_secret) - if not args.installation_only and args.suite == "integration": + if not args.installation_only: for bearer_env, target_workspace, client_id, secret_env in MANAGED_DEFAULTS_TARGETS: secret = os.environ.get(secret_env, "").strip() if args.workspace.rstrip("/") == target_workspace: @@ -855,10 +840,6 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: target_bearers[bearer_env] = mint_m2m_token(target_workspace, client_id, secret) test_dependencies = ["pytest==9.0.3"] - if args.suite == "e2e-cuj": - test_dependencies.append( - next(line for line in freeze.splitlines() if line.startswith("databricks-sdk==")) - ) if os.name == "posix": test_dependencies.extend(["pexpect==4.9.0", "pyte==0.8.2"]) run( @@ -908,17 +889,9 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: runtime_env[f"UG_INTEGRATION_{agent.upper()}_MODEL"] = ( getattr(args, f"{agent}_model") or "" ) - if args.suite == "e2e-cuj": - for key in ("UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET"): - runtime_env[key] = os.environ[key] - suite = ROOT / "tests" / ("e2e_cuj" if args.suite == "e2e-cuj" else "integration") - report["suite"] = args.suite + suite = ROOT / "tests/integration" suite_hash = hashlib.sha256() - suite_files = set(suite.rglob("*.py")) - if args.suite == "e2e-cuj": - suite_files.update((ROOT / "tests/integration/utils").rglob("*.py")) - suite_files.add(ROOT / "tests/integration/conftest.py") - for path in [Path(__file__), *sorted(suite_files), suite / "pytest.ini"]: + for path in [Path(__file__), *sorted(suite.rglob("*.py")), suite / "pytest.ini"]: suite_hash.update(str(path.relative_to(ROOT)).encode() + b"\0" + path.read_bytes()) report["suite_sha256"] = suite_hash.hexdigest() extra = args.pytest_args diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index 00dd2e904..9f5ce9141 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -30,21 +30,18 @@ name the backing model, not the service. Codex evidence joins the completed answ client-selected turn model. None proves the gateway's backing destination. Only live passes establish coverage. -The tests in `test_catalog_discovery.py` subclass Lilly's `BaseCujTest`. -`WORKSPACE_URL` comes from the explicit runner workspace; the SDK authenticates with -`UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET`. Existing integration helpers -provide fresh installed-agent sessions, real TUI driving, and transcript evidence. +The test class selects the CUJ3 workspace, `https://dbc-bbdd5508-648e.cloud.databricks.com`. +The shared `cuj` fixture supplies its authenticated SDK client and isolated local session; +workspace configuration remains read-only. Set `UG_CUJ_SP_CLIENT_ID` and +`UG_CUJ_SP_CLIENT_SECRET` before a live run. -CI runs a serialized Claude/Codex matrix using `UG_CUJ3_WORKSPACE` and the CUJ SP -secrets. Job concurrency prevents simultaneous runs against that workspace. Both -lanes block full/live runs. Run on a clean host/container with the two SP environment -variables set and an explicit dedicated workspace: +CI discovers these tests through the shared `dedicated-cuj` job, which installs both +pinned agents. No catalog-specific workflow or workspace secret is needed. Locally, +install ug, both agents, and the Databricks CLI on a clean POSIX host, then run: ```bash -python3.12 scripts/run_integration.py --suite e2e-cuj \ - --ug-version checkout --claude-version 2.1.280 --codex-version 0.154.0 \ - --workspace "$CATALOG_DISCOVERY_WORKSPACE" \ - -- -m 'managed and catalog_discovery and workspace_isolated' +uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest \ + --confcutdir=tests/e2e_cuj tests/e2e_cuj/test_catalog_discovery.py -v ``` MCP/skills discovery remains separate. diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini index 98b8bea60..008cc4a9e 100644 --- a/tests/e2e_cuj/pytest.ini +++ b/tests/e2e_cuj/pytest.ini @@ -1,11 +1,11 @@ [pytest] testpaths = . -pythonpath = ../.. ../integration +pythonpath = ../.. addopts = --strict-markers --tb=short markers = managed: requires an admin CodingAgentConfig published on the selected workspace catalog_discovery: managed schema-scoped model discovery and inference - workspace_isolated: owns a dedicated workspace with no concurrent runs + workspace_isolated: reads preconfigured CUJ workspace state with isolated local sessions tui: real interactive terminal and completed task claude: only runs when Claude Code is explicitly selected codex: only runs when Codex is explicitly selected diff --git a/tests/e2e_cuj/test_catalog_discovery.py b/tests/e2e_cuj/test_catalog_discovery.py index d683edbe0..cad9f6a5a 100644 --- a/tests/e2e_cuj/test_catalog_discovery.py +++ b/tests/e2e_cuj/test_catalog_discovery.py @@ -1,13 +1,18 @@ """Managed catalog discovery, default launches, and explicit model selection.""" import json -import os import pytest -from utils.agents import claude, codex -from utils.evidence import FileTask, assert_completed_task_model -from utils.model_discovery import assert_picker_inventory -from utils.terminal import AgentTerminal + +from tests.integration.utils.agents import claude, codex +from tests.integration.utils.evidence import FileTask, assert_completed_task_model +from tests.integration.utils.model_discovery import assert_picker_inventory +from tests.integration.utils.provider_catalog import ( + MODEL_SERVICE_PARENT_SCHEMA_HEADER, + parse_anthropic_provider_page, + parse_codex_provider_catalog, +) +from tests.integration.utils.terminal import AgentTerminal from .base import BaseCujTest from .catalog_discovery_expectations import ( @@ -21,10 +26,46 @@ MODEL_SCHEMA, OTHER_MODEL_SCHEMA, ) +from .helpers.constants import CLAUDE, CODEX pytestmark = [pytest.mark.managed, pytest.mark.catalog_discovery, pytest.mark.workspace_isolated] +def _catalog_display_names(workspace, agent, schema): + """Read a scoped catalog; Workspace.model_ids only supports system.ai.""" + headers = {MODEL_SERVICE_PARENT_SCHEMA_HEADER: schema} + if agent == CODEX: + payload = workspace.client.api_client.do( + "GET", "/ai-gateway/codex/v1/models", headers=headers + ) + models = parse_codex_provider_catalog(payload) + return { + entry["slug"]: entry.get("display_name") or entry["slug"] + for entry in payload["models"] + if entry["slug"] in models + } + assert agent == CLAUDE, agent + headers["Anthropic-Version"] = "2023-06-01" + models, seen, cursor = {}, set(), None + for _page in range(20): + query = {"limit": "1000"} + if cursor: + query["after_id"] = cursor + payload = workspace.client.api_client.do( + "GET", "/ai-gateway/anthropic/v1/models", headers=headers, query=query + ) + page = parse_anthropic_provider_page(payload) + for model, display_name in page.models: + assert model not in models, f"Repeated catalog model: {model}" + models[model] = display_name + if not page.has_more: + return models + cursor = page.last_id + assert cursor and cursor not in seen, "Repeated catalog pagination cursor" + seen.add(cursor) + raise AssertionError("Anthropic catalog exceeded 20 pages") + + def _claude_file_task(session): task = FileTask(session) task.prompt = ( @@ -49,44 +90,41 @@ def _assert_claude_headless_model(result, expected): class TestCatalogDiscovery(BaseCujTest): - WORKSPACE_URL = os.environ.get("UCODE_TEST_WORKSPACE", "") + WORKSPACE_URL = "https://dbc-bbdd5508-648e.cloud.databricks.com" @pytest.mark.claude @pytest.mark.tui - def test_catalog_discovery_claude_picker_preserves_default(self, live_session, workspace): + def test_catalog_discovery_claude_picker_preserves_default(self, cuj): """Scenario: configure, launch ug claude, inspect its picker, then submit a task. Expected: only Sonnet/Haiku/Kimi appear; Gemini and decoys are absent. Dismissing the picker without a selection preserves Sonnet for the completed task. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) - bearer = self.workspace.config.authenticate()["Authorization"].removeprefix("Bearer ") - parent_catalog = claude.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) - decoy_catalog = claude.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) - assert set(parent_catalog.model_ids) == { + parent_catalog = _catalog_display_names(workspace, CLAUDE, MODEL_SCHEMA) + decoy_catalog = _catalog_display_names(workspace, CLAUDE, OTHER_MODEL_SCHEMA) + assert set(parent_catalog) == { claude.discovery_model_id(model) for model in CLAUDE_MODELS }, parent_catalog - assert set(decoy_catalog.model_ids) == {claude.discovery_model_id(CLAUDE_DECOY)}, ( - decoy_catalog - ) + assert set(decoy_catalog) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog task = _claude_file_task(session) with AgentTerminal( session, - "claude", - [str(session.binary), "claude"], + CLAUDE, + [str(session.binary), CLAUDE], "catalog-discovery-claude-picker", ) as tui: tui.boot() picker_screen = tui.open_model_picker( model_visible=lambda screen: all( - claude.model_in_picker(screen, model, parent_catalog.display_names[model]) - for model in parent_catalog.model_ids + claude.model_in_picker(screen, model, parent_catalog[model]) + for model in parent_catalog ) ) - assert_picker_inventory(picker_screen, "claude", parent_catalog.display_names) + assert_picker_inventory(picker_screen, CLAUDE, parent_catalog) for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): assert excluded not in picker_screen, picker_screen assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen @@ -94,92 +132,86 @@ def test_catalog_discovery_claude_picker_preserves_default(self, live_session, w tui.submit(task.prompt) tui.wait_for_task(task, timeout=240) tui.exit_normally() - task.assert_completed(session, "claude") + task.assert_completed(session, CLAUDE) session.assert_not_routed() @pytest.mark.codex @pytest.mark.tui - def test_catalog_discovery_codex_picker_preserves_default(self, live_session, workspace): + def test_catalog_discovery_codex_picker_preserves_default(self, cuj): """Scenario: configure, query Codex model/list, inspect its TUI picker, then submit a task. Expected: model/list and the picker expose only GPT Luna/Kimi, not Gemini or decoys. Dismissing the picker preserves GPT Luna for the completed task's selected model. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) - bearer = self.workspace.config.authenticate()["Authorization"].removeprefix("Bearer ") - parent_catalog = codex.fetch_parent_catalog(workspace, bearer, MODEL_SCHEMA) - decoy_catalog = codex.fetch_parent_catalog(workspace, bearer, OTHER_MODEL_SCHEMA) - assert set(parent_catalog.model_ids) == CODEX_MODELS, parent_catalog - assert set(decoy_catalog.model_ids) == {CODEX_DECOY}, decoy_catalog + parent_catalog = _catalog_display_names(workspace, CODEX, MODEL_SCHEMA) + decoy_catalog = _catalog_display_names(workspace, CODEX, OTHER_MODEL_SCHEMA) + assert set(parent_catalog) == CODEX_MODELS, parent_catalog + assert set(decoy_catalog) == {CODEX_DECOY}, decoy_catalog models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) assert len(models) == len(set(models)), models assert set(models) == CODEX_MODELS, models - display_names = { - entry["slug"]: entry.get("display_name") or entry["slug"] - for page in parent_catalog.payloads - for entry in page["models"] - if entry["slug"] in CODEX_MODELS - } + display_names = parent_catalog task = FileTask(session) with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-picker" + session, CODEX, [str(session.binary), CODEX], "catalog-discovery-codex-picker" ) as tui: tui.boot() picker_screen = tui.open_codex_model_picker( model_visible=lambda screen: all( codex.model_in_picker(screen, model, display_names[model]) - for model in parent_catalog.model_ids + for model in parent_catalog ) ) - assert_picker_inventory(picker_screen, "codex", display_names) + assert_picker_inventory(picker_screen, CODEX, display_names) for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): assert excluded not in picker_screen, picker_screen assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen tui.submit(task.prompt) tui.wait_for_task(task, timeout=240) tui.exit_normally() - assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) session.assert_not_routed() @pytest.mark.claude @pytest.mark.tui - def test_catalog_discovery_bare_ug_uses_claude_default(self, live_session, workspace): + def test_catalog_discovery_bare_ug_uses_claude_default(self, cuj): """Scenario: configure the managed workspace, launch bare ug, and submit its first task. Expected: Claude completes the task on Sonnet without an agent or model override. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = _claude_file_task(session) with AgentTerminal( - session, "claude", [str(session.binary)], "catalog-discovery-bare-ug-default" + session, CLAUDE, [str(session.binary)], "catalog-discovery-bare-ug-default" ) as tui: tui.boot() assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible tui.submit(task.prompt) tui.wait_for_task(task, timeout=240) tui.exit_normally() - task.assert_completed(session, "claude") + task.assert_completed(session, CLAUDE) session.assert_not_routed() @pytest.mark.claude @pytest.mark.tui - def test_catalog_discovery_claude_tui_uses_default(self, live_session, workspace): + def test_catalog_discovery_claude_tui_uses_default(self, cuj): """Scenario: configure, launch ug claude, and submit its first task without opening a picker. Expected: Claude completes the task on the configured Sonnet default without overrides. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = _claude_file_task(session) with AgentTerminal( session, - "claude", - [str(session.binary), "claude"], + CLAUDE, + [str(session.binary), CLAUDE], "catalog-discovery-claude-tui-default", ) as tui: tui.boot() @@ -187,40 +219,40 @@ def test_catalog_discovery_claude_tui_uses_default(self, live_session, workspace tui.submit(task.prompt) tui.wait_for_task(task, timeout=240) tui.exit_normally() - task.assert_completed(session, "claude") + task.assert_completed(session, CLAUDE) session.assert_not_routed() @pytest.mark.codex @pytest.mark.tui - def test_catalog_discovery_codex_tui_uses_default(self, live_session, workspace): + def test_catalog_discovery_codex_tui_uses_default(self, cuj): """Scenario: configure, launch ug codex, and submit its first task without opening a picker. Expected: Codex completes the task with GPT Luna selected, without a model override. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = FileTask(session) with AgentTerminal( - session, "codex", [str(session.binary), "codex"], "catalog-discovery-codex-tui-default" + session, CODEX, [str(session.binary), CODEX], "catalog-discovery-codex-tui-default" ) as tui: tui.boot() tui.submit(task.prompt) tui.wait_for_task(task, timeout=240) tui.exit_normally() - assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) session.assert_not_routed() @pytest.mark.claude - def test_catalog_discovery_claude_headless_uses_default(self, live_session, workspace): + def test_catalog_discovery_claude_headless_uses_default(self, cuj): """Scenario: configure, then run ug claude -p with a file task and no model override. Expected: the print task completes with a Sonnet response. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = _claude_file_task(session) result = session.run( - "claude", + CLAUDE, "-p", task.prompt, "--output-format", @@ -229,40 +261,38 @@ def test_catalog_discovery_claude_headless_uses_default(self, live_session, work "Read", timeout=240, ) - task.assert_headless_answer("claude", result) + task.assert_headless_answer(CLAUDE, result) _assert_claude_headless_model(result, CLAUDE_DEFAULT) session.assert_not_routed() @pytest.mark.codex - def test_catalog_discovery_codex_headless_uses_default(self, live_session, workspace): + def test_catalog_discovery_codex_headless_uses_default(self, cuj): """Scenario: configure, then run ug codex -- exec with a file task and no model override. Expected: the exec task completes with GPT Luna selected. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = FileTask(session) result = session.run( - "codex", "--", "exec", "--skip-git-repo-check", "--json", task.prompt, timeout=240 + CODEX, "--", "exec", "--skip-git-repo-check", "--json", task.prompt, timeout=240 ) - task.assert_headless_answer("codex", result) - assert_completed_task_model(session, "codex", task.value, CODEX_DEFAULT) + task.assert_headless_answer(CODEX, result) + assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) session.assert_not_routed() @pytest.mark.claude @pytest.mark.parametrize("model", sorted(CLAUDE_MODELS - {CLAUDE_DEFAULT})) - def test_catalog_discovery_claude_explicit_model_completes_task( - self, live_session, workspace, model - ): + def test_catalog_discovery_claude_explicit_model_completes_task(self, cuj, model): """Scenario: configure, then run a Claude print task with another compatible scoped model. Expected: each Haiku/Kimi task completes with the explicitly requested model. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = _claude_file_task(session) result = session.run( - "claude", + CLAUDE, "--", "-p", task.prompt, @@ -274,24 +304,22 @@ def test_catalog_discovery_claude_explicit_model_completes_task( claude.discovery_model_id(model), timeout=240, ) - task.assert_headless_answer("claude", result) + task.assert_headless_answer(CLAUDE, result) _assert_claude_headless_model(result, claude.discovery_model_id(model)) session.assert_not_routed() @pytest.mark.codex @pytest.mark.parametrize("model", sorted(CODEX_MODELS - {CODEX_DEFAULT})) - def test_catalog_discovery_codex_explicit_model_completes_task( - self, live_session, workspace, model - ): + def test_catalog_discovery_codex_explicit_model_completes_task(self, cuj, model): """Scenario: configure, then run a Codex exec task with another compatible scoped model. Expected: the Kimi task completes with the explicitly requested model selected. """ - session = live_session - session.run("configure", "--workspace", workspace, "--skip-upgrade", timeout=300) + session, workspace, _recorder = cuj + session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) task = FileTask(session) result = session.run( - "codex", + CODEX, "--", "exec", "--skip-git-repo-check", @@ -301,6 +329,6 @@ def test_catalog_discovery_codex_explicit_model_completes_task( model, timeout=240, ) - task.assert_headless_answer("codex", result) - assert_completed_task_model(session, "codex", task.value, model) + task.assert_headless_answer(CODEX, result) + assert_completed_task_model(session, CODEX, task.value, model) session.assert_not_routed() diff --git a/tests/integration/README.md b/tests/integration/README.md index 8cdd60186..2cb2bcbbb 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -9,6 +9,10 @@ is read-only and checked for changes at teardown; concurrent readers need no res CUJ2 adds three separately collected cases for exact MPS/MCP configuration, Codex inference, and Claude inference. +The [catalog discovery journey](../e2e_cuj/README.md) uses the CUJ3 workspace to check +agent-compatible pickers, schema exclusions, configured defaults, and real inference. +CI collects it through the shared `dedicated-cuj` job. + This suite runs the **installed product** through subprocesses, against the same `UCODE_TEST_WORKSPACE` used by the existing e2e tests. It does not import `ucode`, patch application functions, substitute agent executables, run a fake gateway, diff --git a/tests/test_integration_runner.py b/tests/test_integration_runner.py index 69a4c9b0d..49f749c95 100644 --- a/tests/test_integration_runner.py +++ b/tests/test_integration_runner.py @@ -143,52 +143,3 @@ def test_windows_policy_paths_match_pinned_agent_locations(): def test_windows_policy_preflight_fails_closed_without_required_roots(): with pytest.raises(RuntimeError, match="PROGRAMDATA"): runner.managed_policy_paths(["codex"], {}, platform_name="nt", system_platform="win32") - - -def test_e2e_cuj_runner_uses_explicit_sp_auth_without_legacy_live_filter(): - args = runner.arguments( - ["--suite", "e2e-cuj", "--claude-version", "2.1.280"], - platform_name="posix", - environment={ - "UCODE_TEST_WORKSPACE": "https://dedicated.example.test", - "UG_CUJ_SP_CLIENT_ID": "client-id", - "UG_CUJ_SP_CLIENT_SECRET": "client-secret", - }, - ) - - assert args.suite == "e2e-cuj" - assert args.workspace == "https://dedicated.example.test" - assert args.pytest_args == [] - - -@pytest.mark.parametrize("missing", ["UG_CUJ_SP_CLIENT_ID", "UG_CUJ_SP_CLIENT_SECRET"]) -def test_e2e_cuj_runner_requires_scaffold_credentials(missing): - environment = { - "UCODE_TEST_WORKSPACE": "https://dedicated.example.test", - "UG_CUJ_SP_CLIENT_ID": "client-id", - "UG_CUJ_SP_CLIENT_SECRET": "client-secret", - "DATABRICKS_BEARER": "not-a-substitute", - } - del environment[missing] - with pytest.raises(SystemExit): - runner.arguments( - ["--suite", "e2e-cuj", "--claude-version", "2.1.280"], - platform_name="posix", - environment=environment, - ) - - -@pytest.mark.parametrize( - "mode", [["--installation-only"], ["--headless-only"], ["--profile", "dev"]] -) -def test_e2e_cuj_runner_rejects_incompatible_modes(mode): - with pytest.raises(SystemExit): - runner.arguments( - ["--suite", "e2e-cuj", "--claude-version", "2.1.280", *mode], - platform_name="posix", - environment={ - "UCODE_TEST_WORKSPACE": "https://dedicated.example.test", - "UG_CUJ_SP_CLIENT_ID": "client-id", - "UG_CUJ_SP_CLIENT_SECRET": "client-secret", - }, - ) From 3f1ec8231f58fb5065e30a495721dd8b22ae4830 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Tue, 6 Oct 2026 14:04:40 +0000 Subject: [PATCH 15/19] Remove obsolete provider catalog refactor and tests --- tests/integration/utils/provider_catalog.py | 75 +++++++++------------ tests/test_integration_provider_catalog.py | 73 ++------------------ 2 files changed, 37 insertions(+), 111 deletions(-) diff --git a/tests/integration/utils/provider_catalog.py b/tests/integration/utils/provider_catalog.py index d3f1632df..3834be51d 100644 --- a/tests/integration/utils/provider_catalog.py +++ b/tests/integration/utils/provider_catalog.py @@ -12,7 +12,8 @@ _ANTHROPIC_MODELS_PATH = "/ai-gateway/anthropic/v1/models" _CODEX_MODELS_PATH = "/ai-gateway/codex/v1/models" _PROVIDER_HEADER = "Databricks-Model-Provider-Service" -MODEL_SERVICE_PARENT_SCHEMA_HEADER = "Databricks-Model-Service-Parent-Schema" +_PARENT_SCHEMA_HEADER = "Databricks-Model-Service-Parent-Schema" +MODEL_SERVICE_PARENT_SCHEMA_HEADER = _PARENT_SCHEMA_HEADER _ANTHROPIC_VERSION = "2023-06-01" _ANTHROPIC_PAGE_SIZE = 1000 _ANTHROPIC_MAX_PAGES = 20 @@ -33,7 +34,6 @@ class AnthropicProviderCatalog: model_ids: tuple[str, ...] display_names: dict[str, str | None] - payloads: tuple[dict, ...] = () @dataclass(frozen=True) @@ -41,7 +41,6 @@ class CodexProviderCatalog: """The list-visible model slugs from a Codex provider catalog.""" model_ids: tuple[str, ...] - payloads: tuple[dict, ...] = () def _fail(message: str) -> NoReturn: @@ -112,6 +111,8 @@ def parse_codex_provider_catalog(payload: object) -> tuple[str, ...]: if not isinstance(visibility, str): _fail(f"Codex provider model {index} had an invalid visibility") if visibility != "list": + # Hidden/native aliases are part of the provider response but are not exposed by + # Codex's model/list request used as the observable catalog below. continue slug = raw_model.get("slug") if not isinstance(slug, str) or not slug.strip(): @@ -153,62 +154,51 @@ def _get_json(url: str, headers: dict[str, str]) -> object: raise AssertionError(f"GET {url} returned invalid JSON: {error.msg}") from error -def _validate_request_inputs(workspace: str, token: str) -> str: +def _validate_request_inputs(workspace: str, token: str, provider_service: str) -> str: if not isinstance(workspace, str) or not workspace.rstrip("/"): _fail("provider catalog request requires a workspace URL") if not isinstance(token, str) or not token: _fail("provider catalog request requires an explicit bearer token") + if not isinstance(provider_service, str) or not provider_service.strip(): + _fail("provider catalog request requires a provider service") return workspace.rstrip("/") -def _validate_scope(scope: str) -> None: - if not isinstance(scope, str) or not scope.strip(): - _fail("provider catalog request requires an explicit catalog scope") - - def fetch_anthropic_provider_catalog( workspace: str, token: str, provider_service: str ) -> AnthropicProviderCatalog: """Fetch and validate the complete Anthropic provider catalog with bounded pagination.""" - _validate_scope(provider_service) - return _fetch_anthropic_catalog(workspace, token, {_PROVIDER_HEADER: provider_service}) + return _fetch_anthropic_catalog(workspace, token, provider_service, _PROVIDER_HEADER) def fetch_anthropic_parent_catalog( workspace: str, token: str, parent_schema: str ) -> AnthropicProviderCatalog: - """Fetch the complete Anthropic catalog scoped to a Unity Catalog parent schema.""" - _validate_scope(parent_schema) - return _fetch_anthropic_catalog( - workspace, token, {MODEL_SERVICE_PARENT_SCHEMA_HEADER: parent_schema} - ) + """Fetch the Anthropic-compatible models advertised for a Unity Catalog parent schema.""" + return _fetch_anthropic_catalog(workspace, token, parent_schema, _PARENT_SCHEMA_HEADER) def _fetch_anthropic_catalog( - workspace: str, token: str, scope_headers: dict[str, str] + workspace: str, token: str, source: str, source_header: str ) -> AnthropicProviderCatalog: - base_url = _validate_request_inputs(workspace, token) + base_url = _validate_request_inputs(workspace, token, source) headers = { "Authorization": f"Bearer {token}", "Accept": "application/json", "Anthropic-Version": _ANTHROPIC_VERSION, - **scope_headers, + source_header: source, } model_ids: list[str] = [] display_names: dict[str, str | None] = {} - payloads: list[dict] = [] cursor: str | None = None cursors: set[str] = set() for _page_number in range(_ANTHROPIC_MAX_PAGES): query = {"limit": str(_ANTHROPIC_PAGE_SIZE)} if cursor is not None: query["after_id"] = cursor - path = f"{_ANTHROPIC_MODELS_PATH}?{urllib.parse.urlencode(query)}" - payload = _get_json(f"{base_url}{path}", headers) - page = parse_anthropic_provider_page(payload) - assert isinstance(payload, dict), "Expected an Anthropic catalog object" - payloads.append(payload) + url = f"{base_url}{_ANTHROPIC_MODELS_PATH}?{urllib.parse.urlencode(query)}" + page = parse_anthropic_provider_page(_get_json(url, headers)) for model_id, display_name in page.models: if model_id in display_names: _fail(f"Anthropic provider catalog repeated model id {model_id!r}") @@ -216,8 +206,8 @@ def _fetch_anthropic_catalog( display_names[model_id] = display_name if not page.has_more: - return AnthropicProviderCatalog(tuple(model_ids), display_names, tuple(payloads)) - assert page.last_id is not None + return AnthropicProviderCatalog(tuple(model_ids), display_names) + assert page.last_id is not None # checked by parse_anthropic_provider_page if page.last_id in cursors: _fail(f"Anthropic provider catalog repeated pagination cursor {page.last_id!r}") cursors.add(page.last_id) @@ -230,28 +220,29 @@ def fetch_codex_provider_catalog( workspace: str, token: str, provider_service: str ) -> CodexProviderCatalog: """Fetch and validate the list-visible Codex provider catalog.""" - _validate_scope(provider_service) - return _fetch_codex_catalog(workspace, token, {_PROVIDER_HEADER: provider_service}) - - -def _fetch_codex_catalog( - workspace: str, token: str, scope_headers: dict[str, str] -) -> CodexProviderCatalog: - base_url = _validate_request_inputs(workspace, token) + base_url = _validate_request_inputs(workspace, token, provider_service) payload = _get_json( f"{base_url}{_CODEX_MODELS_PATH}", - {"Authorization": f"Bearer {token}", "Accept": "application/json", **scope_headers}, + { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + _PROVIDER_HEADER: provider_service, + }, ) - model_ids = parse_codex_provider_catalog(payload) - assert isinstance(payload, dict), "Expected a Codex catalog object" - return CodexProviderCatalog(model_ids, (payload,)) + return CodexProviderCatalog(parse_codex_provider_catalog(payload)) def fetch_codex_parent_catalog( workspace: str, token: str, parent_schema: str ) -> CodexProviderCatalog: """Fetch the API-compatible models advertised for a Unity Catalog parent schema.""" - _validate_scope(parent_schema) - return _fetch_codex_catalog( - workspace, token, {MODEL_SERVICE_PARENT_SCHEMA_HEADER: parent_schema} + base_url = _validate_request_inputs(workspace, token, parent_schema) + payload = _get_json( + f"{base_url}{_CODEX_MODELS_PATH}", + { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + _PARENT_SCHEMA_HEADER: parent_schema, + }, ) + return CodexProviderCatalog(parse_codex_provider_catalog(payload)) diff --git a/tests/test_integration_provider_catalog.py b/tests/test_integration_provider_catalog.py index d5827c175..41aa1f66d 100644 --- a/tests/test_integration_provider_catalog.py +++ b/tests/test_integration_provider_catalog.py @@ -10,28 +10,6 @@ from tests.integration.utils import provider_catalog as catalog -def _mock_json_boundary(monkeypatch, get_json): - class CatalogOpener: - def open(self, request, *, timeout): - assert request.get_method() == "GET" - assert timeout == 30 - names = ( - "Authorization", - "Accept", - "Anthropic-Version", - "Databricks-Model-Provider-Service", - catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER, - ) - headers = { - name: request.get_header(name.capitalize()) - for name in names - if request.has_header(name.capitalize()) - } - return io.BytesIO(json.dumps(get_json(request.full_url, headers)).encode()) - - monkeypatch.setattr(catalog.urllib.request, "urlopen", CatalogOpener().open) - - def _page(*ids, has_more=False, last_id=None): return { "data": [{"id": model_id, "display_name": model_id} for model_id in ids], @@ -105,7 +83,7 @@ def get_json(url, headers): requests.append((url, headers)) return next(pages) - _mock_json_boundary(monkeypatch, get_json) + monkeypatch.setattr(catalog, "_get_json", get_json) result = catalog.fetch_anthropic_provider_catalog("https://workspace/", "token", "c.s.mps") assert result.model_ids == ("a", "b") assert result.display_names == {"a": "a", "b": "b"} @@ -115,8 +93,6 @@ def get_json(url, headers): for _, headers in requests: assert headers["Authorization"] == "Bearer token" assert headers["Databricks-Model-Provider-Service"] == "c.s.mps" - assert headers["Anthropic-Version"] == "2023-06-01" - assert catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER not in headers pages = iter([_page("a", has_more=True, last_id="a"), _page("a")]) with pytest.raises(AssertionError, match="repeated model id"): @@ -127,14 +103,14 @@ def test_anthropic_provider_fetch_rejects_repeated_cursor(monkeypatch): pages = iter( [_page("a", has_more=True, last_id="cursor"), _page("b", has_more=True, last_id="cursor")] ) - _mock_json_boundary(monkeypatch, lambda *args: next(pages)) + monkeypatch.setattr(catalog, "_get_json", lambda *args: next(pages)) with pytest.raises(AssertionError, match="repeated pagination cursor"): catalog.fetch_anthropic_provider_catalog("https://workspace", "token", "c.s.mps") def test_anthropic_provider_fetch_has_a_page_bound(monkeypatch): pages = iter(_page(str(i), has_more=True, last_id=str(i)) for i in range(20)) - _mock_json_boundary(monkeypatch, lambda *args: next(pages)) + monkeypatch.setattr(catalog, "_get_json", lambda *args: next(pages)) with pytest.raises(AssertionError, match="exceeded 20 pages"): catalog.fetch_anthropic_provider_catalog("https://workspace", "token", "c.s.mps") @@ -151,7 +127,6 @@ def urlopen(request, timeout): assert request.full_url == "https://workspace/ai-gateway/codex/v1/models" assert request.get_header("Authorization") == "Bearer token" assert request.get_header("Databricks-model-provider-service") == "c.s.mps" - assert not request.has_header(catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER.capitalize()) assert timeout == 30 return Response(json.dumps(payload).encode()) @@ -169,7 +144,7 @@ def get_json(url, headers): assert "Databricks-Model-Provider-Service" not in headers return {"models": [{"slug": model, "visibility": "list"} for model in ids]} - _mock_json_boundary(monkeypatch, get_json) + monkeypatch.setattr(catalog, "_get_json", get_json) result = catalog.fetch_codex_parent_catalog("https://workspace/", "token", "c.s") assert result.model_ids == tuple(ids) @@ -182,43 +157,3 @@ def urlopen(request, timeout): monkeypatch.setattr(catalog.urllib.request, "urlopen", urlopen) with pytest.raises(AssertionError, match=f"HTTP {status}"): catalog.fetch_codex_provider_catalog("https://workspace", "token", "c.s.mps") - - -def test_anthropic_parent_catalog_retains_pages_labels_and_scoped_requests(monkeypatch): - pages = [ - { - "data": [{"id": "catalog.models.claude_sonnet", "display_name": "Sonnet"}], - "has_more": True, - "last_id": "cursor with / and ?", - }, - { - "data": [{"id": "anthropic-aigw-12345678-catalog.models.kimi"}], - "has_more": False, - }, - ] - requests = [] - - class CatalogOpener: - def open(self, request, *, timeout): - requests.append(request) - assert timeout == 30 - assert request.get_header("Authorization") == "Bearer test-bearer" - assert ( - request.get_header(catalog.MODEL_SERVICE_PARENT_SCHEMA_HEADER.capitalize()) - == "catalog.models" - ) - assert request.get_header("Anthropic-version") == "2023-06-01" - assert not request.has_header("Databricks-model-provider-service") - return io.BytesIO(json.dumps(pages[len(requests) - 1]).encode()) - - monkeypatch.setattr(catalog.urllib.request, "urlopen", CatalogOpener().open) - result = catalog.fetch_anthropic_parent_catalog( - "https://workspace.invalid/", "test-bearer", "catalog.models" - ) - assert result.payloads == tuple(pages) - assert result.model_ids == (pages[0]["data"][0]["id"], pages[1]["data"][0]["id"]) - assert result.display_names == {result.model_ids[0]: "Sonnet", result.model_ids[1]: None} - assert parse_qs(urlparse(requests[1].full_url).query) == { - "limit": ["1000"], - "after_id": ["cursor with / and ?"], - } From b10f0219d339422fb9699a493f986e8334ffb815 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Wed, 7 Oct 2026 02:55:57 +0000 Subject: [PATCH 16/19] Isolate catalog CUJs and verify live inference evidence --- tests/e2e_cuj/README.md | 15 ++- tests/e2e_cuj/test_catalog_discovery.py | 164 +++++++++++++++++------- 2 files changed, 129 insertions(+), 50 deletions(-) diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index 9f5ce9141..d57d1deae 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -22,17 +22,18 @@ Ten independently configured cases cover picker discovery, default launches, and Picker cases dismiss the menu without changing selection, then complete a task on the default. Separate cases cover bare `ug`, `ug claude`, and `ug codex` TUI first tasks and Claude print/Codex exec defaults. Each additional compatible model gets its own headless task case. -Success depends on native behavior, not generated settings, managed-config JSON, or catalog caches. -Defaults omit model overrides; expected answers are withheld from prompts. Claude headless results -require the requested service/alias in `modelUsage` with nonzero output tokens; TUI cases check the -selected default in the native banner and a completed assistant answer. Claude transcript model IDs -name the backing model, not the service. Codex evidence joins the completed answer to its -client-selected turn model. None proves the gateway's backing destination. Only live passes +Success requires a completed native task and successful recorded gateway inference for the +expected service/alias, not generated settings or catalog caches. Defaults omit model overrides; +expected answers are withheld from prompts. Claude headless results also require nonzero output +tokens in `modelUsage`; Codex joins the completed answer to its client-selected turn model. +Claude transcript model IDs name the backing model, not the service. TUI tasks reject unexpected +permission prompts. These checks do not prove the gateway's backing destination. Only live passes establish coverage. The test class selects the CUJ3 workspace, `https://dbc-bbdd5508-648e.cloud.databricks.com`. The shared `cuj` fixture supplies its authenticated SDK client and isolated local session; -workspace configuration remains read-only. Set `UG_CUJ_SP_CLIENT_ID` and +workspace configuration remains read-only. Each case cleans up with public `ug revert` before +the next case configures, including after failures. Set `UG_CUJ_SP_CLIENT_ID` and `UG_CUJ_SP_CLIENT_SECRET` before a live run. CI discovers these tests through the shared `dedicated-cuj` job, which installs both diff --git a/tests/e2e_cuj/test_catalog_discovery.py b/tests/e2e_cuj/test_catalog_discovery.py index cad9f6a5a..013bfbcf3 100644 --- a/tests/e2e_cuj/test_catalog_discovery.py +++ b/tests/e2e_cuj/test_catalog_discovery.py @@ -12,7 +12,7 @@ parse_anthropic_provider_page, parse_codex_provider_catalog, ) -from tests.integration.utils.terminal import AgentTerminal +from tests.integration.utils.terminal import AgentTerminal, TerminalProcess from .base import BaseCujTest from .catalog_discovery_expectations import ( @@ -26,7 +26,9 @@ MODEL_SCHEMA, OTHER_MODEL_SCHEMA, ) -from .helpers.constants import CLAUDE, CODEX +from .helpers.constants import CLAUDE, CODEX, INFERENCE_PATHS, MANAGED_PATHS +from .helpers.evidence import SessionEvidence, message_text +from .helpers.terminal import Terminal pytestmark = [pytest.mark.managed, pytest.mark.catalog_discovery, pytest.mark.workspace_isolated] @@ -89,6 +91,64 @@ def _assert_claude_headless_model(result, expected): assert usage[expected]["outputTokens"] > 0, usage +def _request_contains_task(request, agent, task): + field = "messages" if agent == CLAUDE else "input" + entries = request.payload.get(field) + if not isinstance(entries, (list, str)): + return False + if isinstance(entries, str): + return task.prompt in entries + return any( + task.prompt in message_text(entry.get("content", entry.get("text", ""))) + for entry in entries + if isinstance(entry, dict) + ) + + +def _assert_inference_evidence(recorder, checkpoint, agent, task, expected): + expected_wire_model = claude.discovery_model_id(expected) if agent == CLAUDE else expected + requests = recorder.requests_after(checkpoint) + inference_requests = [ + request + for request in requests + if request.method == "POST" and request.path == INFERENCE_PATHS[agent] + ] + assert inference_requests, { + "agent": agent, + "path": INFERENCE_PATHS[agent], + "requests": [(request.method, request.path) for request in requests], + } + task_requests = [ + request for request in inference_requests if _request_contains_task(request, agent, task) + ] + assert task_requests, "No inference request contained the submitted task prompt" + for request in task_requests: + assert request.payload["model"] == expected_wire_model, request.payload + response = recorder.response_for(request, timeout=240) + assert response.status_code == 200, response.status_code + assert response.body, "Inference response was empty" + + +@pytest.fixture(autouse=True) +def _revert_catalog_test_state(cuj): + yield + session, _, _ = cuj + state_dir = session.home / ".ucode" + if any( + (state_dir / name).is_file() for name in ("state.json", "managed-backups/manifest.json") + ): + with TerminalProcess( + session, + "ug", + [str(session.binary), "revert"], + "catalog-discovery-cleanup-revert", + ) as terminal: + terminal.finish() + assert not any(path.exists() for path in MANAGED_PATHS), ( + "Catalog CUJ teardown left machine-wide agent settings" + ) + + class TestCatalogDiscovery(BaseCujTest): WORKSPACE_URL = "https://dbc-bbdd5508-648e.cloud.databricks.com" @@ -100,8 +160,8 @@ def test_catalog_discovery_claude_picker_preserves_default(self, cuj): Expected: only Sonnet/Haiku/Kimi appear; Gemini and decoys are absent. Dismissing the picker without a selection preserves Sonnet for the completed task. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, workspace, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) parent_catalog = _catalog_display_names(workspace, CLAUDE, MODEL_SCHEMA) decoy_catalog = _catalog_display_names(workspace, CLAUDE, OTHER_MODEL_SCHEMA) @@ -111,12 +171,10 @@ def test_catalog_discovery_claude_picker_preserves_default(self, cuj): assert set(decoy_catalog) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog task = _claude_file_task(session) - with AgentTerminal( - session, - CLAUDE, - [str(session.binary), CLAUDE], - "catalog-discovery-claude-picker", - ) as tui: + evidence = SessionEvidence(session.home, CLAUDE) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "catalog-discovery-claude-picker", [CLAUDE]) as tui: tui.boot() picker_screen = tui.open_model_picker( model_visible=lambda screen: all( @@ -130,9 +188,10 @@ def test_catalog_discovery_claude_picker_preserves_default(self, cuj): assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) + tui.task(evidence, task) tui.exit_normally() task.assert_completed(session, CLAUDE) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, CLAUDE_DEFAULT) session.assert_not_routed() @pytest.mark.codex @@ -143,8 +202,8 @@ def test_catalog_discovery_codex_picker_preserves_default(self, cuj): Expected: model/list and the picker expose only GPT Luna/Kimi, not Gemini or decoys. Dismissing the picker preserves GPT Luna for the completed task's selected model. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, workspace, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) parent_catalog = _catalog_display_names(workspace, CODEX, MODEL_SCHEMA) decoy_catalog = _catalog_display_names(workspace, CODEX, OTHER_MODEL_SCHEMA) @@ -157,9 +216,10 @@ def test_catalog_discovery_codex_picker_preserves_default(self, cuj): display_names = parent_catalog task = FileTask(session) - with AgentTerminal( - session, CODEX, [str(session.binary), CODEX], "catalog-discovery-codex-picker" - ) as tui: + evidence = SessionEvidence(session.home, CODEX) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "catalog-discovery-codex-picker", [CODEX]) as tui: tui.boot() picker_screen = tui.open_codex_model_picker( model_visible=lambda screen: all( @@ -172,9 +232,10 @@ def test_catalog_discovery_codex_picker_preserves_default(self, cuj): assert excluded not in picker_screen, picker_screen assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) + tui.task(evidence, task) tui.exit_normally() assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) + _assert_inference_evidence(recorder, checkpoint, CODEX, task, CODEX_DEFAULT) session.assert_not_routed() @pytest.mark.claude @@ -184,18 +245,22 @@ def test_catalog_discovery_bare_ug_uses_claude_default(self, cuj): Expected: Claude completes the task on Sonnet without an agent or model override. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) task = _claude_file_task(session) + evidence = SessionEvidence(session.home, CLAUDE) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() with AgentTerminal( session, CLAUDE, [str(session.binary)], "catalog-discovery-bare-ug-default" ) as tui: tui.boot() assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) + Terminal.task(tui, evidence, task) tui.exit_normally() task.assert_completed(session, CLAUDE) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, CLAUDE_DEFAULT) session.assert_not_routed() @pytest.mark.claude @@ -205,21 +270,20 @@ def test_catalog_discovery_claude_tui_uses_default(self, cuj): Expected: Claude completes the task on the configured Sonnet default without overrides. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) task = _claude_file_task(session) - with AgentTerminal( - session, - CLAUDE, - [str(session.binary), CLAUDE], - "catalog-discovery-claude-tui-default", - ) as tui: + evidence = SessionEvidence(session.home, CLAUDE) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "catalog-discovery-claude-tui-default", [CLAUDE]) as tui: tui.boot() assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) + tui.task(evidence, task) tui.exit_normally() task.assert_completed(session, CLAUDE) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, CLAUDE_DEFAULT) session.assert_not_routed() @pytest.mark.codex @@ -229,17 +293,19 @@ def test_catalog_discovery_codex_tui_uses_default(self, cuj): Expected: Codex completes the task with GPT Luna selected, without a model override. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) task = FileTask(session) - with AgentTerminal( - session, CODEX, [str(session.binary), CODEX], "catalog-discovery-codex-tui-default" - ) as tui: + evidence = SessionEvidence(session.home, CODEX) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "catalog-discovery-codex-tui-default", [CODEX]) as tui: tui.boot() tui.submit(task.prompt) - tui.wait_for_task(task, timeout=240) + tui.task(evidence, task) tui.exit_normally() assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) + _assert_inference_evidence(recorder, checkpoint, CODEX, task, CODEX_DEFAULT) session.assert_not_routed() @pytest.mark.claude @@ -248,9 +314,11 @@ def test_catalog_discovery_claude_headless_uses_default(self, cuj): Expected: the print task completes with a Sonnet response. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) task = _claude_file_task(session) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() result = session.run( CLAUDE, "-p", @@ -263,6 +331,7 @@ def test_catalog_discovery_claude_headless_uses_default(self, cuj): ) task.assert_headless_answer(CLAUDE, result) _assert_claude_headless_model(result, CLAUDE_DEFAULT) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, CLAUDE_DEFAULT) session.assert_not_routed() @pytest.mark.codex @@ -271,14 +340,17 @@ def test_catalog_discovery_codex_headless_uses_default(self, cuj): Expected: the exec task completes with GPT Luna selected. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) task = FileTask(session) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() result = session.run( CODEX, "--", "exec", "--skip-git-repo-check", "--json", task.prompt, timeout=240 ) task.assert_headless_answer(CODEX, result) assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) + _assert_inference_evidence(recorder, checkpoint, CODEX, task, CODEX_DEFAULT) session.assert_not_routed() @pytest.mark.claude @@ -288,9 +360,11 @@ def test_catalog_discovery_claude_explicit_model_completes_task(self, cuj, model Expected: each Haiku/Kimi task completes with the explicitly requested model. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) task = _claude_file_task(session) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() result = session.run( CLAUDE, "--", @@ -306,6 +380,7 @@ def test_catalog_discovery_claude_explicit_model_completes_task(self, cuj, model ) task.assert_headless_answer(CLAUDE, result) _assert_claude_headless_model(result, claude.discovery_model_id(model)) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, model) session.assert_not_routed() @pytest.mark.codex @@ -315,9 +390,11 @@ def test_catalog_discovery_codex_explicit_model_completes_task(self, cuj, model) Expected: the Kimi task completes with the explicitly requested model selected. """ - session, workspace, _recorder = cuj - session.configure(["configure", "--workspace", workspace.url, "--skip-upgrade"]) + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) task = FileTask(session) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() result = session.run( CODEX, "--", @@ -331,4 +408,5 @@ def test_catalog_discovery_codex_explicit_model_completes_task(self, cuj, model) ) task.assert_headless_answer(CODEX, result) assert_completed_task_model(session, CODEX, task.value, model) + _assert_inference_evidence(recorder, checkpoint, CODEX, task, model) session.assert_not_routed() From 758a34d156de66a1a3bd846ca7e586f9f43bcc11 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Wed, 7 Oct 2026 03:22:18 +0000 Subject: [PATCH 17/19] Match Claude catalog display labels and default picker control --- tests/e2e_cuj/README.md | 2 + tests/e2e_cuj/test_catalog_discovery.py | 17 ++++-- tests/integration/utils/model_discovery.py | 5 ++ tests/test_integration_model_discovery.py | 61 ++++++++++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index d57d1deae..52f5a9fbb 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -14,6 +14,8 @@ and tracing are disabled. Tests neither provision nor validate fixture definitio | `gemini_flash` | Excluded | Excluded | Independent gateway catalogs, Codex's native model/list, and exact TUI picker inventories must agree. +Claude's built-in Default action is not a catalog entry; banners use catalog display names, +while gateway inference checks require the exact service/alias. Compatible `ug_e2e.other_models.claude_decoy` and `codex_decoy` must be discoverable in their own schema but absent from the scoped picker. Claude discovery aliases must be enabled; Kimi uses `anthropic-aigw-<8-character SHA-256 prefix>-`. diff --git a/tests/e2e_cuj/test_catalog_discovery.py b/tests/e2e_cuj/test_catalog_discovery.py index 013bfbcf3..8a727ce58 100644 --- a/tests/e2e_cuj/test_catalog_discovery.py +++ b/tests/e2e_cuj/test_catalog_discovery.py @@ -169,6 +169,7 @@ def test_catalog_discovery_claude_picker_preserves_default(self, cuj): claude.discovery_model_id(model) for model in CLAUDE_MODELS }, parent_catalog assert set(decoy_catalog) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog + default_display_name = parent_catalog[claude.discovery_model_id(CLAUDE_DEFAULT)] task = _claude_file_task(session) evidence = SessionEvidence(session.home, CLAUDE) @@ -186,7 +187,7 @@ def test_catalog_discovery_claude_picker_preserves_default(self, cuj): for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): assert excluded not in picker_screen, picker_screen assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen - assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible + assert f"{default_display_name} · API Usage Billing" in tui.visible, tui.visible tui.submit(task.prompt) tui.task(evidence, task) tui.exit_normally() @@ -245,8 +246,11 @@ def test_catalog_discovery_bare_ug_uses_claude_default(self, cuj): Expected: Claude completes the task on Sonnet without an agent or model override. """ - session, _, recorder = cuj + session, workspace, recorder = cuj recorder.configure_session(session, ["configure", "--skip-upgrade"]) + default_display_name = _catalog_display_names(workspace, CLAUDE, MODEL_SCHEMA)[ + claude.discovery_model_id(CLAUDE_DEFAULT) + ] task = _claude_file_task(session) evidence = SessionEvidence(session.home, CLAUDE) checkpoint = recorder.checkpoint() @@ -255,7 +259,7 @@ def test_catalog_discovery_bare_ug_uses_claude_default(self, cuj): session, CLAUDE, [str(session.binary)], "catalog-discovery-bare-ug-default" ) as tui: tui.boot() - assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible + assert f"{default_display_name} · API Usage Billing" in tui.visible, tui.visible tui.submit(task.prompt) Terminal.task(tui, evidence, task) tui.exit_normally() @@ -270,15 +274,18 @@ def test_catalog_discovery_claude_tui_uses_default(self, cuj): Expected: Claude completes the task on the configured Sonnet default without overrides. """ - session, _, recorder = cuj + session, workspace, recorder = cuj recorder.configure_session(session, ["configure", "--skip-upgrade"]) + default_display_name = _catalog_display_names(workspace, CLAUDE, MODEL_SCHEMA)[ + claude.discovery_model_id(CLAUDE_DEFAULT) + ] task = _claude_file_task(session) evidence = SessionEvidence(session.home, CLAUDE) checkpoint = recorder.checkpoint() recorder.prepare_launch() with Terminal(session, "catalog-discovery-claude-tui-default", [CLAUDE]) as tui: tui.boot() - assert f"{CLAUDE_DEFAULT} · api usage billing" in tui.visible.casefold(), tui.visible + assert f"{default_display_name} · API Usage Billing" in tui.visible, tui.visible tui.submit(task.prompt) tui.task(evidence, task) tui.exit_normally() diff --git a/tests/integration/utils/model_discovery.py b/tests/integration/utils/model_discovery.py index 941337627..926dc063b 100644 --- a/tests/integration/utils/model_discovery.py +++ b/tests/integration/utils/model_discovery.py @@ -10,6 +10,9 @@ "claude-opus-5": ("Opus", "Opus 5"), "claude-sonnet-5": ("Sonnet", "Sonnet 5"), } +_CLAUDE_DEFAULT_PICKER_ROW = re.compile( + r"^Default(?: \(recommended\))?[ \t]+Use the default model\b" +) _SYSTEM_AI_PREFIX = "system.ai." @@ -103,6 +106,8 @@ def assert_picker_inventory(screen: str, agent: str, display_names: dict[str, st assert rows, f"No numbered {agent} picker rows:\n{screen}" observed: list[str] = [] for label in rows: + if agent == "claude" and _CLAUDE_DEFAULT_PICKER_ROW.match(label): + continue matched = { model for model, display_name in display_names.items() diff --git a/tests/test_integration_model_discovery.py b/tests/test_integration_model_discovery.py index 05d91e083..d9e5ec849 100644 --- a/tests/test_integration_model_discovery.py +++ b/tests/test_integration_model_discovery.py @@ -17,6 +17,11 @@ "codex": frozenset({"catalog.models.gpt_luna", "catalog.models.kimi"}), } +CLAUDE_DEFAULT_CONTROL_ROW = ( + "Default (recommended) Use the default model " + "(currently Opus 5.5 (1M context)) · $4/$20 per Mtok" +) + @pytest.mark.parametrize("cursor", ["", "❯ ", "› ", "> "]) @pytest.mark.parametrize("label", ["system.ai.claude-sonnet-5", "Sonnet Custom"]) @@ -210,3 +215,59 @@ def test_picker_inventory_rejects_duplicate_and_missing_rows(agent): assert_picker_inventory(f"{rows}\n {len(models) + 1}. {models[0]}", agent, labels) with pytest.raises(AssertionError): assert_picker_inventory(f" 1. {models[0]}", agent, labels) + + +@pytest.mark.parametrize( + "control_row", + [ + CLAUDE_DEFAULT_CONTROL_ROW, + "Default Use the default model (currently Sonnet 5 (200k context)) · $3/$15 per Mtok", + "Default (recommended) Use the default model (currently Haiku 4.5 (200k context)) · $1/$5 per Mtok", + ], +) +def test_picker_inventory_ignores_claude_default_control_row(control_row): + models = sorted(PICKER_MODELS["claude"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"{rows}\n {len(models) + 1}. {control_row}" + assert_picker_inventory(screen, "claude", labels) + + +def test_picker_inventory_requires_missing_claude_models_with_default_control_row(): + models = sorted(PICKER_MODELS["claude"]) + labels = dict.fromkeys(models, None) + rows = "\n".join( + f" {position}. {model}" for position, model in enumerate(models[:-1], start=1) + ) + screen = f"{rows}\n {len(models)}. {CLAUDE_DEFAULT_CONTROL_ROW}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, "claude", labels) + + +@pytest.mark.parametrize("default_label", ["Default", "Default (recommended)"]) +def test_picker_inventory_rejects_generic_claude_default_label(default_label): + models = sorted(PICKER_MODELS["claude"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"{rows}\n {len(models) + 1}. {default_label}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, "claude", labels) + + +@pytest.mark.parametrize("extra_label", ["Opus (1M context) Opus 5.5", "Gemini 3 Flash"]) +def test_picker_inventory_rejects_extra_claude_native_rows(extra_label): + models = sorted(PICKER_MODELS["claude"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"{rows}\n {len(models) + 1}. {extra_label}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, "claude", labels) + + +def test_picker_inventory_does_not_ignore_claude_default_control_row_for_codex(): + models = sorted(PICKER_MODELS["codex"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"{rows}\n {len(models) + 1}. {CLAUDE_DEFAULT_CONTROL_ROW}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, "codex", labels) From 3d0e56072ea80d06f472d06e65b93705a63c213c Mon Sep 17 00:00:00 2001 From: Tien Le Date: Wed, 7 Oct 2026 05:12:29 +0000 Subject: [PATCH 18/19] Correlate catalog inference with exact user prompts --- tests/e2e_cuj/test_catalog_discovery.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/e2e_cuj/test_catalog_discovery.py b/tests/e2e_cuj/test_catalog_discovery.py index 8a727ce58..f77c8c236 100644 --- a/tests/e2e_cuj/test_catalog_discovery.py +++ b/tests/e2e_cuj/test_catalog_discovery.py @@ -97,12 +97,18 @@ def _request_contains_task(request, agent, task): if not isinstance(entries, (list, str)): return False if isinstance(entries, str): - return task.prompt in entries - return any( - task.prompt in message_text(entry.get("content", entry.get("text", ""))) - for entry in entries - if isinstance(entry, dict) - ) + return entries == task.prompt + for entry in entries: + if not isinstance(entry, dict) or entry.get("role") != "user": + continue + content = entry.get("content", "") + if isinstance(content, str) and content == task.prompt: + return True + if isinstance(content, list) and any( + message_text([part]) == task.prompt for part in content if isinstance(part, dict) + ): + return True + return False def _assert_inference_evidence(recorder, checkpoint, agent, task, expected): @@ -125,7 +131,11 @@ def _assert_inference_evidence(recorder, checkpoint, agent, task, expected): for request in task_requests: assert request.payload["model"] == expected_wire_model, request.payload response = recorder.response_for(request, timeout=240) - assert response.status_code == 200, response.status_code + assert response.status_code == 200, { + "status": response.status_code, + "model": request.payload["model"], + "output_config": request.payload.get("output_config"), + } assert response.body, "Inference response was empty" From c8de9981144325aa67754ad5e3bdab3f307aa479 Mon Sep 17 00:00:00 2001 From: Tien Le Date: Wed, 7 Oct 2026 05:30:58 +0000 Subject: [PATCH 19/19] Remove unused agent catalog fetch aliases --- tests/integration/utils/agents/claude.py | 2 -- tests/integration/utils/agents/codex.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/tests/integration/utils/agents/claude.py b/tests/integration/utils/agents/claude.py index 4846ceaaa..7947b8b26 100644 --- a/tests/integration/utils/agents/claude.py +++ b/tests/integration/utils/agents/claude.py @@ -2,7 +2,6 @@ from ..model_discovery import claude_discovery_model_id as discovery_model_id from ..model_discovery import claude_model_in_picker as model_in_picker -from ..provider_catalog import fetch_anthropic_parent_catalog as fetch_parent_catalog from . import _evidence_id __all__ = [ @@ -10,7 +9,6 @@ "assistant_answers", "completed_task_models", "discovery_model_id", - "fetch_parent_catalog", "model_in_picker", ] diff --git a/tests/integration/utils/agents/codex.py b/tests/integration/utils/agents/codex.py index 99e505420..ed3ccc902 100644 --- a/tests/integration/utils/agents/codex.py +++ b/tests/integration/utils/agents/codex.py @@ -1,14 +1,12 @@ """Codex-specific integration evidence and model discovery helpers.""" from ..model_discovery import codex_model_in_picker as model_in_picker -from ..provider_catalog import fetch_codex_parent_catalog as fetch_parent_catalog from . import _evidence_id __all__ = [ "EVIDENCE_KIND", "assistant_answers", "completed_task_models", - "fetch_parent_catalog", "model_in_picker", ]