diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md new file mode 100644 index 000000000..52f5a9fbb --- /dev/null +++ b/tests/e2e_cuj/README.md @@ -0,0 +1,57 @@ +# Catalog discovery CUJ + +The dedicated workspace must already publish a CodingAgentConfig enabling both agents +with `ug_e2e.models` as their model source. Claude is the default agent; its default and +Sonnet-family model are `claude_sonnet`, and Codex defaults to `gpt_luna`. Smart routing +and tracing are disabled. Tests neither provision nor validate fixture definitions. + +| Service under `ug_e2e.models` | Claude Code | Codex | +| --- | --- | --- | +| `gpt_luna` | Excluded | Included; default | +| `claude_haiku` | Included | Excluded | +| `claude_sonnet` | Included; default | Excluded | +| `kimi` | Included | Included | +| `gemini_flash` | Excluded | Excluded | + +Independent gateway catalogs, Codex's native model/list, and exact TUI picker inventories must agree. +Claude's built-in Default action is not a catalog entry; banners use catalog display names, +while gateway inference checks require the exact service/alias. +Compatible `ug_e2e.other_models.claude_decoy` and `codex_decoy` must be discoverable in their +own schema but absent from the scoped picker. Claude discovery aliases must be enabled; +Kimi uses `anthropic-aigw-<8-character SHA-256 prefix>-`. + +Ten independently configured cases cover picker discovery, default launches, and explicit models. +Picker cases dismiss the menu without changing selection, then complete a task on the default. +Separate cases cover bare `ug`, `ug claude`, and `ug codex` TUI first tasks and Claude print/Codex +exec defaults. Each additional compatible model gets its own headless task case. +Success requires a completed native task and successful recorded gateway inference for the +expected service/alias, not generated settings or catalog caches. Defaults omit model overrides; +expected answers are withheld from prompts. Claude headless results also require nonzero output +tokens in `modelUsage`; Codex joins the completed answer to its client-selected turn model. +Claude transcript model IDs name the backing model, not the service. TUI tasks reject unexpected +permission prompts. These checks do not prove the gateway's backing destination. Only live passes +establish coverage. + +The test class selects the CUJ3 workspace, `https://dbc-bbdd5508-648e.cloud.databricks.com`. +The shared `cuj` fixture supplies its authenticated SDK client and isolated local session; +workspace configuration remains read-only. Each case cleans up with public `ug revert` before +the next case configures, including after failures. Set `UG_CUJ_SP_CLIENT_ID` and +`UG_CUJ_SP_CLIENT_SECRET` before a live run. + +CI discovers these tests through the shared `dedicated-cuj` job, which installs both +pinned agents. No catalog-specific workflow or workspace secret is needed. Locally, +install ug, both agents, and the Databricks CLI on a clean POSIX host, then run: + +```bash +uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest \ + --confcutdir=tests/e2e_cuj tests/e2e_cuj/test_catalog_discovery.py -v +``` + +MCP/skills discovery remains separate. + +Collection only (no authentication or inference): + +```bash +uv run pytest -c tests/e2e_cuj/pytest.ini --confcutdir=tests/e2e_cuj \ + --collect-only tests/e2e_cuj +``` diff --git a/tests/e2e_cuj/catalog_discovery_expectations.py b/tests/e2e_cuj/catalog_discovery_expectations.py new file mode 100644 index 000000000..59e54c532 --- /dev/null +++ b/tests/e2e_cuj/catalog_discovery_expectations.py @@ -0,0 +1,11 @@ +"""Model fixtures owned by the catalog discovery journey.""" + +MODEL_SCHEMA = "ug_e2e.models" +OTHER_MODEL_SCHEMA = "ug_e2e.other_models" +CLAUDE_DEFAULT = f"{MODEL_SCHEMA}.claude_sonnet" +CODEX_DEFAULT = f"{MODEL_SCHEMA}.gpt_luna" +GEMINI_MODEL = f"{MODEL_SCHEMA}.gemini_flash" +CLAUDE_MODELS = frozenset({CLAUDE_DEFAULT, f"{MODEL_SCHEMA}.claude_haiku", f"{MODEL_SCHEMA}.kimi"}) +CODEX_MODELS = frozenset({CODEX_DEFAULT, f"{MODEL_SCHEMA}.kimi"}) +CLAUDE_DECOY = f"{OTHER_MODEL_SCHEMA}.claude_decoy" +CODEX_DECOY = f"{OTHER_MODEL_SCHEMA}.codex_decoy" diff --git a/tests/e2e_cuj/pytest.ini b/tests/e2e_cuj/pytest.ini new file mode 100644 index 000000000..008cc4a9e --- /dev/null +++ b/tests/e2e_cuj/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +testpaths = . +pythonpath = ../.. +addopts = --strict-markers --tb=short +markers = + managed: requires an admin CodingAgentConfig published on the selected workspace + catalog_discovery: managed schema-scoped model discovery and inference + workspace_isolated: reads preconfigured CUJ workspace state with isolated local sessions + tui: real interactive terminal and completed task + claude: only runs when Claude Code is explicitly selected + codex: only runs when Codex is explicitly selected diff --git a/tests/e2e_cuj/test_catalog_discovery.py b/tests/e2e_cuj/test_catalog_discovery.py new file mode 100644 index 000000000..f77c8c236 --- /dev/null +++ b/tests/e2e_cuj/test_catalog_discovery.py @@ -0,0 +1,429 @@ +"""Managed catalog discovery, default launches, and explicit model selection.""" + +import json + +import pytest + +from tests.integration.utils.agents import claude, codex +from tests.integration.utils.evidence import FileTask, assert_completed_task_model +from tests.integration.utils.model_discovery import assert_picker_inventory +from tests.integration.utils.provider_catalog import ( + MODEL_SERVICE_PARENT_SCHEMA_HEADER, + parse_anthropic_provider_page, + parse_codex_provider_catalog, +) +from tests.integration.utils.terminal import AgentTerminal, TerminalProcess + +from .base import BaseCujTest +from .catalog_discovery_expectations import ( + CLAUDE_DECOY, + CLAUDE_DEFAULT, + CLAUDE_MODELS, + CODEX_DECOY, + CODEX_DEFAULT, + CODEX_MODELS, + GEMINI_MODEL, + MODEL_SCHEMA, + OTHER_MODEL_SCHEMA, +) +from .helpers.constants import CLAUDE, CODEX, INFERENCE_PATHS, MANAGED_PATHS +from .helpers.evidence import SessionEvidence, message_text +from .helpers.terminal import Terminal + +pytestmark = [pytest.mark.managed, pytest.mark.catalog_discovery, pytest.mark.workspace_isolated] + + +def _catalog_display_names(workspace, agent, schema): + """Read a scoped catalog; Workspace.model_ids only supports system.ai.""" + headers = {MODEL_SERVICE_PARENT_SCHEMA_HEADER: schema} + if agent == CODEX: + payload = workspace.client.api_client.do( + "GET", "/ai-gateway/codex/v1/models", headers=headers + ) + models = parse_codex_provider_catalog(payload) + return { + entry["slug"]: entry.get("display_name") or entry["slug"] + for entry in payload["models"] + if entry["slug"] in models + } + assert agent == CLAUDE, agent + headers["Anthropic-Version"] = "2023-06-01" + models, seen, cursor = {}, set(), None + for _page in range(20): + query = {"limit": "1000"} + if cursor: + query["after_id"] = cursor + payload = workspace.client.api_client.do( + "GET", "/ai-gateway/anthropic/v1/models", headers=headers, query=query + ) + page = parse_anthropic_provider_page(payload) + for model, display_name in page.models: + assert model not in models, f"Repeated catalog model: {model}" + models[model] = display_name + if not page.has_more: + return models + cursor = page.last_id + assert cursor and cursor not in seen, "Repeated catalog pagination cursor" + seen.add(cursor) + raise AssertionError("Anthropic catalog exceeded 20 pages") + + +def _claude_file_task(session): + task = FileTask(session) + task.prompt = ( + f"Use the Read tool to read {session.cwd / task.filename}. Reply with only its contents." + ) + return task + + +def _assert_claude_headless_model(result, expected): + final = None + for line in result.stdout.splitlines(): + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict) and payload.get("type") == "result": + final = payload + assert final is not None and not final.get("is_error"), result.stdout + usage = final["modelUsage"] + assert set(usage) == {expected}, {"expected": expected, "observed": sorted(usage)} + assert usage[expected]["outputTokens"] > 0, usage + + +def _request_contains_task(request, agent, task): + field = "messages" if agent == CLAUDE else "input" + entries = request.payload.get(field) + if not isinstance(entries, (list, str)): + return False + if isinstance(entries, str): + return entries == task.prompt + for entry in entries: + if not isinstance(entry, dict) or entry.get("role") != "user": + continue + content = entry.get("content", "") + if isinstance(content, str) and content == task.prompt: + return True + if isinstance(content, list) and any( + message_text([part]) == task.prompt for part in content if isinstance(part, dict) + ): + return True + return False + + +def _assert_inference_evidence(recorder, checkpoint, agent, task, expected): + expected_wire_model = claude.discovery_model_id(expected) if agent == CLAUDE else expected + requests = recorder.requests_after(checkpoint) + inference_requests = [ + request + for request in requests + if request.method == "POST" and request.path == INFERENCE_PATHS[agent] + ] + assert inference_requests, { + "agent": agent, + "path": INFERENCE_PATHS[agent], + "requests": [(request.method, request.path) for request in requests], + } + task_requests = [ + request for request in inference_requests if _request_contains_task(request, agent, task) + ] + assert task_requests, "No inference request contained the submitted task prompt" + for request in task_requests: + assert request.payload["model"] == expected_wire_model, request.payload + response = recorder.response_for(request, timeout=240) + assert response.status_code == 200, { + "status": response.status_code, + "model": request.payload["model"], + "output_config": request.payload.get("output_config"), + } + assert response.body, "Inference response was empty" + + +@pytest.fixture(autouse=True) +def _revert_catalog_test_state(cuj): + yield + session, _, _ = cuj + state_dir = session.home / ".ucode" + if any( + (state_dir / name).is_file() for name in ("state.json", "managed-backups/manifest.json") + ): + with TerminalProcess( + session, + "ug", + [str(session.binary), "revert"], + "catalog-discovery-cleanup-revert", + ) as terminal: + terminal.finish() + assert not any(path.exists() for path in MANAGED_PATHS), ( + "Catalog CUJ teardown left machine-wide agent settings" + ) + + +class TestCatalogDiscovery(BaseCujTest): + WORKSPACE_URL = "https://dbc-bbdd5508-648e.cloud.databricks.com" + + @pytest.mark.claude + @pytest.mark.tui + def test_catalog_discovery_claude_picker_preserves_default(self, cuj): + """Scenario: configure, launch ug claude, inspect its picker, then submit a task. + + Expected: only Sonnet/Haiku/Kimi appear; Gemini and decoys are absent. + Dismissing the picker without a selection preserves Sonnet for the completed task. + """ + session, workspace, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + + parent_catalog = _catalog_display_names(workspace, CLAUDE, MODEL_SCHEMA) + decoy_catalog = _catalog_display_names(workspace, CLAUDE, OTHER_MODEL_SCHEMA) + assert set(parent_catalog) == { + claude.discovery_model_id(model) for model in CLAUDE_MODELS + }, parent_catalog + assert set(decoy_catalog) == {claude.discovery_model_id(CLAUDE_DECOY)}, decoy_catalog + default_display_name = parent_catalog[claude.discovery_model_id(CLAUDE_DEFAULT)] + + task = _claude_file_task(session) + evidence = SessionEvidence(session.home, CLAUDE) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "catalog-discovery-claude-picker", [CLAUDE]) as tui: + tui.boot() + picker_screen = tui.open_model_picker( + model_visible=lambda screen: all( + claude.model_in_picker(screen, model, parent_catalog[model]) + for model in parent_catalog + ) + ) + assert_picker_inventory(picker_screen, CLAUDE, parent_catalog) + for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): + assert excluded not in picker_screen, picker_screen + assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen + assert f"{default_display_name} · API Usage Billing" in tui.visible, tui.visible + tui.submit(task.prompt) + tui.task(evidence, task) + tui.exit_normally() + task.assert_completed(session, CLAUDE) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, CLAUDE_DEFAULT) + session.assert_not_routed() + + @pytest.mark.codex + @pytest.mark.tui + def test_catalog_discovery_codex_picker_preserves_default(self, cuj): + """Scenario: configure, query Codex model/list, inspect its TUI picker, then submit a task. + + Expected: model/list and the picker expose only GPT Luna/Kimi, not Gemini or decoys. + Dismissing the picker preserves GPT Luna for the completed task's selected model. + """ + session, workspace, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + + parent_catalog = _catalog_display_names(workspace, CODEX, MODEL_SCHEMA) + decoy_catalog = _catalog_display_names(workspace, CODEX, OTHER_MODEL_SCHEMA) + assert set(parent_catalog) == CODEX_MODELS, parent_catalog + assert set(decoy_catalog) == {CODEX_DECOY}, decoy_catalog + + models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) + assert len(models) == len(set(models)), models + assert set(models) == CODEX_MODELS, models + display_names = parent_catalog + + task = FileTask(session) + evidence = SessionEvidence(session.home, CODEX) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "catalog-discovery-codex-picker", [CODEX]) as tui: + tui.boot() + picker_screen = tui.open_codex_model_picker( + model_visible=lambda screen: all( + codex.model_in_picker(screen, model, display_names[model]) + for model in parent_catalog + ) + ) + assert_picker_inventory(picker_screen, CODEX, display_names) + for excluded in (GEMINI_MODEL, CLAUDE_DECOY, CODEX_DECOY): + assert excluded not in picker_screen, picker_screen + assert excluded.rsplit(".", 1)[-1] not in picker_screen, picker_screen + tui.submit(task.prompt) + tui.task(evidence, task) + tui.exit_normally() + assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) + _assert_inference_evidence(recorder, checkpoint, CODEX, task, CODEX_DEFAULT) + session.assert_not_routed() + + @pytest.mark.claude + @pytest.mark.tui + def test_catalog_discovery_bare_ug_uses_claude_default(self, cuj): + """Scenario: configure the managed workspace, launch bare ug, and submit its first task. + + Expected: Claude completes the task on Sonnet without an agent or model override. + """ + session, workspace, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + default_display_name = _catalog_display_names(workspace, CLAUDE, MODEL_SCHEMA)[ + claude.discovery_model_id(CLAUDE_DEFAULT) + ] + task = _claude_file_task(session) + evidence = SessionEvidence(session.home, CLAUDE) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with AgentTerminal( + session, CLAUDE, [str(session.binary)], "catalog-discovery-bare-ug-default" + ) as tui: + tui.boot() + assert f"{default_display_name} · API Usage Billing" in tui.visible, tui.visible + tui.submit(task.prompt) + Terminal.task(tui, evidence, task) + tui.exit_normally() + task.assert_completed(session, CLAUDE) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, CLAUDE_DEFAULT) + session.assert_not_routed() + + @pytest.mark.claude + @pytest.mark.tui + def test_catalog_discovery_claude_tui_uses_default(self, cuj): + """Scenario: configure, launch ug claude, and submit its first task without opening a picker. + + Expected: Claude completes the task on the configured Sonnet default without overrides. + """ + session, workspace, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + default_display_name = _catalog_display_names(workspace, CLAUDE, MODEL_SCHEMA)[ + claude.discovery_model_id(CLAUDE_DEFAULT) + ] + task = _claude_file_task(session) + evidence = SessionEvidence(session.home, CLAUDE) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "catalog-discovery-claude-tui-default", [CLAUDE]) as tui: + tui.boot() + assert f"{default_display_name} · API Usage Billing" in tui.visible, tui.visible + tui.submit(task.prompt) + tui.task(evidence, task) + tui.exit_normally() + task.assert_completed(session, CLAUDE) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, CLAUDE_DEFAULT) + session.assert_not_routed() + + @pytest.mark.codex + @pytest.mark.tui + def test_catalog_discovery_codex_tui_uses_default(self, cuj): + """Scenario: configure, launch ug codex, and submit its first task without opening a picker. + + Expected: Codex completes the task with GPT Luna selected, without a model override. + """ + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + task = FileTask(session) + evidence = SessionEvidence(session.home, CODEX) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "catalog-discovery-codex-tui-default", [CODEX]) as tui: + tui.boot() + tui.submit(task.prompt) + tui.task(evidence, task) + tui.exit_normally() + assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) + _assert_inference_evidence(recorder, checkpoint, CODEX, task, CODEX_DEFAULT) + session.assert_not_routed() + + @pytest.mark.claude + def test_catalog_discovery_claude_headless_uses_default(self, cuj): + """Scenario: configure, then run ug claude -p with a file task and no model override. + + Expected: the print task completes with a Sonnet response. + """ + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + task = _claude_file_task(session) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + result = session.run( + CLAUDE, + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + timeout=240, + ) + task.assert_headless_answer(CLAUDE, result) + _assert_claude_headless_model(result, CLAUDE_DEFAULT) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, CLAUDE_DEFAULT) + session.assert_not_routed() + + @pytest.mark.codex + def test_catalog_discovery_codex_headless_uses_default(self, cuj): + """Scenario: configure, then run ug codex -- exec with a file task and no model override. + + Expected: the exec task completes with GPT Luna selected. + """ + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + task = FileTask(session) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + result = session.run( + CODEX, "--", "exec", "--skip-git-repo-check", "--json", task.prompt, timeout=240 + ) + task.assert_headless_answer(CODEX, result) + assert_completed_task_model(session, CODEX, task.value, CODEX_DEFAULT) + _assert_inference_evidence(recorder, checkpoint, CODEX, task, CODEX_DEFAULT) + session.assert_not_routed() + + @pytest.mark.claude + @pytest.mark.parametrize("model", sorted(CLAUDE_MODELS - {CLAUDE_DEFAULT})) + def test_catalog_discovery_claude_explicit_model_completes_task(self, cuj, model): + """Scenario: configure, then run a Claude print task with another compatible scoped model. + + Expected: each Haiku/Kimi task completes with the explicitly requested model. + """ + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + task = _claude_file_task(session) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + result = session.run( + CLAUDE, + "--", + "-p", + task.prompt, + "--output-format", + "json", + "--allowedTools", + "Read", + "--model", + claude.discovery_model_id(model), + timeout=240, + ) + task.assert_headless_answer(CLAUDE, result) + _assert_claude_headless_model(result, claude.discovery_model_id(model)) + _assert_inference_evidence(recorder, checkpoint, CLAUDE, task, model) + session.assert_not_routed() + + @pytest.mark.codex + @pytest.mark.parametrize("model", sorted(CODEX_MODELS - {CODEX_DEFAULT})) + def test_catalog_discovery_codex_explicit_model_completes_task(self, cuj, model): + """Scenario: configure, then run a Codex exec task with another compatible scoped model. + + Expected: the Kimi task completes with the explicitly requested model selected. + """ + session, _, recorder = cuj + recorder.configure_session(session, ["configure", "--skip-upgrade"]) + task = FileTask(session) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + result = session.run( + CODEX, + "--", + "exec", + "--skip-git-repo-check", + "--json", + task.prompt, + "--model", + model, + timeout=240, + ) + task.assert_headless_answer(CODEX, result) + assert_completed_task_model(session, CODEX, task.value, model) + _assert_inference_evidence(recorder, checkpoint, CODEX, task, model) + session.assert_not_routed() diff --git a/tests/integration/README.md b/tests/integration/README.md index 8cdd60186..2cb2bcbbb 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -9,6 +9,10 @@ is read-only and checked for changes at teardown; concurrent readers need no res CUJ2 adds three separately collected cases for exact MPS/MCP configuration, Codex inference, and Claude inference. +The [catalog discovery journey](../e2e_cuj/README.md) uses the CUJ3 workspace to check +agent-compatible pickers, schema exclusions, configured defaults, and real inference. +CI collects it through the shared `dedicated-cuj` job. + This suite runs the **installed product** through subprocesses, against the same `UCODE_TEST_WORKSPACE` used by the existing e2e tests. It does not import `ucode`, patch application functions, substitute agent executables, run a fake gateway, diff --git a/tests/integration/utils/agents/__init__.py b/tests/integration/utils/agents/__init__.py index 1a952fc10..ae5f60cc1 100644 --- a/tests/integration/utils/agents/__init__.py +++ b/tests/integration/utils/agents/__init__.py @@ -1 +1,10 @@ -"""Native agent transcript helpers used by the shared evidence layer.""" +"""Shared validation for agent-specific integration evidence.""" + +import re + + +def _evidence_id(value: object, field: str) -> str: + assert isinstance(value, str) and re.fullmatch(r"\S+", value), ( + f"Missing or malformed completed-task {field}" + ) + return value diff --git a/tests/integration/utils/agents/claude.py b/tests/integration/utils/agents/claude.py index 2c79f661d..7947b8b26 100644 --- a/tests/integration/utils/agents/claude.py +++ b/tests/integration/utils/agents/claude.py @@ -1,6 +1,19 @@ -"""Claude transcript helpers for integration journeys.""" +"""Claude-specific integration evidence and model discovery helpers.""" + +from ..model_discovery import claude_discovery_model_id as discovery_model_id +from ..model_discovery import claude_model_in_picker as model_in_picker +from . import _evidence_id + +__all__ = [ + "EVIDENCE_KIND", + "assistant_answers", + "completed_task_models", + "discovery_model_id", + "model_in_picker", +] SESSION_DIRECTORY = ".claude/projects" +EVIDENCE_KIND = "response-reported model" def is_child_session(path: str, records: list[dict]) -> bool: @@ -20,3 +33,29 @@ def assistant_answers(records: list[dict]) -> list[str]: if part.get("type") == "text" and isinstance(part.get("text"), str) ) return answers + + +def completed_task_models(records: list[dict], answer_value: str) -> set[str]: + """Parse Claude's response-reported model on the assistant's matching answer.""" + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) + found: set[str] = set() + for record in records: + if record.get("type") != "assistant": + continue + message = record.get("message") + assert isinstance(message, dict), "Malformed Claude assistant message" + assert message.get("role") == "assistant", "Missing Claude assistant role" + content = message.get("content") + assert isinstance(content, list), "Malformed Claude assistant content" + for part in content: + assert isinstance(part, dict), "Malformed Claude assistant content block" + if part.get("type") != "text": + continue + text = part.get("text") + assert isinstance(text, str), "Malformed Claude assistant text" + if answer_value in text: + model = _evidence_id(message.get("model"), "model") + found.add(_evidence_id(model.removesuffix("[1m]"), "model")) + return found diff --git a/tests/integration/utils/agents/codex.py b/tests/integration/utils/agents/codex.py index fea968ce8..ed3ccc902 100644 --- a/tests/integration/utils/agents/codex.py +++ b/tests/integration/utils/agents/codex.py @@ -1,6 +1,17 @@ -"""Codex transcript helpers for integration journeys.""" +"""Codex-specific integration evidence and model discovery helpers.""" + +from ..model_discovery import codex_model_in_picker as model_in_picker +from . import _evidence_id + +__all__ = [ + "EVIDENCE_KIND", + "assistant_answers", + "completed_task_models", + "model_in_picker", +] SESSION_DIRECTORY = ".codex/sessions" +EVIDENCE_KIND = "client-selected model" def is_child_session(path: str, records: list[dict]) -> bool: @@ -21,3 +32,33 @@ def assistant_answers(records: list[dict]) -> list[str]: if payload.get("type") == "task_complete" and payload.get("last_agent_message"): answers.append(payload["last_agent_message"]) return answers + + +def completed_task_models(records: list[dict], answer_value: str) -> set[str]: + """Join Codex's completed answer turn to its client-selected context model.""" + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) + completed: set[str] = set() + for record in records: + if record.get("type") != "event_msg": + continue + payload = record.get("payload") + assert isinstance(payload, dict), "Malformed Codex event payload" + if payload.get("type") != "task_complete": + continue + answer = payload.get("last_agent_message") + assert isinstance(answer, str), "Missing or malformed Codex completed answer" + if answer_value in answer: + completed.add(_evidence_id(payload.get("turn_id"), "turn ID")) + models: dict[str, set[str]] = {turn_id: set() for turn_id in completed} + for record in records: + if record.get("type") != "turn_context": + continue + payload = record.get("payload") + assert isinstance(payload, dict), "Malformed Codex turn context" + turn_id = _evidence_id(payload.get("turn_id"), "turn ID") + if turn_id in completed: + models[turn_id].add(_evidence_id(payload.get("model"), "model")) + assert all(models.values()), "Missing Codex model context for completed answer turn" + return {model for turn_models in models.values() for model in turn_models} diff --git a/tests/integration/utils/evidence.py b/tests/integration/utils/evidence.py index b5dff142a..938fb7eba 100644 --- a/tests/integration/utils/evidence.py +++ b/tests/integration/utils/evidence.py @@ -5,7 +5,7 @@ import uuid from pathlib import Path -from .agents import claude, codex +from .agents import _evidence_id, claude, codex _AGENT_HELPERS = {"claude": claude, "codex": codex} @@ -56,6 +56,39 @@ def is_child_session(agent: str, path: str, records: list[dict]) -> bool: return _AGENT_HELPERS.get(agent, codex).is_child_session(path, records) +def completed_task_models(session, agent: str, answer_value: str) -> set[str]: + """Read parent task model evidence, not proof of the gateway's destination.""" + assert agent in _AGENT_HELPERS, f"Unsupported evidence agent: {agent}" + adapter = _AGENT_HELPERS[agent] + assert isinstance(answer_value, str) and answer_value.strip(), ( + "Expected a nonempty answer value" + ) + sessions = agent_sessions(session, agent) + session.record("agent-sessions.json", sessions) + return { + model + for path, records in sessions.items() + if not is_child_session(agent, path, records) + for model in adapter.completed_task_models(records, answer_value) + } + + +def assert_completed_task_model(session, agent: str, answer_value: str, expected: str) -> None: + expected = _evidence_id(expected, "expected model") + observed = completed_task_models(session, agent, answer_value) + adapter = _AGENT_HELPERS[agent] + session.record( + f"completed-task-model-{agent}-{answer_value[:12]}.json", + { + "expected": expected, + "observed": sorted(observed), + "evidence_kind": adapter.EVIDENCE_KIND, + "gateway_destination_proven": False, + }, + ) + assert observed == {expected}, {"expected": expected, "observed": sorted(observed)} + + def assistant_answer_contains(session, agent: str, value: str, *, child: bool = False) -> bool: """Whether a native parent or child assistant answer contains ``value``.""" return any( diff --git a/tests/integration/utils/model_discovery.py b/tests/integration/utils/model_discovery.py index 36171de96..926dc063b 100644 --- a/tests/integration/utils/model_discovery.py +++ b/tests/integration/utils/model_discovery.py @@ -1,5 +1,6 @@ """Validation helpers for model-discovery evidence.""" +import hashlib import re _CLAUDE_GATEWAY_ALIAS = re.compile(r"^anthropic-aigw-[0-9a-fA-F]{8}-(?P.*)$") @@ -9,9 +10,20 @@ "claude-opus-5": ("Opus", "Opus 5"), "claude-sonnet-5": ("Sonnet", "Sonnet 5"), } +_CLAUDE_DEFAULT_PICKER_ROW = re.compile( + r"^Default(?: \(recommended\))?[ \t]+Use the default model\b" +) _SYSTEM_AI_PREFIX = "system.ai." +def claude_discovery_model_id(model_id: str) -> str: + """Encode the exact wire ID expected by Claude's gateway discovery feature.""" + if "claude" in model_id.lower() or "anthropic" in model_id.lower(): + return model_id + checksum = hashlib.sha256(model_id.encode("utf-8")).hexdigest()[:8] + return f"anthropic-aigw-{checksum}-{model_id}" + + def claude_model_in_picker(screen: str, model_id: str, display_name: str | None) -> bool: """Return whether a Claude model appears in a numbered picker row. @@ -70,3 +82,45 @@ def claude_system_model_ids(models: list[dict]) -> list[str]: ), f"Expected a system.ai model id: {model_id!r}" canonical_ids.append(canonical_id) return canonical_ids + + +def codex_model_in_picker(screen: str, model_id: str, display_name: str) -> bool: + """Match native numbered picker rows, not banners, prompt text, or footers.""" + rows = re.findall(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)][ \t]+([^\n]+)$", screen) + return any(_picker_row_matches(row, model_id, display_name) for row in rows) + + +def _picker_row_matches(label: str, model_id: str, display_name: str | None) -> bool: + return any( + re.search(rf"(? None: + """Require exactly one unambiguous numbered native row per expected model.""" + assert agent in {"claude", "codex"}, agent + assert display_names, "Expected a nonempty provider model inventory" + rows = re.findall(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)](?:[ \t]+([^\n]*))?$", screen) + assert rows, f"No numbered {agent} picker rows:\n{screen}" + observed: list[str] = [] + for label in rows: + if agent == "claude" and _CLAUDE_DEFAULT_PICKER_ROW.match(label): + continue + matched = { + model + for model, display_name in display_names.items() + if _picker_row_matches(label, model, display_name) + } + assert len(matched) == 1, ( + f"Unmatched or ambiguous {agent} picker row {label!r}: {sorted(matched)}" + ) + model = matched.pop() + assert model not in observed, f"Duplicate {agent} picker model row: {model}" + observed.append(model) + assert set(observed) == set(display_names), { + "agent": agent, + "expected": sorted(display_names), + "observed": observed, + } diff --git a/tests/integration/utils/provider_catalog.py b/tests/integration/utils/provider_catalog.py index d0a5c00fc..3834be51d 100644 --- a/tests/integration/utils/provider_catalog.py +++ b/tests/integration/utils/provider_catalog.py @@ -13,6 +13,7 @@ _CODEX_MODELS_PATH = "/ai-gateway/codex/v1/models" _PROVIDER_HEADER = "Databricks-Model-Provider-Service" _PARENT_SCHEMA_HEADER = "Databricks-Model-Service-Parent-Schema" +MODEL_SERVICE_PARENT_SCHEMA_HEADER = _PARENT_SCHEMA_HEADER _ANTHROPIC_VERSION = "2023-06-01" _ANTHROPIC_PAGE_SIZE = 1000 _ANTHROPIC_MAX_PAGES = 20 diff --git a/tests/integration/utils/terminal.py b/tests/integration/utils/terminal.py index d9336c341..85240dd59 100644 --- a/tests/integration/utils/terminal.py +++ b/tests/integration/utils/terminal.py @@ -339,6 +339,31 @@ def open_model_picker(self, *, model_visible=None): ) return screen + def open_codex_model_picker(self, *, model_visible): + """Capture Codex's native numbered /model menu, then dismiss it with Escape.""" + assert self.agent == "codex", self.agent + self.submit("/model") + self.wait_for( + lambda text: ( + "select model" in text.lower() + and re.search(r"(?m)^[ \t]*(?:[❯›>][ \t]*)?\d+[.)][ \t]+\S", text) + ), + "Codex's numbered model picker", + timeout=60, + ) + self.wait_for(model_visible, "all scoped models in Codex's picker", timeout=60) + screen = self.visible + self.actions.append({"reason": "codex-model-picker-visible", "screen": screen}) + self.send("\x1b", "close Codex's model picker without changing its default") + self.wait_for( + lambda text: ( + "select model" not in text.lower() + and re.search(r"(?m)^\s*[❯›>]\s*(?!\d+[.)])", text) + ), + "the Codex prompt after closing the model picker", + ) + return screen + def wait_for_task(self, task, timeout=180): permission_in_progress = False diff --git a/tests/test_integration_evidence.py b/tests/test_integration_evidence.py index f618304d1..910984916 100644 --- a/tests/test_integration_evidence.py +++ b/tests/test_integration_evidence.py @@ -4,6 +4,8 @@ import pytest +from tests.integration.utils import evidence +from tests.integration.utils.agents import claude, codex from tests.integration.utils.evidence import ( SubagentCalculation, assert_no_terminal_api_error, @@ -19,6 +21,15 @@ def record(self, _name, _value): pass +def _transcript_session(home, agent, transcripts): + directory = home / {"claude": ".claude/projects", "codex": ".codex/sessions"}[agent] + for name, records in transcripts.items(): + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(record) + "\n" for record in records)) + return _Session(home) + + def _write_answer(home, agent, *, child, value): if agent == "claude": directory = home / ".claude/projects/project" @@ -89,3 +100,119 @@ def test_tagged_calculation_requires_the_native_child_answer(tmp_path, agent): assert task.marker in task.prompt assert f'task name "{task.marker}"' in task.prompt assert "1+1" in task.prompt + + +def test_codex_model_identity_uses_only_the_completed_answer_turn(): + records = [ + { + "type": "turn_context", + "payload": {"turn_id": "other", "model": "catalog.other_models.codex_decoy"}, + }, + { + "type": "turn_context", + "payload": {"turn_id": "matching", "model": "catalog.models.gpt_luna"}, + }, + { + "type": "event_msg", + "payload": { + "type": "task_complete", + "turn_id": "matching", + "last_agent_message": "withheld-file-value", + }, + }, + ] + assert codex.completed_task_models(records, "withheld-file-value") == { + "catalog.models.gpt_luna" + } + + +def test_codex_model_identity_rejects_prompt_only_evidence(): + records = [ + { + "type": "turn_context", + "payload": {"turn_id": "matching", "model": "catalog.models.gpt_luna"}, + }, + { + "type": "response_item", + "payload": {"type": "message", "role": "user", "content": "withheld-file-value"}, + }, + ] + assert codex.completed_task_models(records, "withheld-file-value") == set() + + +def test_claude_model_identity_uses_assistant_answer_not_tool_output(): + records = [ + { + "type": "user", + "message": { + "model": "catalog.other_models.claude_decoy", + "content": [{"type": "text", "text": "value"}], + }, + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "model": "catalog.models.claude_sonnet", + "content": [{"type": "text", "text": "value"}], + }, + }, + ] + assert claude.completed_task_models(records, "value") == {"catalog.models.claude_sonnet"} + + +def completed_records(agent, model, answer="value", turn_id="matching"): + if agent == "claude": + return [ + { + "type": "assistant", + "message": { + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": answer}], + }, + } + ] + return [ + {"type": "turn_context", "payload": {"turn_id": turn_id, "model": model}}, + { + "type": "event_msg", + "payload": {"type": "task_complete", "turn_id": turn_id, "last_agent_message": answer}, + }, + ] + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("parent_answer", [None, "value", "unrelated-answer"]) +def test_completed_task_models_excludes_child_only_and_conflicting_child_evidence( + tmp_path, agent, parent_answer +): + child = completed_records(agent, "child-model") + if agent == "codex": + child.insert(0, {"type": "session_meta", "payload": {"source": {"subagent": "spawn"}}}) + sessions = {"project/subagents/child.jsonl": child} + if parent_answer is not None: + sessions["project/parent.jsonl"] = completed_records(agent, "parent-model", parent_answer) + session = _transcript_session(tmp_path, agent, sessions) + expected = {"parent-model"} if parent_answer == "value" else set() + assert evidence.completed_task_models(session, agent, "value") == expected + + +def test_codex_completed_task_models_requires_context_in_same_session(tmp_path): + context, completion = completed_records("codex", "model") + session = _transcript_session( + tmp_path, + "codex", + {"context-session.jsonl": [context], "answer-session.jsonl": [completion]}, + ) + with pytest.raises(AssertionError, match="Missing Codex model context"): + evidence.completed_task_models(session, "codex", "value") + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("models", [[], ["unexpected"], ["expected", "conflicting"]]) +def test_completed_task_model_assertion_requires_exact_singleton(tmp_path, agent, models): + records = [record for model in models for record in completed_records(agent, model)] + session = _transcript_session(tmp_path, agent, {"parent.jsonl": records}) + with pytest.raises(AssertionError): + evidence.assert_completed_task_model(session, agent, "value", "expected") diff --git a/tests/test_integration_model_discovery.py b/tests/test_integration_model_discovery.py index fd8860e7f..d9e5ec849 100644 --- a/tests/test_integration_model_discovery.py +++ b/tests/test_integration_model_discovery.py @@ -2,7 +2,25 @@ import pytest -from tests.integration.utils.model_discovery import claude_model_in_picker, claude_system_model_ids +from tests.integration.utils.model_discovery import ( + assert_picker_inventory, + claude_discovery_model_id, + claude_model_in_picker, + claude_system_model_ids, + codex_model_in_picker, +) + +PICKER_MODELS = { + "claude": frozenset( + {"catalog.models.claude_sonnet", "catalog.models.claude_haiku", "catalog.models.kimi"} + ), + "codex": frozenset({"catalog.models.gpt_luna", "catalog.models.kimi"}), +} + +CLAUDE_DEFAULT_CONTROL_ROW = ( + "Default (recommended) Use the default model " + "(currently Opus 5.5 (1M context)) · $4/$20 per Mtok" +) @pytest.mark.parametrize("cursor", ["", "❯ ", "› ", "> "]) @@ -131,3 +149,125 @@ def test_claude_system_model_ids_rejects_invalid_or_out_of_scope_ids(model_id): def test_claude_system_model_ids_rejects_empty_missing_and_duplicate_ids(models): with pytest.raises(AssertionError): claude_system_model_ids(models) + + +def test_claude_discovery_requires_exact_gateway_wire_ids(): + assert {claude_discovery_model_id(model) for model in PICKER_MODELS["claude"]} == { + "catalog.models.claude_haiku", + "catalog.models.claude_sonnet", + "anthropic-aigw-69e2a9a0-catalog.models.kimi", + } + assert ( + claude_discovery_model_id("catalog.other_models.claude_decoy") + == "catalog.other_models.claude_decoy" + ) + + +@pytest.mark.parametrize("cursor", ["", "❯ ", "› ", "> "]) +@pytest.mark.parametrize("label", ["catalog.models.gpt_luna", "GPT Luna"]) +def test_codex_picker_matches_model_only_in_numbered_rows(cursor, label): + screen = f"Select Model and Effort\n {cursor}1. {label} (current)\nEnter to confirm" + assert codex_model_in_picker(screen, "catalog.models.gpt_luna", "GPT Luna") + + +@pytest.mark.parametrize( + "screen", + [ + "GPT Luna\nSelect Model and Effort\n 1. Other model", + "Select Model and Effort\n 1. Other model\nCurrent model: GPT Luna", + " 1. catalog.models.gpt_luna_v2", + " 1.\nGPT Luna", + "", + ], +) +def test_codex_picker_rejects_nonrows_and_different_model_ids(screen): + assert not codex_model_in_picker(screen, "catalog.models.gpt_luna", "GPT Luna") + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_picker_inventory_accepts_exact_rows_and_ignores_banner_text(agent): + models = PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"] + labels = {model: model.rsplit(".", 1)[-1].replace("_", " ").title() for model in models} + rows = "\n".join( + f" {position}. {label}" for position, label in enumerate(labels.values(), start=1) + ) + screen = f"Banner: Gemini Flash\nSelect model\n{rows}\nEnter to confirm" + assert_picker_inventory(screen, agent, labels) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +@pytest.mark.parametrize("excluded_label", ["Gemini Flash", "Friendly out-of-scope model"]) +def test_picker_inventory_rejects_extra_friendly_label_rows(agent, excluded_label): + models = PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"] + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"Select model\n{rows}\n {len(models) + 1}. {excluded_label}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, agent, labels) + + +@pytest.mark.parametrize("agent", ["claude", "codex"]) +def test_picker_inventory_rejects_duplicate_and_missing_rows(agent): + models = sorted(PICKER_MODELS["claude"] if agent == "claude" else PICKER_MODELS["codex"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + with pytest.raises(AssertionError): + assert_picker_inventory(f"{rows}\n {len(models) + 1}. {models[0]}", agent, labels) + with pytest.raises(AssertionError): + assert_picker_inventory(f" 1. {models[0]}", agent, labels) + + +@pytest.mark.parametrize( + "control_row", + [ + CLAUDE_DEFAULT_CONTROL_ROW, + "Default Use the default model (currently Sonnet 5 (200k context)) · $3/$15 per Mtok", + "Default (recommended) Use the default model (currently Haiku 4.5 (200k context)) · $1/$5 per Mtok", + ], +) +def test_picker_inventory_ignores_claude_default_control_row(control_row): + models = sorted(PICKER_MODELS["claude"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"{rows}\n {len(models) + 1}. {control_row}" + assert_picker_inventory(screen, "claude", labels) + + +def test_picker_inventory_requires_missing_claude_models_with_default_control_row(): + models = sorted(PICKER_MODELS["claude"]) + labels = dict.fromkeys(models, None) + rows = "\n".join( + f" {position}. {model}" for position, model in enumerate(models[:-1], start=1) + ) + screen = f"{rows}\n {len(models)}. {CLAUDE_DEFAULT_CONTROL_ROW}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, "claude", labels) + + +@pytest.mark.parametrize("default_label", ["Default", "Default (recommended)"]) +def test_picker_inventory_rejects_generic_claude_default_label(default_label): + models = sorted(PICKER_MODELS["claude"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"{rows}\n {len(models) + 1}. {default_label}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, "claude", labels) + + +@pytest.mark.parametrize("extra_label", ["Opus (1M context) Opus 5.5", "Gemini 3 Flash"]) +def test_picker_inventory_rejects_extra_claude_native_rows(extra_label): + models = sorted(PICKER_MODELS["claude"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"{rows}\n {len(models) + 1}. {extra_label}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, "claude", labels) + + +def test_picker_inventory_does_not_ignore_claude_default_control_row_for_codex(): + models = sorted(PICKER_MODELS["codex"]) + labels = dict.fromkeys(models, None) + rows = "\n".join(f" {position}. {model}" for position, model in enumerate(models, start=1)) + screen = f"{rows}\n {len(models) + 1}. {CLAUDE_DEFAULT_CONTROL_ROW}" + with pytest.raises(AssertionError): + assert_picker_inventory(screen, "codex", labels)