diff --git a/src/ucode/agents/claude.py b/src/ucode/agents/claude.py index 68e3f1d00..101901162 100644 --- a/src/ucode/agents/claude.py +++ b/src/ucode/agents/claude.py @@ -210,8 +210,6 @@ def _otel_trace_env(workspace: str) -> dict[str, str]: "sonnet": "ANTHROPIC_DEFAULT_SONNET_MODEL", "haiku": "ANTHROPIC_DEFAULT_HAIKU_MODEL", } -CLAUDE_CUSTOM_MODEL_FAMILIES = ("opus", "sonnet", "haiku") -CLAUDE_CUSTOM_MODEL_SELECTOR = "opus" # Launch-scoped feature flags that ucode may write into Claude settings. These # must be removed again when the corresponding launch flag is absent. CLAUDE_CONDITIONAL_ENV_KEYS = ("CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY",) @@ -1866,15 +1864,6 @@ def _launch_model_args(tool_args: list[str], launch_model: str | None) -> list[s return ["--model", launch_model] -def _launch_custom_model_settings(model: str) -> dict: - """Pin a Databricks model through launch-scoped Claude family aliases.""" - return { - "env": { - CLAUDE_DEFAULT_MODEL_ENV_KEYS[family]: model for family in CLAUDE_CUSTOM_MODEL_FAMILIES - }, - } - - def _resolve_picker_model_id(model: str, settings_env: dict) -> str: """Resolve configured aliases and context suffixes for comparisons only.""" model = re.sub(r"\[(?:1m|200k)\]$", "", model) @@ -1886,24 +1875,6 @@ def _resolve_picker_model_id(model: str, settings_env: dict) -> str: return re.sub(r"\[(?:1m|200k)\]$", "", model) -def _is_managed_launch_model(state: dict, model: str) -> bool: - """Return whether *model* is present in Claude's managed model catalog.""" - picker_models = state.get("_claude_launch_picker_models") - if not isinstance(picker_models, list) or not picker_models: - picker_models = state.get("claude_static_models") - if not isinstance(picker_models, list) or not picker_models: - return False - - settings_env = read_json_safe(CLAUDE_SETTINGS_PATH).get("env") - settings_env = settings_env if isinstance(settings_env, dict) else {} - resolved_model = _resolve_picker_model_id(model, settings_env) - return any( - isinstance(picker_model, str) - and _resolve_picker_model_id(picker_model, settings_env) == resolved_model - for picker_model in picker_models - ) - - def _resolve_launch_binary(binary: str) -> str: """Resolve Claude's native executable without sending arguments through a batch shim.""" if os.name != "nt": @@ -2129,18 +2100,8 @@ def launch( os.environ["OAUTH_TOKEN"] = get_databricks_token(workspace, state.get("profile")) settings_override = None launch_args = list(tool_args) - launch_custom_model = state.get("_claude_launch_custom_model") - if isinstance(launch_custom_model, str) and launch_custom_model: - if _is_managed_launch_model(state, launch_custom_model): - launch_args = [ - *_launch_model_args(tool_args, launch_custom_model), - *tool_args, - ] - else: - # Claude rejects raw model ids outside its catalog. - os.environ["ANTHROPIC_MODEL"] = CLAUDE_CUSTOM_MODEL_SELECTOR - settings_override = _launch_custom_model_settings(launch_custom_model) - elif options.user_pinned_model: + if options.user_pinned_model: + # Pass the exact ID: managed family defaults can override temporary aliases. os.environ["ANTHROPIC_MODEL"] = options.user_pinned_model settings_override = {"env": {"ANTHROPIC_MODEL": options.user_pinned_model}} launch_args = [ diff --git a/src/ucode/cli.py b/src/ucode/cli.py index 656abd746..4aa7e1f6e 100644 --- a/src/ucode/cli.py +++ b/src/ucode/cli.py @@ -2918,9 +2918,6 @@ def _launch_tool( if tool == "claude": if provider: state["_claude_launch_provider"] = provider - elif model: - # Set after configure_tool so the selection stays launch-scoped. - state["_claude_launch_custom_model"] = model elif tool == "codex": if provider: state["_codex_launch_provider"] = provider @@ -3215,8 +3212,8 @@ def claude_cmd( typer.Option( "--model", help="Launch on a specific Databricks model id (e.g. a UC " - "`..`). Pinned via launch-scoped Claude family aliases so the " - "gateway resolves it — unlike Claude Code's own --model, which rejects non-catalog ids. " + "`..`). Passed to Claude Code's native `--model` option " + "for this launch only. " "With --provider, pass a family (opus/sonnet/haiku) or a target the service allows to " "start on that tier instead of Claude Code's opus default. Pass before any `--` separator.", ), diff --git a/tests/README.md b/tests/README.md index dc5cf381e..a99835818 100644 --- a/tests/README.md +++ b/tests/README.md @@ -98,6 +98,8 @@ sign-in or TUI test. Claude picker composition is checked directly through the catalog and renderer functions in `test_agent_claude.py`; focused CLI cases cover source selection and launch precedence. +Explicit Claude/GPT `--model` regressions in `test_cli.py` exercise the real launcher +without a managed catalog and preserve family defaults; these are component checks. Managed UC schema regressions in `test_cli.py` retain non-default catalog models with an overall default, a family default, or both, while preserving startup selection and family mappings. `TestBuildClaudeArgv` also checks that caller permission denies survive ug's technical @@ -185,7 +187,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_codex_headless_prompt_argument`, `test_ug_codex_headless_prompt_stdin`, `test_ug_codex_headless_prompt_after_separator` | Run Codex from a script using each prompt form | Completed turn and final answer contain the file value; exit zero; no routing | | `test_ug_opencode_headless_prompt_argument` | Run OpenCode from a script (`run --format json --auto`) with an argument prompt | Completed Read tool call; final text answer contains the file value; exit zero (non-blocking CI lane) | | `test_ug_claude_exports_trace_to_configured_table`, `test_ug_codex_exports_trace_to_configured_table` | Configure tracing, complete a headless task carrying a unique trace marker, then wait for ingestion | The configured trace table contains an agent span with the same trace-safe marker and requested model | -| `test_ug_claude_headless_explicit_model_bypasses_routing` | Pass `--model VALUE` / `--model=VALUE` with routing enabled | Real file task completes; no routing wrapper | +| `test_ug_claude_headless_explicit_model_bypasses_routing` | Pass `--model VALUE` / `--model=VALUE` before and after ug's separator, without workspace policy and with routing enabled | Real file task completes; JSON `modelUsage` reports the requested model with output tokens; no routing wrapper | | `test_ug_codex_headless_explicit_model_bypasses_routing` | Pass `--model VALUE` / `--model=VALUE` / `-m VALUE` with routing enabled | Real file task completes; no routing wrapper | | `test_ug_claude_preserves_caller_settings_and_hook` | Pass a settings path containing spaces | Real SessionStart hook executes; caller file unchanged; file task completes | | `test_ug_claude_reports_unsupported_short_model_option` | Pass Claude's unsupported `-m` | Actual agent error and exit status preserved | @@ -215,7 +217,7 @@ integration utilities; only CUJ-specific evidence correlation stays in a test fi | `test_ug_and_ucode_auth_helpers_emit_only_the_supplied_bearer` | Run both auth helper commands with the public bearer override, with and without forced refresh | Exact token-only stdout, no warnings or ANSI escapes; no workspace authentication or saved state | | `test_ug_and_ucode_web_search_helpers_preserve_mcp_stdio` | Initialize and list tools through both web-search helper commands | Exactly the MCP JSON-RPC responses; no text/ANSI contamination; existing server/tool identities preserved; no model request | -With Claude and Codex selected there are **62 live cases** (12 marked TUI cases), +With Claude and Codex selected there are **64 live cases** (12 marked TUI cases), **1 two-workspace case** (marker `workspace_switch`), **33 managed-fixture cases** (marker `managed_fixture`, with only the CodingAgentConfig input injected from a JSON file in `fixtures/managed_config/`), and **7 installation checks**. The 14 retained numbered scenarios @@ -279,7 +281,7 @@ dependency graph to reproduce a user's combination. Every relevant same-reposito PR and push to `main` runs both smoke and the full CUJ suite. Smoke covers the Databricks Hosted configure/TUI, custom OAuth CLI TUI, and headless argument journeys for both agents, in two parallel jobs. After smoke finishes, the full -suite runs all 62 live cases across two parallel agent jobs: one Claude VM and one +suite runs all 64 live cases across two parallel agent jobs: one Claude VM and one Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs for the same agent overlap within a run. diff --git a/tests/integration/README.md b/tests/integration/README.md index 9bf85f70b..bc16b34df 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -32,6 +32,11 @@ These component checks do not establish live installer, Isaac, or inference cove The live Claude/Codex headless prompt-argument and stdin journeys require raw stdout to parse as JSON/JSONL, without discarding non-JSON lines before checking the completed file task. +The Claude explicit-model journey covers both `--model VALUE` and `--model=VALUE` +before and after ug's separator, with no workspace policy. It requires a completed +file task, the requested model with output tokens in JSON `modelUsage`, and no routing +wrapper. Use the runner's `--claude-model` option to reproduce a specific model. + The `smart_defaults` wire schema, legacy `spend_tiers` cache reads, and recommendation request gating are covered by unit/component tests listed in `../README.md`. This suite does not yet assert live `recommendModel` request counts for configs with and without tiers. @@ -376,7 +381,7 @@ startup banners and footer text cannot satisfy discovery assertions. Cases 7–1 they only configure, list models, and open/close the picker. Other live CUJs perform real model tasks. -There are **62 live cases** (including 12 marked TUI journeys) and **7 installation +There are **64 live cases** (including 12 marked TUI journeys) and **7 installation checks** with Claude and Codex; selecting OpenCode adds one live headless case. One **`workspace_switch` case** uses two real workspaces and checks skills MCP cleanup and a completed Claude task. A further **33 `managed_fixture` cases** (two of them also `live`) run on the @@ -396,7 +401,7 @@ catalog discovery with overall defaults, family defaults, or both, along with ex selection and preservation of static model lists. Both cases run on the managed workspace's own bearer; no second workspace or extra secret is involved. The 14 retained numbered scenarios comprise 24 explicit journeys: 12 managed and 12 unmanaged -executions; the complete integration suite collects 101 executions. See the named coverage and gaps matrix in +executions; the complete integration suite collects 103 executions. See the named coverage and gaps matrix in [../README.md](../README.md). ```bash @@ -539,12 +544,12 @@ each test; only explicit-model scenarios choose and record a discovered Every same-repository PR and push to `main` runs **Smoke journeys**, followed by **Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, -two agent jobs). Full runs all 62 live cases, including those smoke cases, in two +two agent jobs). Full runs all 64 live cases, including those smoke cases, in two disjoint agent lanes: | Agent lane | Marker | Cases | | --- | --- | --- | -| Claude | `live and claude` | 28 | +| Claude | `live and claude` | 30 | | Codex | `live and codex` | 34 | A non-blocking **OpenCode** job (`live and opencode`, one case) runs alongside them with @@ -849,7 +854,7 @@ uv run --no-project --python 3.12 python scripts/run_integration.py \ unset DATABRICKS_BEARER ``` -This runs all 62 live cases. For the seven installation checks, run the same +This runs all 64 live cases. For the seven installation checks, run the same runner/version/index arguments with `--installation-only` and omit `-- -m live`; no bearer or workspace is needed. Results remain under `.integration-runs/`. Each invocation needs a new output directory; an existing one is rejected. diff --git a/tests/integration/test_ug_claude_headless.py b/tests/integration/test_ug_claude_headless.py index 9ffba1e77..4a41204de 100644 --- a/tests/integration/test_ug_claude_headless.py +++ b/tests/integration/test_ug_claude_headless.py @@ -115,11 +115,14 @@ def test_ug_claude_headless_prompt_after_separator(live_session, workspace): @pytest.mark.parametrize("model_form", ["separate", "equals"]) -def test_ug_claude_headless_explicit_model_bypasses_routing(live_session, workspace, model_form): - """Scenario: choose an explicit model while global smart routing is enabled. +@pytest.mark.parametrize("model_owner", ["ug", "claude"]) +def test_ug_claude_headless_explicit_model_bypasses_routing( + live_session, workspace, model_form, model_owner +): + """Scenario: choose a model before/after ug's separator with no workspace policy. - Expected: the model option is accepted, the real file task completes, and - no routing wrapper overrides the caller's choice. + Expected: with smart routing enabled, the real file task completes on the + requested model, confirmed by JSON modelUsage, without a routing wrapper. """ session = live_session task = FileTask(session) @@ -138,6 +141,7 @@ def test_ug_claude_headless_explicit_model_bypasses_routing(live_session, worksp session.env["ENABLE_SMART_ROUTING_V2"] = "1" result = session.run( "claude", + *(model_args if model_owner == "ug" else []), "--", "-p", task.prompt, @@ -145,10 +149,13 @@ def test_ug_claude_headless_explicit_model_bypasses_routing(live_session, worksp "json", "--allowedTools", "Read", - *model_args, + *(model_args if model_owner == "claude" else []), timeout=180, ) task.assert_headless_answer("claude", result) + usage = json.loads(result.stdout)["modelUsage"] + assert set(usage) == {model}, {"expected": model, "observed": sorted(usage)} + assert usage[model]["outputTokens"] > 0, usage session.assert_not_routed() diff --git a/tests/test_agent_claude.py b/tests/test_agent_claude.py index e9bb3436a..9320341cd 100644 --- a/tests/test_agent_claude.py +++ b/tests/test_agent_claude.py @@ -2636,9 +2636,7 @@ def test_launch_model_is_only_set_for_current_process(self, monkeypatch): settings = json.loads(calls[0][2]) assert settings["env"]["ANTHROPIC_MODEL"] == "cat.schema.model" - def test_launch_custom_model_uses_family_aliases_without_native_model( - self, monkeypatch, tmp_path - ): + def test_launch_custom_model_uses_native_model(self, monkeypatch, tmp_path): calls: list[list[str]] = [] monkeypatch.setenv("ANTHROPIC_MODEL", "stale-model") monkeypatch.setattr(claude, "get_databricks_token", lambda *_args: "token") @@ -2653,42 +2651,45 @@ def test_launch_custom_model_uses_family_aliases_without_native_model( custom_model = "system.ai.claude-opus-4-8" claude.launch( - {"workspace": WS, "_claude_launch_custom_model": custom_model}, + {"workspace": WS}, ["--debug"], options=LaunchOptions(user_pinned_model=custom_model), ) - assert os.environ["ANTHROPIC_MODEL"] == claude.CLAUDE_CUSTOM_MODEL_SELECTOR + assert os.environ["ANTHROPIC_MODEL"] == custom_model assert calls[0][0:2] == ["claude", "--settings"] settings = json.loads(calls[0][2]) assert settings["model"] == saved_model - for family in claude.CLAUDE_CUSTOM_MODEL_FAMILIES: - key = claude.CLAUDE_DEFAULT_MODEL_ENV_KEYS[family] - assert settings["env"][key] == custom_model - assert "ANTHROPIC_DEFAULT_FABLE_MODEL" not in settings["env"] - assert "--model" not in calls[0] - assert calls[0][-1] == "--debug" + assert settings["env"] == {"USER_SETTING": "keep", "ANTHROPIC_MODEL": custom_model} + assert calls[0][3:] == ["--model", custom_model, "--debug"] assert settings_path.read_bytes() == original_settings def test_launch_custom_model_preserves_forwarded_native_model(self, monkeypatch): calls: list[list[str]] = [] + monkeypatch.delenv("ANTHROPIC_MODEL", raising=False) monkeypatch.setattr(claude, "get_databricks_token", lambda *_args: "token") monkeypatch.setattr(claude, "exec_or_spawn", lambda argv: calls.append(argv)) claude.launch( - {"workspace": WS, "_claude_launch_custom_model": "system.ai.claude-opus-4-8"}, + {"workspace": WS}, ["--model", "claude-sonnet-5"], options=LaunchOptions(user_pinned_model="system.ai.claude-opus-4-8"), ) - assert calls[0][-2:] == ["--model", "claude-sonnet-5"] + assert calls[0][3:] == ["--model", "claude-sonnet-5"] def test_launch_managed_custom_model_uses_native_model_over_stale_settings( self, monkeypatch, tmp_path ): calls: list[list[str]] = [] + monkeypatch.delenv("ANTHROPIC_MODEL", raising=False) settings_path = tmp_path / "settings.json" - settings_path.write_text(json.dumps({"model": "system.ai.claude-haiku-4-5"})) + original_settings = { + "model": "system.ai.claude-haiku-4-5", + "availableModels": ["system.ai.claude-opus-4-8", "system.ai.claude-haiku-4-5"], + "enforceAvailableModels": True, + } + settings_path.write_text(json.dumps(original_settings)) monkeypatch.setattr(claude, "CLAUDE_SETTINGS_PATH", settings_path) monkeypatch.setattr(claude, "get_databricks_token", lambda *_args: "token") monkeypatch.setattr(claude, "exec_or_spawn", lambda argv: calls.append(argv)) @@ -2697,14 +2698,18 @@ def test_launch_managed_custom_model_uses_native_model_over_stale_settings( claude.launch( { "workspace": WS, - "_claude_launch_custom_model": custom_model, "claude_static_models": ["system.ai.claude-opus-4-8", "system.ai.claude-haiku-4-5"], }, [], options=LaunchOptions(user_pinned_model=custom_model), ) - assert calls[0][1:3] == ["--settings", str(settings_path)] + assert calls[0][1] == "--settings" + launch_settings = json.loads(calls[0][2]) + assert launch_settings["model"] == original_settings["model"] + assert launch_settings["availableModels"] == original_settings["availableModels"] + assert launch_settings["enforceAvailableModels"] is True + assert json.loads(settings_path.read_text()) == original_settings assert calls[0][-2:] == ["--model", custom_model] def test_launch_default_model_is_inherited_by_smart_routing(self, monkeypatch): diff --git a/tests/test_cli.py b/tests/test_cli.py index 7e5124b8b..5bef5ca53 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -815,10 +815,7 @@ def test_claude_model_location_preserves_explicit_model(self): "main.default.claude-opus-5" ) assert "_claude_launch_default_model" not in calls["launch"].call_args.args[1] - assert ( - calls["launch"].call_args.args[1]["_claude_launch_custom_model"] - == "main.default.claude-opus-5" - ) + assert "_claude_launch_custom_model" not in calls["launch"].call_args.args[1] assert calls["launch"].call_args.args[2] == [] @pytest.mark.parametrize( @@ -1512,10 +1509,7 @@ def test_model_is_launch_scoped_for_claude(self, monkeypatch): assert result.exit_code == 0, result.output assert mock_configure.call_args.kwargs["custom_model"] is None assert mock_configure.call_args.kwargs["route_root_model"] is None - assert ( - mock_launch.call_args.args[1]["_claude_launch_custom_model"] - == "cat.schema.claude-opus-5" - ) + assert "_claude_launch_custom_model" not in mock_launch.call_args.args[1] assert ( mock_launch.call_args.kwargs["options"].user_pinned_model == "cat.schema.claude-opus-5" ) @@ -1532,10 +1526,55 @@ def test_explicit_model_clears_managed_default_route_root(self): assert result.exit_code == 0, result.output assert calls["configure"].call_args.kwargs["route_root_model"] is None assert ( - calls["launch"].call_args.args[1]["_claude_launch_custom_model"] + calls["launch"].call_args.kwargs["options"].user_pinned_model == "system.ai.claude-opus-4-8" ) + @pytest.mark.parametrize("model", ["system.ai.claude-sonnet-4-5", "system.ai.gpt-5-6-luna"]) + @pytest.mark.parametrize("equals_form", [False, True], ids=["separate", "equals"]) + def test_unmanaged_claude_model_reaches_native_launcher( + self, monkeypatch, tmp_path, model, equals_form + ): + """No workspace policy/catalog: explicit IDs reach native --model, not opus aliases.""" + from ucode.agents import claude + + monkeypatch.setenv("ENABLE_SMART_ROUTING_V2", "1") + monkeypatch.setenv("ANTHROPIC_MODEL", "stale-model") + settings_path = tmp_path / "ucode-settings.json" + original_settings = { + "env": { + "ANTHROPIC_DEFAULT_OPUS_MODEL": "system.ai.claude-opus-4-8[1m]", + "ANTHROPIC_DEFAULT_SONNET_MODEL": "system.ai.claude-sonnet-5[1m]", + } + } + settings_path.write_text(json.dumps(original_settings)) + monkeypatch.setattr(claude, "CLAUDE_SETTINGS_PATH", settings_path) + monkeypatch.setattr(claude, "_resolve_launch_binary", lambda binary: binary) + monkeypatch.setattr(claude, "get_databricks_token", lambda *_: "token") + native_calls: list[list[str]] = [] + monkeypatch.setattr(claude, "exec_or_spawn", native_calls.append) + + with _launch_policy_patches(None) as calls: + calls["launch"].side_effect = lambda tool, state, args, *, options: claude.launch( + state, args, options=options + ) + model_args = [f"--model={model}"] if equals_form else ["--model", model] + result = runner.invoke(app, ["claude", *model_args]) + + assert result.exit_code == 0, result.output + assert calls["launch"].call_args.kwargs["options"].launch_smart_routing is False + assert "_claude_launch_picker_models" not in calls["launch"].call_args.args[1] + assert "claude_static_models" not in calls["launch"].call_args.args[1] + assert calls["configure"].call_args.kwargs["custom_model"] is None + calls["list_catalog"].assert_not_called() + assert len(native_calls) == 1 + assert native_calls[0][3:] == ["--model", model] + settings = json.loads(native_calls[0][2]) + assert settings["env"]["ANTHROPIC_MODEL"] == model + for key, value in original_settings["env"].items(): + assert settings["env"][key] == value + assert json.loads(settings_path.read_text()) == original_settings + def test_v2_model_sets_transient_launch_override(self, monkeypatch): monkeypatch.setenv("ENABLE_SMART_ROUTING_V2", "1") state = {**MINIMAL_STATE, "claude_models": {"opus": "system.ai.claude-opus-4-8"}}