From 15de8c725a90bb2d113d19525cc6aa9388e36655 Mon Sep 17 00:00:00 2001 From: "lilly.luo" Date: Fri, 9 Oct 2026 21:32:52 +0000 Subject: [PATCH 1/6] Add a Claude-only subagent routing and orchestration preset --- README.md | 22 ++++++---- src/ucode/smart_routing/config.py | 14 +++++++ tests/README.md | 5 +++ tests/e2e_cuj/README.md | 6 +++ tests/e2e_cuj/test_cuj4_smart_routing.py | 53 +++++++++++++++++++++++- tests/integration/README.md | 7 +++- tests/test_smart_routing_config.py | 29 +++++++++++++ 7 files changed, 125 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index affff4856..6c5621964 100644 --- a/README.md +++ b/README.md @@ -253,15 +253,16 @@ The generated shell hooks expect Git Bash; PowerShell-only setups are not covere Use `SMART_ROUTER_CONFIG_VERSION` at launch to select a smart-routing configuration. Each version defines separate Claude Code and Codex flag mappings; the existing versions -use identical mappings for both agents: +use identical mappings for both agents. The Claude-only version disables routing for Codex: -| Version | Subagent routing | First-prompt routing | Orchestrator | -| --- | --- | --- | --- | -| `first_prompt_and_subagent_no_orch_v0` | On | On | Off | -| `subagent_only_v0` | On | Off | Off | -| `subagent_only_v1` | On | Off | Off | -| `subagent_orch_v0` | On | Off | On | -| `subagent_orch_v1` | On | Off | On | +| Version | Routed agents | Subagent routing | First-prompt routing | Orchestrator | +| --- | --- | --- | --- | --- | +| `first_prompt_and_subagent_no_orch_v0` | Claude, Codex | On | On | Off | +| `subagent_only_v0` | Claude, Codex | On | Off | Off | +| `subagent_only_v1` | Claude, Codex | On | Off | Off | +| `subagent_orch_v0` | Claude, Codex | On | Off | On | +| `subagent_orch_v1` | Claude, Codex | On | Off | On | +| `claude_only_subagent_orch_v0` | Claude | On | Off | On | `first_prompt_and_subagent_no_orch_v0` is the customer configuration for first-prompt and subagent routing without orchestration: `ENABLE_SMART_ROUTING_V2=1`, @@ -274,6 +275,11 @@ so first-prompt routing remains off; orchestration is also off. `subagent_orch_v1` enables all three legacy flags. Like `subagent_only_v1`, it routes subagents rather than the first prompt, and it additionally enables orchestration. +`claude_only_subagent_orch_v0` keeps Claude's `subagent_orch_v0` flags (`0`, `1`, `1`) +and sets all three flags to `"0"` for Codex. Codex uses its ordinary launch and +managed default model, without routing or orchestration. Explicit launch/session +on/off controls retain their existing precedence over presets. + `SMART_ROUTER_NAME` still selects the router independently of the preset. Smart-routed Claude and Codex sessions install `smart-router`. The `subagent_orch_v0` diff --git a/src/ucode/smart_routing/config.py b/src/ucode/smart_routing/config.py index 076daddfd..9fd0541bd 100644 --- a/src/ucode/smart_routing/config.py +++ b/src/ucode/smart_routing/config.py @@ -30,6 +30,9 @@ # Route only subagents with V2, subagent-only, and orchestration all enabled. SUBAGENT_ORCH_V1 = "subagent_orch_v1" +# Preserve subagent_orch_v0 for Claude, with all Codex routing flags disabled. +CLAUDE_ONLY_SUBAGENT_ORCH_V0 = "claude_only_subagent_orch_v0" + _BASE_VERSIONS = { FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0: { ENABLE_SMART_ROUTING_ENV_VAR: "1", @@ -64,6 +67,17 @@ } +# Codex opts out of routing and orchestration; Claude keeps subagent_orch_v0. +_VERSIONS[CLAUDE_ONLY_SUBAGENT_ORCH_V0] = { + AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ORCH_V0], + AGENT_CODEX: { + **_BASE_VERSIONS[SUBAGENT_ORCH_V0], + ENABLE_SUBAGENT_ROUTING_ENV_VAR: "0", + ENABLE_SMART_ROUTER_ORCHESTRATOR_ENV_VAR: "0", + }, +} + + def _validate_versions(versions: Mapping[str, Mapping[str, Mapping[str, str]]]) -> None: """Require both agents and a complete, explicit flag set for each.""" expected_agents = {AGENT_CLAUDE, AGENT_CODEX} diff --git a/tests/README.md b/tests/README.md index e4d8d86da..0519d4486 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,6 +35,11 @@ CUJ4's Claude preset sessions also use it after verifying delegated file tasks: answer does not prove that a resumed child has stopped running. They cover Claude/Codex helper dispatch and rejection of routing decisions without the agent-specific prompt-submission evidence. +The `claude_only_subagent_orch_v0` CUJ also requires a completed routed Claude +subagent and a completed Codex file task using the managed default without routing +requests, new router decisions, a recipe header, or orchestrator context. +Component checks cover conflicting inherited flags and managed routing defaults. + Agent-specific smart-routing preset component checks cover shared base presets, explicit agent mappings, complete per-agent validation, session precedence, and preserving the preset selector until managed-default agent selection. diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index 3382630a2..c31e9f8a6 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -117,3 +117,9 @@ uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest \ --confcutdir=tests/e2e_cuj tests/e2e_cuj/test_cuj4_smart_routing.py \ -k test_smart_router_config_version -v ``` + + +CUJ4 additionally covers `claude_only_subagent_orch_v0`: Claude retains +`subagent_orch_v0` behavior and completes a routed child file task; Codex completes +a file task on the managed default without routing requests, new decisions, +a recipe header, or orchestrator context. Workspace configuration remains read-only. diff --git a/tests/e2e_cuj/test_cuj4_smart_routing.py b/tests/e2e_cuj/test_cuj4_smart_routing.py index c19ab368e..e6bc883ae 100644 --- a/tests/e2e_cuj/test_cuj4_smart_routing.py +++ b/tests/e2e_cuj/test_cuj4_smart_routing.py @@ -13,6 +13,7 @@ read_jsonl, ) from ucode.smart_routing.config import ( + CLAUDE_ONLY_SUBAGENT_ORCH_V0, FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0, SUBAGENT_ONLY_V0, SUBAGENT_ONLY_V1, @@ -237,6 +238,7 @@ class TestCujSmartRouting(BaseCujTest): (SUBAGENT_ONLY_V1, False, False), (SUBAGENT_ORCH_V0, False, True), (SUBAGENT_ORCH_V1, False, True), + (CLAUDE_ONLY_SUBAGENT_ORCH_V0, False, True), ], ) def test_smart_router_config_version( @@ -245,7 +247,8 @@ def test_smart_router_config_version( """Scenario: launch both agents with a preset, then explicitly request a subagent. Expected: first-prompt routing and orchestrator context match the preset; - one routed native child completes the delegated task. + one routed native child completes the delegated task. The Claude-only preset + exercises Claude here; Codex's disabled path has its own file-task case. """ session, workspace, recorder = cuj previous = session.env.get("SMART_ROUTER_CONFIG_VERSION") @@ -253,7 +256,10 @@ def test_smart_router_config_version( try: configs = _assert_published_config_matches_expectations(workspace.config()) recorder.configure_session(session, ["configure", "--disable-databricks-ai-tools"]) - for agent in AGENTS: + agents = ( + (CLAUDE,) if SMART_ROUTER_CONFIG_VERSION == CLAUDE_ONLY_SUBAGENT_ORCH_V0 else AGENTS + ) + for agent in agents: supported = workspace.model_ids(agent) evidence = SessionEvidence(session.home, agent) existing_sessions = set(agent_sessions(session, agent)) @@ -356,6 +362,49 @@ def test_smart_router_config_version( "Smart-router preset cleanup left machine-wide agent settings", ) + def test_claude_only_preset_disables_codex_routing(self, cuj): + """Scenario: select the Claude-only preset and complete a Codex file task. + + Expected: real inference uses the managed default, with no routing requests, + router decisions, recipe header, or orchestrator context. + """ + session, workspace, recorder = cuj + previous = session.env.get("SMART_ROUTER_CONFIG_VERSION") + session.env["SMART_ROUTER_CONFIG_VERSION"] = CLAUDE_ONLY_SUBAGENT_ORCH_V0 + try: + configs = _assert_published_config_matches_expectations(workspace.config()) + recorder.configure_session(session, ["configure", "--disable-databricks-ai-tools"]) + task = _file_task(session, CODEX) + evidence = SessionEvidence(session.home, CODEX) + decisions = session.home / ".ucode" / "codex-smart-routing-decisions.jsonl" + before = read_jsonl(decisions) + checkpoint = recorder.checkpoint() + recorder.prepare_launch() + with Terminal(session, "claude-only-codex", [CODEX]) as tui: + tui.boot(timeout=150) + tui.submit(task.prompt) + tui.task(evidence, task) + tui.exit_normally() + requests = recorder.requests_after(checkpoint) + assert not any(request.path == ROUTING_PATH for request in requests) + assert read_jsonl(decisions) == before + inference = _task_inference_request(requests, CODEX, task.prompt) + inference = served_inference_request(recorder, requests, inference, CODEX) + expected = configs[CODEX]["default_models"]["default_model"] + assert canonical_model(inference.payload["model"]) == canonical_model(expected) + evidence.assert_applied(task, workspace.model_ids(CODEX), expected=expected) + assert "databricks-smart-router-recipe" not in inference.headers + assert ORCHESTRATOR_CONTEXT.encode() not in inference.body + finally: + if previous is None: + session.env.pop("SMART_ROUTER_CONFIG_VERSION", None) + else: + session.env["SMART_ROUTER_CONFIG_VERSION"] = previous + session.revert_machine_wide( + "claude-only-codex-revert", + "Claude-only preset cleanup left machine-wide agent settings", + ) + @pytest.mark.parametrize("agent", AGENTS) def test_agent_completes_real_first_prompt_file_task_without_model_override( self, completed_smart_routing_runs, agent diff --git a/tests/integration/README.md b/tests/integration/README.md index 85e63db89..0f2eaa2d8 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -11,10 +11,15 @@ inference, and Claude inference. The config equality also accounts for the works skill name as data; skill download and invocation are not covered. The dedicated smart-routing CUJ in `../e2e_cuj/test_cuj4_smart_routing.py` leaves the original -managed-default and explicit-model cases unchanged. One additional test runs the five supported +managed-default and explicit-model cases unchanged. One additional test runs the five shared `SMART_ROUTER_CONFIG_VERSION` values, checking both agents' first-prompt routing, orchestrator context in inference input, and completed explicitly requested routed subagents. This is not automatic orchestrator-delegation coverage. No workspace configuration is modified. +The `claude_only_subagent_orch_v0` CUJ also requires a completed routed Claude +subagent and a completed Codex file task using the managed default without routing +requests, new router decisions, a recipe header, or orchestrator context. +Component checks cover conflicting inherited flags and managed routing defaults. + Agent-specific smart-routing preset component checks cover shared base presets, explicit agent mappings, complete per-agent validation, session precedence, and preserving the preset selector until managed-default agent selection. diff --git a/tests/test_smart_routing_config.py b/tests/test_smart_routing_config.py index 4b081402b..3c5f4da3a 100644 --- a/tests/test_smart_routing_config.py +++ b/tests/test_smart_routing_config.py @@ -2,6 +2,7 @@ from __future__ import annotations +from itertools import product import pytest from ucode.constants import ( @@ -11,6 +12,7 @@ ENABLE_SMART_ROUTING_ENV_VAR, ENABLE_SUBAGENT_ROUTING_ENV_VAR, SMART_ROUTER_CONFIG_VERSION_ENV_VAR, + SMART_ROUTING_ENV_KEYS, ) from ucode.smart_routing import config, orchestrator, session_env, v2 @@ -200,3 +202,30 @@ def test_agent_config_selection_and_session_precedence(monkeypatch, agent, enabl session_env.SESSION_ENV_VAR: str(control), "ENABLE_SMART_ROUTER_ORCHESTRATOR": "0", } + + +@pytest.mark.parametrize("agent", [AGENT_CLAUDE, AGENT_CODEX]) +@pytest.mark.parametrize( + "inherited", list(product([None, "0", "1"], repeat=len(SMART_ROUTING_ENV_KEYS))) +) +def test_claude_only_preset_overrides_inherited_flags_and_managed_default(agent, inherited): + source = { + key: value + for key, value in zip(SMART_ROUTING_ENV_KEYS, inherited, strict=True) + if value is not None + } + source[SMART_ROUTER_CONFIG_VERSION_ENV_VAR] = config.CLAUDE_ONLY_SUBAGENT_ORCH_V0 + original = source.copy() + expected = ( + _EXPECTED_PRESETS[config.SUBAGENT_ORCH_V0] + if agent == AGENT_CLAUDE + else dict.fromkeys(SMART_ROUTING_ENV_KEYS, "0") + ) + assert config.resolve_environment(source, agent=agent) == expected + previous = config.apply_config(source, agent=agent) + assert source == expected + assert v2.smart_routing_enabled(source, default=True, agent=agent) == (agent == AGENT_CLAUDE) + assert not v2.first_prompt_routing_enabled(source, agent=agent) + assert orchestrator.feature_enabled(source, agent=agent) == (agent == AGENT_CLAUDE) + v2.restore_smart_routing_env(previous, source) + assert source == original From 20cf69503c8642d45d30c578a6114cfac84c109c Mon Sep 17 00:00:00 2001 From: "lilly.luo" Date: Fri, 9 Oct 2026 22:24:01 +0000 Subject: [PATCH 2/6] Define routing versions in one dictionary --- src/ucode/smart_routing/config.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/ucode/smart_routing/config.py b/src/ucode/smart_routing/config.py index 9fd0541bd..23d82b3eb 100644 --- a/src/ucode/smart_routing/config.py +++ b/src/ucode/smart_routing/config.py @@ -63,17 +63,18 @@ # Agents share the base preset unless explicitly changed; comment any intentional difference. _VERSIONS = { - version: {AGENT_CLAUDE: flags, AGENT_CODEX: flags} for version, flags in _BASE_VERSIONS.items() -} - - -# Codex opts out of routing and orchestration; Claude keeps subagent_orch_v0. -_VERSIONS[CLAUDE_ONLY_SUBAGENT_ORCH_V0] = { - AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ORCH_V0], - AGENT_CODEX: { - **_BASE_VERSIONS[SUBAGENT_ORCH_V0], - ENABLE_SUBAGENT_ROUTING_ENV_VAR: "0", - ENABLE_SMART_ROUTER_ORCHESTRATOR_ENV_VAR: "0", + **{ + version: {AGENT_CLAUDE: flags, AGENT_CODEX: flags} + for version, flags in _BASE_VERSIONS.items() + }, + # Codex opts out of routing and orchestration; Claude keeps subagent_orch_v0. + CLAUDE_ONLY_SUBAGENT_ORCH_V0: { + AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ORCH_V0], + AGENT_CODEX: { + **_BASE_VERSIONS[SUBAGENT_ORCH_V0], + ENABLE_SUBAGENT_ROUTING_ENV_VAR: "0", + ENABLE_SMART_ROUTER_ORCHESTRATOR_ENV_VAR: "0", + }, }, } From 45398a21bb3c7984e7c8cea262236a1ca2f636c1 Mon Sep 17 00:00:00 2001 From: "lilly.luo" Date: Fri, 9 Oct 2026 22:26:27 +0000 Subject: [PATCH 3/6] List routing versions and agent configs explicitly --- src/ucode/smart_routing/config.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/ucode/smart_routing/config.py b/src/ucode/smart_routing/config.py index 23d82b3eb..b42c4e7ca 100644 --- a/src/ucode/smart_routing/config.py +++ b/src/ucode/smart_routing/config.py @@ -63,15 +63,31 @@ # Agents share the base preset unless explicitly changed; comment any intentional difference. _VERSIONS = { - **{ - version: {AGENT_CLAUDE: flags, AGENT_CODEX: flags} - for version, flags in _BASE_VERSIONS.items() + FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0: { + AGENT_CLAUDE: _BASE_VERSIONS[FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0], + AGENT_CODEX: _BASE_VERSIONS[FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0], + }, + SUBAGENT_ONLY_V0: { + AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ONLY_V0], + AGENT_CODEX: _BASE_VERSIONS[SUBAGENT_ONLY_V0], + }, + SUBAGENT_ONLY_V1: { + AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ONLY_V1], + AGENT_CODEX: _BASE_VERSIONS[SUBAGENT_ONLY_V1], + }, + SUBAGENT_ORCH_V0: { + AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ORCH_V0], + AGENT_CODEX: _BASE_VERSIONS[SUBAGENT_ORCH_V0], + }, + SUBAGENT_ORCH_V1: { + AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ORCH_V1], + AGENT_CODEX: _BASE_VERSIONS[SUBAGENT_ORCH_V1], }, # Codex opts out of routing and orchestration; Claude keeps subagent_orch_v0. CLAUDE_ONLY_SUBAGENT_ORCH_V0: { AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ORCH_V0], AGENT_CODEX: { - **_BASE_VERSIONS[SUBAGENT_ORCH_V0], + ENABLE_SMART_ROUTING_ENV_VAR: "0", ENABLE_SUBAGENT_ROUTING_ENV_VAR: "0", ENABLE_SMART_ROUTER_ORCHESTRATOR_ENV_VAR: "0", }, From 674e6ad3a0c6f81eb17388195e2f3af5b5b3c0bf Mon Sep 17 00:00:00 2001 From: "lilly.luo" Date: Fri, 9 Oct 2026 22:27:31 +0000 Subject: [PATCH 4/6] Rename Claude-only preset to subagent_orch_v0_claude_only --- README.md | 4 ++-- src/ucode/smart_routing/config.py | 4 ++-- tests/README.md | 2 +- tests/e2e_cuj/README.md | 2 +- tests/e2e_cuj/test_cuj4_smart_routing.py | 8 ++++---- tests/integration/README.md | 2 +- tests/test_smart_routing_config.py | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 6c5621964..c0ef8a3f6 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ use identical mappings for both agents. The Claude-only version disables routing | `subagent_only_v1` | Claude, Codex | On | Off | Off | | `subagent_orch_v0` | Claude, Codex | On | Off | On | | `subagent_orch_v1` | Claude, Codex | On | Off | On | -| `claude_only_subagent_orch_v0` | Claude | On | Off | On | +| `subagent_orch_v0_claude_only` | Claude | On | Off | On | `first_prompt_and_subagent_no_orch_v0` is the customer configuration for first-prompt and subagent routing without orchestration: `ENABLE_SMART_ROUTING_V2=1`, @@ -275,7 +275,7 @@ so first-prompt routing remains off; orchestration is also off. `subagent_orch_v1` enables all three legacy flags. Like `subagent_only_v1`, it routes subagents rather than the first prompt, and it additionally enables orchestration. -`claude_only_subagent_orch_v0` keeps Claude's `subagent_orch_v0` flags (`0`, `1`, `1`) +`subagent_orch_v0_claude_only` keeps Claude's `subagent_orch_v0` flags (`0`, `1`, `1`) and sets all three flags to `"0"` for Codex. Codex uses its ordinary launch and managed default model, without routing or orchestration. Explicit launch/session on/off controls retain their existing precedence over presets. diff --git a/src/ucode/smart_routing/config.py b/src/ucode/smart_routing/config.py index b42c4e7ca..37e6a91f5 100644 --- a/src/ucode/smart_routing/config.py +++ b/src/ucode/smart_routing/config.py @@ -31,7 +31,7 @@ SUBAGENT_ORCH_V1 = "subagent_orch_v1" # Preserve subagent_orch_v0 for Claude, with all Codex routing flags disabled. -CLAUDE_ONLY_SUBAGENT_ORCH_V0 = "claude_only_subagent_orch_v0" +SUBAGENT_ORCH_V0_CLAUDE_ONLY = "subagent_orch_v0_claude_only" _BASE_VERSIONS = { FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0: { @@ -84,7 +84,7 @@ AGENT_CODEX: _BASE_VERSIONS[SUBAGENT_ORCH_V1], }, # Codex opts out of routing and orchestration; Claude keeps subagent_orch_v0. - CLAUDE_ONLY_SUBAGENT_ORCH_V0: { + SUBAGENT_ORCH_V0_CLAUDE_ONLY: { AGENT_CLAUDE: _BASE_VERSIONS[SUBAGENT_ORCH_V0], AGENT_CODEX: { ENABLE_SMART_ROUTING_ENV_VAR: "0", diff --git a/tests/README.md b/tests/README.md index 0519d4486..c1a0311bd 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,7 +35,7 @@ CUJ4's Claude preset sessions also use it after verifying delegated file tasks: answer does not prove that a resumed child has stopped running. They cover Claude/Codex helper dispatch and rejection of routing decisions without the agent-specific prompt-submission evidence. -The `claude_only_subagent_orch_v0` CUJ also requires a completed routed Claude +The `subagent_orch_v0_claude_only` CUJ also requires a completed routed Claude subagent and a completed Codex file task using the managed default without routing requests, new router decisions, a recipe header, or orchestrator context. Component checks cover conflicting inherited flags and managed routing defaults. diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index c31e9f8a6..8ed179b47 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -119,7 +119,7 @@ uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest \ ``` -CUJ4 additionally covers `claude_only_subagent_orch_v0`: Claude retains +CUJ4 additionally covers `subagent_orch_v0_claude_only`: Claude retains `subagent_orch_v0` behavior and completes a routed child file task; Codex completes a file task on the managed default without routing requests, new decisions, a recipe header, or orchestrator context. Workspace configuration remains read-only. diff --git a/tests/e2e_cuj/test_cuj4_smart_routing.py b/tests/e2e_cuj/test_cuj4_smart_routing.py index e6bc883ae..d078d1840 100644 --- a/tests/e2e_cuj/test_cuj4_smart_routing.py +++ b/tests/e2e_cuj/test_cuj4_smart_routing.py @@ -13,11 +13,11 @@ read_jsonl, ) from ucode.smart_routing.config import ( - CLAUDE_ONLY_SUBAGENT_ORCH_V0, FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0, SUBAGENT_ONLY_V0, SUBAGENT_ONLY_V1, SUBAGENT_ORCH_V0, + SUBAGENT_ORCH_V0_CLAUDE_ONLY, SUBAGENT_ORCH_V1, ) @@ -238,7 +238,7 @@ class TestCujSmartRouting(BaseCujTest): (SUBAGENT_ONLY_V1, False, False), (SUBAGENT_ORCH_V0, False, True), (SUBAGENT_ORCH_V1, False, True), - (CLAUDE_ONLY_SUBAGENT_ORCH_V0, False, True), + (SUBAGENT_ORCH_V0_CLAUDE_ONLY, False, True), ], ) def test_smart_router_config_version( @@ -257,7 +257,7 @@ def test_smart_router_config_version( configs = _assert_published_config_matches_expectations(workspace.config()) recorder.configure_session(session, ["configure", "--disable-databricks-ai-tools"]) agents = ( - (CLAUDE,) if SMART_ROUTER_CONFIG_VERSION == CLAUDE_ONLY_SUBAGENT_ORCH_V0 else AGENTS + (CLAUDE,) if SMART_ROUTER_CONFIG_VERSION == SUBAGENT_ORCH_V0_CLAUDE_ONLY else AGENTS ) for agent in agents: supported = workspace.model_ids(agent) @@ -370,7 +370,7 @@ def test_claude_only_preset_disables_codex_routing(self, cuj): """ session, workspace, recorder = cuj previous = session.env.get("SMART_ROUTER_CONFIG_VERSION") - session.env["SMART_ROUTER_CONFIG_VERSION"] = CLAUDE_ONLY_SUBAGENT_ORCH_V0 + session.env["SMART_ROUTER_CONFIG_VERSION"] = SUBAGENT_ORCH_V0_CLAUDE_ONLY try: configs = _assert_published_config_matches_expectations(workspace.config()) recorder.configure_session(session, ["configure", "--disable-databricks-ai-tools"]) diff --git a/tests/integration/README.md b/tests/integration/README.md index 0f2eaa2d8..317a5be92 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -15,7 +15,7 @@ managed-default and explicit-model cases unchanged. One additional test runs the `SMART_ROUTER_CONFIG_VERSION` values, checking both agents' first-prompt routing, orchestrator context in inference input, and completed explicitly requested routed subagents. This is not automatic orchestrator-delegation coverage. No workspace configuration is modified. -The `claude_only_subagent_orch_v0` CUJ also requires a completed routed Claude +The `subagent_orch_v0_claude_only` CUJ also requires a completed routed Claude subagent and a completed Codex file task using the managed default without routing requests, new router decisions, a recipe header, or orchestrator context. Component checks cover conflicting inherited flags and managed routing defaults. diff --git a/tests/test_smart_routing_config.py b/tests/test_smart_routing_config.py index 3c5f4da3a..b4aa26865 100644 --- a/tests/test_smart_routing_config.py +++ b/tests/test_smart_routing_config.py @@ -214,7 +214,7 @@ def test_claude_only_preset_overrides_inherited_flags_and_managed_default(agent, for key, value in zip(SMART_ROUTING_ENV_KEYS, inherited, strict=True) if value is not None } - source[SMART_ROUTER_CONFIG_VERSION_ENV_VAR] = config.CLAUDE_ONLY_SUBAGENT_ORCH_V0 + source[SMART_ROUTER_CONFIG_VERSION_ENV_VAR] = config.SUBAGENT_ORCH_V0_CLAUDE_ONLY original = source.copy() expected = ( _EXPECTED_PRESETS[config.SUBAGENT_ORCH_V0] From 674f3ccb624ea627fb4b269f9732f813a2dc5856 Mon Sep 17 00:00:00 2001 From: "lilly.luo" Date: Fri, 9 Oct 2026 22:32:58 +0000 Subject: [PATCH 5/6] Derive shared smart routing CUJ assertions from agent flags --- tests/README.md | 7 +- tests/e2e_cuj/README.md | 17 ++- tests/e2e_cuj/test_cuj4_smart_routing.py | 146 ++++++++++------------- tests/integration/README.md | 11 +- 4 files changed, 77 insertions(+), 104 deletions(-) diff --git a/tests/README.md b/tests/README.md index c1a0311bd..97d9fb262 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,9 +35,10 @@ CUJ4's Claude preset sessions also use it after verifying delegated file tasks: answer does not prove that a resumed child has stopped running. They cover Claude/Codex helper dispatch and rejection of routing decisions without the agent-specific prompt-submission evidence. -The `subagent_orch_v0_claude_only` CUJ also requires a completed routed Claude -subagent and a completed Codex file task using the managed default without routing -requests, new router decisions, a recipe header, or orchestrator context. +The shared preset CUJ runs both agents for every version and derives routing and +orchestration assertions from each agent's resolved flags. Both must complete a native +child task. `subagent_orch_v0_claude_only` routes Claude's child; Codex uses the managed +default without routing requests, new router decisions, recipe headers, or orchestrator context. Component checks cover conflicting inherited flags and managed routing defaults. Agent-specific smart-routing preset component checks cover shared base presets, explicit agent mappings, diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index 8ed179b47..6fea6f7b8 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -65,10 +65,11 @@ uv run pytest -c tests/e2e_cuj/pytest.ini --confcutdir=tests/e2e_cuj \ ## Smart-routing CUJ `test_cuj4_smart_routing.py` uses its own read-only workspace with managed smart routing enabled. -It runs the original routed/explicit-model journeys and five +It runs the original routed/explicit-model journeys and six `SMART_ROUTER_CONFIG_VERSION` cases using the shared version constants; each launches Claude and Codex once. -The independent expectation table is: +The CUJ derives routing and orchestration assertions from each agent's resolved preset flags. +Component tests independently verify the preset values. The resulting expectations are: | Selector | First prompt routed | Orchestrator context | Child on first prompt | Explicitly requested child | | --- | --- | --- | --- | --- | @@ -77,6 +78,7 @@ The independent expectation table is: | `subagent_only_v1` | No | No | No | Yes, routed | | `subagent_orch_v0` | No | Yes | No | Yes, routed | | `subagent_orch_v1` | No | Yes | No | Yes, routed | +| `subagent_orch_v0_claude_only` | No | Claude only | No | Yes for both; only Claude routed | First prompts explicitly forbid delegation to isolate first-prompt routing. Assertions require the expected presence/absence of a prompt-correlated router request, successful inference on @@ -91,8 +93,9 @@ only known display/safeguards rejections followed by native removal of the rejec and a final non-empty 200 are accepted. Model, prompt, budget, and effort must remain unchanged. Each preset session then explicitly requests one child for a separate hidden-value file task. Assertions require a new native child transcript containing the value, -a correlated spawn-routing decision, and successful child inference on the -router's selected model. This tests requested delegation, not automatic orchestrator delegation. +a correlated spawn-routing decision and successful child inference on the +router's selected model when routing is enabled. Disabled routing requires no router +requests, new routing decisions, recipe headers, or orchestrator context. This tests requested delegation, not automatic orchestrator delegation. The wait uses the child's answer directly; parent-answer reconstruction is reserved for first-prompt tasks. Offline checks reject parent-only answers and accept child answers before the parent replies. Offline evidence regressions do not establish a live CUJ pass. @@ -117,9 +120,3 @@ uv run --with pexpect==4.9.0 --with pyte==0.8.2 pytest \ --confcutdir=tests/e2e_cuj tests/e2e_cuj/test_cuj4_smart_routing.py \ -k test_smart_router_config_version -v ``` - - -CUJ4 additionally covers `subagent_orch_v0_claude_only`: Claude retains -`subagent_orch_v0` behavior and completes a routed child file task; Codex completes -a file task on the managed default without routing requests, new decisions, -a recipe header, or orchestrator context. Workspace configuration remains read-only. diff --git a/tests/e2e_cuj/test_cuj4_smart_routing.py b/tests/e2e_cuj/test_cuj4_smart_routing.py index d078d1840..37d3cd5d3 100644 --- a/tests/e2e_cuj/test_cuj4_smart_routing.py +++ b/tests/e2e_cuj/test_cuj4_smart_routing.py @@ -19,6 +19,7 @@ SUBAGENT_ORCH_V0, SUBAGENT_ORCH_V0_CLAUDE_ONLY, SUBAGENT_ORCH_V1, + resolve_environment, ) from .base import BaseCujTest @@ -231,24 +232,22 @@ class TestCujSmartRouting(BaseCujTest): WORKSPACE_URL = "https://dbc-1a9622fc-2e91.cloud.databricks.com/" @pytest.mark.parametrize( - "SMART_ROUTER_CONFIG_VERSION, first_prompt_routed, orchestrator_enabled", + "SMART_ROUTER_CONFIG_VERSION", [ - (FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0, True, False), - (SUBAGENT_ONLY_V0, False, False), - (SUBAGENT_ONLY_V1, False, False), - (SUBAGENT_ORCH_V0, False, True), - (SUBAGENT_ORCH_V1, False, True), - (SUBAGENT_ORCH_V0_CLAUDE_ONLY, False, True), + FIRST_PROMPT_AND_SUBAGENT_NO_ORCH_V0, + SUBAGENT_ONLY_V0, + SUBAGENT_ONLY_V1, + SUBAGENT_ORCH_V0, + SUBAGENT_ORCH_V1, + SUBAGENT_ORCH_V0_CLAUDE_ONLY, ], ) - def test_smart_router_config_version( - self, cuj, SMART_ROUTER_CONFIG_VERSION, first_prompt_routed, orchestrator_enabled - ): + def test_smart_router_config_version(self, cuj, SMART_ROUTER_CONFIG_VERSION): """Scenario: launch both agents with a preset, then explicitly request a subagent. - Expected: first-prompt routing and orchestrator context match the preset; - one routed native child completes the delegated task. The Claude-only preset - exercises Claude here; Codex's disabled path has its own file-task case. + Expected: each agent's resolved flags determine first-prompt routing, + orchestration, and child routing. One native child completes the delegated + task even when routing is disabled. """ session, workspace, recorder = cuj previous = session.env.get("SMART_ROUTER_CONFIG_VERSION") @@ -256,13 +255,22 @@ def test_smart_router_config_version( try: configs = _assert_published_config_matches_expectations(workspace.config()) recorder.configure_session(session, ["configure", "--disable-databricks-ai-tools"]) - agents = ( - (CLAUDE,) if SMART_ROUTER_CONFIG_VERSION == SUBAGENT_ORCH_V0_CLAUDE_ONLY else AGENTS - ) - for agent in agents: + for agent in AGENTS: + expected_env = resolve_environment( + {"SMART_ROUTER_CONFIG_VERSION": SMART_ROUTER_CONFIG_VERSION}, agent=agent + ) + v2_enabled = expected_env["ENABLE_SMART_ROUTING_V2"] == "1" + subagent_only = expected_env["ENABLE_SMART_ROUTING_SUBAGENT_ONLY"] == "1" + subagent_routed = v2_enabled or subagent_only + first_prompt_routed = v2_enabled and not subagent_only + orchestrator_enabled = ( + expected_env["ENABLE_SMART_ROUTER_ORCHESTRATOR"] == "1" and subagent_routed + ) supported = workspace.model_ids(agent) evidence = SessionEvidence(session.home, agent) existing_sessions = set(agent_sessions(session, agent)) + decisions_path = session.home / ".ucode" / f"{agent}-smart-routing-decisions.jsonl" + original_decisions = read_jsonl(decisions_path) task = FileTask(session) task.prompt += " Do not delegate." checkpoint = recorder.checkpoint() @@ -288,6 +296,9 @@ def test_smart_router_config_version( expected_model ) evidence.assert_applied(task, supported, expected=expected_model) + assert ( + "databricks-smart-router-recipe" in inference.headers + ) == subagent_routed, agent assert ( ORCHESTRATOR_CONTEXT.encode() in inference.body ) == orchestrator_enabled, agent @@ -299,9 +310,6 @@ def test_smart_router_config_version( child_task = FileTask(session) child_task.prompt = child_task.delegate_prompt - decisions_path = ( - session.home / ".ucode" / f"{agent}-smart-routing-decisions.jsonl" - ) decision_count = len(read_jsonl(decisions_path)) checkpoint = recorder.checkpoint() existing_sessions = set(agent_sessions(session, agent)) @@ -324,31 +332,40 @@ def test_smart_router_config_version( for answer in assistant_answers(agent, records) ), agent decisions = read_jsonl(decisions_path)[decision_count:] - assert_subagent_routed( - session, - agent, - child_task, - decision_ids={decision["decision_id"] for decision in decisions}, - ) requests = recorder.requests_after(checkpoint) routes = [request for request in requests if request.path == ROUTING_PATH] - assert len(routes) == 1, (agent, routes) - route_prompt = routes[0].payload["task"]["prompt"] - assert child_task.filename in route_prompt - response = recorder.response_for(routes[0]) - assert response.status_code == 200 - selections = response.payload["route_selection"] - assert len(selections) == 1 - inference = _task_inference_request( - requests, - agent, - route_prompt, - after=routes[0].sequence, - ) - inference = served_inference_request(recorder, requests, inference, agent) - assert canonical_model(inference.payload["model"]) == canonical_model( - selections[0]["route_option"]["model"] - ) + assert len(routes) == int(subagent_routed), (agent, routes) + if subagent_routed: + assert_subagent_routed( + session, + agent, + child_task, + decision_ids={decision["decision_id"] for decision in decisions}, + ) + route_prompt = routes[0].payload["task"]["prompt"] + assert child_task.filename in route_prompt + response = recorder.response_for(routes[0]) + assert response.status_code == 200 + selections = response.payload["route_selection"] + assert len(selections) == 1 + inference = _task_inference_request( + requests, + agent, + route_prompt, + after=routes[0].sequence, + ) + inference = served_inference_request(recorder, requests, inference, agent) + assert canonical_model(inference.payload["model"]) == canonical_model( + selections[0]["route_option"]["model"] + ) + assert "databricks-smart-router-recipe" in inference.headers + else: + assert read_jsonl(decisions_path) == original_decisions + assert not any( + "databricks-smart-router-recipe" in request.headers + or ORCHESTRATOR_CONTEXT.encode() in request.body + for request in requests + ), agent if agent == CLAUDE: tui.wait_for_background_tasks() tui.exit_normally() @@ -362,49 +379,6 @@ def test_smart_router_config_version( "Smart-router preset cleanup left machine-wide agent settings", ) - def test_claude_only_preset_disables_codex_routing(self, cuj): - """Scenario: select the Claude-only preset and complete a Codex file task. - - Expected: real inference uses the managed default, with no routing requests, - router decisions, recipe header, or orchestrator context. - """ - session, workspace, recorder = cuj - previous = session.env.get("SMART_ROUTER_CONFIG_VERSION") - session.env["SMART_ROUTER_CONFIG_VERSION"] = SUBAGENT_ORCH_V0_CLAUDE_ONLY - try: - configs = _assert_published_config_matches_expectations(workspace.config()) - recorder.configure_session(session, ["configure", "--disable-databricks-ai-tools"]) - task = _file_task(session, CODEX) - evidence = SessionEvidence(session.home, CODEX) - decisions = session.home / ".ucode" / "codex-smart-routing-decisions.jsonl" - before = read_jsonl(decisions) - checkpoint = recorder.checkpoint() - recorder.prepare_launch() - with Terminal(session, "claude-only-codex", [CODEX]) as tui: - tui.boot(timeout=150) - tui.submit(task.prompt) - tui.task(evidence, task) - tui.exit_normally() - requests = recorder.requests_after(checkpoint) - assert not any(request.path == ROUTING_PATH for request in requests) - assert read_jsonl(decisions) == before - inference = _task_inference_request(requests, CODEX, task.prompt) - inference = served_inference_request(recorder, requests, inference, CODEX) - expected = configs[CODEX]["default_models"]["default_model"] - assert canonical_model(inference.payload["model"]) == canonical_model(expected) - evidence.assert_applied(task, workspace.model_ids(CODEX), expected=expected) - assert "databricks-smart-router-recipe" not in inference.headers - assert ORCHESTRATOR_CONTEXT.encode() not in inference.body - finally: - if previous is None: - session.env.pop("SMART_ROUTER_CONFIG_VERSION", None) - else: - session.env["SMART_ROUTER_CONFIG_VERSION"] = previous - session.revert_machine_wide( - "claude-only-codex-revert", - "Claude-only preset cleanup left machine-wide agent settings", - ) - @pytest.mark.parametrize("agent", AGENTS) def test_agent_completes_real_first_prompt_file_task_without_model_override( self, completed_smart_routing_runs, agent diff --git a/tests/integration/README.md b/tests/integration/README.md index 317a5be92..ffec7d7e2 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -11,13 +11,14 @@ inference, and Claude inference. The config equality also accounts for the works skill name as data; skill download and invocation are not covered. The dedicated smart-routing CUJ in `../e2e_cuj/test_cuj4_smart_routing.py` leaves the original -managed-default and explicit-model cases unchanged. One additional test runs the five shared +managed-default and explicit-model cases unchanged. One parameterized test runs all six `SMART_ROUTER_CONFIG_VERSION` values, checking both agents' first-prompt routing, orchestrator -context in inference input, and completed explicitly requested routed subagents. +context in inference input, and completed explicitly requested native subagents. This is not automatic orchestrator-delegation coverage. No workspace configuration is modified. -The `subagent_orch_v0_claude_only` CUJ also requires a completed routed Claude -subagent and a completed Codex file task using the managed default without routing -requests, new router decisions, a recipe header, or orchestrator context. +The shared preset CUJ runs both agents for every version and derives routing and +orchestration assertions from each agent's resolved flags. Both must complete a native +child task. `subagent_orch_v0_claude_only` routes Claude's child; Codex uses the managed +default without routing requests, new router decisions, recipe headers, or orchestrator context. Component checks cover conflicting inherited flags and managed routing defaults. Agent-specific smart-routing preset component checks cover shared base presets, explicit agent mappings, From d81c15dd46f8643a39cbfa6503dba9a13363c0d0 Mon Sep 17 00:00:00 2001 From: "lilly.luo" Date: Fri, 9 Oct 2026 22:36:22 +0000 Subject: [PATCH 6/6] Remove Claude-only preset precedence test matrix --- tests/README.md | 1 - tests/integration/README.md | 1 - tests/test_smart_routing_config.py | 29 ----------------------------- 3 files changed, 31 deletions(-) diff --git a/tests/README.md b/tests/README.md index 97d9fb262..c8b155fb7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -39,7 +39,6 @@ The shared preset CUJ runs both agents for every version and derives routing and orchestration assertions from each agent's resolved flags. Both must complete a native child task. `subagent_orch_v0_claude_only` routes Claude's child; Codex uses the managed default without routing requests, new router decisions, recipe headers, or orchestrator context. -Component checks cover conflicting inherited flags and managed routing defaults. Agent-specific smart-routing preset component checks cover shared base presets, explicit agent mappings, complete per-agent validation, session precedence, and preserving the preset selector diff --git a/tests/integration/README.md b/tests/integration/README.md index ffec7d7e2..dfa969e64 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -19,7 +19,6 @@ The shared preset CUJ runs both agents for every version and derives routing and orchestration assertions from each agent's resolved flags. Both must complete a native child task. `subagent_orch_v0_claude_only` routes Claude's child; Codex uses the managed default without routing requests, new router decisions, recipe headers, or orchestrator context. -Component checks cover conflicting inherited flags and managed routing defaults. Agent-specific smart-routing preset component checks cover shared base presets, explicit agent mappings, complete per-agent validation, session precedence, and preserving the preset selector diff --git a/tests/test_smart_routing_config.py b/tests/test_smart_routing_config.py index b4aa26865..4b081402b 100644 --- a/tests/test_smart_routing_config.py +++ b/tests/test_smart_routing_config.py @@ -2,7 +2,6 @@ from __future__ import annotations -from itertools import product import pytest from ucode.constants import ( @@ -12,7 +11,6 @@ ENABLE_SMART_ROUTING_ENV_VAR, ENABLE_SUBAGENT_ROUTING_ENV_VAR, SMART_ROUTER_CONFIG_VERSION_ENV_VAR, - SMART_ROUTING_ENV_KEYS, ) from ucode.smart_routing import config, orchestrator, session_env, v2 @@ -202,30 +200,3 @@ def test_agent_config_selection_and_session_precedence(monkeypatch, agent, enabl session_env.SESSION_ENV_VAR: str(control), "ENABLE_SMART_ROUTER_ORCHESTRATOR": "0", } - - -@pytest.mark.parametrize("agent", [AGENT_CLAUDE, AGENT_CODEX]) -@pytest.mark.parametrize( - "inherited", list(product([None, "0", "1"], repeat=len(SMART_ROUTING_ENV_KEYS))) -) -def test_claude_only_preset_overrides_inherited_flags_and_managed_default(agent, inherited): - source = { - key: value - for key, value in zip(SMART_ROUTING_ENV_KEYS, inherited, strict=True) - if value is not None - } - source[SMART_ROUTER_CONFIG_VERSION_ENV_VAR] = config.SUBAGENT_ORCH_V0_CLAUDE_ONLY - original = source.copy() - expected = ( - _EXPECTED_PRESETS[config.SUBAGENT_ORCH_V0] - if agent == AGENT_CLAUDE - else dict.fromkeys(SMART_ROUTING_ENV_KEYS, "0") - ) - assert config.resolve_environment(source, agent=agent) == expected - previous = config.apply_config(source, agent=agent) - assert source == expected - assert v2.smart_routing_enabled(source, default=True, agent=agent) == (agent == AGENT_CLAUDE) - assert not v2.first_prompt_routing_enabled(source, agent=agent) - assert orchestrator.feature_enabled(source, agent=agent) == (agent == AGENT_CLAUDE) - v2.restore_smart_routing_env(previous, source) - assert source == original