From ee953dc1fce592abb78587a0c08c7181a05db8fc Mon Sep 17 00:00:00 2001 From: David Liu Date: Wed, 7 Oct 2026 17:24:52 +0000 Subject: [PATCH] [AIGTWY-4977] Cover managed skills in CUJ 2, 3 and 6 The CUJ plan's skills checks were skipped because Default Storage rejects skill uploads. The ug_e2e.skills fixtures now live on customer S3 storage, so the journeys cover them. - CUJ 2 (skills.names = [ug_e2e.skills.fixture-summary]): - only that bundle downloads, byte-identical to the live fixture; - it is listed in /skills without its siblings; - both agents invoke it, answering with a token that exists only in the bundle's reference file. - CUJ 3 (skills.unity_catalog_location = ug_e2e.skills): - every skill in the schema downloads and is listed; - the ug_e2e.other_skills decoy stays absent; - ug status counts the skills; - both agents invoke one. - CUJ 6: - Phase A downloads the named skill once to both roots, with no terminal launch, since one would make ug own /etc and block phase B's headless configure; - Phases B and C drop the selector, and the skill disappears from disk and /skills, including after relaunch; - the hand-authored user skill is the positive control that the /skills view loaded. Shared helpers are in tests/e2e_cuj/helpers/skills.py. Co-authored-by: Isaac --- tests/README.md | 2 +- tests/e2e_cuj/AGENTS.md | 3 +- tests/e2e_cuj/helpers/constants.py | 6 ++ tests/e2e_cuj/helpers/skills.py | 94 ++++++++++++++++++++ tests/e2e_cuj/test_cuj2_mps_mcp.py | 2 + tests/e2e_cuj/test_cuj2_skills.py | 76 ++++++++++++++++ tests/e2e_cuj/test_cuj3_skills.py | 101 ++++++++++++++++++++++ tests/e2e_cuj/test_cuj_repeated_config.py | 74 ++++++++++++++-- 8 files changed, 347 insertions(+), 11 deletions(-) create mode 100644 tests/e2e_cuj/helpers/skills.py create mode 100644 tests/e2e_cuj/test_cuj2_skills.py create mode 100644 tests/e2e_cuj/test_cuj3_skills.py diff --git a/tests/README.md b/tests/README.md index 5459ca961..578fc73ff 100644 --- a/tests/README.md +++ b/tests/README.md @@ -327,7 +327,7 @@ These are unit/component checks; they do not establish live sudo password-prompt | Scenario | Status / requirement | | --- | --- | | Broad live MCP functionality | CUJ2 covers sandbox MCP configuration and generated client listings. Live sandbox execution is deferred because MAS cannot downscope the CI service principal. Installation tests cover the local web-search MCP handshake and tool listing, not upstream proxying or a real search request. Other MCP services, live parent/child search, and permission decisions remain deferred | -| Dedicated CUJ Skill coverage | Deferred because of a backend storage-path issue; CUJ2 has no Skill configuration, download, invocation, or assertion path | +| Dedicated CUJ Skill coverage | CUJ6 covers the named managed skill `ug_e2e.skills.fixture-summary`: download to both skill roots, an unchanged repeat configure, and removal from disk and from both agents' `/skills` once the selector is dropped, including after relaunch. Skill fixtures need a schema on customer S3 storage because Default Storage rejects skill uploads. CUJ2 checks that only the named skill downloads, byte-identical to the live fixture, is listed in `/skills` without its siblings, and is invoked by both agents. CUJ3 checks that every skill in `ug_e2e.skills` downloads and is listed, the `ug_e2e.other_skills` decoy stays absent, `ug status` counts the skills, and both agents invoke one | | Broad configure flags, multiple workspaces, and PAT flows | Deferred while focusing on basic CUJs | | Workspace-switch MCP cleanup | The `workspace_switch` CUJ covers real registration, cleanup, repeat configure, and a completed Claude task. Unit/component tests cover duplicate attempts and injected removal failures; the CUJ does not force an agent timeout. It runs in the required managed CI lane for full/live runs. | | Relayed/subscription MPS discovery | Not covered by the scoped discovery journeys | diff --git a/tests/e2e_cuj/AGENTS.md b/tests/e2e_cuj/AGENTS.md index cfdab7bfe..767198f54 100644 --- a/tests/e2e_cuj/AGENTS.md +++ b/tests/e2e_cuj/AGENTS.md @@ -49,7 +49,8 @@ Model evidence combines observed gateway requests/responses with native complete records; neither routing banners nor native records alone prove an applied decision. `test_cuj2_mps_mcp.py` runs three independently collected cases against a permanently -preconfigured workspace: configuration, Codex inference, and Claude inference. Each case +preconfigured workspace: configuration, Codex inference, and Claude inference. `test_cuj2_skills.py` +adds the named-skill cases against the same workspace. Each case reads and validates the exact two-agent MPS/MCP shape. No case creates, updates, deletes, or restores CodingAgentConfig. The fixture's final read-only equality check is the only workspace mutation guard. The test uses the shared `UserSession`, `Workspace`, and diff --git a/tests/e2e_cuj/helpers/constants.py b/tests/e2e_cuj/helpers/constants.py index e05da54ff..3981e01f3 100644 --- a/tests/e2e_cuj/helpers/constants.py +++ b/tests/e2e_cuj/helpers/constants.py @@ -41,3 +41,9 @@ class CodingAgent(StrEnum): ) UC_MODEL_LOCATION_FIXTURE = ("ug_e2e.models", "ug_e2e.models.codex_primary") FIXTURE_READER_MCP_SERVICE_NAME = "ug_e2e.tools.fixture_reader" +# UC skill names reject underscores, so the plan's `fixture_summary` is hyphenated. +FIXTURE_SUMMARY_SKILL_NAME = "ug_e2e.skills.fixture-summary" +FIXTURE_NOTES_SKILL_NAME = "ug_e2e.skills.fixture-notes" +FIXTURE_CATALOG_LOOKUP_SKILL_NAME = "ug_e2e.skills.fixture-catalog-lookup" +FIXTURE_DECOY_SKILL_NAME = "ug_e2e.other_skills.fixture-decoy" +SKILLS_LOCATION = "ug_e2e.skills" diff --git a/tests/e2e_cuj/helpers/skills.py b/tests/e2e_cuj/helpers/skills.py new file mode 100644 index 000000000..9364b0c1c --- /dev/null +++ b/tests/e2e_cuj/helpers/skills.py @@ -0,0 +1,94 @@ +"""Managed-skill checks shared by the CUJs that publish a skills selector.""" + +from tests.integration.utils.evidence import assistant_answer_contains +from tests.integration.utils.terminal import AgentTerminal + +from .constants import CLAUDE, CODEX, FIXTURE_SUMMARY_SKILL_NAME, INFERENCE_PATHS + +# Claude reads the first root and Codex the second; ug writes managed skills to both. +SKILL_ROOTS = (".claude/skills", ".agents/skills") + + +def skill_name(fqn: str) -> str: + """The securable leaf, which is also the bundle directory and the name agents list.""" + return fqn.rsplit(".", 1)[1] + + +FIXTURE_SUMMARY = skill_name(FIXTURE_SUMMARY_SKILL_NAME) +# Only reference/fixture-facts.md holds the token, so a reply with it proves the bundle was read. +FIXTURE_SUMMARY_TOKEN = "ugsum-4c1e9a" + + +def skills_view( + session, + recorder, + agent: str, + listed: tuple[str, ...], + name: str | None = None, + codex_filter: str = "", +) -> str: + """The agent's /skills screen once every ``listed`` skill renders, so absence checks are real. + + Codex's skill popup shows eight rows, so ``codex_filter`` narrows it; it must also match any + skill the caller asserts is absent. + """ + recorder.prepare_launch() + with AgentTerminal( + session, agent, [str(session.binary), agent], name or f"{agent}-skills" + ) as tui: + tui.boot() + tui.send("/skills", "type the /skills command") + tui.send("\r", "open the skills list") + if agent == CODEX: + tui.send("\r", "confirm the skills selection so the list renders") + if codex_filter: + tui.send(codex_filter, "filter the skills popup") + tui.wait_for( + lambda s: all(name in s for name in listed), + "the /skills view to list skills", + timeout=60, + ) + return tui.visible + + +def bundles_on_disk(session, root: str) -> set[str]: + return {path.name for path in (session.home / root).iterdir()} + + +def bundle_files(session, root: str, bundle: str) -> dict[str, bytes]: + base = session.home / root / bundle + return { + str(path.relative_to(base)): path.read_bytes() for path in base.rglob("*") if path.is_file() + } + + +def assert_fixture_summary_invoked(session, recorder, agent: str) -> None: + """Run a headless prompt that must load fixture-summary and answer from its reference file.""" + recorder.prepare_launch() + # Codex's `$name` mention loads the skill explicitly; small models skip implicit skill matches. + skill = f"${FIXTURE_SUMMARY}" if agent == CODEX else f"the {FIXTURE_SUMMARY} skill" + prompt = ( + f"Use {skill} and reply with only the one-line summary it produces. Do not use subagents." + ) + checkpoint = recorder.checkpoint() + if agent == CLAUDE: + session.run( + CLAUDE, + "--", + "-p", + prompt, + "--output-format", + "json", + "--allowedTools", + "Skill,Read", + timeout=240, + ) + else: + session.run(CODEX, "--", "exec", "--skip-git-repo-check", "--json", prompt, timeout=240) + assert assistant_answer_contains(session, agent, FIXTURE_SUMMARY_TOKEN), ( + f"{agent} never answered with the token from the skill's reference file" + ) + request = recorder.expect_request( + method="POST", path=INFERENCE_PATHS[agent], after=checkpoint, timeout=240 + ) + assert recorder.response_for(request, timeout=240).status_code == 200 diff --git a/tests/e2e_cuj/test_cuj2_mps_mcp.py b/tests/e2e_cuj/test_cuj2_mps_mcp.py index 0af6311df..481c9ebd9 100644 --- a/tests/e2e_cuj/test_cuj2_mps_mcp.py +++ b/tests/e2e_cuj/test_cuj2_mps_mcp.py @@ -22,6 +22,7 @@ CLAUDE, CODEX, CODING_AGENT_BY_CLI_NAME, + FIXTURE_SUMMARY_SKILL_NAME, INFERENCE_PATHS, MODEL_PROVIDER_SERVICE_FIXTURES, SANDBOX_MCP_SERVICE_NAME, @@ -53,6 +54,7 @@ def _assert_cuj2_config(config: dict) -> None: "default_agent": CODING_AGENT_BY_CLI_NAME[CODEX], "enabled_agents": [expected_agent_config(CLAUDE), expected_agent_config(CODEX)], "mcp_servers": {"names": [SANDBOX_MCP_SERVICE_NAME]}, + "skills": {"names": [FIXTURE_SUMMARY_SKILL_NAME]}, } assert actual == expected, config diff --git a/tests/e2e_cuj/test_cuj2_skills.py b/tests/e2e_cuj/test_cuj2_skills.py new file mode 100644 index 000000000..fa38df4c9 --- /dev/null +++ b/tests/e2e_cuj/test_cuj2_skills.py @@ -0,0 +1,76 @@ +"""CUJ2 skills: the one named managed skill is downloaded, listed, and invoked by both agents.""" + +from __future__ import annotations + +import pytest + +from ucode.skills_api import fetch_skill_bundle + +from .helpers.constants import ( + CLAUDE, + CODEX, + FIXTURE_CATALOG_LOOKUP_SKILL_NAME, + FIXTURE_NOTES_SKILL_NAME, + SKILLS_LOCATION, +) +from .helpers.skills import ( + FIXTURE_SUMMARY, + SKILL_ROOTS, + assert_fixture_summary_invoked, + bundle_files, + bundles_on_disk, + skill_name, + skills_view, +) +from .test_cuj2_mps_mcp import _assert_cuj2_config, _Cuj2Base + +OTHER_SKILLS = tuple(map(skill_name, (FIXTURE_NOTES_SKILL_NAME, FIXTURE_CATALOG_LOOKUP_SKILL_NAME))) + + +class TestCuj2Skills(_Cuj2Base): + @pytest.fixture(scope="class") + def configured(self, cuj): + session, workspace, recorder = cuj + _assert_cuj2_config(workspace.config()) + recorder.configure_session( + session, ["configure", "--skip-upgrade", "--disable-databricks-ai-tools"] + ) + return cuj + + def test_named_skill_matches_live_fixture_and_siblings_are_absent(self, configured): + """Scenario: configure with `skills.names` selecting only fixture-summary. + + Expected: both roots hold exactly that bundle, byte-identical to the live fixture. + """ + session, workspace, _ = configured + live, reason = fetch_skill_bundle( + workspace.url, + session.env["DATABRICKS_BEARER"], + *SKILLS_LOCATION.split("."), + FIXTURE_SUMMARY, + ) + assert live and "reference/fixture-facts.md" in live, reason + for root in SKILL_ROOTS: + assert bundles_on_disk(session, root) == {FIXTURE_SUMMARY} + assert bundle_files(session, root, FIXTURE_SUMMARY) == live + + @pytest.mark.parametrize("agent", [CLAUDE, CODEX]) + def test_agent_lists_only_the_named_skill(self, configured, agent): + """Scenario: open /skills in each agent. + + Expected: fixture-summary is listed and the schema's other skills are not. + """ + session, _, recorder = configured + screen = skills_view(session, recorder, agent, (FIXTURE_SUMMARY,)) + for other in OTHER_SKILLS: + assert other not in screen, f"{agent} /skills:\n{screen}" + + @pytest.mark.parametrize("agent", [CLAUDE, CODEX]) + def test_agent_invokes_the_named_skill(self, configured, agent): + """Scenario: ask each agent, headless, to use fixture-summary. + + Expected: the reply carries the token found only in the bundle's reference file, after a + 200 inference response through the recorder. + """ + session, _, recorder = configured + assert_fixture_summary_invoked(session, recorder, agent) diff --git a/tests/e2e_cuj/test_cuj3_skills.py b/tests/e2e_cuj/test_cuj3_skills.py new file mode 100644 index 000000000..b071321be --- /dev/null +++ b/tests/e2e_cuj/test_cuj3_skills.py @@ -0,0 +1,101 @@ +"""CUJ3 skills: a `unity_catalog_location` downloads every skill in the schema and nothing else.""" + +from __future__ import annotations + +import re + +import pytest + +from .base import BaseCujTest +from .helpers.constants import ( + CLAUDE, + CODEX, + FIXTURE_CATALOG_LOOKUP_SKILL_NAME, + FIXTURE_DECOY_SKILL_NAME, + FIXTURE_NOTES_SKILL_NAME, + FIXTURE_SUMMARY_SKILL_NAME, + SKILLS_LOCATION, +) +from .helpers.skills import ( + SKILL_ROOTS, + assert_fixture_summary_invoked, + bundle_files, + bundles_on_disk, + skill_name, + skills_view, +) + +SCHEMA_SKILLS = tuple( + map( + skill_name, + (FIXTURE_SUMMARY_SKILL_NAME, FIXTURE_NOTES_SKILL_NAME, FIXTURE_CATALOG_LOOKUP_SKILL_NAME), + ) +) +DECOY = skill_name(FIXTURE_DECOY_SKILL_NAME) +STATUS_PANEL_TITLES = {CLAUDE: "Claude Code", CODEX: "Codex"} + + +class TestCuj3Skills(BaseCujTest): + WORKSPACE_URL = "https://dbc-bbdd5508-648e.cloud.databricks.com" + + @pytest.fixture(scope="class") + def configured(self, cuj): + session, workspace, recorder = cuj + assert workspace.config()["skills"] == {"unity_catalog_location": SKILLS_LOCATION} + session.configure( + [ + "configure", + "--workspace", + workspace.url, + "--skip-upgrade", + "--disable-databricks-ai-tools", + ] + ) + # `ug status` takes no workspace flag, so read it before retargeting to the recorder. + status = session.run("status", timeout=120).stdout + recorder.retarget_session(session.home) + return session, recorder, status + + def test_every_schema_skill_is_on_disk_and_the_decoy_is_not(self, configured): + """Scenario: configure with `skills.unity_catalog_location` set to the skills schema. + + Expected: both roots hold exactly the schema's three skills with their supporting files, + and nothing from the other schema. + """ + session, _, _ = configured + for root in SKILL_ROOTS: + assert bundles_on_disk(session, root) == set(SCHEMA_SKILLS) + assert "reference/fixture-facts.md" in bundle_files(session, root, SCHEMA_SKILLS[0]) + assert "scripts/lookup_schema.py" in bundle_files(session, root, SCHEMA_SKILLS[2]) + + @pytest.mark.parametrize("agent", [CLAUDE, CODEX]) + def test_agent_lists_every_schema_skill_and_not_the_decoy(self, configured, agent): + """Scenario: open /skills in each agent. + + Expected: all three schema skills are listed and the out-of-scope decoy is not. + """ + session, recorder, _ = configured + screen = skills_view(session, recorder, agent, SCHEMA_SKILLS, codex_filter="fixture") + assert DECOY not in screen, f"{agent} /skills:\n{screen}" + + @pytest.mark.parametrize("agent", [CLAUDE, CODEX]) + def test_status_counts_the_skills(self, configured, agent): + """Scenario: run `ug status` after configure. + + Expected: each agent's panel reports one Skills row counting the three schema skills. + """ + *_, status = configured + panel = re.search( + rf"╭[─ ]*{STATUS_PANEL_TITLES[agent]}[─ ]*╮.*?Skills:\s+(\d+)", status, re.DOTALL + ) + assert panel and int(panel.group(1)) == len(SCHEMA_SKILLS), status + + @pytest.mark.parametrize("agent", [CLAUDE, CODEX]) + def test_agent_invokes_a_downloaded_skill(self, configured, agent): + """Scenario: ask each agent, headless, to use fixture-summary. + + Expected: the reply carries the token found only in the bundle's reference file, so + listing alone is not the pass condition. + """ + session, recorder, _ = configured + assert_fixture_summary_invoked(session, recorder, agent) diff --git a/tests/e2e_cuj/test_cuj_repeated_config.py b/tests/e2e_cuj/test_cuj_repeated_config.py index 9b18cb87d..7368a9132 100644 --- a/tests/e2e_cuj/test_cuj_repeated_config.py +++ b/tests/e2e_cuj/test_cuj_repeated_config.py @@ -6,10 +6,8 @@ Workspace configs are read only, so each phase is the single published config of its own workspace and the shared home moves from A to B to C. All three share one metastore with the ``ug_e2e`` fixtures (the Bedrock Claude MPS ``ug_e2e.providers.bedrock``, the Codex model -services in ``ug_e2e.models``, and the ``fixture_reader`` MCP) and one trace table. - -Managed skills are not covered: UC skill bundles cannot be uploaded on these workspaces' -Default Storage catalog (Files API HTTP 501). +services in ``ug_e2e.models``, the ``fixture_reader`` MCP, and the ``fixture-summary`` skill) +and one trace table. """ import hashlib @@ -39,12 +37,14 @@ CLAUDE, CODEX, FIXTURE_READER_MCP_SERVICE_NAME, + FIXTURE_SUMMARY_SKILL_NAME, INFERENCE_PATHS, MANAGED_PATHS, UC_MODEL_LOCATION_FIXTURE, CodingAgent, ) from .helpers.evidence import SessionEvidence, canonical_model +from .helpers.skills import SKILL_ROOTS, skill_name, skills_view from .helpers.tui_request_recorder import TuiRequestRecorder from .helpers.workspace import Workspace @@ -69,6 +69,7 @@ TRACE_INGESTION_SECONDS = 300 USER_MCP = "ug-e2e-user-echo" USER_SKILL = "ug-e2e-user-notes" +MANAGED_SKILL = skill_name(FIXTURE_SUMMARY_SKILL_NAME) USER_MCP_SERVER = """\ import json, sys @@ -145,6 +146,15 @@ def mcp_tool_names(registration: dict) -> tuple[str, ...]: return tuple(tool["name"] for tool in responses[2]["result"]["tools"]) +def skill_bundles(files: "AgentFiles", root: str) -> set[str]: + """The skill directories under one root, from the files captured there.""" + return { + Path(path).relative_to(root).parts[0] + for path in files.skill_files + if Path(path).is_relative_to(root) + } + + def workspace_client(url: str) -> WorkspaceClient: """The base class authenticates only WORKSPACE_URL; phases B and C need their own clients.""" return WorkspaceClient( @@ -206,6 +216,7 @@ class PhaseB: os_managed_codex: dict codex_span_count: int claude_span_count: int + skills: dict[str, str] @dataclass(frozen=True) @@ -221,6 +232,7 @@ class PhaseC: bare_ug: AgentTask phase_b_span_count: int codex_span_count: int + skills: dict[str, str] @dataclass(frozen=True) @@ -293,7 +305,7 @@ def mcp_list(self, agent: str) -> tuple[str, ...]: def skill_files(self) -> dict[str, str]: files = {} - for root in (self.home / ".claude/skills", self.home / ".agents/skills"): + for root in (self.home / name for name in SKILL_ROOTS): for path in sorted(root.rglob("*")) if root.is_dir() else []: if path.is_file(): digest = hashlib.sha256(path.read_bytes()).hexdigest() @@ -381,6 +393,16 @@ def run_task(self, phase: str, agent: str, *, marker: str | None = None) -> Agen assert requests, f"The recorder saw no {agent} inference request" return AgentTask(requests=requests, turn_models=tuple(turn.models)) + def skills_view(self, phase: str, agent: str) -> str: + # The user's own skill rendering proves the list loaded before absence is checked. + return skills_view( + self.session, + self.recorders[phase], + agent, + (USER_SKILL,), + name=f"phase-{phase}-{agent}-skills", + ) + def bare_ug_task(self, phase: str) -> AgentTask: """Bare `ug` in a real terminal; helpers.terminal.Terminal only drives `ug `.""" recorder = self.recorders[phase] @@ -430,9 +452,14 @@ def seed_user_state(self) -> SeededUserState: """Ordinary user-owned prefs, MCP registration, and skill; no ug state.""" claude_settings = self.home / ".claude/settings.json" claude_settings.parent.mkdir(parents=True) - claude_settings.write_text( - json.dumps({"cleanupPeriodDays": 45, "env": {"UG_E2E_USER_PREF": self.run_id}}) - ) + # A user who already picked a theme, in Claude's own format: Claude's TUI saves the theme + # here on first launch, so that write is a no-op and any diff is ug's. + user_settings = { + "cleanupPeriodDays": 45, + "env": {"UG_E2E_USER_PREF": self.run_id}, + "theme": "dark", + } + claude_settings.write_text(json.dumps(user_settings, indent=2) + "\n") codex_config = self.home / ".codex/config.toml" codex_config.parent.mkdir(parents=True) codex_config.write_text( @@ -448,7 +475,7 @@ def seed_user_state(self) -> SeededUserState: (CODEX, ["mcp", "add", USER_MCP, "--", sys.executable, str(server)]), ): self.session.run(*args, binary=self.agent_binary(agent)) - for root in (".claude/skills", ".agents/skills"): + for root in SKILL_ROOTS: skill = self.home / root / USER_SKILL / "SKILL.md" skill.parent.mkdir(parents=True) skill.write_text( @@ -501,6 +528,7 @@ def journey(self, cuj): def phase_a(self, journey): # Configure A directly twice: through the recorder, the repeat would be a switch back # from the recorder URL rather than a repeat of the same workspace. + # No terminal launch in A: it makes ug own /etc, which phase B's headless configure refuses. journey.enter("a") upstream = ["--workspace", journey.workspaces["a"].url] journey.session.configure([*CONFIGURE_ARGS, *upstream]) @@ -534,6 +562,7 @@ def phase_b(self, journey, phase_a): os_managed_codex=read_toml(CODEX_OS_MANAGED_CONFIG), codex_span_count=codex_span_count, claude_span_count=journey.trace_count(claude_marker, resource=True), + skills={agent: journey.skills_view("b", agent) for agent in (CLAUDE, CODEX)}, ) @pytest.fixture(scope="class") @@ -555,6 +584,7 @@ def phase_c(self, journey, phase_b): codex_marker = f"{journey.run_id}-codex-c" codex = journey.run_task("c", CODEX, marker=codex_marker) final = journey.agent_files() + skills = {CODEX: journey.skills_view("c", CODEX)} # Absence can't be polled; wait out the ingestion window before counting. time.sleep(TRACE_INGESTION_SECONDS) return PhaseC( @@ -569,6 +599,7 @@ def phase_c(self, journey, phase_b): bare_ug=bare_ug, phase_b_span_count=journey.trace_count(f"{journey.run_id}-codex-b", resource=False), codex_span_count=journey.trace_count(codex_marker, resource=False), + skills=skills, ) # Phase A: CUJ 1's config plus a named managed MCP. @@ -582,10 +613,18 @@ def test_phase_a_workspace_publishes_plan_config(self, journey): assert codex["models"]["model_services"] == [CODEX_SOL, CODEX_LUNA] assert tracing_enabled(claude) and tracing_enabled(codex) assert config["mcp_servers"]["names"] == [FIXTURE_READER_MCP_SERVICE_NAME] + assert config["skills"]["names"] == [FIXTURE_SUMMARY_SKILL_NAME] def test_phase_a_repeat_configure_changes_nothing(self, phase_a): assert phase_a.second == phase_a.first + def test_phase_a_downloads_the_managed_skill_once(self, phase_a): + files = phase_a.second + for root in SKILL_ROOTS: + assert skill_bundles(files, root) == {MANAGED_SKILL, USER_SKILL} + for bundle_file in ("SKILL.md", "reference/fixture-facts.md"): + assert f"{root}/{MANAGED_SKILL}/{bundle_file}" in files.skill_files + def test_phase_a_registers_managed_and_user_mcp_once(self, phase_a): files = phase_a.second for registrations in (files.claude_mcp, files.codex_mcp): @@ -627,6 +666,7 @@ def test_phase_b_workspace_publishes_plan_config(self, journey): assert lower_keys(codex["http_headers"]) == {RUN_HEADER: "cuj6-phase-b"} assert tracing_enabled(codex) assert not config.get("mcp_servers", {}).get("names") + assert not config.get("skills") def test_phase_b_claude_routes_through_the_provider(self, phase_b): assert CLAUDE_PROVIDER_MODEL in phase_b.claude.request_models @@ -692,6 +732,14 @@ def test_phase_b_managed_mcp_stays_removed_across_launches(self, phase_b): ): assert MANAGED_MCP_ENTRY not in registered + def test_phase_b_managed_skill_stays_removed_across_launches(self, phase_b): + # The /skills views open after the task launches, so they also cover restarts. + for files in (phase_b.files, phase_b.restarted): + for root in SKILL_ROOTS: + assert skill_bundles(files, root) == {USER_SKILL} + for agent, screen in phase_b.skills.items(): + assert MANAGED_SKILL not in screen, f"{agent} /skills:\n{screen}" + def test_phase_b_traces_codex_but_not_claude(self, phase_b): # Codex's span proves the destination is up before Claude's absence counts. assert phase_b.codex_span_count > 0 @@ -706,6 +754,14 @@ def test_phase_c_workspace_publishes_plan_config(self, journey): assert codex["models"] == {"model_services": [CODEX_LUNA]} assert codex["default_models"]["default_model"] == CODEX_LUNA assert not tracing_enabled(codex) and not codex.get("http_headers") + assert not config.get("skills") + + def test_phase_c_managed_skill_stays_removed(self, journey, phase_c): + for root in SKILL_ROOTS: + assert skill_bundles(phase_c.final, root) == {USER_SKILL} + # skill_files lists files only; a leftover empty bundle directory would hide there. + assert not (journey.home / root / MANAGED_SKILL).exists() + assert MANAGED_SKILL not in phase_c.skills[CODEX], phase_c.skills[CODEX] def test_phase_c_terminal_configure_clears_phase_b_machine_wide_headers(self, phase_c): provider = phase_c.os_managed_codex["model_providers"]["Databricks"]