diff --git a/tests/README.md b/tests/README.md index e4d8d86da..8f26faba6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -31,8 +31,10 @@ running" or a native task menu containing only completed rows before sending `/e stopping tasks or confirming an exit dialog. Running/scheduled sections, incomplete row counts, and completed-task text outside the native menu cannot satisfy the wait. The live Claude subagent skill-toggle journey uses this wait after its final calculation. -CUJ4's Claude preset sessions also use it after verifying delegated file tasks: a completed -answer does not prove that a resumed child has stopped running. +CUJ4's preset sessions verify the child's answer and routed inference, then require +the parent's completed native turn containing the delegated result and a normal `/exit`. +They do not parse the `/tasks` menu. Offline evidence checks reject child answers, +notifications, and parent tool calls as substitutes for the parent's final result. They cover Claude/Codex helper dispatch and rejection of routing decisions without the agent-specific prompt-submission evidence. Agent-specific smart-routing preset component checks cover shared base presets, explicit agent mappings, diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index 3382630a2..b54be1e4c 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -93,7 +93,8 @@ Each preset session then explicitly requests one child for a separate hidden-val file task. Assertions require a new native child transcript containing the value, a correlated spawn-routing decision, and successful child inference on the router's selected model. This tests requested delegation, not automatic orchestrator delegation. -The wait uses the child's answer directly; parent-answer reconstruction is reserved for first-prompt tasks. +Child inference evidence uses the child's answer directly. Both agents also wait for the +parent's completed native turn containing the delegated result before exiting. Offline checks reject parent-only answers and accept child answers before the parent replies. Offline evidence regressions do not establish a live CUJ pass. Selector cases have separate TUI artifact names, and the session environment is restored afterward. @@ -102,9 +103,10 @@ Claude sessions start in auto mode; unless a routed first prompt switches the se Claude's informational auto-mode classifier billing notice over the transcript. The CUJ terminal waits for that exact notice to render stably, presses Enter (continue), and observes dismissal before continuing. Later occurrences are handled the same way; any other dialog still fails. -After verifying the delegated task, Claude sessions also wait in the native `/tasks` view -until no background work remains, including a child resumed after its initial answer. -Only then does the test close the task view and submit `/exit`. +After verifying the delegated task, both agents require the parent's final answer with +the file value and a normal `/exit`, without visiting `/tasks`. Native task notifications +can precede that answer but cannot satisfy it; parent tool calls do not count as completion. +This covers the delegation-and-reply journey rather than the background-task management UI. Each preset also runs public `ug revert` in cleanup, including after a failed assertion, so interactive launches' OS-managed settings cannot contaminate the next preset's configuration. The existing Claude explicit-model precedence case remains skipped; the routing-disabled case diff --git a/tests/e2e_cuj/helpers/evidence.py b/tests/e2e_cuj/helpers/evidence.py index 73a401474..9eafa5d72 100644 --- a/tests/e2e_cuj/helpers/evidence.py +++ b/tests/e2e_cuj/helpers/evidence.py @@ -194,7 +194,11 @@ def _completed_turn(records, task): responses = [] for row in records[prompts[0] + 1 :]: msg = row.get("message", {}) - if row.get("type") == "user" and isinstance(msg.get("content"), str): + if ( + row.get("type") == "user" + and isinstance(msg.get("content"), str) + and (row.get("origin") or {}).get("kind") != "task-notification" + ): break if row.get("type") == "assistant": assert row.get("sessionId") == session_id and session_id diff --git a/tests/e2e_cuj/test_cuj4_smart_routing.py b/tests/e2e_cuj/test_cuj4_smart_routing.py index c19ab368e..a11651167 100644 --- a/tests/e2e_cuj/test_cuj4_smart_routing.py +++ b/tests/e2e_cuj/test_cuj4_smart_routing.py @@ -245,7 +245,8 @@ def test_smart_router_config_version( """Scenario: launch both agents with a preset, then explicitly request a subagent. Expected: first-prompt routing and orchestrator context match the preset; - one routed native child completes the delegated task. + one routed native child completes the delegated task, the parent returns + its result, and the interactive session exits normally. """ session, workspace, recorder = cuj previous = session.env.get("SMART_ROUTER_CONFIG_VERSION") @@ -306,6 +307,7 @@ def test_smart_router_config_version( ), "completed native subagent file task", ) + tui.task(evidence, child_task) children = { path: records for path, records in agent_sessions(session, agent).items() @@ -343,8 +345,6 @@ def test_smart_router_config_version( assert canonical_model(inference.payload["model"]) == canonical_model( selections[0]["route_option"]["model"] ) - if agent == CLAUDE: - tui.wait_for_background_tasks() tui.exit_normally() finally: if previous is None: diff --git a/tests/integration/README.md b/tests/integration/README.md index 85e63db89..af06eadf4 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -64,8 +64,10 @@ after Escape before typing `/exit`. Offline PTY checks reject active/scheduled t lists, and completed-task text outside the native menu. It neither stops tasks nor confirms an exit dialog. Process exit retains its 30-second timeout. Offline PTY checks cover this ordering. -The shared wait also runs before CUJ4's Claude preset sessions exit, after delegated-task -evidence is checked, because Claude can resume a child after its first completed answer. +CUJ4's preset journey instead verifies delegated child evidence, waits for the +parent's completed native turn containing the result, and requires normal `/exit`. +Its routing assertions are independent of `/tasks` presentation. Offline checks reject +child-only results, completion notifications, and parent tool calls as final parent answers. Agent-specific bundled instruction selection and prompt/compaction hook content are covered in `../test_skills.py`; those component checks do not run live agents. diff --git a/tests/test_cuj_evidence.py b/tests/test_cuj_evidence.py index 40f84ba74..43a9384bf 100644 --- a/tests/test_cuj_evidence.py +++ b/tests/test_cuj_evidence.py @@ -121,6 +121,40 @@ def test_cuj_evidence_claude_real_user_message_ends_parent_turn(): assert completed_turn(CLAUDE, [prompt, user, answer], task) is None +def test_claude_delegated_journey_requires_parent_result_after_child_notification(tmp_path): + task = FileTask(SimpleNamespace(cwd=tmp_path)) + task.prompt = task.delegate_prompt + evidence = SessionEvidence(tmp_path, CLAUDE) + parent = evidence.directory / "session.jsonl" + child = evidence.directory / "session" / "subagents" / "agent-child.jsonl" + prompt, answer = records(CLAUDE, task, "system.ai.claude-sonnet-4-6") + child_rows = records(CLAUDE, task, "system.ai.claude-haiku-4-5") + for row in child_rows: + row["isSidechain"] = True + write_rows(child, child_rows) + notification = { + "type": "user", + "sessionId": "session", + "origin": {"kind": "task-notification", "producer": "session-task"}, + "message": { + "content": f"{task.value}" + }, + } + write_rows(parent, [prompt, notification]) + assert evidence.completed(task) is None + + working = copy.deepcopy(answer) + working["message"]["stop_reason"] = "tool_use" + working["message"]["content"].append({"type": "tool_use", "name": "SendMessage"}) + write_rows(parent, [prompt, notification, working]) + assert evidence.completed(task) is None + + write_rows(parent, [prompt, notification, working, answer]) + turn = evidence.completed(task) + assert turn is not None + assert turn.answer == task.value + + @pytest.mark.parametrize("agent", [CLAUDE, CODEX]) @pytest.mark.parametrize( "failure",