diff --git a/tests/acceptance/conftest.py b/tests/acceptance/conftest.py index 9e7b4bc..90ae771 100644 --- a/tests/acceptance/conftest.py +++ b/tests/acceptance/conftest.py @@ -398,7 +398,7 @@ def hand_started_session( # noqa: ANN201 session = hand_started.HandStartedSession( lane=lane.name, binary=binary, - arguments=lane.arguments, + arguments=hand_started.launch_arguments(lane.arguments, lane.boot), workspace=config.workspace, environment=terminal_environment, journal=journal, diff --git a/tests/acceptance/hand_started.py b/tests/acceptance/hand_started.py index 43cdf8a..ef89ade 100644 --- a/tests/acceptance/hand_started.py +++ b/tests/acceptance/hand_started.py @@ -33,6 +33,38 @@ written to the run's artifacts as evidence a human can read; every assertion in `journey.py` rests on the roster, the rollout, the filesystem, `engine.log` or the chat. + +A fourth, measured on #110 (2026-08-27, `codex-cli` 0.150.0): + +* **The Codex lane launches with an initial prompt, and that argument is the + whole update gate.** From the instant `~/.codex/version.json` learns of a + release, every hand-started `codex` stops at `1. Update now / 2. Skip / 3. Skip + until next version` **before the Session registers** — no thread on the daemon, + no rollout, indefinitely; #105 watched one probe boot normally and the three + that started after the check landed die there, 90 seconds apart. Codex skips + that prompt outright when it is handed a non-empty `PROMPT` argument + (`codex [OPTIONS] [PROMPT]`), decided at `tui/src/lib.rs:995-996` and + byte-identical at 0.149.1 and 0.150.0 (read on #107): + + let skip_update_prompt = cli.prompt.as_ref().is_some_and(|p| !p.is_empty()); + + So `journey.CODEX` passes the words its first turn would have typed anyway. + **The emptiness test is the mechanism** — an empty string is not a skip — which + is why the argument is the journey's own instruction and not a placeholder that + could quietly shrink to `""`. + + The alternative, `-c check_for_update_on_startup=false`, suppresses the prompt + *and* makes the TUI refuse the shared daemon (`can_reuse_implicit_local_daemon` + requires `cli_kv_overrides.is_empty()`, `tui/src/lib.rs:919-921`): it would + silently destroy the attachment the run exists to measure. **Never reach for + `-c` to solve a boot gate.** + + This gate is *skipped*, not arranged, and that is what makes it unlike + `TrustGate`: nothing of the user's is edited, `version.json` is not written, and + there is nothing for the run to revoke afterwards. What it *is* unlike an + ordinary flag in is that **a prompt on the command line is a turn**, running + before the walk has asked for anything — so `journey.Walk.settle_boot_turn` + waits it out before a word is typed or a chat mark is taken. """ from __future__ import annotations @@ -51,6 +83,7 @@ import threading import time from collections import deque +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path @@ -114,6 +147,90 @@ def resolve(binary: str, path_value: str) -> Path | None: return Path(found) if found else None +@dataclass(frozen=True) +class BootPrompt: + """Words a lane's TUI is *launched* with, and how to know that turn is over. + + The two travel together because either alone is a trap. A prompt on the + command line is what carries the Codex lane past the update gate (#110), and + it is also a **turn**, running before the walk has asked for anything; the + reading is how `journey.Walk.settle_boot_turn` knows it may stop waiting. A + lane that gained the first without the second is exactly the false green the + Codex review of #110 found, so the type does not let it. + """ + + words: str + #: Whether the agent's own record shows **no turn in flight**, given where + #: that record is now (or None when there is not one yet). + turn_over: Callable[[Path | None], bool] + + +#: How Codex brackets a turn in its own record. Measured 2026-08-27 against a +#: real rollout on this machine (`rollout-2026-08-27T10-17-38-01a04026…jsonl`, +#: codex-cli 0.150.0): two turns, two `task_started`, two `task_complete`, and +#: the file's last record is the second `task_complete`. +TURN_STARTED = "task_started" +TURN_COMPLETE = "task_complete" + + +def codex_turn_over(rollout: Path | None) -> bool: + """Whether Codex's own record shows no turn in flight: all started, all complete. + + **Not** "the record stopped growing for a while", which is what `_drive_turn` + settles for and what the Codex review of #110 rejected for this use: a turn + waiting on the model appends nothing, so silence is a turn that may be over + and may be thinking. For a graded turn that ambiguity costs a slow reading; + for the boot turn it costs the run its meaning, because a boot turn wrongly + called over is a Stop landing where a later step is looking for a different + one. Codex says which it is, so this asks Codex. + """ + if rollout is None: + return False + started = complete = 0 + try: + with rollout.open() as lines: + for line in lines: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue # a line half-written while this read it + if record.get("type") != "event_msg": + continue + payload = record.get("payload") + kind = payload.get("type") if isinstance(payload, dict) else None + started += kind == TURN_STARTED + complete += kind == TURN_COMPLETE + except OSError: + return False + return started > 0 and complete >= started + + +def launch_arguments(flags: tuple[str, ...], boot: BootPrompt | None) -> tuple[str, ...]: + """A lane's argv: its flags, then its boot prompt — last, and never empty. + + The two rules this holds are both load-bearing and neither is visible in a + tuple literal. **Last**, because both agents take the prompt as a positional + (`codex [OPTIONS] [PROMPT]`), so a prompt written before a flag is read as + that flag's value. **Never empty**, because emptiness is what the update gate + tests (see the fourth measurement above): `""` is a prompt that starts no + turn and skips no gate, and the TUI would stop at the update prompt exactly + as if nothing had been passed — silently, and only in the weeks after a + release, which is the worst way for a harness to be wrong. + + A lane with no boot prompt passes `None` and gets its flags back. That is the + Claude lane, and it is the ordinary case: a Session nobody has typed into is + what the walk's first step wants to find. + """ + if boot is None: + return flags + if not boot.words.strip(): + raise ValueError( + "a boot prompt must be non-empty: an empty PROMPT is not a skipped update gate, " + "it is a launch that stops at the gate with nothing to show for it" + ) + return (*flags, boot.words) + + class SessionRefused(RuntimeError): """The command would not start, so there is no Session to judge anything on.""" diff --git a/tests/acceptance/journey.py b/tests/acceptance/journey.py index f72e64c..ab54814 100644 --- a/tests/acceptance/journey.py +++ b/tests/acceptance/journey.py @@ -36,6 +36,12 @@ words arriving and the permission that turn raises — because a relayed instruction that needs a permission is exactly the shape the product has to survive, and running it twice would prove less at twice the cost. + +The Codex lane runs a sixth that no step drives: it is *launched* with a prompt, +because that is what carries it past Codex's update gate (#110), and a prompt on +the command line is a turn. No step observes it and `Walk.settle_boot_turn` waits +it out before the walk begins — a turn still running when the first step types is +a turn whose Stop lands where a later step is looking for a different one. """ from __future__ import annotations @@ -78,6 +84,22 @@ #: cadence once there is one. DISCOVERY_SECONDS = 30.0 +#: How long a boot turn gets, as a multiple of the far side's turn figure. +#: Derived rather than guessed: `settle_boot_turn` waits out a TUI's **boot and** +#: its first turn, and boot alone has been measured at the whole turn figure on +#: this machine — a `codex` sat in `starting MCP servers` for a full 180s +#: ground-truth wait on 2026-08-26 (`hand_started.codex_ground_truth`). So two of +#: them, one for each half, and a lane still unsettled after that is blocked +#: rather than typed into. Blocked, not merely slow: everything after it would be +#: measuring a Session with a turn still running underneath. +BOOT_TURN_ALLOWANCE = 2.0 + +#: The engine's own line for a Stop it announced (`core/bridge.py:_session_stopped`). +#: `stop notice` matches a looser pattern for its own purposes; `drain_boot_notice` +#: wants the announcement itself, because "was a notice raised for the boot turn" +#: is exactly the question it is asking. +ENGINE_STOP_LINE = r"(?i)Session stopped:" + #: What the escalated permission announcement says (`core/approvals.py:43-46`). #: Matched rather than quoted whole: the tool name and the detail are the agent's, #: and this run does not get to predict them. @@ -214,6 +236,13 @@ class Lane: binary: str #: Arguments the *person* would not normally type, and why each is here. arguments: tuple[str, ...] + #: What the lane's TUI is **launched with**, or None when it is launched + #: silent. Not one of `arguments`, because it is not a flag: it starts a turn + #: nobody drove, before the walk has asked for anything. `hand_started. + #: launch_arguments` puts its words last on the command line and refuses an + #: empty one; `Walk.settle_boot_turn` waits the turn out, through the reading + #: the value carries, before a word is typed. + boot: hand_started.BootPrompt | None #: The words that make this lane's agent spawn a Child Process. "subagent" #: and "sub-agent" appear here on purpose: this string is spoken *to* the #: agent, where it is the agent's own mechanism word and the thing that makes @@ -267,6 +296,10 @@ class Lane: agent="claude", binary="claude", arguments=("--permission-mode", "default"), + # Launched silent. No boot gate of the Codex kind has been measured here — + # `claude` boots into an empty composer — and a Session nobody has typed into + # is what `roster` and `stable name`'s three reads want to find. + boot=None, relayed=lambda workspace: writing(RELAY_FILE, RELAY_WORD), # The flag the harness passes *is* the whole policy on this lane, and Claude # publishes no per-turn readback of it, so there is nothing to read back and @@ -301,11 +334,36 @@ class Lane: #: a trusted workspace already gives (measured 2026-08-26 on the failing run's #: own rollout, `turn_context.sandbox_policy = workspace-write`), so the flag #: fixes the ground rather than moving it. +#: +#: The **boot prompt** is not a flag and is not here to ask for anything: it is +#: what gets this lane past the update gate (#110; the measurement is in +#: `hand_started`'s module docstring). Three things follow, and each is stated +#: because a reader will otherwise meet it as a surprise: +#: +#: * **It is an extra turn, not a replacement.** `stable name` still types +#: `ACKNOWLEDGE` itself, because it requires a name held across a Stop it +#: drives, and `stop notice` marks the chat immediately before that turn — a +#: Stop crossed at launch would predate the mark and the notice would be +#: unfindable. The words are `ACKNOWLEDGE`'s so the boot turn asks the Session +#: nothing the run does not already ask, and it uses no tools, so it cannot +#: raise a permission before `approval` is there to answer it, and it leaves +#: `codex_turn_policy` reading the same ground: the sandbox is this lane's pin +#: and the `turn_context` that step grades is `relay`'s, the last one written. +#: * **The walk waits it out first** (`Walk.settle_boot_turn`). Two turns of the +#: same words are not two turns the harness can tell apart: a boot turn still +#: running when `stop notice`'s mark is taken puts *its* Stop Notice after the +#: mark, and the step would pass on the notice for a turn nobody drove. +#: * **The rollout now exists before `roster` runs.** Codex writes its rollout +#: when the first *turn* starts, so this lane's `ground_truth` carries a real +#: `session_id` at the first read instead of the `""` it used to carry — the +#: evidence line `roster` prints changes shape, and the pid join it rests on +#: does not. CODEX = Lane( name="codex", agent="codex", binary="codex", arguments=("--sandbox", "workspace-write"), + boot=hand_started.BootPrompt(words=ACKNOWLEDGE.words, turn_over=hand_started.codex_turn_over), # Measured 2026-08-27 through the shared daemon with the product's own pin # and no sandbox override, on codex-cli 0.149.1 and again on 0.150.0 over a # 0.149.1 app-server: a write to a path outside the workspace raises @@ -391,6 +449,9 @@ def __init__( self.before_first_turn: int | None = None #: Held for `approval`: what `relay`'s turn raised and how it was answered. self.approval_evidence: str | None = None + #: Held for `drain_boot_notice`, which writes the one `boot turn` + #: observation once both halves of the arrangement are done. + self.boot_turn: Turn | None = None self.turns: list[Turn] = [] # --- the walk --------------------------------------------------------- @@ -404,7 +465,9 @@ def walk(self) -> None: "the revoke. It is not a step: the run cannot both arrange this and judge it.", ) try: + boot_mark = self.settle_boot_turn() self.arm_switches() + self.drain_boot_notice(boot_mark) except LaneBlocked as unarmed: self.journey.skip_rest(str(unarmed)) return @@ -423,6 +486,109 @@ def walk(self) -> None: or "no turn ran", ) + def settle_boot_turn(self) -> int | None: + """Wait out the turn the *launch* started, and mark the chat behind it. + + Not a step, for `arm_switches`' reason: it is how this lane is put in the + state the walk assumes, not a claim about the product. But it is not + optional either, and what it prevents is a **false green** rather than a + red. + + A lane with a `boot` prompt is running a turn from the moment it starts + (#110 — a non-empty prompt is what carries it past the update gate). + Nothing may be typed into a Session that is mid-turn, and no chat mark may + be taken while one is in flight: `stable name` drives the walk's first + turn and hands `stop notice` the mark from just before it, so a boot turn + that ends *after* that mark puts its own Stop Notice on the far side of it + — and `stop notice` passes on a notice for a turn nobody drove, having + proved nothing. The two turns carry the same words, so no reader of the + chat could tell them apart afterwards either. + + This waits on the agent's **own** turn boundary rather than on the record + going quiet. `_drive_turn` settles for nine seconds of silence, which for + a graded turn costs a slow reading and here would cost the run its + meaning, because a turn waiting on the model appends nothing either. + Codex says which it is (`hand_started.codex_turn_over`), so this asks + Codex. + + The mark it returns is the second half, and `drain_boot_notice` spends + it. Nothing is typed and nothing is asked — this only watches. + """ + boot = self.lane.boot + if boot is None: + return None + started = time.monotonic() + # Resolves the record this waits on. The ordinary first call: every step + # reads ground truth through here, and it is cached after the first. + truth = self._ground_truth() + allowed = self.far_side.agent_turn_seconds * BOOT_TURN_ALLOWANCE + over = support.wait_for( + lambda: boot.turn_over(self._record_now()), + deadline_seconds=allowed, + poll_seconds=2.0, + ) + self.boot_turn = self._measured("boot prompt", started, bool(over)) + if not self.boot_turn.ended: + raise LaneBlocked( + f"the turn this lane was launched with had not ended after " + f"{self.boot_turn.seconds:.0f}s, so the walk cannot type into this Session " + f"without racing it. The agent reports {truth.describe()}; its record is " + f"{self._record_size()} bytes. Screen tail: {self.session.screen_tail()[-600:]!r}" + ) + return self.person.latest_message_id() + + def drain_boot_notice(self, mark: int | None) -> None: + """Let the boot turn's Stop Notice land before the walk marks the chat for a later one. + + **Turning an outlet on is what delivers what was held.** A notice with no + route open is retained rather than dropped (`core/escalation.py:230-232`), + and `flip_switch(on=True)` sweeps the retained ones + (`core/bridge.py:361-372`) — so `arm_switches`, which runs between + `settle_boot_turn` and this, is where a held boot notice goes out. That + much is already before `stable name`'s mark and needs nothing from here. + + What needs this is the other path. The rollout's `task_complete` and the + engine's own observation of the Stop are not synchronised, so an engine + that observes it after the arming escalates it straight out, and a green + `stop notice` would then rest on that message losing a race — which is + the one thing this harness may not do. So the walk waits here, on a mark + taken behind the boot turn, until either the notice arrives or the window + `stop notice` itself trusts has passed. + + **It records and never asserts**, because *no notice* is a legitimate + answer: a Stop is only raised on a transition out of `active`, and the + first `idle` a thread reports is it sitting there having done nothing + (`adapters/agent/codex/adapter.py:965-985`). An engine that first saw this + thread already idle raised nothing for the boot turn, and there is nothing + to drain — which is also the case that pays the full window. + """ + if mark is None or self.boot_turn is None: + return + arrived = self.person.await_message( + mark, deadline_seconds=self.far_side.telegram_round_trip_seconds + ) + announced = support.matching_lines(self.engine.log_lines(), ENGINE_STOP_LINE) + boot = self.lane.boot + assert boot is not None # there is no mark to spend without one + self.journey.observe( + "boot turn", + f"the Session was launched with {boot.words!r} — the words " + f"`stable name` types anyway — because a non-empty prompt is what skips Codex's " + f"update gate (#110). Waited out on Codex's own `task_started`/`task_complete` " + f"bracketing, with every outlet still off: {self.boot_turn.seconds:.1f}s. Its Stop " + f"Notice was then drained behind chat mark {mark} so that nothing after " + f"`stable name`'s later mark can be it: " + + ( + f"chat message {arrived.id} ({arrived.text[:80]!r})" + if arrived is not None + else f"nothing arrived in " + f"{self.far_side.telegram_round_trip_seconds:.0f}s, which is what an engine " + f"that first saw this thread already idle would do" + ) + + f"; engine.log Stop lines: {announced[-2:] or 'none'}. Arranged by the harness, " + f"not judged by it.", + ) + def arm_switches(self) -> None: """Voice off, Message on, Duty on — the text-only mode this whole run exercises. @@ -1031,6 +1197,10 @@ def _drive_turn( if size > 0 and settled_for >= 9.0: ended = True break + return self._measured(what, started, ended) + + def _measured(self, what: str, started: float, ended: bool) -> Turn: + """Record one turn on the walk, whoever asked for it.""" turn = Turn(what, time.monotonic() - started, ended) self.turns.append(turn) self.journal("turn", lane=self.lane.name, what=what, seconds=turn.seconds, ended=turn.ended)