Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/acceptance/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ def hand_started_session( # noqa: ANN201
session = hand_started.HandStartedSession(
lane=lane.name,
binary=binary,
arguments=lane.arguments,
arguments=hand_started.launch_arguments(lane.arguments, lane.boot),
workspace=config.workspace,
environment=terminal_environment,
journal=journal,
Expand Down
117 changes: 117 additions & 0 deletions tests/acceptance/hand_started.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,38 @@
written to the run's artifacts as evidence a human can read; every assertion in
`journey.py` rests on the roster, the rollout, the filesystem, `engine.log` or
the chat.

A fourth, measured on #110 (2026-08-27, `codex-cli` 0.150.0):

* **The Codex lane launches with an initial prompt, and that argument is the
whole update gate.** From the instant `~/.codex/version.json` learns of a
release, every hand-started `codex` stops at `1. Update now / 2. Skip / 3. Skip
until next version` **before the Session registers** — no thread on the daemon,
no rollout, indefinitely; #105 watched one probe boot normally and the three
that started after the check landed die there, 90 seconds apart. Codex skips
that prompt outright when it is handed a non-empty `PROMPT` argument
(`codex [OPTIONS] [PROMPT]`), decided at `tui/src/lib.rs:995-996` and
byte-identical at 0.149.1 and 0.150.0 (read on #107):

let skip_update_prompt = cli.prompt.as_ref().is_some_and(|p| !p.is_empty());

So `journey.CODEX` passes the words its first turn would have typed anyway.
**The emptiness test is the mechanism** — an empty string is not a skip — which
is why the argument is the journey's own instruction and not a placeholder that
could quietly shrink to `""`.

The alternative, `-c check_for_update_on_startup=false`, suppresses the prompt
*and* makes the TUI refuse the shared daemon (`can_reuse_implicit_local_daemon`
requires `cli_kv_overrides.is_empty()`, `tui/src/lib.rs:919-921`): it would
silently destroy the attachment the run exists to measure. **Never reach for
`-c` to solve a boot gate.**

This gate is *skipped*, not arranged, and that is what makes it unlike
`TrustGate`: nothing of the user's is edited, `version.json` is not written, and
there is nothing for the run to revoke afterwards. What it *is* unlike an
ordinary flag in is that **a prompt on the command line is a turn**, running
before the walk has asked for anything — so `journey.Walk.settle_boot_turn`
waits it out before a word is typed or a chat mark is taken.
"""

from __future__ import annotations
Expand All @@ -51,6 +83,7 @@
import threading
import time
from collections import deque
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path

Expand Down Expand Up @@ -114,6 +147,90 @@ def resolve(binary: str, path_value: str) -> Path | None:
return Path(found) if found else None


@dataclass(frozen=True)
class BootPrompt:
"""Words a lane's TUI is *launched* with, and how to know that turn is over.

The two travel together because either alone is a trap. A prompt on the
command line is what carries the Codex lane past the update gate (#110), and
it is also a **turn**, running before the walk has asked for anything; the
reading is how `journey.Walk.settle_boot_turn` knows it may stop waiting. A
lane that gained the first without the second is exactly the false green the
Codex review of #110 found, so the type does not let it.
"""

words: str
#: Whether the agent's own record shows **no turn in flight**, given where
#: that record is now (or None when there is not one yet).
turn_over: Callable[[Path | None], bool]


#: How Codex brackets a turn in its own record. Measured 2026-08-27 against a
#: real rollout on this machine (`rollout-2026-08-27T10-17-38-01a04026…jsonl`,
#: codex-cli 0.150.0): two turns, two `task_started`, two `task_complete`, and
#: the file's last record is the second `task_complete`.
TURN_STARTED = "task_started"
TURN_COMPLETE = "task_complete"


def codex_turn_over(rollout: Path | None) -> bool:
"""Whether Codex's own record shows no turn in flight: all started, all complete.

**Not** "the record stopped growing for a while", which is what `_drive_turn`
settles for and what the Codex review of #110 rejected for this use: a turn
waiting on the model appends nothing, so silence is a turn that may be over
and may be thinking. For a graded turn that ambiguity costs a slow reading;
for the boot turn it costs the run its meaning, because a boot turn wrongly
called over is a Stop landing where a later step is looking for a different
one. Codex says which it is, so this asks Codex.
"""
if rollout is None:
return False
started = complete = 0
try:
with rollout.open() as lines:
for line in lines:
try:
record = json.loads(line)
except json.JSONDecodeError:
continue # a line half-written while this read it
if record.get("type") != "event_msg":
continue
payload = record.get("payload")
kind = payload.get("type") if isinstance(payload, dict) else None
started += kind == TURN_STARTED
complete += kind == TURN_COMPLETE
except OSError:
return False
return started > 0 and complete >= started


def launch_arguments(flags: tuple[str, ...], boot: BootPrompt | None) -> tuple[str, ...]:
"""A lane's argv: its flags, then its boot prompt — last, and never empty.

The two rules this holds are both load-bearing and neither is visible in a
tuple literal. **Last**, because both agents take the prompt as a positional
(`codex [OPTIONS] [PROMPT]`), so a prompt written before a flag is read as
that flag's value. **Never empty**, because emptiness is what the update gate
tests (see the fourth measurement above): `""` is a prompt that starts no
turn and skips no gate, and the TUI would stop at the update prompt exactly
as if nothing had been passed — silently, and only in the weeks after a
release, which is the worst way for a harness to be wrong.

A lane with no boot prompt passes `None` and gets its flags back. That is the
Claude lane, and it is the ordinary case: a Session nobody has typed into is
what the walk's first step wants to find.
"""
if boot is None:
return flags
if not boot.words.strip():
raise ValueError(
"a boot prompt must be non-empty: an empty PROMPT is not a skipped update gate, "
"it is a launch that stops at the gate with nothing to show for it"
)
return (*flags, boot.words)


class SessionRefused(RuntimeError):
"""The command would not start, so there is no Session to judge anything on."""

Expand Down
170 changes: 170 additions & 0 deletions tests/acceptance/journey.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@
words arriving and the permission that turn raises — because a relayed
instruction that needs a permission is exactly the shape the product has to
survive, and running it twice would prove less at twice the cost.

The Codex lane runs a sixth that no step drives: it is *launched* with a prompt,
because that is what carries it past Codex's update gate (#110), and a prompt on
the command line is a turn. No step observes it and `Walk.settle_boot_turn` waits
it out before the walk begins — a turn still running when the first step types is
a turn whose Stop lands where a later step is looking for a different one.
"""

from __future__ import annotations
Expand Down Expand Up @@ -78,6 +84,22 @@
#: cadence once there is one.
DISCOVERY_SECONDS = 30.0

#: How long a boot turn gets, as a multiple of the far side's turn figure.
#: Derived rather than guessed: `settle_boot_turn` waits out a TUI's **boot and**
#: its first turn, and boot alone has been measured at the whole turn figure on
#: this machine — a `codex` sat in `starting MCP servers` for a full 180s
#: ground-truth wait on 2026-08-26 (`hand_started.codex_ground_truth`). So two of
#: them, one for each half, and a lane still unsettled after that is blocked
#: rather than typed into. Blocked, not merely slow: everything after it would be
#: measuring a Session with a turn still running underneath.
BOOT_TURN_ALLOWANCE = 2.0

#: The engine's own line for a Stop it announced (`core/bridge.py:_session_stopped`).
#: `stop notice` matches a looser pattern for its own purposes; `drain_boot_notice`
#: wants the announcement itself, because "was a notice raised for the boot turn"
#: is exactly the question it is asking.
ENGINE_STOP_LINE = r"(?i)Session stopped:"

#: What the escalated permission announcement says (`core/approvals.py:43-46`).
#: Matched rather than quoted whole: the tool name and the detail are the agent's,
#: and this run does not get to predict them.
Expand Down Expand Up @@ -214,6 +236,13 @@ class Lane:
binary: str
#: Arguments the *person* would not normally type, and why each is here.
arguments: tuple[str, ...]
#: What the lane's TUI is **launched with**, or None when it is launched
#: silent. Not one of `arguments`, because it is not a flag: it starts a turn
#: nobody drove, before the walk has asked for anything. `hand_started.
#: launch_arguments` puts its words last on the command line and refuses an
#: empty one; `Walk.settle_boot_turn` waits the turn out, through the reading
#: the value carries, before a word is typed.
boot: hand_started.BootPrompt | None
#: The words that make this lane's agent spawn a Child Process. "subagent"
#: and "sub-agent" appear here on purpose: this string is spoken *to* the
#: agent, where it is the agent's own mechanism word and the thing that makes
Expand Down Expand Up @@ -267,6 +296,10 @@ class Lane:
agent="claude",
binary="claude",
arguments=("--permission-mode", "default"),
# Launched silent. No boot gate of the Codex kind has been measured here —
# `claude` boots into an empty composer — and a Session nobody has typed into
# is what `roster` and `stable name`'s three reads want to find.
boot=None,
relayed=lambda workspace: writing(RELAY_FILE, RELAY_WORD),
# The flag the harness passes *is* the whole policy on this lane, and Claude
# publishes no per-turn readback of it, so there is nothing to read back and
Expand Down Expand Up @@ -301,11 +334,36 @@ class Lane:
#: a trusted workspace already gives (measured 2026-08-26 on the failing run's
#: own rollout, `turn_context.sandbox_policy = workspace-write`), so the flag
#: fixes the ground rather than moving it.
#:
#: The **boot prompt** is not a flag and is not here to ask for anything: it is
#: what gets this lane past the update gate (#110; the measurement is in
#: `hand_started`'s module docstring). Three things follow, and each is stated
#: because a reader will otherwise meet it as a surprise:
#:
#: * **It is an extra turn, not a replacement.** `stable name` still types
#: `ACKNOWLEDGE` itself, because it requires a name held across a Stop it
#: drives, and `stop notice` marks the chat immediately before that turn — a
#: Stop crossed at launch would predate the mark and the notice would be
#: unfindable. The words are `ACKNOWLEDGE`'s so the boot turn asks the Session
#: nothing the run does not already ask, and it uses no tools, so it cannot
#: raise a permission before `approval` is there to answer it, and it leaves
#: `codex_turn_policy` reading the same ground: the sandbox is this lane's pin
#: and the `turn_context` that step grades is `relay`'s, the last one written.
#: * **The walk waits it out first** (`Walk.settle_boot_turn`). Two turns of the
#: same words are not two turns the harness can tell apart: a boot turn still
#: running when `stop notice`'s mark is taken puts *its* Stop Notice after the
#: mark, and the step would pass on the notice for a turn nobody drove.
#: * **The rollout now exists before `roster` runs.** Codex writes its rollout
#: when the first *turn* starts, so this lane's `ground_truth` carries a real
#: `session_id` at the first read instead of the `""` it used to carry — the
#: evidence line `roster` prints changes shape, and the pid join it rests on
#: does not.
CODEX = Lane(
name="codex",
agent="codex",
binary="codex",
arguments=("--sandbox", "workspace-write"),
boot=hand_started.BootPrompt(words=ACKNOWLEDGE.words, turn_over=hand_started.codex_turn_over),
# Measured 2026-08-27 through the shared daemon with the product's own pin
# and no sandbox override, on codex-cli 0.149.1 and again on 0.150.0 over a
# 0.149.1 app-server: a write to a path outside the workspace raises
Expand Down Expand Up @@ -391,6 +449,9 @@ def __init__(
self.before_first_turn: int | None = None
#: Held for `approval`: what `relay`'s turn raised and how it was answered.
self.approval_evidence: str | None = None
#: Held for `drain_boot_notice`, which writes the one `boot turn`
#: observation once both halves of the arrangement are done.
self.boot_turn: Turn | None = None
self.turns: list[Turn] = []

# --- the walk ---------------------------------------------------------
Expand All @@ -404,7 +465,9 @@ def walk(self) -> None:
"the revoke. It is not a step: the run cannot both arrange this and judge it.",
)
try:
boot_mark = self.settle_boot_turn()
self.arm_switches()
self.drain_boot_notice(boot_mark)
except LaneBlocked as unarmed:
self.journey.skip_rest(str(unarmed))
return
Expand All @@ -423,6 +486,109 @@ def walk(self) -> None:
or "no turn ran",
)

def settle_boot_turn(self) -> int | None:
"""Wait out the turn the *launch* started, and mark the chat behind it.

Not a step, for `arm_switches`' reason: it is how this lane is put in the
state the walk assumes, not a claim about the product. But it is not
optional either, and what it prevents is a **false green** rather than a
red.

A lane with a `boot` prompt is running a turn from the moment it starts
(#110 — a non-empty prompt is what carries it past the update gate).
Nothing may be typed into a Session that is mid-turn, and no chat mark may
be taken while one is in flight: `stable name` drives the walk's first
turn and hands `stop notice` the mark from just before it, so a boot turn
that ends *after* that mark puts its own Stop Notice on the far side of it
— and `stop notice` passes on a notice for a turn nobody drove, having
proved nothing. The two turns carry the same words, so no reader of the
chat could tell them apart afterwards either.

This waits on the agent's **own** turn boundary rather than on the record
going quiet. `_drive_turn` settles for nine seconds of silence, which for
a graded turn costs a slow reading and here would cost the run its
meaning, because a turn waiting on the model appends nothing either.
Codex says which it is (`hand_started.codex_turn_over`), so this asks
Codex.

The mark it returns is the second half, and `drain_boot_notice` spends
it. Nothing is typed and nothing is asked — this only watches.
"""
boot = self.lane.boot
if boot is None:
return None
started = time.monotonic()
# Resolves the record this waits on. The ordinary first call: every step
# reads ground truth through here, and it is cached after the first.
truth = self._ground_truth()
allowed = self.far_side.agent_turn_seconds * BOOT_TURN_ALLOWANCE
over = support.wait_for(
lambda: boot.turn_over(self._record_now()),
deadline_seconds=allowed,
poll_seconds=2.0,
)
self.boot_turn = self._measured("boot prompt", started, bool(over))
if not self.boot_turn.ended:
raise LaneBlocked(
f"the turn this lane was launched with had not ended after "
f"{self.boot_turn.seconds:.0f}s, so the walk cannot type into this Session "
f"without racing it. The agent reports {truth.describe()}; its record is "
f"{self._record_size()} bytes. Screen tail: {self.session.screen_tail()[-600:]!r}"
)
return self.person.latest_message_id()

def drain_boot_notice(self, mark: int | None) -> None:
"""Let the boot turn's Stop Notice land before the walk marks the chat for a later one.

**Turning an outlet on is what delivers what was held.** A notice with no
route open is retained rather than dropped (`core/escalation.py:230-232`),
and `flip_switch(on=True)` sweeps the retained ones
(`core/bridge.py:361-372`) — so `arm_switches`, which runs between
`settle_boot_turn` and this, is where a held boot notice goes out. That
much is already before `stable name`'s mark and needs nothing from here.

What needs this is the other path. The rollout's `task_complete` and the
engine's own observation of the Stop are not synchronised, so an engine
that observes it after the arming escalates it straight out, and a green
`stop notice` would then rest on that message losing a race — which is
the one thing this harness may not do. So the walk waits here, on a mark
taken behind the boot turn, until either the notice arrives or the window
`stop notice` itself trusts has passed.

**It records and never asserts**, because *no notice* is a legitimate
answer: a Stop is only raised on a transition out of `active`, and the
first `idle` a thread reports is it sitting there having done nothing
(`adapters/agent/codex/adapter.py:965-985`). An engine that first saw this
thread already idle raised nothing for the boot turn, and there is nothing
to drain — which is also the case that pays the full window.
"""
if mark is None or self.boot_turn is None:
return
arrived = self.person.await_message(
mark, deadline_seconds=self.far_side.telegram_round_trip_seconds
)
announced = support.matching_lines(self.engine.log_lines(), ENGINE_STOP_LINE)
boot = self.lane.boot
assert boot is not None # there is no mark to spend without one
self.journey.observe(
"boot turn",
f"the Session was launched with {boot.words!r} — the words "
f"`stable name` types anyway — because a non-empty prompt is what skips Codex's "
f"update gate (#110). Waited out on Codex's own `task_started`/`task_complete` "
f"bracketing, with every outlet still off: {self.boot_turn.seconds:.1f}s. Its Stop "
f"Notice was then drained behind chat mark {mark} so that nothing after "
f"`stable name`'s later mark can be it: "
+ (
f"chat message {arrived.id} ({arrived.text[:80]!r})"
if arrived is not None
else f"nothing arrived in "
f"{self.far_side.telegram_round_trip_seconds:.0f}s, which is what an engine "
f"that first saw this thread already idle would do"
)
+ f"; engine.log Stop lines: {announced[-2:] or 'none'}. Arranged by the harness, "
f"not judged by it.",
)

def arm_switches(self) -> None:
"""Voice off, Message on, Duty on — the text-only mode this whole run exercises.

Expand Down Expand Up @@ -1031,6 +1197,10 @@ def _drive_turn(
if size > 0 and settled_for >= 9.0:
ended = True
break
return self._measured(what, started, ended)

def _measured(self, what: str, started: float, ended: bool) -> Turn:
"""Record one turn on the walk, whoever asked for it."""
turn = Turn(what, time.monotonic() - started, ended)
self.turns.append(turn)
self.journal("turn", lane=self.lane.name, what=what, seconds=turn.seconds, ended=turn.ended)
Expand Down
Loading