From f6387295036c84e5974f4d6a6f35aa4c2ce1f93c Mon Sep 17 00:00:00 2001 From: Coditan-XO Date: Sun, 9 Aug 2026 03:09:24 +0000 Subject: [PATCH 1/4] watch: hold the wedge ladder for a run parked at a gate whose worker is alive The fleet already knew. On 2026-08-08 two workers escalated as "idle 254s, possible wedge, escalation 1" while bin/fm-crew-state.sh, run against the same tasks at the same moment, returned "state: parked - source: run-step - parked at review: 3 finding(s) (ask-user: captain decision)". Both panes were idle because the worker had asked its question and stopped, exactly as its brief requires. crew_absorb_class collapsed that authoritative `parked` reading into `none`, so the stale path had no way to tell "stopped on purpose at a gate" from "stopped, reason unknown" and climbed the escalation ladder against it. `parked` is now its own absorb class, restricted to source: run-step - a status-log-sourced gate is an arbitrarily old append-only EVENT and reads the same whether the crew is at the gate or died an hour ago. The class alone never absorbs anything. A worker that crashed one second after printing its gate prompt leaves the run parked identically and forever, so the watcher corroborates with the one signal that changes when the worker dies: the agent PROCESS, via the same fm_backend_agent_alive probe the codex static-pane backstop uses. Only `alive` holds the ladder; `dead` and `unknown` fall through to the unchanged surface and the unchanged escalation timings. The liveness reading is re-taken at every escalation window rather than trusted once, and the hold still earns one bounded recheck per FM_PAUSE_RESURFACE_SECS, so a gate nobody ever answers cannot rot invisibly. Deliberately out of scope, and left at today's behavior: - pi (its launcher execs into a generic `node`) and every backend past tmux/herdr read `unknown`, so they get no hold at all. - The declared-pause and captain-held branch keeps `parked` mapped to `none`; its own dead-agent rule already owns that idle pane. - bin/fm-mark-parked.sh is untouched. A mid-pipeline gate is neither terminal nor an external wait, and a hand-placed marker would keep muting the pane after the gate was answered. Costs no extra crew-state read: both call sites already held the verdict, and only the `parked` branch adds a liveness probe. Nothing new runs per poll. The two hold sites are now one function, ladder_hold, so the timer refresh, the frozen-hash age anchor, and the throttle marker cannot drift apart. Tests cover all four states plus both failure directions: parked+alive absorbed with the wedge timer armed, parked+dead surfaced immediately, parked+dead escalating on the unchanged ladder schedule, unknown liveness surfacing as today, and a status-log-sourced gate earning no class. --- bin/fm-classify-lib.sh | 25 ++++- bin/fm-mark-parked.sh | 5 + bin/fm-watch.sh | 176 ++++++++++++++++++++++++++++----- docs/architecture.md | 3 + docs/configuration.md | 4 +- tests/fm-watch-triage.test.sh | 177 +++++++++++++++++++++++++++++++++- 6 files changed, 359 insertions(+), 31 deletions(-) diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index 64ee51465f..9544a53a07 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -363,15 +363,29 @@ signal_reason_is_actionable() { # ... # (e.g. waiting on CI); # paused - the crew's authoritative current state is a declared external-wait # pause (paused:), which is EXPECTED to idle; +# parked - the crew's authoritative RUN-STEP state is a decision gate +# (no-mistakes awaiting_approval/fix_review). The pipeline stopped on +# purpose and is waiting for an answer, so an idle pane is the +# CORRECT behavior rather than a wedge. Deliberately restricted to +# source: run-step - a status-log-sourced `parked` is the same +# append-only needs-decision EVENT the stale path already reasons +# about, is arbitrarily old, and reads identically whether the crew is +# at the gate or died an hour ago. +# This token says the RUN is parked; it says NOTHING about the worker +# still being there, because a crashed worker leaves the run parked +# forever. A caller must corroborate with an interface-text- +# independent liveness signal before absorbing on it +# (bin/fm-watch.sh's parked_gate_liveness_class); # degraded- the reader could not consult its authoritative source at all: a # required tool is absent, or the call it makes could not be made or # did not answer (fm-crew-state.sh's `degraded` verdict, whose cause # token says which). NOT a statement about the crew: the supervisor # must report the broken instrument rather than draw either # conclusion from it; -# none - neither, so the wake must surface (a stopped/finished/parked/failed/ -# torn-down/unknown crew, or an unreadable verdict). -# One fm-crew-state.sh read serves BOTH absorb reasons at once. Reading the state +# none - none of the above, so the wake must surface (a stopped/finished/ +# failed/torn-down/unknown crew, a gate reported only by the status +# log, or an unreadable verdict). +# One fm-crew-state.sh read serves EVERY absorb reason at once. Reading the state # authoritatively (not the status log) is what keeps run-step precedence: a crew # that appended paused: but then STARTED a run reports working, never paused. # NOT a pure read: fm-crew-state.sh may make a bounded no-mistakes call, so callers @@ -386,10 +400,13 @@ crew_absorb_class() { # state=${line#state: }; state=${state%% *} if [ "$state" = paused ]; then printf 'paused'; return; fi if [ "$state" = degraded ]; then printf 'degraded'; return; fi + src=${line#*source: }; src=${src%% *} if [ "$state" = working ]; then - src=${line#*source: }; src=${src%% *} case "$src" in run-step|pane) printf 'working'; return ;; esac fi + if [ "$state" = parked ]; then + case "$src" in run-step) printf 'parked'; return ;; esac + fi printf 'none' } diff --git a/bin/fm-mark-parked.sh b/bin/fm-mark-parked.sh index eb08d6e2f3..5c456d9c41 100755 --- a/bin/fm-mark-parked.sh +++ b/bin/fm-mark-parked.sh @@ -3,6 +3,11 @@ # terminal outcome has been relayed and only external human action remains. # bin/fm-watch.sh owns window validation, secondmate rejection, key derivation, # and marker creation; this wrapper only provides a seatbelt-safe command shape. +# NOT for a mid-pipeline worker waiting at a no-mistakes decision gate: that one +# is neither terminal nor waiting on anything external, the watcher recognizes it +# on its own from the authoritative run-step plus agent liveness +# (bin/fm-watch.sh's parked_gate_liveness_class), and a hand-placed marker here +# would keep muting that pane after the gate was answered. set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 6fa36bd163..52021efb6d 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -9,8 +9,13 @@ # otherwise, so a crew that finishes (or stops and waits) without a current # working signal is never silently swallowed. A declared external-wait pause or # a firstmate-declared parked terminal task is the separate idle absorb case and -# re-surfaces only on its long bounded cadence. A new status write still surfaces -# immediately in normal mode and clears parked tracking. +# re-surfaces only on its long bounded cadence. A run the authoritative run-step +# reports stopped at a DECISION GATE is the third: idling there is the correct +# behavior, so the stale path holds its wedge ladder - but only for as long as +# the worker's agent PROCESS is confirmed alive, because a crashed worker leaves +# the run parked identically and forever (parked_gate_liveness_class). +# A new status write still surfaces immediately in normal mode and clears parked +# tracking. # While state/.afk exists, the away daemon owns triage and this watcher queues # every actionable wake without running the more expensive normal-mode # classifiers, though signal records are collapsed to one per task in away mode @@ -26,8 +31,10 @@ # firstmate hands it to a no-mistakes validation. A declared # external-wait pause or firstmate-declared parked terminal # wait is absorbed instead with its own long re-surface -# cadence, never as a wedge. Only when neither absorb class -# applies does the log's last line decide: +# cadence, never as a wedge, and a run parked at a decision +# gate whose worker is confirmed alive holds the ladder on +# the same bounded-recheck terms as an active run. Only when +# no absorb class applies does the log's last line decide: # terminal (captain-relevant) or non-terminal (no verb), # both surfaced at once. A provably-working stale past the # wedge threshold also surfaces, with an "escalation N" @@ -358,6 +365,32 @@ wake() { # below). FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3} +# ladder_hold: the one implementation of "this stale pane has a positive reason not +# to climb the wedge ladder". Refreshes the escalation timer so the reason is +# RE-READ every window rather than trusted once, and surfaces one bounded recheck +# whenever the pane has been frozen for a full PAUSE_RESURFACE_SECS - measured on +# the frozen hash's own age, not on the timer this hold keeps refreshing, so a hold +# can never postpone its own recheck. It never touches the escalation counter, so a +# held pane never climbs toward demand-deep-inspection. +# names the evidence in the wake text and the triage log; +# is what firstmate should confirm. Both callers pass their own, because "the run +# is running" and "the run is parked at a gate" need different confirmations. +ladder_hold() { # + local win=$1 since_file=$2 label=$3 situation=$4 next_step=$5 wkey hold_age rf reason + date +%s > "$since_file" + wkey=$(window_state_key "$win") + hold_age=$(age_of "$STATE/.stale-$wkey") + rf="$STATE/.wedgeheld-$wkey" + if [ "$hold_age" -ge "$PAUSE_RESURFACE_SECS" ] && [ "$(age_of "$rf")" -ge "$PAUSE_RESURFACE_SECS" ]; then + reason="stale: $win (pane unchanged ${hold_age}s while $situation - bounded recheck on a long cadence, not a wedge escalation; $next_step)" + fm_wake_append stale "$win" "$reason" || exit 1 + date +%s > "$rf" + wake "$reason" + return + fi + triage_log "absorbed $label ($situation, ladder held): $win" +} + # Repeat-poll wedge-timer bookkeeping for an already-classified stale hash # absorbed as provably-working - repairs a missing/corrupt timer (self-heals a # watcher restart between recording the hash and recording the timer), or @@ -378,8 +411,13 @@ FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3} # read is what distinguishes them, it runs once per interval rather than once per # poll, and the moment a run ends or dies the crew stops reading as working and # the next elapse escalates on the unchanged schedule. +# +# A run PARKED at a decision gate holds the ladder on the same terms, with one +# extra requirement the active-run case does not need: the gate reading is only +# trusted while agent liveness confirms a worker is still there to answer it +# (parked_gate_liveness_class). Both holds are one function - ladder_hold above. wedge_timer_check() { # - local win=$1 since_file=$2 label=$3 escalation_file=$4 since age n reason wkey hold_age rf wtask wclass + local win=$1 since_file=$2 label=$3 escalation_file=$4 since age n reason wtask wclass since=$(cat "$since_file" 2>/dev/null || true) case "$since" in ''|*[!0-9]*) @@ -391,6 +429,15 @@ wedge_timer_check() { # "$since_file" # Bounded insurance against the one way an active run can lie: a run # whose agent died mid-step keeps reporting `running` indefinitely, and # holding the ladder on that reading alone would let a dead crew rot - # invisibly. So a hold that has lasted a full PAUSE_RESURFACE_SECS - the - # fleet's existing "recheck a known-idle wait" cadence, measured on the - # frozen pane hash's own age, not on the timer this hold keeps - # refreshing - surfaces ONE bounded recheck and then waits out another - # full window. It never touches the escalation counter, so an active run - # still never climbs the wedge ladder toward demand-deep-inspection. - wkey=$(window_state_key "$win") - hold_age=$(age_of "$STATE/.stale-$wkey") - rf="$STATE/.wedgeheld-$wkey" - if [ "$hold_age" -ge "$PAUSE_RESURFACE_SECS" ] && [ "$(age_of "$rf")" -ge "$PAUSE_RESURFACE_SECS" ]; then - reason="stale: $win (pane unchanged ${hold_age}s while the run still reports active - bounded recheck on a long cadence, not a wedge escalation; confirm the run is really progressing)" - fm_wake_append stale "$win" "$reason" || exit 1 - date +%s > "$rf" - wake "$reason" - return - fi - triage_log "absorbed $label (run still active at escalation, ladder held): $win" + # invisibly - so the hold earns one bounded recheck per window. + ladder_hold "$win" "$since_file" "$label" \ + "the run still reports active" \ + "confirm the run is really progressing" + return + fi + if [ "$wclass" = parked ]; then + # The run is stopped at a decision gate AND the worker is confirmed + # alive: the pane is idle because the crew asked a question and waited, + # which is the behavior its brief requires. Hold the ladder on the same + # terms as an active run - including the bounded recheck, so a gate + # nobody ever answers still cannot rot invisibly. + ladder_hold "$win" "$since_file" "$label" \ + "the run is parked at a decision gate and this worker is still alive" \ + "answer the gate or confirm the decision is still pending" return fi n=$(( $(cat "$escalation_file" 2>/dev/null || echo 0) + 1 )) @@ -726,6 +770,69 @@ codex_static_pane_upgrade() { # printf '%s' "$fallback" } +# parked_gate_liveness_class: decide what a `parked` absorb class (crew_absorb_class +# in bin/fm-classify-lib.sh - the authoritative run-step reports the pipeline stopped +# at a decision gate) is worth on the STALE path, by corroborating it with a signal +# that does not read interface text: the harness-agent PROCESS. +# +# The gate answers the wedge question outright for the crew that is still there. A +# worker parked at an ask-user finding is idle because it did exactly what its brief +# requires - ask, then stop. The 2026-08-08 evidence: two such workers escalated as +# `idle 254s, possible wedge, escalation 1` while fm-crew-state.sh, run against the +# same tasks at the same moment, returned `state: parked - source: run-step - parked +# at review: 3 finding(s) (ask-user: captain decision)`. The fleet already knew. +# +# But `parked` alone is NOT that answer, and absorbing on it would be the failure +# this whole absorb path exists to avoid. A worker that crashed one second after +# printing its gate prompt leaves the run parked in exactly the same way, forever: +# no-mistakes has no idea its agent is gone, so the run-step reading is IDENTICAL +# for a healthy gate and a dead one. Only `alive` - a confirmed agent process, the +# one piece of evidence that changes when the worker dies - licenses the absorb: +# alive -> parked (hold the ladder on the long bounded cadence) +# dead -> none (the caller's ordinary surface path: a run parked with +# nobody left to answer it is a real failure, and it must +# reach firstmate on the unchanged timings, never be +# swallowed by the parked case) +# unknown -> none (fm_backend_agent_alive's contract: never license an action +# from unknown. Today that covers pi, whose launcher execs +# into a generic `node`, and every backend past tmux/herdr - +# those crews keep exactly today's behavior) +# Secondmates are excluded like the codex backstop excludes them: they never drive +# a run, so a `parked` verdict cannot be theirs to begin with. +# +# COST: this adds no crew-state read anywhere. Both call sites already had the +# fm-crew-state.sh verdict in hand - the first sighting of a stale hash, and the +# wedge escalation - and the only new work is one liveness probe on the branch +# where the verdict is `parked`. Nothing here runs per poll per task, which is the +# budget the watcher's whole absorb design is built around. +parked_gate_liveness_class() { # + local win=$1 alive + [ "$(window_kind "$win")" != secondmate ] || { printf 'none'; return; } + alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || alive=unknown + case "$alive" in + alive) printf 'parked' ;; + *) printf 'none' ;; + esac +} + +# stale_absorb_class: crew_absorb_class as the FIRST-SIGHT stale path should read +# it, with the two interface-text-independent corroborations layered on. They are +# mutually exclusive by construction and each probes agent liveness at most once: +# `parked` is a positive run-step reading that needs a live worker behind it, while +# codex_static_pane_upgrade only ever fires on `none` (no run-step at all), so a +# crew can never be both. Keeping them separate keeps each backstop's coverage its +# own: the codex upgrade stays confined to a harness whose busy row lies, and the +# parked gate stays confined to a run that is deliberately stopped. +stale_absorb_class() { # + local win=$1 task=$2 class + class=$(crew_absorb_class "$task") + if [ "$class" = parked ]; then + parked_gate_liveness_class "$win" + return + fi + codex_static_pane_upgrade "$win" "$class" +} + pause_state_class() { # local win=$1 task=$2 key last recheck_file class agent_alive key=${win//:/_} @@ -735,7 +842,7 @@ pause_state_class() { # recheck_file="$STATE/.paused-rechecked-$key" if ! status_is_paused_or_captain_held "$last"; then rm -f "$recheck_file" - codex_static_pane_upgrade "$win" "$(crew_absorb_class "$task")" + stale_absorb_class "$win" "$task" return fi if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then @@ -751,6 +858,12 @@ pause_state_class() { # return fi class=$(crew_absorb_class "$task") + # Below this line the crew has DECLARED a pause or a verified captain hold, and + # the branch's own dead-agent rule already owns that idle pane. The run-step gate + # reading adds nothing here and its liveness corroboration would collide with the + # declaration's, so `parked` keeps the exact `none` handling it had before the + # gate class existed; only the undeclared path above consults it. + [ "$class" = parked ] && class=none if [ "$class" = working ]; then rm -f "$recheck_file" printf 'working' @@ -1678,6 +1791,11 @@ EOF # - paused: the crew declared an external wait, or a declared pause or # captain hold is paired with a confidently dead agent, so absorb on # the long PAUSE_RESURFACE_SECS recheck cadence instead of wedge-escalating; + # - parked: the authoritative run-step reports the pipeline stopped at a + # decision gate AND the worker's agent process is confirmed alive, so + # absorb and start the wedge timer exactly as `working` does - the gate + # is why the pane is idle, and the liveness read is what stops that + # answer from covering for a worker that crashed at the gate; # - none: no running pipeline, idle pane, no busy signature, no declared # pause - the crew has STOPPED. Surface immediately so firstmate peeks # (it may be done via an interactive menu that wrote no done: status, @@ -1692,6 +1810,16 @@ EOF date +%s > "$ssf" triage_log "absorbed non-terminal stale (provably working): $w" ;; + parked) + # The run stopped at a decision gate and the worker is confirmed + # alive. Absorbed exactly like an active run - suppressor advanced, + # wedge timer armed - so the moment that worker stops being alive + # the very next escalation surfaces it on the unchanged schedule. + clear_pause_tracking "$w" + printf '%s' "$h" > "$sf" + date +%s > "$ssf" + triage_log "absorbed non-terminal stale (parked at a decision gate, worker alive): $w" + ;; paused) handle_paused_stale "$w" "$task" "$h" ;; diff --git a/docs/architecture.md b/docs/architecture.md index 5231c68e22..0a540f4a38 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -19,6 +19,9 @@ For an ordinary crew that has stopped, the normal-mode watcher first surfaces on Live or inconclusive liveness remains fail-open at that initial surface, and the secondmate idle-endpoint exemption is unchanged. Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck. A terminal task firstmate marks parked (`bin/fm-mark-parked.sh `) after relaying its outcome and confirming only external human action remains is absorbed the same way and on the same cadence, keyed off the marker instead of a status verb; a later status write or metadata change clears the marker and returns the task to normal classification. +A mid-pipeline run that `bin/fm-crew-state.sh` reports stopped at a no-mistakes decision gate is a distinct case from that operator-declared marker, and the watcher recognizes it without one: a worker parked at an ask-user finding is idle precisely because it followed its instructions, so its stale pane holds the wedge ladder on the bounded-recheck cadence instead of escalating. +That hold rests on `fm_backend_agent_alive` confirming the worker's agent process and on nothing else, because a worker that crashed at the gate leaves the run parked identically and forever; a `dead` or unreadable liveness verdict keeps the unchanged escalation, and the reading is re-taken at every escalation window rather than trusted once. +A gate reported only by the append-only status log earns no hold at all, since that event is arbitrarily old and reads the same either way. Fresh stale panes use the same current-state read before trusting the status log, so an active run or busy pane outranks an old captain-relevant status-log line left behind before validation. No-change heartbeats are also benign. Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. diff --git a/docs/configuration.md b/docs/configuration.md index d5a60e7b3e..9c85ad6dab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -750,8 +750,8 @@ FM_WATCHER_STALE_GRACE=300 # defaults to FM_GUARD_GRACE; seconds a live watche FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals into one wake FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked -FM_STALE_ESCALATE_SECS=240 # idle seconds before an absorbed stale pane escalates, re-checked against the crew state at that moment: an active run holds the ladder (bounded recheck once per FM_PAUSE_RESURFACE_SECS), a crew with no active run escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb -FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon; also the cadence of the bounded recheck for a wedge ladder held by an active run +FM_STALE_ESCALATE_SECS=240 # idle seconds before an absorbed stale pane escalates, re-checked against the crew state at that moment: an active run, or a run parked at a decision gate whose worker's agent process is confirmed alive, holds the ladder (bounded recheck once per FM_PAUSE_RESURFACE_SECS), while a crew with neither escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb +FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon; also the cadence of the bounded recheck for a wedge ladder held by an active run or by a decision gate whose worker is confirmed alive FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WATCH_TRIAGE_LOG_MAX_BYTES=262144 # size cap for the watcher's absorbed-wake debug log FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT= # optional seconds allowed for bootstrap's best-effort clone refresh; unset/blank defaults to max(20, 5 + 3 * origin-backed-project-count) diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index c8192f5bda..371fad5568 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -268,12 +268,22 @@ test_crew_absorb_class_classifier() { ! crew_is_provably_working a || fail "a paused crew was treated as provably working" FM_FAKE_CREW_STATE='state: working · source: status-log · working: compiling' [ "$(crew_absorb_class a)" = none ] || fail "stale working: status-log classed absorbable" + FM_FAKE_CREW_STATE='state: parked · source: run-step · parked at review: 3 finding(s) (ask-user: captain decision)' + [ "$(crew_absorb_class a)" = parked ] || fail "an authoritative run-step decision gate not classed parked" + ! crew_is_provably_working a || fail "a parked run was treated as provably working" + ! crew_is_paused a || fail "a parked run was conflated with a declared external-wait pause" + ! crew_is_degraded a || fail "a parked run was conflated with an unread crew state" + # A gate known only from the append-only status log is an arbitrarily old EVENT + # that reads the same whether the crew is at the gate or died an hour ago, so it + # earns no absorb class of its own. + FM_FAKE_CREW_STATE='state: parked · source: status-log · needs-decision: pick A or B' + [ "$(crew_absorb_class a)" = none ] || fail "a status-log-sourced gate was classed absorbable" FM_FAKE_CREW_STATE='state: unknown · source: none · worktree gone' [ "$(crew_absorb_class a)" = none ] || fail "unknown crew classed absorbable" ! crew_is_paused a || fail "unknown crew classed paused" [ "$(crew_absorb_class "")" = none ] || fail "empty id not classed none" unset FM_FAKE_CREW_STATE - pass "crew_absorb_class: working/paused/none from one read; crew_is_paused and crew_is_provably_working agree" + pass "crew_absorb_class: working/paused/parked/none from one read, and the predicates keep their own meanings" } # signal_crew_provably_working: a no-verb "signal:" wake is benign ONLY when EVERY @@ -933,6 +943,166 @@ test_codex_backstop_scoped_to_codex() { pass "the agent-liveness absorb is scoped to codex: a claude worker with an alive process still surfaces immediately" } +# --- parked-at-a-decision-gate liveness gate --------------------------------- +# A worker parked at an ask-user finding is idle because it did exactly what its +# brief requires: ask, then stop. fm-crew-state.sh answers that outright +# ("state: parked · source: run-step · parked at review: 3 finding(s) (ask-user: +# captain decision)") and the watcher escalated it as a possible wedge anyway, +# every four minutes, because crew_absorb_class collapsed `parked` into `none`. +# The gate absorb is corroborated by the agent PROCESS and by nothing else: a +# worker that crashed one second after printing its gate prompt leaves the run +# parked identically and forever, so `dead` and `unknown` keep today's behavior +# exactly. harness=claude throughout, so nothing here can be the codex backstop. +# +# The authoritative reading the watcher used to ignore. Passed into each watcher +# launch rather than exported by the fixture, because the fixture runs inside a +# command substitution and an export there would never reach the watcher. +PARKED_GATE_STATE='state: parked · source: run-step · parked at review: 3 finding(s) (ask-user: captain decision)' + +_parked_gate_case() { # + local dir state fakebin capture_file window key pane_hash sig statusf + dir=$(make_case "$1"); state="$dir/state"; fakebin="$dir/fakebin" + capture_file="$dir/pane.txt"; window=$2; statusf="$state/gate.status" + printf '%s' "$4" > "$capture_file" + printf 'window=%s\nkind=ship\nharness=claude\n' "$window" > "$state/gate.meta" + printf '%s\n' "$3" > "$statusf" + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "$4") + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + printf '%s\t%s\t%s\t%s\n' "$dir" "$state" "$fakebin" "$key" +} + +test_parked_gate_alive_absorbed() { + local fields dir state fakebin key out window pid pane_hash + window="test:fm-gatealive" + fields=$(_parked_gate_case parked-gate-alive "$window" 'working: running the pipeline' 'idle at the review gate') + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out"; pane_hash=$(hash_text "idle at the review gate") + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "a worker parked at a decision gate with a live agent still woke firstmate: $(cat "$out")" + fi + [ ! -s "$state/.wake-queue" ] || fail "the parked-gate absorb enqueued a wake: $(cat "$state/.wake-queue")" + [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on the parked-gate absorb" + [ -s "$state/.stale-since-$key" ] || fail "the parked-gate absorb did not arm the wedge timer, so a dying worker could never escalate" + grep -F "parked at a decision gate, worker alive" "$state/.watch-triage.log" >/dev/null \ + || fail "the absorb did not record WHY it believed the worker was fine: $(cat "$state/.watch-triage.log" 2>/dev/null)" + reap "$pid" + pass "a worker parked at a decision gate with a live agent is absorbed, and says why, with the wedge timer still armed" +} + +test_parked_gate_dead_surfaces() { + local fields dir state fakebin key out drain_out window pid + window="test:fm-gatedead" + fields=$(_parked_gate_case parked-gate-dead "$window" 'working: running the pipeline' 'idle at the review gate') + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out"; drain_out="$dir/drain.out" + # Identical run-step reading, agent gone (bare shell). The gate can no longer be + # answered by anyone, so the parked case must NOT swallow it. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=bash FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a run parked at a gate whose worker died was never surfaced" + grep -Fx "stale: $window" "$out" >/dev/null || fail "the dead parked worker did not print the immediate stale wake: $(cat "$out")" + [ ! -e "$state/.stale-since-$key" ] || fail "a dead parked worker started the wedge timer instead of surfacing at once" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the dead parked-worker surface failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the dead parked worker's wake was not queued" + pass "a run parked at a gate whose worker has died surfaces immediately (the parked case never masks a crash)" +} + +test_parked_gate_unknown_liveness_surfaces() { + local fields dir state fakebin key out window pid + window="test:fm-gateunknown" + # pi's launcher execs into a generic `node`, so its liveness reads unknown + # (docs/tmux-backend.md "Known gaps"). Unknown never licenses an absorb: these + # crews keep exactly the behavior they have today. + fields=$(_parked_gate_case parked-gate-unknown "$window" 'working: running the pipeline' 'idle at the review gate') + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=node FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "an unreadable agent liveness was treated as proof the parked worker was fine" + grep -Fx "stale: $window" "$out" >/dev/null || fail "the unknown-liveness parked worker did not surface: $(cat "$out")" + [ ! -e "$state/.stale-since-$key" ] || fail "an unknown-liveness parked worker was absorbed onto the wedge timer" + pass "a parked worker whose agent liveness cannot be read keeps today's behavior and still surfaces" +} + +# The captain's exact repeat: the crew appended needs-decision:, the signal path +# already relayed it, and each poll thereafter climbed the wedge ladder against a +# run that fm-crew-state.sh reported parked at an ask-user gate. +_parked_gate_ladder_case() { # + local fields dir state fakebin key last pane_hash + fields=$(_parked_gate_case "$1" "$2" 'needs-decision: review raised an ask-user finding for the captain' 'idle at the review gate') + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + last='needs-decision: review raised an ask-user finding for the captain' + printf '%s' "$last" > "$state/.hb-surfaced-gate" + pane_hash=$(hash_text "idle at the review gate") + printf '%s' "$pane_hash" > "$state/.stale-$key" + echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key" + printf '%s\t%s\t%s\t%s\n' "$dir" "$state" "$fakebin" "$key" +} + +test_parked_gate_alive_holds_the_wedge_ladder() { + local fields dir state fakebin key out window pid + window="test:fm-ladderalive" + fields=$(_parked_gate_ladder_case parked-gate-ladder-alive "$window") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "a live worker parked at an ask-user gate still escalated as a possible wedge: $(cat "$out")" + fi + grep -F "possible wedge" "$out" >/dev/null && { reap "$pid"; fail "the parked gate was reported as a possible wedge: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "the held ladder still enqueued a wake: $(cat "$state/.wake-queue")"; } + [ ! -e "$state/.wedge-escalations-$key" ] || { reap "$pid"; fail "a parked-gate hold climbed the wedge-escalation ladder"; } + [ -s "$state/.stale-since-$key" ] || { reap "$pid"; fail "the parked-gate hold dropped the wedge timer instead of refreshing it"; } + grep -F "ladder held" "$state/.watch-triage.log" >/dev/null \ + || { reap "$pid"; fail "the parked-gate ladder-hold branch never executed: $(cat "$state/.watch-triage.log" 2>/dev/null)"; } + reap "$pid" + pass "a live worker parked at an ask-user gate holds the wedge ladder instead of escalating" +} + +# The failure direction that matters. Same frozen pane, same parked run-step, same +# already-relayed needs-decision line - only the agent is gone. The escalation must +# fire on the unchanged schedule; a parked case that swallowed this would be silence +# nobody notices. +test_parked_gate_dead_escalates_on_the_ladder() { + local fields dir state fakebin key out drain_out window pid + window="test:fm-ladderdead" + fields=$(_parked_gate_ladder_case parked-gate-ladder-dead "$window") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out"; drain_out="$dir/drain.out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=bash FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a parked run whose worker died was held by the ladder instead of escalating: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null || fail "the dead parked worker did not escalate: $(cat "$out")" + grep -F "escalation 1" "$out" >/dev/null || fail "the dead parked worker escalated off the normal schedule: $(cat "$out")" + [ "$(cat "$state/.wedge-escalations-$key" 2>/dev/null || true)" = 1 ] || fail "the dead parked worker did not climb the wedge ladder" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the dead parked-worker escalation failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the dead parked worker's escalation was not queued" + pass "a run parked at a gate whose worker died escalates on the unchanged wedge schedule" +} + # --- non-terminal stale, crew NOT provably working: surfaced immediately ------ # The key requirement: a crew with no running pipeline that has gone quiet (and is # not busy) has stopped - it may be done via interactive menus, waiting, or wedged. @@ -2262,6 +2432,11 @@ test_nonterminal_stale_provably_working_absorbed_then_escalated test_codex_static_pane_alive_absorbed test_codex_static_pane_dead_surfaces test_codex_backstop_scoped_to_codex +test_parked_gate_alive_absorbed +test_parked_gate_dead_surfaces +test_parked_gate_unknown_liveness_surfaces +test_parked_gate_alive_holds_the_wedge_ladder +test_parked_gate_dead_escalates_on_the_ladder test_terminal_stale_already_surfaced_absorbed_then_escalates test_terminal_stale_changed_line_still_surfaces test_wedge_escalation_marks_demand_deep_inspection_after_threshold From b3f45649ff594d7180ba203bc70a5d1c8fc5279f Mon Sep 17 00:00:00 2001 From: Coditan-XO Date: Sun, 9 Aug 2026 04:17:53 +0000 Subject: [PATCH 2/4] no-mistakes(review): surface a parked gate's first sighting instead of absorbing it --- bin/fm-watch.sh | 78 ++++++++++--------------- docs/architecture.md | 3 +- tests/fm-watch-triage.test.sh | 103 ++++++++++++++++++++++------------ 3 files changed, 99 insertions(+), 85 deletions(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 52021efb6d..061c67ae3b 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -10,10 +10,11 @@ # working signal is never silently swallowed. A declared external-wait pause or # a firstmate-declared parked terminal task is the separate idle absorb case and # re-surfaces only on its long bounded cadence. A run the authoritative run-step -# reports stopped at a DECISION GATE is the third: idling there is the correct -# behavior, so the stale path holds its wedge ladder - but only for as long as -# the worker's agent PROCESS is confirmed alive, because a crashed worker leaves -# the run parked identically and forever (parked_gate_liveness_class). +# reports stopped at a DECISION GATE still surfaces on first sight - nothing has +# relayed that gate yet - but does not ESCALATE as a wedge afterwards, since +# idling at a gate is the correct behavior; that hold lasts only as long as the +# worker's agent PROCESS is confirmed alive, because a crashed worker leaves the +# run parked identically and forever (parked_gate_liveness_class). # A new status write still surfaces immediately in normal mode and clears parked # tracking. # While state/.afk exists, the away daemon owns triage and this watcher queues @@ -31,8 +32,9 @@ # firstmate hands it to a no-mistakes validation. A declared # external-wait pause or firstmate-declared parked terminal # wait is absorbed instead with its own long re-surface -# cadence, never as a wedge, and a run parked at a decision -# gate whose worker is confirmed alive holds the ladder on +# cadence, never as a wedge. A run parked at a decision gate +# whose worker is confirmed alive surfaces its first sighting +# like any other stopped crew, then holds the wedge ladder on # the same bounded-recheck terms as an active run. Only when # no absorb class applies does the log's last line decide: # terminal (captain-relevant) or non-terminal (no verb), @@ -772,8 +774,13 @@ codex_static_pane_upgrade() { # # parked_gate_liveness_class: decide what a `parked` absorb class (crew_absorb_class # in bin/fm-classify-lib.sh - the authoritative run-step reports the pipeline stopped -# at a decision gate) is worth on the STALE path, by corroborating it with a signal -# that does not read interface text: the harness-agent PROCESS. +# at a decision gate) is worth on the WEDGE-ESCALATION path, by corroborating it with +# a signal that does not read interface text: the harness-agent PROCESS. +# +# Scoped to the ladder on purpose: a gate only ever holds an escalation on a stale +# pane firstmate has ALREADY been woken for. The first sighting of that pane still +# surfaces at once, whatever the run-step says, because nothing has told anyone +# about the gate yet. # # The gate answers the wedge question outright for the crew that is still there. A # worker parked at an ask-user finding is idle because it did exactly what its brief @@ -789,7 +796,7 @@ codex_static_pane_upgrade() { # # for a healthy gate and a dead one. Only `alive` - a confirmed agent process, the # one piece of evidence that changes when the worker dies - licenses the absorb: # alive -> parked (hold the ladder on the long bounded cadence) -# dead -> none (the caller's ordinary surface path: a run parked with +# dead -> none (the caller's ordinary escalation path: a run parked with # nobody left to answer it is a real failure, and it must # reach firstmate on the unchanged timings, never be # swallowed by the parked case) @@ -800,11 +807,11 @@ codex_static_pane_upgrade() { # # Secondmates are excluded like the codex backstop excludes them: they never drive # a run, so a `parked` verdict cannot be theirs to begin with. # -# COST: this adds no crew-state read anywhere. Both call sites already had the -# fm-crew-state.sh verdict in hand - the first sighting of a stale hash, and the -# wedge escalation - and the only new work is one liveness probe on the branch -# where the verdict is `parked`. Nothing here runs per poll per task, which is the -# budget the watcher's whole absorb design is built around. +# COST: this adds no crew-state read anywhere. The wedge escalation already had the +# fm-crew-state.sh verdict in hand, and the only new work is one liveness probe on +# the branch where that verdict is `parked`, once per escalation window. Nothing +# here runs per poll per task, which is the budget the watcher's whole absorb design +# is built around. parked_gate_liveness_class() { # local win=$1 alive [ "$(window_kind "$win")" != secondmate ] || { printf 'none'; return; } @@ -815,24 +822,6 @@ parked_gate_liveness_class() { # esac } -# stale_absorb_class: crew_absorb_class as the FIRST-SIGHT stale path should read -# it, with the two interface-text-independent corroborations layered on. They are -# mutually exclusive by construction and each probes agent liveness at most once: -# `parked` is a positive run-step reading that needs a live worker behind it, while -# codex_static_pane_upgrade only ever fires on `none` (no run-step at all), so a -# crew can never be both. Keeping them separate keeps each backstop's coverage its -# own: the codex upgrade stays confined to a harness whose busy row lies, and the -# parked gate stays confined to a run that is deliberately stopped. -stale_absorb_class() { # - local win=$1 task=$2 class - class=$(crew_absorb_class "$task") - if [ "$class" = parked ]; then - parked_gate_liveness_class "$win" - return - fi - codex_static_pane_upgrade "$win" "$class" -} - pause_state_class() { # local win=$1 task=$2 key last recheck_file class agent_alive key=${win//:/_} @@ -842,7 +831,15 @@ pause_state_class() { # recheck_file="$STATE/.paused-rechecked-$key" if ! status_is_paused_or_captain_held "$last"; then rm -f "$recheck_file" - stale_absorb_class "$win" "$task" + class=$(crew_absorb_class "$task") + # A gate reading never absorbs a FIRST sighting: at first sight nothing has + # relayed the gate to firstmate yet (this path is reached precisely when the + # crew's last status line is not captain-relevant), so swallowing it would + # leave the decision waiting on the long bounded cadence with nobody told. + # The gate only ever holds the wedge LADDER, once the pane is a known stale + # firstmate has already seen (wedge_timer_check). + [ "$class" = parked ] && class=none + codex_static_pane_upgrade "$win" "$class" return fi if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then @@ -1791,11 +1788,6 @@ EOF # - paused: the crew declared an external wait, or a declared pause or # captain hold is paired with a confidently dead agent, so absorb on # the long PAUSE_RESURFACE_SECS recheck cadence instead of wedge-escalating; - # - parked: the authoritative run-step reports the pipeline stopped at a - # decision gate AND the worker's agent process is confirmed alive, so - # absorb and start the wedge timer exactly as `working` does - the gate - # is why the pane is idle, and the liveness read is what stops that - # answer from covering for a worker that crashed at the gate; # - none: no running pipeline, idle pane, no busy signature, no declared # pause - the crew has STOPPED. Surface immediately so firstmate peeks # (it may be done via an interactive menu that wrote no done: status, @@ -1810,16 +1802,6 @@ EOF date +%s > "$ssf" triage_log "absorbed non-terminal stale (provably working): $w" ;; - parked) - # The run stopped at a decision gate and the worker is confirmed - # alive. Absorbed exactly like an active run - suppressor advanced, - # wedge timer armed - so the moment that worker stops being alive - # the very next escalation surfaces it on the unchanged schedule. - clear_pause_tracking "$w" - printf '%s' "$h" > "$sf" - date +%s > "$ssf" - triage_log "absorbed non-terminal stale (parked at a decision gate, worker alive): $w" - ;; paused) handle_paused_stale "$w" "$task" "$h" ;; diff --git a/docs/architecture.md b/docs/architecture.md index 0a540f4a38..40610f547a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -19,7 +19,8 @@ For an ordinary crew that has stopped, the normal-mode watcher first surfaces on Live or inconclusive liveness remains fail-open at that initial surface, and the secondmate idle-endpoint exemption is unchanged. Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck. A terminal task firstmate marks parked (`bin/fm-mark-parked.sh `) after relaying its outcome and confirming only external human action remains is absorbed the same way and on the same cadence, keyed off the marker instead of a status verb; a later status write or metadata change clears the marker and returns the task to normal classification. -A mid-pipeline run that `bin/fm-crew-state.sh` reports stopped at a no-mistakes decision gate is a distinct case from that operator-declared marker, and the watcher recognizes it without one: a worker parked at an ask-user finding is idle precisely because it followed its instructions, so its stale pane holds the wedge ladder on the bounded-recheck cadence instead of escalating. +A mid-pipeline run that `bin/fm-crew-state.sh` reports stopped at a no-mistakes decision gate is a distinct case from that operator-declared marker, and the watcher recognizes it without one: a worker parked at an ask-user finding is idle precisely because it followed its instructions, so once its stale pane has been surfaced it holds the wedge ladder on the bounded-recheck cadence instead of escalating. +The first sighting of that pane is surfaced like any other stopped crew, because a gate the crew never wrote a captain-relevant status for has reached nobody yet, and an absorb there would leave the decision waiting silently. That hold rests on `fm_backend_agent_alive` confirming the worker's agent process and on nothing else, because a worker that crashed at the gate leaves the run parked identically and forever; a `dead` or unreadable liveness verdict keeps the unchanged escalation, and the reading is re-taken at every escalation window rather than trusted once. A gate reported only by the append-only status log earns no hold at all, since that event is arbitrarily old and reads the same either way. Fresh stale panes use the same current-state read before trusting the status log, so an active run or busy pane outranks an old captain-relevant status-log line left behind before validation. diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 371fad5568..f065f39493 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -949,10 +949,12 @@ test_codex_backstop_scoped_to_codex() { # ("state: parked · source: run-step · parked at review: 3 finding(s) (ask-user: # captain decision)") and the watcher escalated it as a possible wedge anyway, # every four minutes, because crew_absorb_class collapsed `parked` into `none`. -# The gate absorb is corroborated by the agent PROCESS and by nothing else: a -# worker that crashed one second after printing its gate prompt leaves the run -# parked identically and forever, so `dead` and `unknown` keep today's behavior -# exactly. harness=claude throughout, so nothing here can be the codex backstop. +# The gate buys no silence at a FIRST sighting - that wake is how firstmate learns +# the pane is idle at all - only a hold on the wedge LADDER afterwards, corroborated +# by the agent PROCESS and by nothing else: a worker that crashed one second after +# printing its gate prompt leaves the run parked identically and forever, so `dead` +# and `unknown` keep today's escalation exactly. harness=claude throughout, so +# nothing here can be the codex backstop. # # The authoritative reading the watcher used to ignore. Passed into each watcher # launch rather than exported by the fixture, because the fixture runs inside a @@ -974,27 +976,54 @@ _parked_gate_case() { # printf '%s\t%s\t%s\t%s\n' "$dir" "$state" "$fakebin" "$key" } -test_parked_gate_alive_absorbed() { - local fields dir state fakebin key out window pid pane_hash +# The gate never buys silence on a pane nobody has been told about yet: this crew's +# last status line is not captain-relevant, so no needs-decision/blocked line ever +# reached firstmate, and a first-sight absorb would leave the decision waiting out +# the long bounded cadence unannounced. The gate only ever holds the wedge LADDER, +# from the second sighting of that already-surfaced pane onward. +test_parked_gate_first_sight_surfaces_then_holds_the_ladder() { + local fields dir state fakebin key out drain_out window pid pane_hash window="test:fm-gatealive" fields=$(_parked_gate_case parked-gate-alive "$window" 'working: running the pipeline' 'idle at the review gate') IFS=$'\t' read -r dir state fakebin key <<< "$fields" - out="$dir/watch.out"; pane_hash=$(hash_text "idle at the review gate") + out="$dir/watch.out"; drain_out="$dir/drain.out"; pane_hash=$(hash_text "idle at the review gate") + + # Phase A: first sighting. A live agent at a parked gate changes nothing here. PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! + wait_for_exit "$pid" 40 || fail "the first sighting of a run parked at an unrelayed gate was swallowed: $(cat "$out")" + grep -Fx "stale: $window" "$out" >/dev/null || fail "the first sighting did not print the immediate stale wake: $(cat "$out")" + [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on the first-sight surface" + [ ! -e "$state/.stale-since-$key" ] || fail "the first sighting started a wedge timer instead of surfacing at once" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the parked-gate first surface failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the first sighting's wake was not queued" + + # Phase B: same pane, already surfaced, now past the escalation threshold. THIS + # is where the gate earns its hold - the alarm firstmate has already seen must + # not climb the wedge ladder against a worker that is alive and waiting for an + # answer. + echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! if ! wait_live "$pid" 30; then - reap "$pid"; fail "a worker parked at a decision gate with a live agent still woke firstmate: $(cat "$out")" + reap "$pid"; fail "a live worker parked at a gate still escalated on an already-surfaced pane: $(cat "$out")" fi - [ ! -s "$state/.wake-queue" ] || fail "the parked-gate absorb enqueued a wake: $(cat "$state/.wake-queue")" - [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on the parked-gate absorb" - [ -s "$state/.stale-since-$key" ] || fail "the parked-gate absorb did not arm the wedge timer, so a dying worker could never escalate" - grep -F "parked at a decision gate, worker alive" "$state/.watch-triage.log" >/dev/null \ - || fail "the absorb did not record WHY it believed the worker was fine: $(cat "$state/.watch-triage.log" 2>/dev/null)" + grep -F "possible wedge" "$out" >/dev/null && { reap "$pid"; fail "the parked gate was reported as a possible wedge: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "the held ladder enqueued a wake: $(cat "$state/.wake-queue")"; } + [ ! -e "$state/.wedge-escalations-$key" ] || { reap "$pid"; fail "a parked-gate hold climbed the wedge-escalation ladder"; } + [ -s "$state/.stale-since-$key" ] || { reap "$pid"; fail "the parked-gate hold dropped the wedge timer instead of refreshing it"; } + grep -F "the run is parked at a decision gate and this worker is still alive" "$state/.watch-triage.log" >/dev/null \ + || { reap "$pid"; fail "the hold did not record WHY it believed the worker was fine: $(cat "$state/.watch-triage.log" 2>/dev/null)"; } reap "$pid" - pass "a worker parked at a decision gate with a live agent is absorbed, and says why, with the wedge timer still armed" + pass "a run parked at an unrelayed gate surfaces its first sighting, then holds the wedge ladder while its worker is alive" } test_parked_gate_dead_surfaces() { @@ -1018,26 +1047,6 @@ test_parked_gate_dead_surfaces() { pass "a run parked at a gate whose worker has died surfaces immediately (the parked case never masks a crash)" } -test_parked_gate_unknown_liveness_surfaces() { - local fields dir state fakebin key out window pid - window="test:fm-gateunknown" - # pi's launcher execs into a generic `node`, so its liveness reads unknown - # (docs/tmux-backend.md "Known gaps"). Unknown never licenses an absorb: these - # crews keep exactly the behavior they have today. - fields=$(_parked_gate_case parked-gate-unknown "$window" 'working: running the pipeline' 'idle at the review gate') - IFS=$'\t' read -r dir state fakebin key <<< "$fields" - out="$dir/watch.out" - PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ - FM_FAKE_TMUX_CURRENT_COMMAND=node FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ - FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ - FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & - pid=$! - wait_for_exit "$pid" 40 || fail "an unreadable agent liveness was treated as proof the parked worker was fine" - grep -Fx "stale: $window" "$out" >/dev/null || fail "the unknown-liveness parked worker did not surface: $(cat "$out")" - [ ! -e "$state/.stale-since-$key" ] || fail "an unknown-liveness parked worker was absorbed onto the wedge timer" - pass "a parked worker whose agent liveness cannot be read keeps today's behavior and still surfaces" -} - # The captain's exact repeat: the crew appended needs-decision:, the signal path # already relayed it, and each poll thereafter climbed the wedge ladder against a # run that fm-crew-state.sh reported parked at an ask-user gate. @@ -1103,6 +1112,28 @@ test_parked_gate_dead_escalates_on_the_ladder() { pass "a run parked at a gate whose worker died escalates on the unchanged wedge schedule" } +# pi's launcher execs into a generic `node`, so its liveness reads unknown +# (docs/tmux-backend.md "Known gaps"). Unknown is not evidence of anything and +# never licenses a hold: these crews keep exactly the escalation they have today. +test_parked_gate_unknown_liveness_escalates_on_the_ladder() { + local fields dir state fakebin key out window pid + window="test:fm-ladderunknown" + fields=$(_parked_gate_ladder_case parked-gate-ladder-unknown "$window") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=node FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "an unreadable agent liveness was treated as proof the parked worker was fine: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null || fail "the unknown-liveness parked worker did not escalate: $(cat "$out")" + grep -F "escalation 1" "$out" >/dev/null || fail "the unknown-liveness parked worker escalated off the normal schedule: $(cat "$out")" + [ "$(cat "$state/.wedge-escalations-$key" 2>/dev/null || true)" = 1 ] || fail "the unknown-liveness parked worker did not climb the wedge ladder" + pass "a parked worker whose agent liveness cannot be read keeps today's escalation" +} + # --- non-terminal stale, crew NOT provably working: surfaced immediately ------ # The key requirement: a crew with no running pipeline that has gone quiet (and is # not busy) has stopped - it may be done via interactive menus, waiting, or wedged. @@ -2432,11 +2463,11 @@ test_nonterminal_stale_provably_working_absorbed_then_escalated test_codex_static_pane_alive_absorbed test_codex_static_pane_dead_surfaces test_codex_backstop_scoped_to_codex -test_parked_gate_alive_absorbed +test_parked_gate_first_sight_surfaces_then_holds_the_ladder test_parked_gate_dead_surfaces -test_parked_gate_unknown_liveness_surfaces test_parked_gate_alive_holds_the_wedge_ladder test_parked_gate_dead_escalates_on_the_ladder +test_parked_gate_unknown_liveness_escalates_on_the_ladder test_terminal_stale_already_surfaced_absorbed_then_escalates test_terminal_stale_changed_line_still_surfaces test_wedge_escalation_marks_demand_deep_inspection_after_threshold From 0c6c7192c00a3e1c481620d4bda086185fe3e83c Mon Sep 17 00:00:00 2001 From: Coditan-XO Date: Sun, 9 Aug 2026 04:48:17 +0000 Subject: [PATCH 3/4] no-mistakes(review): never route a parked gate through the codex backstop --- bin/fm-watch.sh | 23 ++++++++---- docs/architecture.md | 1 + tests/fm-watch-triage.test.sh | 71 +++++++++++++++++++++++------------ 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 061c67ae3b..3d56582db5 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -832,13 +832,22 @@ pause_state_class() { # if ! status_is_paused_or_captain_held "$last"; then rm -f "$recheck_file" class=$(crew_absorb_class "$task") - # A gate reading never absorbs a FIRST sighting: at first sight nothing has - # relayed the gate to firstmate yet (this path is reached precisely when the - # crew's last status line is not captain-relevant), so swallowing it would - # leave the decision waiting on the long bounded cadence with nobody told. - # The gate only ever holds the wedge LADDER, once the pane is a known stale - # firstmate has already seen (wedge_timer_check). - [ "$class" = parked ] && class=none + # A gate reading never absorbs a FIRST sighting, for ANY harness: at first + # sight nothing has relayed the gate to firstmate yet (this path is reached + # precisely when the crew's last status line is not captain-relevant), so + # swallowing it would leave the decision waiting on the long bounded cadence + # with nobody told. The gate only ever holds the wedge LADDER, once the pane + # is a known stale firstmate has already seen (wedge_timer_check). + # It short-circuits BEFORE codex_static_pane_upgrade rather than being folded + # into its `none` fallback: that backstop answers "no run-step says anything, + # is the process still there?" for a codex worker mid-turn on a static pane, + # and a run-step that authoritatively reports the run STOPPED at a gate is not + # that question. Routing the gate through it would hand codex alone the + # first-sight absorb every other harness just lost. + if [ "$class" = parked ]; then + printf 'none' + return + fi codex_static_pane_upgrade "$win" "$class" return fi diff --git a/docs/architecture.md b/docs/architecture.md index 40610f547a..39cf179aa2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,6 +21,7 @@ Its initial normal-mode status signal still surfaces through the no-verb path, w A terminal task firstmate marks parked (`bin/fm-mark-parked.sh `) after relaying its outcome and confirming only external human action remains is absorbed the same way and on the same cadence, keyed off the marker instead of a status verb; a later status write or metadata change clears the marker and returns the task to normal classification. A mid-pipeline run that `bin/fm-crew-state.sh` reports stopped at a no-mistakes decision gate is a distinct case from that operator-declared marker, and the watcher recognizes it without one: a worker parked at an ask-user finding is idle precisely because it followed its instructions, so once its stale pane has been surfaced it holds the wedge ladder on the bounded-recheck cadence instead of escalating. The first sighting of that pane is surfaced like any other stopped crew, because a gate the crew never wrote a captain-relevant status for has reached nobody yet, and an absorb there would leave the decision waiting silently. +That is true for every harness including codex: an authoritative run-step gate is not the no-current-state-proof case the codex static-pane backstop covers, so it never reaches that backstop and codex gets no first-sight absorb the other harnesses lack. That hold rests on `fm_backend_agent_alive` confirming the worker's agent process and on nothing else, because a worker that crashed at the gate leaves the run parked identically and forever; a `dead` or unreadable liveness verdict keeps the unchanged escalation, and the reading is re-taken at every escalation window rather than trusted once. A gate reported only by the append-only status log earns no hold at all, since that event is arbitrarily old and reads the same either way. Fresh stale panes use the same current-state read before trusting the status log, so an active run or busy pane outranks an old captain-relevant status-log line left behind before validation. diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index f065f39493..bb2b043c78 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -953,20 +953,22 @@ test_codex_backstop_scoped_to_codex() { # the pane is idle at all - only a hold on the wedge LADDER afterwards, corroborated # by the agent PROCESS and by nothing else: a worker that crashed one second after # printing its gate prompt leaves the run parked identically and forever, so `dead` -# and `unknown` keep today's escalation exactly. harness=claude throughout, so -# nothing here can be the codex backstop. +# and `unknown` keep today's escalation exactly. The first-sight rule is harness- +# independent: codex is covered explicitly, because its static-pane liveness +# backstop answers a different question (no run-step at all) and must not hand +# codex alone the absorb every other harness gives up here. # # The authoritative reading the watcher used to ignore. Passed into each watcher # launch rather than exported by the fixture, because the fixture runs inside a # command substitution and an export there would never reach the watcher. PARKED_GATE_STATE='state: parked · source: run-step · parked at review: 3 finding(s) (ask-user: captain decision)' -_parked_gate_case() { # - local dir state fakebin capture_file window key pane_hash sig statusf +_parked_gate_case() { # [harness] + local dir state fakebin capture_file window key pane_hash sig statusf harness dir=$(make_case "$1"); state="$dir/state"; fakebin="$dir/fakebin" - capture_file="$dir/pane.txt"; window=$2; statusf="$state/gate.status" + capture_file="$dir/pane.txt"; window=$2; statusf="$state/gate.status"; harness=${5:-claude} printf '%s' "$4" > "$capture_file" - printf 'window=%s\nkind=ship\nharness=claude\n' "$window" > "$state/gate.meta" + printf 'window=%s\nkind=ship\nharness=%s\n' "$window" "$harness" > "$state/gate.meta" printf '%s\n' "$3" > "$statusf" sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" key=$(printf '%s' "$window" | tr ':/.' '___') @@ -980,26 +982,28 @@ _parked_gate_case() { # # last status line is not captain-relevant, so no needs-decision/blocked line ever # reached firstmate, and a first-sight absorb would leave the decision waiting out # the long bounded cadence unannounced. The gate only ever holds the wedge LADDER, -# from the second sighting of that already-surfaced pane onward. -test_parked_gate_first_sight_surfaces_then_holds_the_ladder() { - local fields dir state fakebin key out drain_out window pid pane_hash - window="test:fm-gatealive" - fields=$(_parked_gate_case parked-gate-alive "$window" 'working: running the pipeline' 'idle at the review gate') +# from the second sighting of that already-surfaced pane onward. Driven once per +# harness, since the harness is exactly what decides whether a `none` verdict gets +# a liveness absorb of its own. +_assert_parked_gate_surfaces_then_holds() { # + local fields dir state fakebin key out drain_out window harness agent pid pane_hash + window=$2; harness=$3; agent=$4 + fields=$(_parked_gate_case "$1" "$window" 'working: running the pipeline' 'idle at the review gate' "$harness") IFS=$'\t' read -r dir state fakebin key <<< "$fields" out="$dir/watch.out"; drain_out="$dir/drain.out"; pane_hash=$(hash_text "idle at the review gate") # Phase A: first sighting. A live agent at a parked gate changes nothing here. PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ - FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_FAKE_TMUX_CURRENT_COMMAND="$agent" FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! - wait_for_exit "$pid" 40 || fail "the first sighting of a run parked at an unrelayed gate was swallowed: $(cat "$out")" - grep -Fx "stale: $window" "$out" >/dev/null || fail "the first sighting did not print the immediate stale wake: $(cat "$out")" - [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on the first-sight surface" - [ ! -e "$state/.stale-since-$key" ] || fail "the first sighting started a wedge timer instead of surfacing at once" - FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the parked-gate first surface failed" - grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the first sighting's wake was not queued" + wait_for_exit "$pid" 40 || fail "$harness: the first sighting of a run parked at an unrelayed gate was swallowed: $(cat "$out")" + grep -Fx "stale: $window" "$out" >/dev/null || fail "$harness: the first sighting did not print the immediate stale wake: $(cat "$out")" + [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "$harness: stale suppressor not advanced on the first-sight surface" + [ ! -e "$state/.stale-since-$key" ] || fail "$harness: the first sighting started a wedge timer instead of surfacing at once" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "$harness: drain after the parked-gate first surface failed" + grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "$harness: the first sighting's wake was not queued" # Phase B: same pane, already surfaced, now past the escalation threshold. THIS # is where the gate earns its hold - the alarm firstmate has already seen must @@ -1008,24 +1012,40 @@ test_parked_gate_first_sight_surfaces_then_holds_the_ladder() { echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key" : > "$out" PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ - FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_FAKE_TMUX_CURRENT_COMMAND="$agent" FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & pid=$! if ! wait_live "$pid" 30; then - reap "$pid"; fail "a live worker parked at a gate still escalated on an already-surfaced pane: $(cat "$out")" + reap "$pid"; fail "$harness: a live worker parked at a gate still escalated on an already-surfaced pane: $(cat "$out")" fi - grep -F "possible wedge" "$out" >/dev/null && { reap "$pid"; fail "the parked gate was reported as a possible wedge: $(cat "$out")"; } - [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "the held ladder enqueued a wake: $(cat "$state/.wake-queue")"; } - [ ! -e "$state/.wedge-escalations-$key" ] || { reap "$pid"; fail "a parked-gate hold climbed the wedge-escalation ladder"; } - [ -s "$state/.stale-since-$key" ] || { reap "$pid"; fail "the parked-gate hold dropped the wedge timer instead of refreshing it"; } + grep -F "possible wedge" "$out" >/dev/null && { reap "$pid"; fail "$harness: the parked gate was reported as a possible wedge: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "$harness: the held ladder enqueued a wake: $(cat "$state/.wake-queue")"; } + [ ! -e "$state/.wedge-escalations-$key" ] || { reap "$pid"; fail "$harness: a parked-gate hold climbed the wedge-escalation ladder"; } + [ -s "$state/.stale-since-$key" ] || { reap "$pid"; fail "$harness: the parked-gate hold dropped the wedge timer instead of refreshing it"; } grep -F "the run is parked at a decision gate and this worker is still alive" "$state/.watch-triage.log" >/dev/null \ - || { reap "$pid"; fail "the hold did not record WHY it believed the worker was fine: $(cat "$state/.watch-triage.log" 2>/dev/null)"; } + || { reap "$pid"; fail "$harness: the hold did not record WHY it believed the worker was fine: $(cat "$state/.watch-triage.log" 2>/dev/null)"; } reap "$pid" +} + +test_parked_gate_first_sight_surfaces_then_holds_the_ladder() { + _assert_parked_gate_surfaces_then_holds parked-gate-alive "test:fm-gatealive" claude claude pass "a run parked at an unrelayed gate surfaces its first sighting, then holds the wedge ladder while its worker is alive" } +# Same run, same live process, harness=codex. codex is the one harness whose +# otherwise-`none` static pane earns a liveness absorb of its own +# (codex_static_pane_upgrade), and an authoritative run-step gate must not be +# routed into it: that would give codex alone the 3600s bounded recheck where +# every other harness surfaces at once, on the very decision nobody has been told +# about. The gate short-circuits first, so codex surfaces like the rest and still +# holds the ladder afterwards. +test_parked_gate_codex_first_sight_surfaces_then_holds_the_ladder() { + _assert_parked_gate_surfaces_then_holds parked-gate-codex "test:fm-gatecodex" codex codex + pass "a codex worker parked at an unrelayed gate surfaces its first sighting too, then holds the wedge ladder" +} + test_parked_gate_dead_surfaces() { local fields dir state fakebin key out drain_out window pid window="test:fm-gatedead" @@ -2464,6 +2484,7 @@ test_codex_static_pane_alive_absorbed test_codex_static_pane_dead_surfaces test_codex_backstop_scoped_to_codex test_parked_gate_first_sight_surfaces_then_holds_the_ladder +test_parked_gate_codex_first_sight_surfaces_then_holds_the_ladder test_parked_gate_dead_surfaces test_parked_gate_alive_holds_the_wedge_ladder test_parked_gate_dead_escalates_on_the_ladder From cf2beb18ce4a98d170a2814de423e83f05e367c5 Mon Sep 17 00:00:00 2001 From: Coditan-XO Date: Sun, 9 Aug 2026 05:37:18 +0000 Subject: [PATCH 4/4] no-mistakes(document): document decision-gate wedge-ladder hold in watcher docs --- .agents/skills/harness-adapters/SKILL.md | 1 + docs/architecture.md | 4 +- tests/fm-watch-triage.test.sh | 56 ++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/.agents/skills/harness-adapters/SKILL.md b/.agents/skills/harness-adapters/SKILL.md index 5d8971cf09..654b51dc98 100644 --- a/.agents/skills/harness-adapters/SKILL.md +++ b/.agents/skills/harness-adapters/SKILL.md @@ -198,6 +198,7 @@ The false-wedge exposure is real and was reproduced live (docs/codex-busy-detect The busy-signature matching in `fm-watch.sh` and `fm-tmux-lib.sh` stays deliberately unchanged - the row still matches whenever it renders - but the absorb path now carries an interface-text-INDEPENDENT backstop scoped to codex. `codex_static_pane_upgrade` in `bin/fm-watch.sh` treats an otherwise-stale codex pane as provably working (absorb plus wedge timer) whenever the codex agent PROCESS is confidently alive: `fm_backend_agent_alive` reads `alive` because the pane's foreground command stays `codex` for the entire turn (verified 100/100 samples), a signal that does not depend on the busy row at all. A crashed codex reads `dead` (bare shell) and still surfaces at once, and a genuinely wedged codex still escalates past `STALE_ESCALATE_SECS` and on to demand-deep-inspection, so the backstop hides a healthy worker without hiding a stuck one. +One stale pane never reaches the backstop at all: when `bin/fm-crew-state.sh`'s run-step authoritatively reports the run stopped at a no-mistakes decision gate, that pane surfaces on first sight for every harness alike, so codex gets no first-sight absorb the others lack (docs/architecture.md owns the gate rule). This also covers the keymap-remap blind spot below, because it does not read the interrupt-hint text. The keymap case is the one that requires an operator to have changed something. diff --git a/docs/architecture.md b/docs/architecture.md index 39cf179aa2..7b96558276 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ firstmate's always-loaded operating contract and routing index for conditional p ## Event-driven supervision A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies detected wakes in bash, and wakes the first mate only when something is actionable. -Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, confirmed unhealthy certsync health or cannot-run certsync status reads from the heartbeat path, stale panes whose crew is not provably working and whose captain-relevant status the signal path has not already surfaced, whether their status log looks terminal or non-terminal, absorbed stale panes that persist past `FM_STALE_ESCALATE_SECS` and no longer read as provably working at that moment, declared external waits (a crew's `paused:` status or a firstmate-declared `.parked-` marker) that remain past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. +Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, confirmed unhealthy certsync health or cannot-run certsync status reads from the heartbeat path, stale panes whose crew is not provably working and whose captain-relevant status the signal path has not already surfaced, whether their status log looks terminal or non-terminal, absorbed stale panes that persist past `FM_STALE_ESCALATE_SECS` and at that moment neither read as provably working nor sit at a decision gate a confirmed-alive worker can still answer, declared external waits (a crew's `paused:` status or a firstmate-declared `.parked-` marker) that remain past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. Repeated stale escalations on the same unchanged pane add an escalation count to the wake reason and, at `FM_WEDGE_DEMAND_INSPECT_COUNT`, a `demand-deep-inspection` marker. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) before detector state advances, so losing a model-facing delivery wait cannot lose the wake. No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step for that crew's branch or a backend busy signature. @@ -85,7 +85,7 @@ A presence-gated sub-supervisor (`bin/fm-supervise-daemon.sh`) extends this for The watcher and daemon share `bin/fm-classify-lib.sh` for captain-relevant status verbs, declared-external-wait vocabulary, and status-scan primitives. Terminal verbs remain captain-relevant, while a nonterminal progress verb cannot become terminal merely because its prose contains a legacy free-text token such as `merged`; bare legacy free-text lines remain compatible. The always-on watcher also uses that library's absorb classification on no-verb signals and first-sighting stale panes before status-log terminality is trusted, while the daemon maintains distinct wedge and declared-pause recheck cadences. -The watcher reads that classification once more when a wedge timer would escalate, so an idle pane whose run is still active never climbs the escalation ladder, and a hold lasting a full pause-recheck window still surfaces one bounded recheck. +The watcher reads that classification once more when a wedge timer would escalate, so an idle pane whose run is still active - or parked at the decision gate above with its worker confirmed alive - never climbs the escalation ladder, and a hold lasting a full pause-recheck window still surfaces one bounded recheck. A captain-relevant status the signal path already surfaced does not wake firstmate a second time when the same crew's pane then settles stale; the suppression is keyed on the surfaced status content, never on the window, and it still starts the wedge timer. In away mode, seen-status dedupe does not clear possible-wedge aging for nonterminal progress, so housekeeping still re-escalates an unchanged idle pane at the configured bound. The daemon escalates captain-relevant events, plus a bounded recheck for a declared pause that remains idle, as one batched, single-line digest using the canonical `away-supervisor` syntax from `bin/fm-operational-input.sh`. diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index bb2b043c78..94eba99086 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1154,6 +1154,61 @@ test_parked_gate_unknown_liveness_escalates_on_the_ladder() { pass "a parked worker whose agent liveness cannot be read keeps today's escalation" } +# The other half of the "a stopped alarm is worse than a noisy one" rule: the hold +# is a HOLD, not silence. A gate nobody ever answers - the crew is alive and will +# wait forever - must still reach the captain on the long bounded cadence, in the +# recheck's own words rather than as a wedge, and without ever climbing the ladder. +# The anchor is the frozen hash's own mtime, which the hold refreshes nowhere, so a +# permanent hold cannot postpone its own recheck. +test_parked_gate_hold_gets_bounded_recheck() { + local fields dir state fakebin key out window pid back stale_mtime + window="test:fm-gaterot" + fields=$(_parked_gate_ladder_case parked-gate-hold-recheck "$window") + IFS=$'\t' read -r dir state fakebin key <<< "$fields" + out="$dir/watch.out" + # This pane has been frozen at the gate for a full recheck window. + back=$(( $(date +%s) - 500 )) + if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$state/.stale-$key" + else touch -m -d "@$back" "$state/.stale-$key"; fi + stale_mtime=$(file_mtime "$state/.stale-$key") + + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a gate held all day never got its bounded recheck: $(cat "$out")" + grep -F "bounded recheck" "$out" >/dev/null || fail "the recheck did not identify itself as a bounded recheck: $(cat "$out")" + grep -F "the run is parked at a decision gate and this worker is still alive" "$out" >/dev/null \ + || fail "the recheck did not tell the captain WHY the pane was held: $(cat "$out")" + grep -F "possible wedge" "$out" >/dev/null && fail "a parked-gate recheck was mislabeled a wedge escalation: $(cat "$out")" + [ ! -e "$state/.wedge-escalations-$key" ] || fail "a parked-gate bounded recheck climbed the wedge-escalation ladder" + [ -s "$state/.wedgeheld-$key" ] || fail "the parked-gate bounded recheck did not record its throttle marker" + [ "$(file_mtime "$state/.stale-$key")" = "$stale_mtime" ] \ + || fail "the parked hold refreshed the .stale- anchor, which would postpone its own recheck forever" + grep "$(printf '\tstale\t')" "$state/.wake-queue" | grep -F "$window" >/dev/null \ + || fail "the parked-gate bounded recheck was not queued for the captain: $(cat "$state/.wake-queue")" + + # Throttled: the next poll inside the same recheck window holds again silently. + echo "$back" > "$state/.stale-since-$key" + : > "$out" + : > "$state/.wake-queue" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$dir/pane.txt" \ + FM_FAKE_TMUX_CURRENT_COMMAND=claude FM_FAKE_CREW_STATE="$PARKED_GATE_STATE" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \ + FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "the parked-gate bounded recheck repeated inside its own cadence: $(cat "$out")" + fi + [ ! -s "$state/.wake-queue" ] || fail "a throttled parked-gate recheck enqueued another wake" + [ ! -e "$state/.wedge-escalations-$key" ] || fail "a throttled parked-gate recheck climbed the wedge-escalation ladder" + reap "$pid" + pass "a parked gate nobody answers still earns one bounded recheck per window, never a wedge escalation" +} + # --- non-terminal stale, crew NOT provably working: surfaced immediately ------ # The key requirement: a crew with no running pipeline that has gone quiet (and is # not busy) has stopped - it may be done via interactive menus, waiting, or wedged. @@ -2489,6 +2544,7 @@ test_parked_gate_dead_surfaces test_parked_gate_alive_holds_the_wedge_ladder test_parked_gate_dead_escalates_on_the_ladder test_parked_gate_unknown_liveness_escalates_on_the_ladder +test_parked_gate_hold_gets_bounded_recheck test_terminal_stale_already_surfaced_absorbed_then_escalates test_terminal_stale_changed_line_still_surfaces test_wedge_escalation_marks_demand_deep_inspection_after_threshold