From 703a6ddb019495a912ef12273664bb9e712296a0 Mon Sep 17 00:00:00 2001 From: thirtysevend Date: Thu, 6 Aug 2026 18:54:48 +0200 Subject: [PATCH] fix(watch): honor declared pause cadence --- bin/fm-watch.sh | 36 +++++++++++---- docs/architecture.md | 7 +-- tests/fm-watch-triage.test.sh | 86 +++++++++++++++++++++++------------ 3 files changed, 88 insertions(+), 41 deletions(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 2f150af60d..4be3c3bd4f 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -153,8 +153,10 @@ STALE_ESCALATE_SECS=${FM_STALE_ESCALATE_SECS:-240} # idle secs before a provabl BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600} # A crew that declared a pause is idling on a known external wait, so its stale # pane is absorbed rather than wedge-escalated. -# A captain-held or paused crew whose agent has confidently exited uses the same -# bounded cadence, while a live or ambiguously read agent still surfaces once. +# A declared pause uses the bounded cadence unless an authoritative active run +# proves work resumed. +# A captain-held crew uses that cadence only after its agent confidently exits, +# while a live or ambiguously read captain-held agent still surfaces once. # These cases re-surface once for a recheck every PAUSE_RESURFACE_SECS - far # longer than the wedge threshold, but finite so a forgotten hold cannot rot invisibly. PAUSE_RESURFACE_SECS=${FM_PAUSE_RESURFACE_SECS:-$FM_PAUSE_RESURFACE_SECS_DEFAULT} @@ -363,8 +365,10 @@ clear_pause_tracking() { # } # Reconcile a declared pause or captain-held status with authoritative crew state. -# Only a confidently dead ordinary crew may recover paused classification after -# fm-crew-state has fallen back to stopped or unknown. +# The latest explicit paused event remains authoritative while the endpoint idles, +# but an active run overrides it as working and a later status event clears it. +# A captain-held transfer requires a confidently dead ordinary crew before it can +# recover paused classification after fm-crew-state falls back to stopped or unknown. pause_state_class() { # local win=$1 task=$2 key last recheck_file class agent_alive key=${win//:/_} @@ -372,9 +376,25 @@ pause_state_class() { # key=${key//./_} last=$(last_status_line "$STATE/$task.status") recheck_file="$STATE/.paused-rechecked-$key" + if status_is_paused "$last"; then + if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then + printf 'paused' + return + fi + class=$(crew_absorb_class "$task") + if [ "$class" = working ]; then + rm -f "$recheck_file" + printf 'working' + return + fi + date +%s > "$recheck_file" + printf 'paused' + return + fi if ! status_is_paused_or_captain_held "$last"; then rm -f "$recheck_file" - crew_absorb_class "$task" + class=$(crew_absorb_class "$task") + [ "$class" = working ] && printf 'working' || printf 'none' return fi if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then @@ -1014,9 +1034,9 @@ EOF # - working: an actively-running pipeline legitimately sits on a static # pane (e.g. waiting on CI), so absorb and start the wedge timer so a # genuinely frozen run still escalates past STALE_ESCALATE_SECS; - # - paused: the crew declared an external wait, or a declared pause or - # captain hold is paired with a confidently dead agent, so absorb on - # the long PAUSE_RESURFACE_SECS cadence instead of wedge-escalating; + # - paused: the latest status declares an external wait, or a captain + # hold is paired with a confidently dead agent, so absorb on the long + # PAUSE_RESURFACE_SECS cadence instead of wedge-escalating; # - none: no running pipeline, no exact busy verdict, no declared pause. # Surface immediately so firstmate inspects the inconclusive state # (it may be done via an interactive menu that wrote no done: status, diff --git a/docs/architecture.md b/docs/architecture.md index 60ee3a5020..87aab4a36b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -18,9 +18,10 @@ The receipt makes retirement safely retryable across restarts: fixed-path recove A concurrent replacement remains armed, every non-merged or invalid observation remains unchanged, and retirement never performs task or persistent-secondmate cleanup. `bin/fm-pr-lib.sh` owns the receipt format and strict identity mechanics, while `bin/fm-watch.sh` owns queue-before-retirement ordering. No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step attributed to that crew's current code, or an exact busy verdict from the semantic busy-state contract. -A crew that declares `paused:` for a known external wait is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge. -For an ordinary crew that has stopped, the normal-mode watcher first surfaces one stale wake, then applies that same cadence to an unchanged `paused:` or durable `captain-held` endpoint only when the backend confidently reports its agent dead. -Live or inconclusive liveness remains fail-open at that initial surface, and the secondmate idle-endpoint exemption is unchanged. +A crew whose latest status event declares `paused:` for a known external wait is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge. +An authoritative active run overrides that pause as working, while any later non-pause status event clears the pause classification and restores normal stale handling. +For an ordinary crew with a durable `captain-held` endpoint, the normal-mode watcher first surfaces one stale wake and applies the bounded cadence only when the backend confidently reports its agent dead. +Live or inconclusive liveness for `captain-held` remains fail-open at that initial surface, and the secondmate idle-endpoint exemption is unchanged. Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck. Fresh stale panes use the same current-state read before trusting the status log, so an active run or a proven busy worker outranks an old captain-relevant status-log line left behind before validation. No-change heartbeats are also benign. diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index c10565bc8a..9413c78bbe 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -679,13 +679,13 @@ test_nonterminal_stale_paused_absorbed_then_resurfaced() { # A captain-held crew can leave a stable backend endpoint after its agent exits. # fm-crew-state then authoritatively reports stopped rather than paused, but the -# confirmed-dead agent plus the declared wait or captain-held transfer must retain -# bounded pause handling. -# A still-live agent at an external-decision gate is the disconfirming case: it -# must surface once, while the unchanged hash must not append the same wake on -# every watcher re-arm. -test_exited_declared_pause_is_bounded_but_live_gate_surfaces() { - local dir state fakebin out capture_file statusf window key pane_hash sig pid back round wakes bare +# confirmed-dead agent plus the captain-held transfer must retain bounded pause +# handling. +# A declared paused status is independently authoritative for an idle live agent: +# changing pane content must not produce a bare stale wake before the long cadence, +# and a later status event must clear that pause classification. +test_declared_pause_and_exited_captain_hold_use_bounded_cadence() { + local dir state fakebin out capture_file statusf window key pane_hash sig pid back round wakes bare i dir=$(make_case exited-declared-pause); state="$dir/state"; fakebin="$dir/fakebin" out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/held.status" window="test:fm-held" @@ -746,44 +746,70 @@ test_exited_declared_pause_is_bounded_but_live_gate_surfaces() { window="test:fm-gate" printf 'idle external-decision gate\n' > "$capture_file" printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/gate.meta" - printf 'paused: waiting at an active external-decision gate\n' > "$statusf" + printf 'paused [key=route]: waiting on an external dependency\n' > "$statusf" sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" key=$(printf '%s' "$window" | tr ':/.' '___') pane_hash=$(hash_text "idle external-decision gate") printf '%s' "$pane_hash" > "$state/.hash-$key" printf '1\n' > "$state/.count-$key" - # First sight must surface promptly so a live external-decision gate is not - # hidden behind the pause cadence. + # A fresh declared pause must absorb even while the agent endpoint is live. PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ - FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting at an active external-decision gate' \ + FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting on an external dependency' \ FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" & pid=$! - wait_for_exit "$pid" 40 || fail "live external-decision gate did not surface immediately" + wait_live "$pid" 30 || { reap "$pid"; fail "live declared pause surfaced before its long cadence: $(cat "$out")"; } + [ -e "$state/.paused-$key" ] || { reap "$pid"; fail "live declared pause did not record pause tracking"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "live declared pause queued a wake before its long cadence"; } + printf 'idle external-decision gate, changing display\n' > "$capture_file" + pane_hash=$(hash_text "idle external-decision gate, changing display") + i=0 + while [ "$i" -lt 30 ] && kill -0 "$pid" 2>/dev/null; do + [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] && break + sleep 0.1 + i=$((i + 1)) + done + kill -0 "$pid" 2>/dev/null || { reap "$pid"; fail "changing live pause emitted a bare stale wake: $(cat "$out")"; } + [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] \ + || { reap "$pid"; fail "changing live pause did not advance its paused stale suppressor"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "changing live pause queued a wake before its long cadence"; } + reap "$pid" - # Re-arm with the stale timer already beyond the wedge threshold. This is the - # exact unchanged-hash fallback after the immediate surface: it must retain - # the pause cadence and discard any residual wedge timer instead of emitting - # a second possible-wedge wake. - printf '%s\n' $(( $(date +%s) - 500 )) > "$state/.stale-since-$key" + # Backdate the status and change the pane hash to reproduce a live idle pane + # whose rendered content drifts between watcher re-arms. + back=$(( $(date +%s) - 500 )) + if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf" + else touch -m -d "@$back" "$statusf"; fi + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" + printf 'idle external-decision gate, refreshed display\n' > "$capture_file" PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ - FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting at an active external-decision gate' \ - FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting on an external dependency' \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" & pid=$! - if ! wait_live "$pid" 30; then - reap "$pid" - fail "live external-decision gate escalated on the wedge timer after its immediate surface: $(cat "$out")" - fi - [ -e "$state/.paused-$key" ] || { reap "$pid"; fail "live external-decision gate lost its pause cadence marker"; } - [ ! -e "$state/.stale-since-$key" ] || { reap "$pid"; fail "live external-decision gate retained the wedge timer"; } - reap "$pid" + wait_for_exit "$pid" 40 || fail "live declared pause did not re-surface on the bounded cadence" wakes=$(awk -F '\t' -v w="$window" '$3 == "stale" && $4 == w { n++ } END { print n + 0 }' "$state/.wake-queue") bare=$(awk -F '\t' -v w="$window" '$3 == "stale" && $4 == w && $5 == "stale: " w { n++ } END { print n + 0 }' "$state/.wake-queue") - [ "$wakes" -eq 1 ] || fail "live external-decision gate should surface once, got $wakes wakes" - [ "$bare" -eq 1 ] || fail "live external-decision gate lost its immediate bare stale surface" - pass "exited declared-pause and captain-held panes use bounded pause cadence while a live decision gate still surfaces once" + [ "$wakes" -eq 1 ] || fail "live declared pause should re-surface once, got $wakes wakes" + [ "$bare" -eq 0 ] || fail "live declared pause surfaced as $bare bare stale wakes" + grep -F "declared pause, rechecked on a long cadence" "$state/.wake-queue" >/dev/null \ + || fail "live declared pause recheck omitted its labeled cadence reason" + + printf 'resolved [key=route]: external dependency cleared\n' >> "$statusf" + sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status" + printf 'idle after external dependency cleared\n' > "$capture_file" + : > "$state/.wake-queue" + : > "$out" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: stopped · source: pane · idle after dependency cleared' \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "resolved pause did not resume normal stale behavior" + grep -Fx "stale: $window" "$out" >/dev/null || fail "resolved pause did not emit the normal bare stale wake" + [ ! -e "$state/.paused-$key" ] || fail "resolved pause retained pause tracking" + pass "declared pauses use bounded labeled cadence and a later status resumes normal stale behavior" } test_secondmate_paused_resurfaces_in_normal_mode() { @@ -1825,7 +1851,7 @@ test_busy_pane_repeated_escalation_reaches_demand_deep_inspection test_busy_pane_default_turn_age_bound_is_3600s test_nonterminal_stale_not_working_surfaced test_nonterminal_stale_paused_absorbed_then_resurfaced -test_exited_declared_pause_is_bounded_but_live_gate_surfaces +test_declared_pause_and_exited_captain_hold_use_bounded_cadence test_secondmate_paused_resurfaces_in_normal_mode test_secondmate_nonpaused_stale_remains_suppressed test_secondmate_unpause_clears_pause_tracking