diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 095030983c..1615e29f55 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -148,6 +148,7 @@ CHECK_INTERVAL=${FM_CHECK_INTERVAL:-300} # seconds between *.check.sh sweeps CHECK_TIMEOUT=${FM_CHECK_TIMEOUT:-30} # seconds allowed per *.check.sh CERTSYNC_HEALTH_TIMEOUT=${FM_CERTSYNC_HEALTH_TIMEOUT:-5} # seconds allowed for certsync heartbeat health CERTSYNC_HEALTH_RESURFACE=${FM_CERTSYNC_HEALTH_RESURFACE:-3600} # seconds before repeating unchanged unhealthy or cannot-run certsync +CERTSYNC_HEARTBEAT_MAX_AGE=${FM_CERTSYNC_HEARTBEAT_MAX_AGE:-7200} # certsync heartbeat older than this reads as unhealthy (daemon stopped / syncs failing); 2x the 3600s max sync interval; 0 disables CONTEXT_CHECK_INTERVAL=${FM_CONTEXT_CHECK_INTERVAL:-300} # seconds between context-ceiling reads # How long an UNCHANGED context-ceiling report stays quiet before it says so # again. Long, because every branch of that check describes a standing condition @@ -1041,36 +1042,42 @@ certsync_health_mark_surfaced() { # or its compose file is the one legitimate "nothing to check here" case: it # means certsync is not deployed on this host, not that a deployed check failed. certsync_health_reason() { - local project compose graph_compose marker previous timeout_previous - local out out_rc healthy summary + local project compose src state_db heartbeat_file daemon_state marker previous timeout_previous + local py out out_rc healthy summary hb_age FM_CERTSYNC_HEALTH_REASON= FM_CERTSYNC_HEALTH_SIGNATURE= project=${FM_CERTSYNC_PROJECT:-$FM_HOME/projects/hlr-certsync} compose=${FM_CERTSYNC_COMPOSE_FILE:-$project/docker-compose.yml} - graph_compose=${FM_CERTSYNC_GRAPH_COMPOSE_FILE:-$project/docker-compose.graph-pem.yml} + src=${FM_CERTSYNC_SRC:-$project/src} + state_db=${FM_CERTSYNC_STATE_DB:-/var/lib/hlr-certsync/certsync-state.sqlite3} + heartbeat_file=${FM_CERTSYNC_HEARTBEAT_FILE:-/var/lib/hlr-certsync/certsync-heartbeat.json} + daemon_state=${FM_CERTSYNC_DAEMON_STATE:-running} marker="$STATE/.certsync-health-surfaced" [ -d "$project" ] || return 1 [ -f "$compose" ] || return 1 - if ! command -v docker >/dev/null 2>&1; then - FM_CERTSYNC_HEALTH_REASON="check: certsync health: cannot run: docker missing" + if ! command -v python3 >/dev/null 2>&1; then + FM_CERTSYNC_HEALTH_REASON="check: certsync health: cannot run: python3 missing" elif ! command -v jq >/dev/null 2>&1; then FM_CERTSYNC_HEALTH_REASON="check: certsync health: cannot run: jq missing" + elif [ ! -f "$src/hlr_certsync/status.py" ]; then + FM_CERTSYNC_HEALTH_REASON="check: certsync health: cannot run: certsync source unavailable at $src" else + # Read certsync's status directly off the heartbeat file and state DB on the + # host - no docker socket, no `exec`, no docker-group membership. certsync + # exposes both under a readable host bind mount (certsync repo's + # docs/deploy.md, "State host path"). build_status computes healthy/reason + # purely from those two files plus the daemon-state argument, so invoking it + # here reproduces exactly what `docker compose exec certsync certsync status` + # produced, byte for byte, but needs no docker access at all. + py='import json,sys +from hlr_certsync.status import build_status +from hlr_certsync.state import StateStore +print(json.dumps(build_status(StateStore(sys.argv[1]), sys.argv[2], daemon_state=sys.argv[3]), sort_keys=True))' timeout_previous=$CHECK_TIMEOUT CHECK_TIMEOUT=$CERTSYNC_HEALTH_TIMEOUT - if [ -f "$graph_compose" ]; then - out=$(run_bounded docker compose -f "$compose" -f "$graph_compose" exec -T certsync certsync status \ - --state-db "${FM_CERTSYNC_STATE_DB:-/var/lib/hlr-certsync/certsync-state.sqlite3}" \ - --heartbeat-file "${FM_CERTSYNC_HEARTBEAT_FILE:-/var/lib/hlr-certsync/certsync-heartbeat.json}" \ - --daemon-state "${FM_CERTSYNC_DAEMON_STATE:-running}") - else - out=$(run_bounded docker compose -f "$compose" exec -T certsync certsync status \ - --state-db "${FM_CERTSYNC_STATE_DB:-/var/lib/hlr-certsync/certsync-state.sqlite3}" \ - --heartbeat-file "${FM_CERTSYNC_HEARTBEAT_FILE:-/var/lib/hlr-certsync/certsync-heartbeat.json}" \ - --daemon-state "${FM_CERTSYNC_DAEMON_STATE:-running}") - fi + out=$(run_bounded env PYTHONPATH="$src" python3 -c "$py" "$state_db" "$heartbeat_file" "$daemon_state") out_rc=$? CHECK_TIMEOUT=$timeout_previous @@ -1083,8 +1090,21 @@ certsync_health_reason() { if [ -z "$healthy" ]; then FM_CERTSYNC_HEALTH_REASON="check: certsync health: cannot run: invalid or missing status JSON" elif [ "$healthy" = true ]; then - rm -f "$marker" 2>/dev/null || true - return 1 + # A running daemon rewrites the heartbeat on every successful sync pass + # (<=3600s apart). Reading frozen files off the host cannot, on its own, + # tell a live healthy daemon from a stopped container whose last-written + # files still say "success" - the old docker-exec check caught that only + # because exec itself failed when the container was down. Reinstate that + # liveness signal here: a heartbeat older than the bound reads as + # unhealthy, never healthy, so "cannot confirm well" never collapses into + # "is well". Set FM_CERTSYNC_HEARTBEAT_MAX_AGE=0 to disable. + hb_age=$(age_of "$heartbeat_file") + if [ "${CERTSYNC_HEARTBEAT_MAX_AGE:-0}" -gt 0 ] 2>/dev/null && [ "$hb_age" -gt "$CERTSYNC_HEARTBEAT_MAX_AGE" ]; then + FM_CERTSYNC_HEALTH_REASON="check: certsync health: unhealthy: heartbeat stale (${hb_age}s > ${CERTSYNC_HEARTBEAT_MAX_AGE}s); daemon may be stopped or syncs failing" + else + rm -f "$marker" 2>/dev/null || true + return 1 + fi else summary=$(printf '%s' "$out" | jq -r '.reason // "unhealthy"' 2>/dev/null \ | tr '\n\t' ' ' \ diff --git a/docs/configuration.md b/docs/configuration.md index c00cc98ea9..d75c963279 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -603,12 +603,12 @@ Both paths share the same signature and marker implementation, so the slow fallb ## Certsync health check (FM_CERTSYNC_*) `bin/fm-watch.sh` folds certsync health into the ordinary heartbeat path when a certsync deployment is present under the home. -The default deployment path is `$FM_HOME/projects/hlr-certsync` with `docker-compose.yml`; `FM_CERTSYNC_PROJECT`, `FM_CERTSYNC_COMPOSE_FILE`, and `FM_CERTSYNC_GRAPH_COMPOSE_FILE` override the project and compose files. -If the optional graph compose file exists, the watcher adds it to the `docker compose` command; if it is absent, the base compose file is enough. -The heartbeat runs `docker compose ... exec -T certsync certsync status` with `FM_CERTSYNC_STATE_DB`, `FM_CERTSYNC_HEARTBEAT_FILE`, and `FM_CERTSYNC_DAEMON_STATE` mapped to the status command's `--state-db`, `--heartbeat-file`, and `--daemon-state` arguments. +The default deployment path is `$FM_HOME/projects/hlr-certsync` with `docker-compose.yml`; `FM_CERTSYNC_PROJECT` and `FM_CERTSYNC_COMPOSE_FILE` override the project directory and compose file that mark certsync as deployed on this host. +The check reads certsync's status **directly off the host filesystem** — it needs no docker socket, no `docker compose exec`, and no `docker`-group membership. certsync exposes its heartbeat JSON and sqlite state DB under a readable host bind mount (see the certsync repo's `docs/deploy.md`, "State host path"); the watcher runs certsync's own `build_status` against those two files via `python3` with `PYTHONPATH=$FM_CERTSYNC_SRC` (default `$FM_CERTSYNC_PROJECT/src`), passing `FM_CERTSYNC_STATE_DB`, `FM_CERTSYNC_HEARTBEAT_FILE`, and `FM_CERTSYNC_DAEMON_STATE` as the `--state-db`, `--heartbeat-file`, and `--daemon-state` inputs. Because `build_status` computes `healthy`/`reason` purely from those two files plus the daemon-state argument, this reproduces exactly the JSON the former `docker compose exec certsync certsync status` produced, with no docker access at all. A confirmed JSON object with `healthy: false` becomes a durable `check` wake keyed as `certsync-health`, with the `reason` field trimmed and bounded in the wake text. -A `healthy: true` reading clears the unchanged marker and stays quiet. -Missing `docker` or `jq`, a failed status command (including a docker permission denial), empty output, invalid JSON, and a missing boolean `healthy` field all produce their own `check` wake carrying a `cannot run: ...` reason, so an inability to read certsync's status can never read the same as a confirmed-healthy status; only a missing project directory or missing compose file (certsync not deployed on this host at all) stays quiet. +A `healthy: true` reading clears the unchanged marker and stays quiet — **provided** the heartbeat is fresh (see below). +Because the files are read off the host rather than through a container `exec`, a stopped container (or a daemon whose syncs have been failing) no longer fails the read the way `exec` did; instead its heartbeat goes stale. To keep "cannot confirm well" from collapsing into "is well", a `healthy: true` reading whose heartbeat file is older than `FM_CERTSYNC_HEARTBEAT_MAX_AGE` is reported as `unhealthy: heartbeat stale (...)` rather than staying quiet. The daemon rewrites the heartbeat on every successful sync pass (at most 3600s apart), so the 7200s default has margin; set `FM_CERTSYNC_HEARTBEAT_MAX_AGE=0` to restore the raw `build_status` verdict with no freshness gate. +Missing `python3` or `jq`, an unreadable certsync source tree (`$FM_CERTSYNC_SRC/hlr_certsync/status.py` absent), a failed status computation, empty output, invalid JSON, and a missing boolean `healthy` field all produce their own `check` wake carrying a `cannot run: ...` reason, so an inability to read certsync's status can never read the same as a confirmed-healthy status; only a missing project directory or missing compose file (certsync not deployed on this host at all) stays quiet. The command is bounded by `FM_CERTSYNC_HEALTH_TIMEOUT` rather than the general check timeout, and an unchanged unhealthy or unchanged cannot-run reading re-surfaces only after `FM_CERTSYNC_HEALTH_RESURFACE`. ## Environment variables @@ -649,14 +649,15 @@ FM_HEARTBEAT=600 # base seconds between heartbeat scans; no-change heartb FM_HEARTBEAT_MAX=7200 # heartbeat backoff cap FM_CHECK_INTERVAL=300 # seconds between slow checks (authenticated merge polls, custom checks, or X-mode dispatch) FM_CHECK_TIMEOUT=30 # seconds allowed per slow check script -FM_CERTSYNC_PROJECT=$FM_HOME/projects/hlr-certsync # certsync deployment directory watched from the heartbeat path when its compose file exists -FM_CERTSYNC_COMPOSE_FILE=$FM_CERTSYNC_PROJECT/docker-compose.yml # base Docker Compose file for the certsync status command -FM_CERTSYNC_GRAPH_COMPOSE_FILE=$FM_CERTSYNC_PROJECT/docker-compose.graph-pem.yml # optional extra certsync graph compose file; used only when present -FM_CERTSYNC_STATE_DB=/var/lib/hlr-certsync/certsync-state.sqlite3 # certsync status --state-db argument -FM_CERTSYNC_HEARTBEAT_FILE=/var/lib/hlr-certsync/certsync-heartbeat.json # certsync status --heartbeat-file argument -FM_CERTSYNC_DAEMON_STATE=running # certsync status --daemon-state argument -FM_CERTSYNC_HEALTH_TIMEOUT=5 # seconds allowed for the heartbeat's certsync status command +FM_CERTSYNC_PROJECT=$FM_HOME/projects/hlr-certsync # certsync deployment directory; presence of its compose file marks certsync as deployed on this host +FM_CERTSYNC_COMPOSE_FILE=$FM_CERTSYNC_PROJECT/docker-compose.yml # compose file whose presence marks certsync as deployed (not exec'd; the check reads state off the host) +FM_CERTSYNC_SRC=$FM_CERTSYNC_PROJECT/src # certsync source tree put on PYTHONPATH to run build_status on the host without docker or an install +FM_CERTSYNC_STATE_DB=/var/lib/hlr-certsync/certsync-state.sqlite3 # host path to certsync's state DB, read as build_status --state-db +FM_CERTSYNC_HEARTBEAT_FILE=/var/lib/hlr-certsync/certsync-heartbeat.json # host path to certsync's heartbeat, read as build_status --heartbeat-file +FM_CERTSYNC_DAEMON_STATE=running # build_status --daemon-state argument +FM_CERTSYNC_HEALTH_TIMEOUT=5 # seconds allowed for the heartbeat's certsync status read FM_CERTSYNC_HEALTH_RESURFACE=3600 # seconds before an unchanged unhealthy or cannot-run certsync status is queued again +FM_CERTSYNC_HEARTBEAT_MAX_AGE=7200 # a healthy reading whose heartbeat is older than this reads as unhealthy (daemon stopped / syncs failing); 2x the 3600s max sync interval; 0 disables the freshness gate FM_CONTEXT_CEILING=300000 # captain-decided token ceiling for the primary session's own context; above it, at a quiet boundary, the watcher queues a reset, ask, or blocked wake; unmeasurable running sessions surface as unenforced (docs/context-reset.md) FM_CONTEXT_CAPTAIN_IDLE_SECS=1800 # silence since the last genuine captain prompt below which the captain counts as in live conversation: the watcher asks instead of ordering a reset, and bin/fm-context-reset.sh refuses FM_CONTEXT_RECEIPT_MAX_AGE=900 # seconds a state/.stow-receipt stays fresh; the receipt and the reset are meant to happen in one turn diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 3fd917a2d1..7a41067a9e 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1931,53 +1931,65 @@ test_heartbeat_backstop_surfaces_unsurfaced_status() { pass "heartbeat backstop fail-safe surfaces a captain-relevant status the per-wake path missed" } -setup_certsync_health_case() { # - local dir=$1 project +# The certsync health check reads certsync's status directly off the host +# filesystem (no docker, no `exec`, no docker-group membership) by running +# certsync's own build_status via python3 against the exposed heartbeat/state +# files. These fakes stand in for that source tree and the heartbeat file so the +# firstmate-side wiring is exercised hermetically: +# - setup_certsync_health_case installs a fake `hlr_certsync` package under the +# project's src/ whose build_status returns the JSON dict named by +# FM_FAKE_CERTSYNC_PAYLOAD (or raises on the RAISE sentinel), and seeds a +# heartbeat file whose mtime the caller controls. +# - certsync_health_env prints the env the watcher needs to find them. +setup_certsync_health_case() { # [heartbeat-age-seconds] + local dir=$1 age=${2:-0} project src hb back project="$dir/home/projects/hlr-certsync" - mkdir -p "$dir/root" - mkdir -p "$project" + src="$project/src" + mkdir -p "$dir/root" "$project" "$src/hlr_certsync" : > "$project/docker-compose.yml" - : > "$project/docker-compose.graph-pem.yml" + : > "$src/hlr_certsync/__init__.py" + cat > "$src/hlr_certsync/state.py" <<'PY' +class StateStore: + def __init__(self, path): + self.path = path +PY + cat > "$src/hlr_certsync/status.py" <<'PY' +import json, os +def build_status(store, heartbeat_path, *, daemon_state="unknown"): + with open(os.environ["FM_FAKE_CERTSYNC_PAYLOAD"]) as fh: + text = fh.read().strip() + if text == "RAISE": + raise RuntimeError("state DB unavailable") + return json.loads(text) +PY + # Seed the heartbeat file the freshness gate stats; default mtime is now. + hb="$dir/certsync-heartbeat.json" + printf '{"last_successful_sync":"2026-08-05T12:00:00Z","last_run_state":"success"}\n' > "$hb" + if [ "$age" -gt 0 ]; then + back=$(( $(date +%s) - age )) + if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$hb" + else touch -m -d "@$back" "$hb"; fi + fi } -install_fake_certsync_docker() { # - local fakebin=$1 payload=$2 - cat > "$fakebin/docker" <<'SH' -#!/usr/bin/env bash -set -u -[ "$#" -ge 1 ] && [ "$1" = compose ] || exit 1 -cat "$FM_FAKE_CERTSYNC_PAYLOAD" -SH - chmod +x "$fakebin/docker" - printf '%s\n' "$payload" > "$fakebin/certsync-payload.json" -} - -# A fake `docker` that fails the way the real one does on this fleet's own -# accounts: no root, no sudo, no docker-group membership, so `docker compose -# exec` denies the connection before certsync ever gets a chance to answer. -# Reproduces the 2026-08-04 defect's exact shape (nonzero exit, empty stdout, -# stderr message) so a regression here is pinned at the same layer it happened. -install_fake_certsync_docker_permission_denied() { # - local fakebin=$1 - cat > "$fakebin/docker" <<'SH' -#!/usr/bin/env bash -set -u -[ "$#" -ge 1 ] && [ "$1" = compose ] || exit 1 -echo "permission denied while trying to connect to the docker API at unix:///var/run/docker.sock" >&2 -exit 1 -SH - chmod +x "$fakebin/docker" +# Env the watcher needs to run the file-read check against the fakes above. +certsync_health_env() { # + local dir=$1 + printf '%s\n' \ + "FM_ROOT_OVERRIDE=$dir/root" \ + "FM_HOME=$dir/home" \ + "FM_CERTSYNC_STATE_DB=$dir/certsync-state.sqlite3" \ + "FM_CERTSYNC_HEARTBEAT_FILE=$dir/certsync-heartbeat.json" \ + "FM_FAKE_CERTSYNC_PAYLOAD=$dir/certsync-payload.json" } test_heartbeat_certsync_healthy_absorbed() { - local dir state fakebin out payload pid - dir=$(make_case heartbeat-certsync-healthy); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out" + local dir state out pid + dir=$(make_case heartbeat-certsync-healthy); state="$dir/state"; out="$dir/watch.out" setup_certsync_health_case "$dir" - payload="$fakebin/certsync-payload.json" - install_fake_certsync_docker "$fakebin" "$payload" - printf '{"healthy":true,"reason":"ok"}\n' > "$payload" - PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$dir/root" FM_HOME="$dir/home" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ - FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 FM_FAKE_CERTSYNC_PAYLOAD="$payload" "$WATCH" > "$out" & + printf '{"healthy":true,"reason":"ok"}\n' > "$dir/certsync-payload.json" + env $(certsync_health_env "$dir") FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" & pid=$! if ! wait_live "$pid" 30; then reap "$pid"; fail "watcher exited for healthy certsync (should absorb): $(cat "$out")" @@ -1985,19 +1997,41 @@ test_heartbeat_certsync_healthy_absorbed() { [ ! -s "$out" ] || fail "healthy certsync printed a wake reason: $(cat "$out")" [ ! -s "$state/.wake-queue" ] || fail "healthy certsync enqueued a durable wake record" reap "$pid" - pass "heartbeat absorbs healthy certsync status" + pass "heartbeat absorbs healthy certsync status read directly off the host files" +} + +# The liveness signal the docker-exec check got for free (exec failed when the +# container was down) is reinstated here as a heartbeat-freshness bound: a +# healthy:true payload whose heartbeat file has gone stale (container stopped, or +# syncs failing so no fresh success) must read as unhealthy, never quiet, so +# "cannot confirm well" never collapses into "is well" off frozen files. +test_heartbeat_certsync_healthy_but_stale_surfaces_check_wake() { + local dir state out drain_out pid + dir=$(make_case heartbeat-certsync-stale); state="$dir/state" + out="$dir/watch.out"; drain_out="$dir/drain.out" + setup_certsync_health_case "$dir" 10800 # heartbeat 3h old, past the 7200s bound + printf '{"healthy":true,"reason":"ok"}\n' > "$dir/certsync-payload.json" + env $(certsync_health_env "$dir") FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "heartbeat did not surface a stale-heartbeat certsync as unhealthy: $(cat "$out")" + grep -F "check: certsync health: unhealthy: heartbeat stale (" "$out" >/dev/null \ + && grep -F "> 7200s); daemon may be stopped or syncs failing" "$out" >/dev/null \ + || fail "stale-heartbeat certsync wake reason was wrong: $(cat "$out")" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after stale certsync wake failed" + grep -F "$(printf '\tcheck\tcertsync-health\tcheck: certsync health: unhealthy: heartbeat stale (')" "$drain_out" >/dev/null \ + || fail "stale-heartbeat certsync check wake was not queued: $(cat "$drain_out")" + pass "a healthy payload with a stale heartbeat surfaces as unhealthy instead of reading as healthy off frozen files" } test_heartbeat_certsync_unhealthy_surfaces_check_wake() { - local dir state fakebin out drain_out payload pid - dir=$(make_case heartbeat-certsync-unhealthy); state="$dir/state"; fakebin="$dir/fakebin" + local dir state out drain_out pid + dir=$(make_case heartbeat-certsync-unhealthy); state="$dir/state" out="$dir/watch.out"; drain_out="$dir/drain.out" setup_certsync_health_case "$dir" - payload="$fakebin/certsync-payload.json" - install_fake_certsync_docker "$fakebin" "$payload" - printf '{"healthy":false,"reason":"state DB missing"}\n' > "$payload" - PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$dir/root" FM_HOME="$dir/home" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ - FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 FM_FAKE_CERTSYNC_PAYLOAD="$payload" "$WATCH" > "$out" & + printf '{"healthy":false,"reason":"state DB missing"}\n' > "$dir/certsync-payload.json" + env $(certsync_health_env "$dir") FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 40 || fail "heartbeat did not surface unhealthy certsync" grep -Fx "check: certsync health: unhealthy: state DB missing" "$out" >/dev/null \ @@ -2009,16 +2043,14 @@ test_heartbeat_certsync_unhealthy_surfaces_check_wake() { } test_afk_heartbeat_certsync_unhealthy_surfaces_check_wake() { - local dir state fakebin out drain_out payload pid - dir=$(make_case afk-heartbeat-certsync-unhealthy); state="$dir/state"; fakebin="$dir/fakebin" + local dir state out drain_out pid + dir=$(make_case afk-heartbeat-certsync-unhealthy); state="$dir/state" out="$dir/watch.out"; drain_out="$dir/drain.out" setup_certsync_health_case "$dir" - payload="$fakebin/certsync-payload.json" - install_fake_certsync_docker "$fakebin" "$payload" - printf '{"healthy":false,"reason":"replication stalled"}\n' > "$payload" + printf '{"healthy":false,"reason":"replication stalled"}\n' > "$dir/certsync-payload.json" : > "$state/.afk" - PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$dir/root" FM_HOME="$dir/home" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ - FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 FM_FAKE_CERTSYNC_PAYLOAD="$payload" "$WATCH" > "$out" & + env $(certsync_health_env "$dir") FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 40 || fail "afk heartbeat did not surface unhealthy certsync" grep -Fx "check: certsync health: unhealthy: replication stalled" "$out" >/dev/null \ @@ -2031,21 +2063,19 @@ test_afk_heartbeat_certsync_unhealthy_surfaces_check_wake() { pass "afk heartbeat surfaces confirmed unhealthy certsync through the check wake path" } -# Regression for the 2026-08-04 defect: an unreadable certsync status (invalid -# JSON, here) used to collapse into the SAME silent no-wake outcome as a -# confirmed-healthy read (test_heartbeat_certsync_healthy_absorbed above), so a -# check that could not run reported exactly like a check that ran and passed. -# It must now surface as its own distinct "cannot run" check wake instead. +# Regression for the 2026-08-04 defect: an unreadable certsync status (here a +# payload with no boolean healthy field) used to collapse into the SAME silent +# no-wake outcome as a confirmed-healthy read, so a check that could not run +# reported exactly like a check that ran and passed. It must surface as its own +# distinct "cannot run" check wake instead. test_heartbeat_certsync_invalid_json_surfaces_check_wake() { - local dir state fakebin out drain_out payload pid - dir=$(make_case heartbeat-certsync-invalid-json); state="$dir/state"; fakebin="$dir/fakebin" + local dir state out drain_out pid + dir=$(make_case heartbeat-certsync-invalid-json); state="$dir/state" out="$dir/watch.out"; drain_out="$dir/drain.out" setup_certsync_health_case "$dir" - payload="$fakebin/certsync-payload.json" - install_fake_certsync_docker "$fakebin" "$payload" - printf 'not-json\n' > "$payload" - PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$dir/root" FM_HOME="$dir/home" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ - FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 FM_FAKE_CERTSYNC_PAYLOAD="$payload" "$WATCH" > "$out" & + printf '{"reason":"cannot tell"}\n' > "$dir/certsync-payload.json" + env $(certsync_health_env "$dir") FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" & pid=$! wait_for_exit "$pid" 40 || fail "heartbeat did not surface an unreadable certsync status as cannot-run: $(cat "$out")" grep -Fx "check: certsync health: cannot run: invalid or missing status JSON" "$out" >/dev/null \ @@ -2056,27 +2086,47 @@ test_heartbeat_certsync_invalid_json_surfaces_check_wake() { pass "heartbeat surfaces an unreadable certsync status as its own cannot-run check wake, distinct from healthy" } -# The literal reported defect: this fleet's own accounts have no root, no sudo, -# and no docker-group membership by design, so `docker compose exec` denies the -# connection before certsync is ever asked. run_bounded used to swallow that -# failure's exit code (`|| true`), so "cannot run" and "healthy" were the exact -# same observable outcome for 6.5 hours before an audit caught it. -test_heartbeat_certsync_permission_denied_surfaces_check_wake() { - local dir state fakebin out drain_out pid - dir=$(make_case heartbeat-certsync-permission-denied); state="$dir/state"; fakebin="$dir/fakebin" +# The status read itself failing (build_status raising - e.g. an unreadable or +# corrupt state DB) is the file-read analogue of the old docker-exec failure: it +# must exit nonzero and surface a "cannot run" wake, never collapse into healthy. +test_heartbeat_certsync_status_read_failure_surfaces_check_wake() { + local dir state out drain_out pid + dir=$(make_case heartbeat-certsync-read-failure); state="$dir/state" out="$dir/watch.out"; drain_out="$dir/drain.out" setup_certsync_health_case "$dir" - install_fake_certsync_docker_permission_denied "$fakebin" - PATH="$fakebin:$PATH" FM_ROOT_OVERRIDE="$dir/root" FM_HOME="$dir/home" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + printf 'RAISE\n' > "$dir/certsync-payload.json" + env $(certsync_health_env "$dir") FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" & pid=$! - wait_for_exit "$pid" 40 || fail "heartbeat did not surface a docker-permission-denied certsync status: $(cat "$out")" + wait_for_exit "$pid" 40 || fail "heartbeat did not surface a failing certsync status read: $(cat "$out")" grep -Fx "check: certsync health: cannot run: status command failed (exit 1)" "$out" >/dev/null \ - || fail "permission-denied certsync wake reason was wrong: $(cat "$out")" - FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after permission-denied certsync wake failed" + || fail "status-read-failure certsync wake reason was wrong: $(cat "$out")" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after status-read-failure certsync wake failed" grep "$(printf '\tcheck\tcertsync-health\tcheck: certsync health: cannot run: status command failed (exit 1)')" "$drain_out" >/dev/null \ - || fail "permission-denied certsync check wake was not queued: $(cat "$drain_out")" - pass "heartbeat surfaces a docker-permission-denied certsync status instead of silently reporting healthy" + || fail "status-read-failure certsync check wake was not queued: $(cat "$drain_out")" + pass "heartbeat surfaces a failing certsync status read as its own cannot-run check wake, not as healthy" +} + +# certsync deployed (compose present) but its source tree absent means the check +# genuinely cannot read status: it must say so, never fall silent as healthy. +test_heartbeat_certsync_source_unavailable_surfaces_check_wake() { + local dir state out drain_out pid src + dir=$(make_case heartbeat-certsync-no-src); state="$dir/state" + out="$dir/watch.out"; drain_out="$dir/drain.out" + setup_certsync_health_case "$dir" + src="$dir/home/projects/hlr-certsync/src" + rm -rf "$src" # deployed, but no importable certsync source + printf '{"healthy":true,"reason":"ok"}\n' > "$dir/certsync-payload.json" + env $(certsync_health_env "$dir") FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "heartbeat did not surface missing certsync source as cannot-run: $(cat "$out")" + grep -F "check: certsync health: cannot run: certsync source unavailable at " "$out" >/dev/null \ + || fail "source-unavailable certsync wake reason was wrong: $(cat "$out")" + FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after source-unavailable certsync wake failed" + grep -F "$(printf '\tcheck\tcertsync-health\tcheck: certsync health: cannot run: certsync source unavailable at ')" "$drain_out" >/dev/null \ + || fail "source-unavailable certsync check wake was not queued: $(cat "$drain_out")" + pass "heartbeat surfaces a certsync deployment with no readable source as its own cannot-run check wake" } # --- beacon stays fresh while absorbing ------------------------------------- @@ -2217,10 +2267,12 @@ test_triage_log_size_cap_accepts_spaced_wc_counts test_heartbeat_no_change_absorbed test_heartbeat_backstop_surfaces_unsurfaced_status test_heartbeat_certsync_healthy_absorbed +test_heartbeat_certsync_healthy_but_stale_surfaces_check_wake test_heartbeat_certsync_unhealthy_surfaces_check_wake test_afk_heartbeat_certsync_unhealthy_surfaces_check_wake test_heartbeat_certsync_invalid_json_surfaces_check_wake -test_heartbeat_certsync_permission_denied_surfaces_check_wake +test_heartbeat_certsync_status_read_failure_surfaces_check_wake +test_heartbeat_certsync_source_unavailable_surfaces_check_wake test_beacon_stays_fresh_while_absorbing test_afk_present_reverts_watcher_to_one_shot test_afk_paused_changed_pane_hands_off_plain_stale