diff --git a/scripts/hetzner/deploy.sh b/scripts/hetzner/deploy.sh index e9eaa273..515c2842 100755 --- a/scripts/hetzner/deploy.sh +++ b/scripts/hetzner/deploy.sh @@ -104,7 +104,12 @@ box "set -e if [ "$FORCE_ENV" = "--env" ] || ! box "test -f /opt/$NAME/shared/.env"; then [ -f "$SRC/.env.selfhost.local" ] || { echo "ERROR: $SRC/.env.selfhost.local missing"; exit 1; } scp -o BatchMode=yes "$SRC/.env.selfhost.local" "$BOX:/opt/$NAME/shared/.env" - box "chmod 600 /opt/$NAME/shared/.env" + # Ownership, not just mode. The app sources this file as its unit's User=; + # a .env it cannot read is a total outage (twice on 2026-08-28), and the + # correct owner is never in doubt. Assert it here as well as in + # host-check's repair loop, so the invariant is stated where the file is + # written and not only where it is later found broken. + box "sudo chown ubuntu:ubuntu /opt/$NAME/shared/.env && chmod 600 /opt/$NAME/shared/.env" echo "uploaded .env → shared/" fi diff --git a/scripts/hetzner/install-host-alerts.sh b/scripts/hetzner/install-host-alerts.sh index 416a25dd..787be951 100755 --- a/scripts/hetzner/install-host-alerts.sh +++ b/scripts/hetzner/install-host-alerts.sh @@ -10,7 +10,16 @@ # 2. Host checks — /opt/monitoring/host-check.sh (own timer) alerts, on # TRANSITION only, on: disk >85% (recovering only under 80%, so a disk # parked on the mark can't flap), mem-available <400MB OR swap >90%, -# `systemctl --failed` non-empty, postgres not accepting connections. +# each failed unit KEYED SEPARATELY, postgres not accepting connections, +# and any app whose .env its own unit User= cannot read — which it repairs +# itself and reports, instead of asking a human for the one right answer. +# +# The governing rule for everything below: a message is worth sending only if a +# human must act on it AND nothing else can. One incident is one message +# (alert_once + a duplicate-text floor in lib-alert.sh); anything with a +# knowable remedy is applied, not announced; test runs set ALERT_DRY_RUN=1 and +# reach the journal only. Suppressed is never invisible — the journal always +# gets every alert. # # Logic here is covered by scripts/hetzner/test-host-alerts.sh (npm run test:ops), # which extracts the heredoc payloads below and drives them with stubbed tools. @@ -32,29 +41,111 @@ cat > "$MON/lib-alert.sh" <<'LIB' #!/usr/bin/env bash # Sourced by host-check.sh and notify-failure.sh. Provides: # alert — send now (Telegram if configured, always journal) +# alert_once +# — send at most once per seconds for , so one incident is +# one message however many times it is detected +# alert_clear — forget 's cooldown; call on recovery # alert_transition # — send only when flips state (state file under $MON/state) # MON is overridable so test-host-alerts.sh can exercise this exact code against # a temp dir; prod never sets it and gets /opt/monitoring. +# +# DELIVERY vs VISIBILITY. Every function here ALWAYS writes to the journal; +# only the Telegram send is ever suppressed. Quiet on the phone must never mean +# invisible in the logs — a suppressed alert nobody can find afterwards is a +# worse failure than the noise it saved. MON="${MON:-/opt/monitoring}" [ -f "$MON/telegram.env" ] && . "$MON/telegram.env" || true -alert() { - local text="$1 $2" + +_alert_key() { printf '%s' "$1" | tr -c 'a-zA-Z0-9' '_'; } + +# Identical-text floor, under every other guard. The keyed cooldowns below are +# the deliberate mechanism, but they only protect callers that remember to pass +# a key: on 2026-08-28 the fleet register check sent the same four-line failure +# twice inside 60 seconds because it had its own private Telegram call and no +# state at all. This floor means a caller that forgets — today's or one written +# next year — still cannot repeat itself. Short enough (5 min) that a genuinely +# new occurrence of the same condition is never swallowed. +ALERT_DEDUPE_SEC="${ALERT_DEDUPE_SEC:-300}" + +# ALERT_DRY_RUN=1 → journal only, nothing delivered. Every test probe and +# every "does the wiring still work" run MUST set it. On 2026-08-28 a live test +# of a new check and a planted zz-audit-probe.service both rang George's actual +# phone at 22:14 — a message about nothing, which is the exact failure this +# file exists to prevent. Testing the alerter must not page anyone. +# The one place anything is delivered. Journal first, always — every caller, +# every path, including the ones that decide not to deliver. +_alert_deliver() { + local text="$1" logger -t watchdog "$text" + if [ -n "${ALERT_DRY_RUN:-}" ]; then + logger -t watchdog "ALERT dry-run, not delivered: $text" + return 0 + fi if [ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "${TELEGRAM_CHAT_ID:-}" ]; then curl -fsS -m 10 "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ --data-urlencode "text=${text}" -o /dev/null \ || logger -t watchdog "ALERT telegram send failed" fi + # Unconditional 0, for the same reason alert_transition ends that way. + return 0 +} + +# Unkeyed send. This is the path with no idea whether it has said this before, +# so it — and ONLY it — gets the identical-text floor. alert_once and +# alert_transition have already made a deliberate, state-backed decision to +# speak; running them through the floor as well would mean two mechanisms +# arguing about one message, and the quieter one silently winning. +alert() { + local text="$1 $2" + mkdir -p "$MON/state" + # Expire old dedupe stamps here rather than in a timer: this is the only code + # that creates them, so it is the only code that has to remember them. + find "$MON/state" -maxdepth 1 -name 'dedupe_*' -mmin +1440 -delete 2>/dev/null || true + local h sf now last + h=$(printf '%s' "$text" | md5sum | cut -c1-16) + sf="$MON/state/dedupe_$h" + now=$(date +%s); last=$(cat "$sf" 2>/dev/null | tr -dc '0-9') + if [ -n "$last" ] && [ "$((now - last))" -lt "$ALERT_DEDUPE_SEC" ]; then + logger -t watchdog "ALERT duplicate suppressed (${ALERT_DEDUPE_SEC}s window): $text" + return 0 + fi + printf '%s' "$now" > "$sf" + _alert_deliver "$text" + return 0 +} + +# One incident is one message. The caller names the subject; repeated detections +# of the SAME subject inside go to the journal only, and a re-page +# after the cooldown is a deliberate still-broken reminder. alert_clear on +# recovery is the other half and is load-bearing: without it the next genuine +# outage of that subject inherits the last one's silence. +alert_once() { # key cooldown emoji text + local key="$1" cd="$2" emoji="$3" text="$4" + local sf="$MON/state/paged_$(_alert_key "$key")" + local now last + now=$(date +%s); last=$(cat "$sf" 2>/dev/null | tr -dc '0-9') + if [ -n "$last" ] && [ "$((now - last))" -lt "$cd" ]; then + logger -t watchdog "ALERT held for ${key}: paged $((now - last))s ago, cooldown ${cd}s" + return 0 + fi + mkdir -p "$MON/state" + printf '%s' "$now" > "$sf" + _alert_deliver "$emoji $text" + return 0 } +alert_clear() { rm -f "$MON/state/paged_$(_alert_key "$1")"; return 0; } alert_transition() { # key state emoji text local key="$1" state="$2" emoji="$3" text="$4" local sf="$MON/state/host_$(printf '%s' "$key" | tr -c 'a-zA-Z0-9' '_')" local prev="ok"; [ -f "$sf" ] && prev=$(cat "$sf") [ "$state" = "$prev" ] && return 0 printf '%s' "$state" > "$sf" - if [ "$state" = "bad" ]; then alert "$emoji" "$text"; else alert "✅" "RECOVERED: $key"; fi + # A transition IS the deliberate decision, so deliver it directly rather than + # through alert()'s floor: a unit that fails, recovers and fails again inside + # five minutes is genuinely three events, and the floor would eat the third. + if [ "$state" = "bad" ]; then _alert_deliver "$emoji $text"; else _alert_deliver "✅ RECOVERED: $key"; fi # MUST return 0 unconditionally. A trailing `[ "$state" = "ok" ] && alert ...` # here returned 1 on the bad path, so a caller written as # check && alert_transition k bad ... || alert_transition k ok ... @@ -84,39 +175,47 @@ MON="${MON:-/opt/monitoring}" unit="${1:-unknown.unit}" utype=$(systemctl show "$unit" -p Type --value 2>/dev/null) -# Per-unit page cooldown. The recovery guard below only silences a unit that -# comes BACK; a unit that cannot start at all fires OnFailure on every restart -# forever, and each one used to be a separate Telegram. On 2026-08-28 -# vitareba-app could not read its .env (root-owned after a chown --reference), -# so Restart=on-failure + RestartSec=3 produced 18 restarts and SIX identical -# "UNIT DOWN" messages in 60 seconds — for ONE incident that no amount of -# paging would fix any faster. A crash loop is not new information every 3 -# seconds: page once, hold for COOLDOWN, then re-page as a reminder while it is -# still down. Recovery clears the stamp, so the next genuine outage pages -# immediately instead of inheriting the last one's silence. +# Per-unit page cooldown, via lib-alert's alert_once. The recovery guard below +# only silences a unit that comes BACK; a unit that cannot start at all fires +# OnFailure on every restart forever, and each one used to be a separate +# Telegram. On 2026-08-28 vitareba-app could not read its .env (root-owned +# after a chown --reference), so Restart=on-failure + RestartSec=3 produced 18 +# restarts and SIX identical "UNIT DOWN" messages in 60 seconds — for ONE +# incident that no amount of paging would fix any faster. A crash loop is not +# new information every 3 seconds: page once, hold for COOLDOWN, then re-page +# as a reminder while it is still down. alert_clear on recovery is the other +# half, so the next genuine outage pages immediately instead of inheriting the +# last one's silence. COOLDOWN=${NOTIFY_COOLDOWN_SEC:-1800} -stamp="$MON/state/paged_$(printf '%s' "$unit" | tr -c 'a-zA-Z0-9' '_')" if [ "$utype" != "oneshot" ]; then sleep 8 if systemctl is-active --quiet "$unit"; then - rm -f "$stamp" + alert_clear "$unit" logger -t watchdog "unit ${unit} failed but recovered (restart/transient) — not paging" exit 0 fi +else + # A oneshot that a retry has ALREADY fixed is not an incident either. The + # weekly restic-check failed at 18:41:42 on a 25-day-old stale lock left by + # the laptop, was re-run at 18:42:28 and finished clean at 18:44:46 — and + # still paged, because OnFailure fires on the failing run and nothing ever + # looked again. So look again: wait out a retry window, then ask whether the + # unit has since run. A retry in flight or a successful result means the + # answer is already on its way and no human is needed. Nothing is lost by + # waiting — if the retry fails too, its own OnFailure fires and we page then. + sleep "${ONESHOT_GRACE_SEC:-90}" + state=$(systemctl is-active "$unit" 2>/dev/null) + result=$(systemctl show "$unit" -p Result --value 2>/dev/null) + if [ "$state" = "active" ] || [ "$state" = "activating" ] || [ "$result" = "success" ]; then + alert_clear "$unit" + logger -t watchdog "oneshot ${unit} failed but a later run is active/succeeded — not paging" + exit 0 + fi fi -now=$(date +%s) -last=$(cat "$stamp" 2>/dev/null | tr -dc '0-9') -if [ -n "$last" ] && [ "$((now - last))" -lt "$COOLDOWN" ]; then - logger -t watchdog "unit ${unit} still failing — paged $((now - last))s ago, holding for ${COOLDOWN}s" - exit 0 -fi -mkdir -p "$MON/state" -printf '%s' "$now" > "$stamp" - tail=$(journalctl -u "$unit" -n 4 --no-pager -o cat 2>/dev/null | tr '\n' ' ' | cut -c1-300) -alert "🔴" "UNIT DOWN: ${unit} — ${tail:-}" +alert_once "$unit" "$COOLDOWN" "🔴" "UNIT DOWN: ${unit} — ${tail:-}" NF chmod +x "$MON/notify-failure.sh" @@ -125,6 +224,12 @@ cat > /etc/systemd/system/notify-failure@.service <<'SVC' Description=Telegram alert for failed unit %i [Service] Type=oneshot +# The notifier deliberately waits (8s for a service, ONESHOT_GRACE_SEC=90s for a +# oneshot) before deciding to page. DefaultTimeoutStartSec is 90s, so without an +# explicit timeout systemd would SIGTERM the notifier inside its own grace window +# and the page would silently never be sent — an anti-noise guard that turns into +# an anti-alert bug. Give it room for the longest wait plus the journal read. +TimeoutStartSec=300 # %i is the failed unit name (systemd-escaped); notify-failure.sh unescapes for display. ExecStart=/opt/monitoring/notify-failure.sh %i SVC @@ -207,6 +312,54 @@ done # Retire the old aggregate latch so it cannot linger at `bad` forever. rm -f "$MON/state/host_units" +# App .env readability — repair it, don't report it. +# +# Every *-app unit runs as User= and sources /opt//shared/.env at start. +# If that file is not readable by that user the app cannot boot AT ALL: it +# crash-loops until a human notices. This happened TWICE on 2026-08-28 — +# vitareba at 18:38 (root-owned after a `chown --reference` of a root:root +# backup) and botsmann at 20:16 (root-owned by an ad-hoc edit four hours +# later). Both times the only signal was a wall of identical UNIT DOWN +# messages, and both times the remedy was the same one deterministic command. +# +# A failure whose correct answer is knowable is not worth a human's attention. +# The owner here is not a guess — it is the unit's own User=. So fix it, verify +# the fix, bring the app back, and send ONE message saying what was repaired: a +# report, not a request. The alert that remains is the one worth having, because +# it is the only way a recurring corruption ever becomes visible. +# APPROOT is overridable for the same reason MON is: so test-host-alerts.sh can +# drive this exact loop against a temp tree. Prod never sets it. +APPROOT="${APPROOT:-/opt}" +for unit in $(systemctl list-unit-files --no-legend --plain '*-app.service' 2>/dev/null | awk '{print $1}'); do + app=${unit%-app.service} + envf="$APPROOT/$app/shared/.env" + [ -f "$envf" ] || continue + user=$(systemctl show "$unit" -p User --value 2>/dev/null); user=${user:-root} + if sudo -n -u "$user" test -r "$envf" 2>/dev/null; then + # Healthy: drop any past repair stamp so a recurrence is reported again + # rather than inheriting the previous repair's cooldown. + alert_clear "envfix_$app" + continue + fi + owner=$(stat -c '%U:%G' "$envf" 2>/dev/null) + if chown "$user:$user" "$envf" 2>/dev/null && chmod 600 "$envf" 2>/dev/null \ + && sudo -n -u "$user" test -r "$envf" 2>/dev/null; then + # The app is definitionally down at this point (it cannot have read its own + # env), so a restart is not a risk — it is the rest of the repair. Clear the + # start-limit first or systemd refuses to try again. + systemctl reset-failed "$unit" >/dev/null 2>&1 || true + systemctl restart "$unit" >/dev/null 2>&1 || true + sleep 5 + up=$(systemctl is-active "$unit" 2>/dev/null) + alert_once "envfix_$app" 3600 "🔧" \ + "FIXED (no action needed): $envf was $owner, unreadable by $user — re-owned to $user:$user and restarted $unit (now: $up). It could not have booted until this was done." + else + # Could not repair: this one IS a question, so ask it, once. + alert_once "envbad_$app" 3600 "🔑" \ + "$envf is not readable by $user ($unit) and auto-repair failed — the app cannot start. Fix: chown $user:$user $envf && systemctl restart $unit" + fi +done + # Postgres accepting connections if pg_isready -q 2>/dev/null; then alert_transition postgres ok "" "" @@ -247,6 +400,21 @@ EOF wired=$((wired+1)) done +# ── Retire a unit that can only ever be noise ──────────────────────────────── +# cloud-init-hotplugd has been `failed` since 2026-07-22 on a box with no +# hotplug events to handle. It is not an incident, it never recovers, and it is +# the member that pinned the OLD aggregate failed-units latch at `bad` for six +# weeks — hiding every real failure behind it (including vitareba's 25 crons +# dying). Now that the check is per-unit it re-pages this instead, which is the +# other failure mode of the same fact. Neither is right: a unit that can only +# produce noise should not be in the failed set at all. Mask it (reversible +# with `systemctl unmask`) rather than teach the checker to look away, so that +# `systemctl --failed` keeps meaning exactly "something is wrong here". +systemctl stop cloud-init-hotplugd.socket >/dev/null 2>&1 || true +systemctl mask cloud-init-hotplugd.socket cloud-init-hotplugd.service >/dev/null 2>&1 || true +systemctl reset-failed cloud-init-hotplugd.service >/dev/null 2>&1 || true +rm -f "$MON/state/host_unit_cloud_init_hotplugd_service" + systemctl daemon-reload systemctl enable --now host-check.timer >/dev/null 2>&1 || true echo "[host-alerts] wired OnFailure into $wired unit(s); host-check.timer active" diff --git a/scripts/hetzner/install-watchdog.sh b/scripts/hetzner/install-watchdog.sh index 2346faf6..bd955d35 100755 --- a/scripts/hetzner/install-watchdog.sh +++ b/scripts/hetzner/install-watchdog.sh @@ -70,18 +70,20 @@ set -uo pipefail MON=/opt/monitoring STATE="$MON/state" mkdir -p "$STATE" -[ -f "$MON/telegram.env" ] && . "$MON/telegram.env" || true - -alert() { # $1=emoji $2=message - local text="$1 $2" - logger -t watchdog "$text" - if [ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "${TELEGRAM_CHAT_ID:-}" ]; then - curl -fsS -m 10 "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ - --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ - --data-urlencode "text=${text}" -o /dev/null \ - || logger -t watchdog "ALERT telegram send failed" - fi -} +# One delivery point for the whole box. This file used to carry its own byte- +# identical copy of alert(), which meant every property added to the shared one +# — the duplicate floor, ALERT_DRY_RUN, the journal-always guarantee — silently +# did not apply here. A second copy of a send path is how the fleet register +# check ended up able to page twice in sixty seconds with no state at all: +# nothing is wrong with the copy on the day it is made, and nothing updates it +# afterwards. lib-alert.sh is installed by install-host-alerts.sh; if it is +# somehow absent, fall back to the journal rather than going silent. +if [ -f "$MON/lib-alert.sh" ]; then + . "$MON/lib-alert.sh" +else + [ -f "$MON/telegram.env" ] && . "$MON/telegram.env" || true + alert() { logger -t watchdog "$1 $2"; logger -t watchdog "ALERT lib-alert.sh missing — journal only"; } +fi check() { # label url (targets.conf 3rd field is ignored — redirects are followed) local label="$1" url="$2" diff --git a/scripts/hetzner/lib.sh b/scripts/hetzner/lib.sh index 76027ae2..3f6056a7 100755 --- a/scripts/hetzner/lib.sh +++ b/scripts/hetzner/lib.sh @@ -5,7 +5,13 @@ set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "$HERE/_box-env.sh" # SSOT: HETZNER_IP, BOX_ROOT, BOX_UBUNTU BOX="$BOX_UBUNTU" -MANIFEST="$HERE/apps.conf" +# Overridable so a caller can judge a PRISTINE register instead of the working +# tree. On the workstation ~15 agent sessions share these checkouts, so the +# tree is a scratchpad: on 2026-08-28 the daily register check read apps.conf +# mid-edit (substrata listed twice) and paged about a duplicate port that had +# never been committed. CI still judges the working tree, which is correct +# there — in CI the working tree IS the commit under review. +MANIFEST="${MANIFEST:-$HERE/apps.conf}" # default_branch [repo_dir] — the remote's default branch name, resolved not guessed. # diff --git a/scripts/hetzner/test-host-alerts.sh b/scripts/hetzner/test-host-alerts.sh index 13e7940d..80889ab4 100755 --- a/scripts/hetzner/test-host-alerts.sh +++ b/scripts/hetzner/test-host-alerts.sh @@ -61,11 +61,27 @@ cat > "$TMP/bin/systemctl" <<'STUB' # units skip the recovery grace period) and whether it came back. Both are # driven by env so a test can stage a crash loop or a deploy blip. if [ "${1:-}" = "show" ]; then - printf '%s\n' "${UNIT_TYPE:-simple}"; exit 0 + # Answer the property actually asked for. A stub that returns the same string + # for every -p is how a Result check can silently read a Type. + prop=""; for a in "$@"; do case "$a" in -p) prop="NEXT";; *) [ "$prop" = "NEXT" ] && { prop="$a"; break; };; esac; done + case "$prop" in + Type) printf '%s\n' "${UNIT_TYPE:-simple}" ;; + Result) printf '%s\n' "${UNIT_RESULT:-exit-code}" ;; + User) printf '%s\n' "${UNIT_USER:-}" ;; + *) printf '%s\n' "${UNIT_TYPE:-simple}" ;; + esac + exit 0 fi if [ "${1:-}" = "is-active" ]; then + # Print the state as well as exiting with it: --quiet callers ignore stdout, + # but the oneshot retry-grace and the env repair both READ this value. + [ "${UNIT_ACTIVE:-1}" = 0 ] && echo active || echo failed exit "${UNIT_ACTIVE:-1}" # default 1 = still down, the crash-loop case fi +if [ "${1:-}" = "list-unit-files" ]; then + for u in ${APP_UNITS:-}; do echo "$u enabled enabled"; done + exit 0 +fi if [ "${1:-}" = "list-units" ]; then # Reproduce systemd's REAL shape: a failed unit is printed with a leading # "●" unless --plain is passed. A stub without it let a bug ship that turned @@ -82,6 +98,29 @@ cat > "$TMP/bin/pg_isready" <<'STUB' #!/usr/bin/env bash exit 0 STUB +# DELIVERY capture. logger records what reaches the JOURNAL; curl records what +# reaches the PHONE. Every suppression in lib-alert.sh is defined as "journal +# yes, phone no", so a suite that can only observe the journal cannot tell a +# correctly-suppressed duplicate from a delivered one — and a phone receiving +# six copies of one fact is the entire bug being fixed here. +cat > "$TMP/bin/curl" <<'STUB' +#!/usr/bin/env bash +for a in "$@"; do case "$a" in text=*) printf '%s\n' "${a#text=}" >> "$SEND_LOG";; esac; done +exit 0 +STUB +# Fake creds so the delivery branch is actually entered. The curl stub above is +# where they land, so nothing leaves this machine. +printf 'TELEGRAM_BOT_TOKEN=test-token\nTELEGRAM_CHAT_ID=test-chat\n' > "$TMP/telegram.env" +# host-check asks "can this unit's User read its .env?" via sudo. Here we run as +# ourselves, so the answer is the real filesystem's — which is the point: the +# repair is tested against actual permission bits, not against a mock's opinion. +cat > "$TMP/bin/sudo" <<'STUB' +#!/usr/bin/env bash +while [ $# -gt 0 ]; do + case "$1" in -n) shift;; -u) shift 2;; *) break;; esac +done +exec "$@" +STUB cat > "$TMP/bin/journalctl" <<'STUB' #!/usr/bin/env bash echo "stub journal line" @@ -96,7 +135,8 @@ chmod +x "$TMP/bin"/* export PATH="$TMP/bin:$PATH" export MON="$TMP" export ALERT_LOG="$TMP/alerts.log" -: > "$ALERT_LOG" +export SEND_LOG="$TMP/sent.log" +: > "$ALERT_LOG"; : > "$SEND_LOG" pass=0 fail=0 check() { # name condition-as-exit-status @@ -107,6 +147,10 @@ run_check() { : > "$ALERT_LOG"; DISK_PCT="$1" bash "$TMP/host-check.sh" >/dev/nu # wc, not `grep -c || echo 0` — grep -c prints 0 *and* exits 1 on no match, so # the fallback would emit a second 0 and every numeric compare would blow up. alerts() { wc -l < "$ALERT_LOG" | tr -d '[:space:]'; } +# What actually reached the phone, as opposed to the journal. Noise assertions +# must use this one: the journal deliberately keeps a line for every suppressed +# alert, so counting journal lines scores correct suppression as a message sent. +sent() { wc -l < "$SEND_LOG" | tr -d '[:space:]'; } disk_state() { cat "$TMP/state/host_disk" 2>/dev/null || echo ""; } echo "host-alert transition tests" @@ -265,7 +309,7 @@ check "notifier: and the one after that (got $(pages))" \ # Silence in Telegram must not mean silence in the journal — the suppressed # restarts still have to be visible to whoever reads the logs afterwards. check "notifier: suppressed restarts are still journalled, not dropped" \ - "$(grep -q 'still failing' "$ALERT_LOG" && echo 0 || echo 1)" + "$(grep -q 'ALERT held for vitareba-app.service' "$ALERT_LOG" && echo 0 || echo 1)" # A different unit is a different incident — one loop must not mute the fleet. notify restic-check.service @@ -298,5 +342,167 @@ UNIT_TYPE=oneshot notify appcron-vitareba-reminders.service check "notifier: a failed oneshot cron pages on the first failure" \ "$([ "$(pages)" -eq 1 ] && echo 0 || echo 1)" +# ── 7. The duplicate floor: identical text cannot reach the phone twice ────── +# On 2026-08-28 the fleet register check sent the SAME four-line failure at +# 22:15 and again at 22:16, because it had its own private Telegram call and no +# state of any kind. Keyed cooldowns cannot help a caller that passes no key, +# so lib-alert puts a floor under every unkeyed send. Note what is asserted: +# delivered ONCE, journalled BOTH times. Suppressed must never mean invisible. +# export, NOT a `VAR=x . lib.sh` prefix: bash discards assignments made in front +# of the `.` builtin as soon as it returns, so the library would be sourced with +# the right values and then RUN with them unset — which under `set -u` kills the +# call outright and looks exactly like successful suppression. Cost an hour once; +# it is why the tests below assert the journal as well as the delivery. +lib() { ( export MON="$TMP" ALERT_DEDUPE_SEC="${DEDUPE:-300}" ALERT_DRY_RUN="${DRY:-}" + . "$TMP/lib-alert.sh"; "$@" ) >/dev/null 2>&1 || true; } +reset_logs() { : > "$ALERT_LOG"; : > "$SEND_LOG"; rm -f "$TMP"/state/dedupe_* "$TMP"/state/paged_*; } + +reset_logs +lib alert "🔴" "register check failed: substrata port 4022 already taken" +lib alert "🔴" "register check failed: substrata port 4022 already taken" +check "floor: the same message is delivered once, not twice (got $(sent))" \ + "$([ "$(sent)" -eq 1 ] && echo 0 || echo 1)" +check "floor: the suppressed copy is still in the journal" \ + "$(grep -q 'duplicate suppressed' "$ALERT_LOG" && echo 0 || echo 1)" + +# The floor must not become a gag: a DIFFERENT message still gets through. +reset_logs +lib alert "🔴" "first thing" +lib alert "🔴" "a genuinely different thing" +check "floor: a different message still reaches the phone (got $(sent))" \ + "$([ "$(sent)" -eq 2 ] && echo 0 || echo 1)" + +# And it must expire — same text, window elapsed, is news again. +reset_logs +DEDUPE=0 lib alert "🔴" "recurring condition" +DEDUPE=0 lib alert "🔴" "recurring condition" +check "floor: expires, so a later recurrence is not swallowed (got $(sent))" \ + "$([ "$(sent)" -eq 2 ] && echo 0 || echo 1)" + +# The floor applies ONLY to the unkeyed path. A keyed sender has already made a +# deliberate, state-backed decision to speak; running it through the floor too +# would mean two mechanisms arguing about one message and the quieter one +# silently winning — e.g. the notifier's still-broken reminder disappearing +# because the text matched the page from half an hour earlier. +reset_logs +lib alert_once svc 1800 "🔴" "UNIT DOWN: identical text" +lib alert_clear svc +lib alert_once svc 1800 "🔴" "UNIT DOWN: identical text" +check "floor: a keyed sender is not gagged by the unkeyed floor (got $(sent))" \ + "$([ "$(sent)" -eq 2 ] && echo 0 || echo 1)" + +# ── 8. ALERT_DRY_RUN: testing the alerter must not page a human ────────────── +# At 22:14 and 22:15 on 2026-08-28 a one-time test of a new check and a planted +# zz-audit-probe.service both rang George's real phone — messages about +# nothing, from work that was going fine. Probes journal; they do not deliver. +reset_logs +DRY=1 lib alert "🔴" "one-time test of the new scheduled check" +check "dry-run: a test send delivers nothing (got $(sent))" \ + "$([ "$(sent)" -eq 0 ] && echo 0 || echo 1)" +check "dry-run: but is still visible in the journal" \ + "$(grep -q 'not delivered' "$ALERT_LOG" && echo 0 || echo 1)" + +# ── 9. alert_once / alert_clear contract ───────────────────────────────────── +reset_logs +lib alert_once k 1800 "🔴" "x"; lib alert_once k 1800 "🔴" "x" +check "alert_once: second call inside the cooldown is silent (got $(sent))" \ + "$([ "$(sent)" -eq 1 ] && echo 0 || echo 1)" +( set +e + MON="$TMP" . "$TMP/lib-alert.sh" + alert_once rc 1800 "x" "y"; a=$? + alert_once rc 1800 "x" "y"; b=$? + alert_clear rc; c=$? + [ "$a" -eq 0 ] && [ "$b" -eq 0 ] && [ "$c" -eq 0 ] +) >/dev/null 2>&1 +check "alert_once/alert_clear return 0 on every path" $? + +# ── 10. A oneshot a retry already fixed is not an incident ─────────────────── +# restic-check failed at 18:41:42 on a 25-day-old stale lock left by the laptop, +# was re-run at 18:42:28 and finished clean at 18:44:46 — and still paged, +# because OnFailure fires on the failing run and nothing ever looked again. +rm -f "$TMP"/state/paged_* +UNIT_TYPE=oneshot UNIT_ACTIVE=0 notify restic-check.service +check "oneshot: a retry already running does not page (got $(pages))" \ + "$([ "$(pages)" -eq 0 ] && echo 0 || echo 1)" + +rm -f "$TMP"/state/paged_* +UNIT_TYPE=oneshot UNIT_ACTIVE=1 UNIT_RESULT=success notify restic-check.service +check "oneshot: a retry that already succeeded does not page (got $(pages))" \ + "$([ "$(pages)" -eq 0 ] && echo 0 || echo 1)" +check "oneshot: the grace path leaves no stamp to mute the next real failure" \ + "$([ ! -e "$(stamp_of restic-check.service)" ] && echo 0 || echo 1)" + +# The half that must survive: a oneshot that is STILL broken after the grace +# window is a real failure and must page. A guard against noise that swallows +# the signal is the worse bug of the two. +rm -f "$TMP"/state/paged_* +UNIT_TYPE=oneshot UNIT_ACTIVE=1 UNIT_RESULT=exit-code notify restic-check.service +check "oneshot: one that is still failing after the grace DOES page (got $(pages))" \ + "$([ "$(pages)" -eq 1 ] && echo 0 || echo 1)" + +# ── 11. Unreadable .env: repaired and reported, not asked about ────────────── +# Twice on 2026-08-28 an app's /opt//shared/.env ended up root-owned while +# the unit runs as ubuntu — vitareba at 18:38, botsmann at 20:16. Neither could +# boot, both crash-looped, and the only signal was a wall of UNIT DOWN messages +# whose remedy was one deterministic command. The owner is not a guess: it is +# the unit's own User=. So the check repairs it and reports what it did. +mkdir -p "$TMP/opt/demo/shared" +cat > "$TMP/bin/chown" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$CHOWN_LOG" +[ -n "${CHOWN_FAIL:-}" ] && exit 1 +exit 0 +STUB +chmod +x "$TMP/bin/chown" +export CHOWN_LOG="$TMP/chown.log" + +env_run() { # run host-check with only the env check able to say anything + : > "$ALERT_LOG"; : > "$SEND_LOG"; : > "$CHOWN_LOG" + # Pin the other checks to their quiet state. An earlier section leaves + # host_mem at `bad`, and its RECOVERED would otherwise be counted here as a + # second message from the env check — the same trap the disk band already + # documents above. + printf '%s' ok > "$TMP/state/host_mem"; printf '%s' ok > "$TMP/state/host_disk" + # Likewise the failed-unit keys: section 1b leaves z.service at `bad`, and an + # empty FAILED_UNITS here would emit its RECOVERED into this section's count. + rm -f "$TMP"/state/host_unit_* + APPROOT="$TMP/opt" APP_UNITS="demo-app.service" UNIT_USER="$(id -un)" \ + DISK_PCT=82 bash "$TMP/host-check.sh" >/dev/null 2>&1 || true +} + +printf 'SECRET=1\n' > "$TMP/opt/demo/shared/.env"; chmod 000 "$TMP/opt/demo/shared/.env" +env_run +check "env: an unreadable .env is repaired, not merely reported" \ + "$([ -r "$TMP/opt/demo/shared/.env" ] && echo 0 || echo 1)" +check "env: the repair targets the unit's own User=, not a backup's owner" \ + "$(grep -q "$(id -un):$(id -un) $TMP/opt/demo/shared/.env" "$CHOWN_LOG" && echo 0 || echo 1)" +check "env: exactly one message, and it says it is already fixed (got $(sent))" \ + "$([ "$(sent)" -eq 1 ] && grep -q 'FIXED (no action needed)' "$SEND_LOG" && echo 0 || echo 1)" + +# Healthy is silent — the repair must not become its own recurring alert. +env_run +check "env: a healthy .env says nothing at all (got $(sent))" \ + "$([ "$(sent)" -eq 0 ] && echo 0 || echo 1)" + +# A recurrence AFTER a healthy pass reports again: the healthy path clears the +# stamp, so a second corruption is not swallowed by the first repair's cooldown. +chmod 000 "$TMP/opt/demo/shared/.env" +env_run +check "env: a fresh recurrence is reported again, not muted by the last repair" \ + "$([ "$(sent)" -eq 1 ] && echo 0 || echo 1)" + +# When the repair CANNOT be made, it becomes a question — asked once, with the +# exact command in it, because that is the only case a human is needed for. +chmod 000 "$TMP/opt/demo/shared/.env" +: > "$ALERT_LOG"; : > "$SEND_LOG"; : > "$CHOWN_LOG" +printf '%s' ok > "$TMP/state/host_mem"; printf '%s' ok > "$TMP/state/host_disk" +rm -f "$TMP"/state/host_unit_* +rm -f "$TMP"/state/paged_envfix_demo "$TMP"/state/paged_envbad_demo +CHOWN_FAIL=1 APPROOT="$TMP/opt" APP_UNITS="demo-app.service" UNIT_USER="$(id -un)" \ + DISK_PCT=82 bash "$TMP/host-check.sh" >/dev/null 2>&1 || true +check "env: an unrepairable .env asks, and carries the exact fix command" \ + "$([ "$(sent)" -eq 1 ] && grep -q 'chown .* && systemctl restart demo-app.service' "$SEND_LOG" && echo 0 || echo 1)" +chmod 600 "$TMP/opt/demo/shared/.env" + printf '\n %d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ] diff --git a/scripts/local/fleet-register-check b/scripts/local/fleet-register-check new file mode 100755 index 00000000..7ec8efa9 --- /dev/null +++ b/scripts/local/fleet-register-check @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# +# Runs fleetcrown's own deploy-readiness gate — the half that can ONLY run +# here, because it needs the actual fleet checkouts under ~/dev, which no CI +# runner has. Nothing else schedules this: CI can only verify apps.conf's +# shape, never whether a registered repo_path still points at something real. +# That gap is exactly what let `sink`'s path go stale (aslan-tattoo -> s-ink) +# for an unknown number of days until someone happened to run this by hand. +# +# Scheduled by the user timer fleet-register-check.timer (daily 09:15). +# Lived untracked in ~/.local/bin until 2026-08-28 — which is why nothing +# reviewed either of the two bugs below. ~/.local/bin/fleet-register-check is +# now a symlink to this file. +# +# ── What it judges: the COMMITTED register, not the working tree ───────────── +# +# On 2026-08-28 at 22:15 and 22:16 this sent George two identical Telegrams +# saying `substrata: port 4022 already taken`. Nothing was wrong with the +# fleet. A different agent session was mid-edit in ~/dev/fleetcrown, and the +# check read apps.conf from that half-saved working tree. Around fifteen agent +# sessions share these checkouts, so the working tree here is a scratchpad, not +# a statement about production — judging it means paging about someone's +# unsaved keystrokes. The register that describes what actually runs is the one +# on origin/main, and a genuinely broken register is CI's job to reject in the +# PR that introduces it. +# +# ── How it alerts: at most one message per finding per day ────────────────── +# +# The two messages above were also identical, 60 seconds apart, because this +# script had its own private Telegram call and no state of any kind. The rule +# on the box (/opt/monitoring/lib-alert.sh) is one incident, one message; the +# same rule applies here, with a local stamp, because the box's alert state +# lives behind root and this runs on the laptop as an unprivileged user. +# +# --dry-run prints what it would send and delivers nothing. Every "does this +# still work" run must use it: on 2026-08-28 a bare test of this very script +# rang George's phone with a message that said, in full, that it was a test. +set -uo pipefail + +DRY_RUN="" +[ "${1:-}" = "--dry-run" ] && DRY_RUN=1 + +FLEETCROWN="${FLEETCROWN_DIR:-$HOME/dev/fleetcrown}" +BOX_SSH="ssh -o ConnectTimeout=8 -i $HOME/.ssh/fleetcrown_ci_deploy ubuntu@167.233.22.31" +# A dry run keeps its own state dir, so it exercises the cooldown exactly like +# the real thing (including "same finding, not repeating") without writing the +# production stamps or delivering anything. A test path that skips the state is +# a test of something other than what runs. +STATE="${XDG_STATE_HOME:-$HOME/.local/state}/fleet-register-check${DRY_RUN:+/dry-run}" +COOLDOWN=${REGISTER_COOLDOWN_SEC:-79200} # 22h: at most one message per daily run +mkdir -p "$STATE" + +[ -d "$FLEETCROWN" ] || { echo "fleet-register-check: $FLEETCROWN missing, skipping" >&2; exit 0; } + +# The committed register, from the remote's default branch. A fetch failure is +# not fatal — a slightly stale origin/main is still a real commit, whereas the +# working tree may be nobody's intention at all. Never fall back to the tree. +TMPD=$(mktemp -d); trap 'rm -rf "$TMPD"' EXIT +git -C "$FLEETCROWN" fetch --quiet origin main 2>/dev/null || true +# REGISTER_REF exists so the failure path can actually be exercised against a +# deliberately broken register; unset, it is origin/main and nothing else. +REF="${REGISTER_REF:-origin/main}" +git -C "$FLEETCROWN" rev-parse --verify --quiet "$REF" >/dev/null || REF=HEAD +git -C "$FLEETCROWN" show "$REF:scripts/hetzner/apps.conf" > "$TMPD/apps.conf" 2>/dev/null || { + echo "fleet-register-check: cannot read apps.conf from $REF" >&2; exit 0; } + +OUT=$(cd "$FLEETCROWN" && MANIFEST="$TMPD/apps.conf" bash scripts/ci/check-deploy-ready.sh 2>&1) +RC=$? + +echo "$OUT" +[ "$RC" -eq 0 ] && exit 0 + +MSG="🔴 fleet register check failed on $(hostname) (register as committed on $REF): +$(echo "$OUT" | tail -15)" + +# One message per distinct finding per cooldown. Keyed on the CONTENT, so a new +# and different problem is never muted by an old one still standing. +KEY=$(printf '%s' "$MSG" | md5sum | cut -c1-16) +STAMP="$STATE/paged_$KEY" +NOW=$(date +%s) +LAST=$(cat "$STAMP" 2>/dev/null | tr -dc '0-9') +if [ -n "$LAST" ] && [ "$((NOW - LAST))" -lt "$COOLDOWN" ]; then + echo "fleet-register-check: same finding paged $((NOW - LAST))s ago — not repeating" >&2 + exit "$RC" +fi +find "$STATE" -maxdepth 1 -name 'paged_*' -mtime +7 -delete 2>/dev/null || true + +printf '%s' "$NOW" > "$STAMP" +if [ -n "$DRY_RUN" ]; then + echo "fleet-register-check: --dry-run, would have sent:" >&2 + printf '%s\n' "$MSG" >&2 + exit "$RC" +fi + +notify-send -u critical -i dialog-warning "Fleet Register Check FAILED" "$(echo "$OUT" | tail -6)" 2>/dev/null || true +printf '%s' "$MSG" | $BOX_SSH ' + cd /opt/fleetcrown/app + set -a; source .env 2>/dev/null; set +a + text=$(cat) + # The app deliberately does not read the bare TELEGRAM_CHAT_ID name (see + # src/lib/env.ts) — it is namespaced per-app to avoid collisions on a box + # that runs several apps in the same env-var space. Check every alias. + CHAT_ID="${APP_TELEGRAM_CHAT_ID:-${FLEETCROWN_TELEGRAM_CHAT_ID:-${COCKPIT_TELEGRAM_CHAT_ID:-${TELEGRAM_CHAT_ID:-}}}}" + [ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "$CHAT_ID" ] || { echo "no telegram creds on box" >&2; exit 0; } + python3 - "$text" "$CHAT_ID" <&1 || echo "fleet-register-check: telegram send failed" >&2 + +exit "$RC"