From 4f72dbf259ed1bbded37fb8f953ac1b8fb013c83 Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:02:02 +0200 Subject: [PATCH] feat(ops): watch the certificates, three weeks before they take the box down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing watched TLS expiry. Caddy renews at 30 days remaining and has never missed, which is exactly why: the check felt unnecessary because it had never been needed. But renewal breaks for reasons no app ever sees — an ACME rate limit, port 80 closed by a firewall change, a DNS record moved — and the first symptom is every site on the box going dark at once. Twenty domains, one failure. The warning exists for weeks; nobody was reading it. Measured today, all twenty are fine: 65-88 days, staggered. That is the moment to add the check, not after the outage that proves it was needed. Reuses the uptime sweep's target list rather than taking a copy — verified identical, 20 targets and 20 certificates. A second list of domains is the gap that hid botsmann for weeks, and it would open here in exactly the same way: a new app registered in apps.conf would be probed for HTTP and silently unprotected against expiry. Thresholds are Caddy's, not arbitrary. ok at 21+ days, because Caddy renews at 30 and has therefore had nine days of attempts; warn from 7 to 20, when renewal should have happened and did not; critical under a week. Unreadable is CRITICAL, never ok: "we could not check the certificate" and "the certificate is fine" must not share an outcome — that conflation is how botsmann's 503 read as healthy for weeks. Daily, not every fifteen minutes. A certificate does not change between sweeps, and twenty TLS handshakes a quarter-hour to relearn the same number is noise that earns nothing — and noise gets muted. No issue/Telegram machinery: fleet-uptime.yml owns that, and a second alerting path to keep in step is not worth it for a check with three weeks of slack. Stated in the workflow rather than left to be discovered. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018waGt1ieA9TjpscqrbrnGb --- .github/workflows/fleet-certs.yml | 62 +++++++++++++++++++++++++ scripts/hetzner/test-uptime-sweep.sh | 12 +++++ scripts/hetzner/uptime-sweep.sh | 67 ++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 .github/workflows/fleet-certs.yml diff --git a/.github/workflows/fleet-certs.yml b/.github/workflows/fleet-certs.yml new file mode 100644 index 00000000..1d384470 --- /dev/null +++ b/.github/workflows/fleet-certs.yml @@ -0,0 +1,62 @@ +# Is any TLS certificate on the box about to expire? +# +# Caddy renews at 30 days remaining and has never missed, which is exactly why +# nothing watched it. A renewal can break for reasons no app ever sees — an ACME +# rate limit, port 80 closed by a firewall change, a DNS record moved — and the +# first symptom is every site on the box going dark at once, with no prior +# signal anywhere. Weeks of warning exist; nobody was reading them. +# +# Daily, not every 15 minutes like the uptime sweep next door: a certificate +# does not change between sweeps, and 20 TLS handshakes every quarter hour to +# learn the same number is noise that earns nothing. +# +# It probes the SAME targets as the uptime sweep, from the same manifest, via +# `uptime-sweep.sh --certs`. A second list of domains is the gap that hid +# botsmann for weeks — there is deliberately no second list. +name: Fleet certs + +on: + schedule: + # 06:41 UTC. Before the working day, and off the hour to spread fleet load. + - cron: '41 6 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: fleet-certs + cancel-in-progress: true + +jobs: + certs: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + + # Pure helpers, no network. If cert_verdict's boundaries break, the sweep + # below would still print a confident table. + - name: The verdicts still hold + run: bash scripts/hetzner/test-uptime-sweep.sh + + - name: Every certificate has time left + run: | + set -uo pipefail + { + echo '## Fleet certs' + echo + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + set +e + bash scripts/hetzner/uptime-sweep.sh --certs 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + rc=${PIPESTATUS[0]} + set -e + echo '```' >> "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # No issue/Telegram machinery here, deliberately. fleet-uptime.yml owns + # that, and copying it would be a second alerting path to keep in step for + # a check that fires with three weeks of slack. A red run on a daily + # schedule is warning enough; if that ever proves too quiet, call the + # existing alerting rather than growing a second one. diff --git a/scripts/hetzner/test-uptime-sweep.sh b/scripts/hetzner/test-uptime-sweep.sh index 17ee3336..e5278e84 100755 --- a/scripts/hetzner/test-uptime-sweep.sh +++ b/scripts/hetzner/test-uptime-sweep.sh @@ -208,6 +208,18 @@ extra_targets | grep -q "^bridge bridge.orangecat.ch /healthz$" \ && ok "a HAND-LISTED app picks up its declared path" \ || no "extra_targets ignored HEALTH_PATHS" +echo +echo "cert_verdict — a broken renewal is silent until every site goes dark" +eq ok "$(cert_verdict 88)" "88 days: Caddy is renewing normally" +eq ok "$(cert_verdict 21)" "21 days is still ok — Caddy renews at 30, so it has had nine days of tries" +eq warn "$(cert_verdict 20)" "20 days: renewal should have happened by now" +eq warn "$(cert_verdict 7)" "7 days: still a working week to fix it" +eq critical "$(cert_verdict 6)" "under a week is an outage with a date on it" +eq critical "$(cert_verdict 0)" "expires today" +eq critical "$(cert_verdict -3)" "already expired reads critical, not as a huge number" +eq critical "$(cert_verdict '')" "unreadable is CRITICAL — 'could not check' must never share an outcome with 'fine'" +eq critical "$(cert_verdict 'x')" "garbage is critical too, for the same reason" + echo if [ "$FAIL" -gt 0 ]; then echo "FAILED: $FAIL failed, $PASS passed" diff --git a/scripts/hetzner/uptime-sweep.sh b/scripts/hetzner/uptime-sweep.sh index f90cf096..f9246003 100755 --- a/scripts/hetzner/uptime-sweep.sh +++ b/scripts/hetzner/uptime-sweep.sh @@ -41,6 +41,9 @@ # uptime-sweep.sh # report every app # uptime-sweep.sh --check # exit 1 if any app is DOWN # uptime-sweep.sh --json # machine-readable, for the alerting workflow +# uptime-sweep.sh --certs # TLS expiry for the same targets; exit 1 if any +# # is critical. Daily, not every 15 minutes — a +# # certificate does not change between sweeps. # # Env: UPTIME_TRIES (default 3), UPTIME_TIMEOUT secs (default 15), # UPTIME_SLEEP secs between tries (default 5), MANIFEST (default apps.conf) @@ -228,6 +231,33 @@ same_path_redirect() { [ "$got" = "$want" ] } +# cert_verdict — how much warning is left? +# +# Caddy renews at 30 days remaining and has never missed. That is exactly why +# nothing watches it, and why a failure would be silent: the first symptom of a +# broken renewal is every site on the box going dark at once, with no prior +# signal anywhere. Renewal can break for reasons the app never sees — an ACME +# rate limit, port 80 closed by a firewall change, a DNS record moved. +# +# ok >= 21 days. Caddy renews at 30, so it has had nine days of tries. +# warn 7-20. Renewal should have happened and did not; look now, while +# looking is cheap. +# critical < 7, or unreadable. An outage with a date on it. +# +# Unknown is critical, never ok: "we could not read the certificate" and "the +# certificate is fine" must not share an outcome. That conflation is what let +# botsmann's 503 read as healthy for weeks. +cert_verdict() { + case "$1" in + ''|*[!0-9-]*) echo critical ;; + -*) echo critical ;; + *) if [ "$1" -lt 7 ]; then echo critical + elif [ "$1" -lt 21 ]; then echo warn + else echo ok + fi ;; + esac +} + # root_verdict — the weaker fallback question: does it serve at all? # 3xx counts as serving: several apps redirect `/` to a locale or a canonical # host (revampit, petvity, vitareba all do) and that is a working app. @@ -265,6 +295,7 @@ MODE=report case "${1:-}" in --check) MODE=check ;; --json) MODE=json ;; + --certs) MODE=certs ;; "") MODE=report ;; *) echo "unknown argument: $1" >&2; exit 2 ;; esac @@ -331,6 +362,42 @@ probe_app() { echo "down health:$code" } + +# cert_days — days until the TLS certificate expires, or empty if we +# could not read one. Deliberately the SAME target list as the HTTP sweep: a +# second list of domains is the gap that hid botsmann. +cert_days() { + local end epoch + end=$(echo | timeout "$TIMEOUT" openssl s_client -servername "$1" -connect "$1:443" 2>/dev/null \ + | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) + [ -n "$end" ] || return 0 + epoch=$(date -d "$end" +%s 2>/dev/null) || return 0 + echo $(( (epoch - $(date +%s)) / 86400 )) +} + +if [ "$MODE" = certs ]; then + command -v openssl >/dev/null 2>&1 || { echo "openssl not found" >&2; exit 2; } + n_ok=0; n_warn=0; n_crit=0; bad="" + printf '%-34s %6s %s\n' DOMAIN DAYS STATE + while IFS=$'\t' read -r _name domain _hp; do + [ -n "$domain" ] || continue + days=$(cert_days "$domain") + verdict=$(cert_verdict "$days") + printf '%-34s %6s %s\n' "$domain" "${days:-?}" "$verdict" + case "$verdict" in + ok) n_ok=$((n_ok + 1)) ;; + warn) n_warn=$((n_warn + 1)); bad="$bad $domain(${days:-?}d)" ;; + critical) n_crit=$((n_crit + 1)); bad="$bad $domain(${days:-?}d)" ;; + esac + done <<<"$( { manifest_targets "$MANIFEST"; extra_targets; } | sort -u )" + + echo + echo "ok=$n_ok warn=$n_warn critical=$n_crit" + [ -n "$bad" ] && echo "NEEDS A LOOK:$bad" + [ "$n_crit" -gt 0 ] && exit 1 + exit 0 +fi + targets=$( { manifest_targets "$MANIFEST"; extra_targets; } | sort -u ) [ -n "$targets" ] || { echo "no targets found in $MANIFEST" >&2; exit 2; }