diff --git a/scripts/hetzner/test-uptime-sweep.sh b/scripts/hetzner/test-uptime-sweep.sh index 1be648e2..17ee3336 100755 --- a/scripts/hetzner/test-uptime-sweep.sh +++ b/scripts/hetzner/test-uptime-sweep.sh @@ -143,9 +143,70 @@ done <<<"$(extra_targets)" || no "now in apps.conf — delete from EXTRA_TARGETS: $dupes" # And they must actually be covered, or the four apps apps.conf excludes stay -# exactly as unwatched as they were before this script existed. -extras="$(extra_targets | grep -c . || true)" -eq 4 "$extras" "the four pre-existing services apps.conf documents are covered" +# exactly as unwatched as they were before this script existed. Asserted BY +# NAME rather than by counting: the list now holds two categories, and a count +# would go green if one of the four were swapped for something else entirely. +for svc in bridge fleetcrown orangecat revampit; do + extra_targets | grep -q "^${svc} " \ + && ok "$svc is covered — apps.conf documents it as deliberately absent" \ + || no "$svc is in no manifest and now in no hand-list either" +done + +echo +echo "url_path / url_host — reading a Location header" +eq /api/health "$(url_path https://aoz.orangecat.ch/api/health)" "a plain path" +eq /api/health "$(url_path 'https://h/api/health?x=1')" "the query is not part of the path" +eq /api/health "$(url_path https://h/api/health/)" "a trailing slash is not a different route" +eq /login "$(url_path 'https://petvity.orangecat.ch/login?returnTo=%2Fapi%2Fhealth')" \ + "the auth wall's path is /login, whatever its query smuggles" +eq / "$(url_path https://h)" "no path at all reads as /" +eq aoz.orangecat.ch "$(url_host https://aoz.orangecat.ch/api/health)" "the host, for saying where we ended up" + +echo +echo "is_redirect" +is_redirect 308 && ok "308 is a redirect" || no "308 should be a redirect" +is_redirect 307 && ok "307 is a redirect" || no "307 should be a redirect" +is_redirect 200 && no "200 is not a redirect" || ok "200 is not a redirect" +is_redirect 404 && no "404 is not a redirect" || ok "404 is not a redirect" + +echo +echo "same_path_redirect — a host move, or an auth wall wearing the same code?" +same_path_redirect /api/health https://aoz.orangecat.ch/api/health \ + && ok "aoz-wohnen -> aoz keeps the path: follow it, the health route is real" \ + || no "a canonical-host redirect must be followed" +same_path_redirect /api/health 'https://petvity.orangecat.ch/login?returnTo=%2Fapi%2Fhealth' \ + && no "petvity's auth wall must NOT be followed — 200 from a login page is a false green" \ + || ok "an auth wall is not a health route, however inviting its 200 looks" +same_path_redirect /api/health https://h/api/health/ \ + && ok "a trailing slash is the same route" || no "trailing slash should match" +same_path_redirect /api/health https://h/en/api/health \ + && no "a locale prefix is a different route" || ok "a locale prefix is not the same route" +same_path_redirect /api/health "" \ + && no "no Location means nothing to follow" || ok "an empty Location is not a redirect target" + +echo +echo "annushka — served, has a process, and can never be in apps.conf" +extra_targets | grep -q "^annushka annushka.orangecat.ch /api/health$" \ + && ok "annushka is watched: its static pages serve on while its api dies" \ + || no "annushka should be in EXTRA_TARGETS" + +echo "HEALTH_PATHS — apps that answer somewhere other than /api/health" +eq /api/healthz "$(health_path_for petvity)" "petvity's real check; /api/health is its pet health-RECORDS api, behind auth" +eq /healthz "$(health_path_for bridge)" "bridge is an SSE service, not a Next app" +eq /api/health "$(health_path_for kivvi)" "an app that follows the convention needs no entry" +eq /api/health "$(health_path_for '')" "an empty name falls back to the default rather than emptying the URL" + +# Both target builders must consult the table. manifest_targets passed it to awk +# as a file-argument assignment, which awk applies only when it REACHES that +# argument — long after BEGIN built the lookup. bridge (hand-listed, -v) worked; +# petvity (manifest) silently kept /api/health and stayed blind. Syntax was +# fine and the sweep still ran; only the emitted path was wrong. +manifest_targets "$MANIFEST" | grep -q "^petvity petvity.orangecat.ch /api/healthz$" \ + && ok "a MANIFEST app picks up its declared path" \ + || no "manifest_targets ignored HEALTH_PATHS — check awk gets it via -v, not a file-arg assignment" +extra_targets | grep -q "^bridge bridge.orangecat.ch /healthz$" \ + && ok "a HAND-LISTED app picks up its declared path" \ + || no "extra_targets ignored HEALTH_PATHS" echo if [ "$FAIL" -gt 0 ]; then diff --git a/scripts/hetzner/uptime-sweep.sh b/scripts/hetzner/uptime-sweep.sh index 76cfea14..f90cf096 100755 --- a/scripts/hetzner/uptime-sweep.sh +++ b/scripts/hetzner/uptime-sweep.sh @@ -23,9 +23,16 @@ # Hence: probe /api/health, and treat 5xx as DOWN. An app with no health route # can only be asked "do you serve anything at all", which is a weaker question — # so it is reported as LIMITED, never as a pass. That distinction is the point. -# Nine apps have a health route; the rest are monitored with one hand tied, and -# saying so is what keeps the number honest (see scripts/ci/shared-inventory.sh, -# which ratchets that health-route count). +# Twelve targets answer a health route; the rest are monitored with one hand +# tied, and saying so is what keeps the number honest (see +# scripts/ci/shared-inventory.sh, which ratchets that health-route count). +# +# THREE of those twelve were counted among "the rest" until 2026-08-28, and none +# of them was ever broken — the sweep was asking the wrong URL and believing the +# answer. aoz-wohnen and revampit answer on the canonical host they redirect to; +# petvity answers at /api/healthz, while the /api/health we asked for is its pet +# health-RECORDS api sitting behind auth. Each looked exactly like an app with +# no health route, which is the one shape this script must never get wrong. # # Runs off-box on GitHub's infrastructure, so it still reports when bitbaum is # dead — which is exactly when an on-box check tells you nothing. @@ -61,20 +68,49 @@ DEFAULT_HEALTH_PATH=/api/health # test-uptime-sweep.sh fails on the duplicate and forces its removal — a # hand-list that cannot quietly outlive its reason. # -# Third field = health path, when the service does not use the fleet's -# /api/health convention. bridge is a ~300-line SSE fan-out service, not a Next -# app: it answers /healthz and returns 404 for `/` by design. Probing it the -# standard way reported a perfectly healthy service as DOWN, which is how a -# monitor teaches people to ignore it. +# Health paths for both lists live in HEALTH_PATHS below, so there is one place +# to look when an app does not follow the convention. +# +# annushka is here for a different reason and will NOT die the same way: it is a +# static concept site with no repo, deployed by hand, so it can never appear in +# a manifest whose other fields are a repo path and a deploy target. It does +# have a process — annushka-api on 4030 serves /api/* — which can fall over +# while the static pages carry on serving perfectly. Exactly botsmann's shape. EXTRA_TARGETS=' -bridge|bridge.orangecat.ch|/healthz +bridge|bridge.orangecat.ch fleetcrown|fleetcrown.orangecat.ch orangecat|orangecat.ch revampit|revampit.orangecat.ch +annushka|annushka.orangecat.ch +' + +# Apps whose health route is not at the fleet's /api/health. One table for +# manifest apps and hand-listed ones alike — an app that answers somewhere else +# is the same problem whichever list it came from. +# +# bridge is a ~300-line SSE fan-out service, not a Next app: it answers /healthz +# and 404s on `/` by design. Probed the standard way it read DOWN — a perfectly +# healthy service, which is how a monitor teaches people to ignore it. +# +# petvity is worse, because its /api/health looks like it exists. That path is +# the pet HEALTH-RECORDS api, behind auth, so it 307s to /login. The sweep read +# "no health route" and settled for asking whether the homepage renders — while +# /api/healthz was there the whole time answering {"ok":true,"db":true}, the +# database check we actually wanted. Measured 2026-08-28. +HEALTH_PATHS=' +bridge|/healthz +petvity|/api/healthz ' # ── Pure helpers (no network, no box) — exercised by test-uptime-sweep.sh ───── +# health_path_for — the declared health path, or the fleet default. +health_path_for() { + local declared + declared=$(printf '%s\n' "$HEALTH_PATHS" | awk -F'|' -v n="$1" '$1 == n { print $2; exit }') + printf '%s' "${declared:-$DEFAULT_HEALTH_PATH}" +} + # manifest_targets — echo "namedomain" per monitorable app. # # Skips comments, blanks, internal-only apps ('-' domain), and apps whose status @@ -82,16 +118,23 @@ revampit|revampit.orangecat.ch # same way selfhost-deploy.yml resolves ${DOMAINS%%,*} — www aliases are the # same app and a second probe would only double the noise. manifest_targets() { - awk -F'|' ' + awk -F'|' -v HEALTH="$DEFAULT_HEALTH_PATH" -v PATHS="$HEALTH_PATHS" ' + BEGIN { + n = split(PATHS, lines, "\n") + for (i = 1; i <= n; i++) { + if (split(lines[i], f, "|") == 2 && f[1] != "") declared[f[1]] = f[2] + } + } + function health_for(name) { return (name in declared) ? declared[name] : HEALTH } /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } { name = $1; domains = $3; status = $9 if (domains == "-" || domains == "") next if (status == "archived" || status == "handed-over") next sub(/,.*/, "", domains) - print name "\t" domains "\t" HEALTH + print name "\t" domains "\t" health_for(name) } - ' HEALTH="$DEFAULT_HEALTH_PATH" "$1" + ' "$1" } # manifest_skipped — echo "namereason" for every app NOT probed, so @@ -110,8 +153,15 @@ manifest_skipped() { # extra_targets — the documented-absent four, same "namedomain" shape. extra_targets() { printf '%s\n' "$EXTRA_TARGETS" \ - | awk -F'|' -v HEALTH="$DEFAULT_HEALTH_PATH" \ - 'NF>=2 { print $1 "\t" $2 "\t" (NF>=3 && $3 != "" ? $3 : HEALTH) }' + | awk -F'|' -v HEALTH="$DEFAULT_HEALTH_PATH" -v PATHS="$HEALTH_PATHS" ' + BEGIN { + n = split(PATHS, lines, "\n") + for (i = 1; i <= n; i++) { + if (split(lines[i], f, "|") == 2 && f[1] != "") declared[f[1]] = f[2] + } + } + function health_for(name) { return (name in declared) ? declared[name] : HEALTH } + NF>=2 { print $1 "\t" $2 "\t" health_for($1) }' } # health_verdict — what /api/health told us. @@ -129,6 +179,55 @@ health_verdict() { esac } +# is_redirect — 3xx, the code that means "ask somewhere else". +is_redirect() { case "$1" in 3??) return 0 ;; *) return 1 ;; esac; } + +# url_path — the path, without query or fragment, trailing slash trimmed. +# `https://h/api/health?x=1` -> `/api/health`; `https://h` -> `/`. +url_path() { + local rest="${1#*://}" + case "$rest" in + */*) rest="/${rest#*/}" ;; + *) rest="/" ;; + esac + rest="${rest%%\?*}" + rest="${rest%%#*}" + [ "$rest" = / ] || rest="${rest%/}" + printf '%s' "$rest" +} + +# url_host — the hostname, for saying WHERE we ended up in the report. +url_host() { + local rest="${1#*://}" + printf '%s' "${rest%%/*}" +} + +# same_path_redirect — did this redirect only +# move HOSTS, keeping the path we asked for? +# +# A 3xx on the health path is two different things wearing one status code, and +# they need opposite treatment: +# +# aoz-wohnen.orangecat.ch 308s to aoz.orangecat.ch — same path, canonical +# host. The health route exists and answers 200; we were asking the wrong +# hostname and reporting LIMITED for an app that has a working check. Worse, +# had its database died the redirect would still have been a 308 and the +# sweep would still have said "serving". +# +# petvity 307s /api/health to /login?returnTo=%2Fapi%2Fhealth — an auth wall. +# Following that returns 200 from a LOGIN PAGE. A blanket `curl -L` would +# report petvity UP on the strength of a form: a false green, which is worse +# than the blind spot it set out to fix. +# +# So: follow it only when the path survives. +same_path_redirect() { + local want="$1" got + [ -n "${2:-}" ] || return 1 + got="$(url_path "$2")" + [ "$want" = / ] || want="${want%/}" + [ "$got" = "$want" ] +} + # root_verdict — the weaker fallback question: does it serve at all? # 3xx counts as serving: several apps redirect `/` to a locale or a canonical # host (revampit, petvity, vitareba all do) and that is a working app. @@ -178,6 +277,11 @@ probe_code() { normalize_code "$(curl -sS -o /dev/null -w '%{http_code}' --max-time "$TIMEOUT" "$1" 2>/dev/null || true)" } +# probe_redirect — echo the Location this URL redirects to, or nothing. +probe_redirect() { + curl -sS -o /dev/null -w '%{redirect_url}' --max-time "$TIMEOUT" "$1" 2>/dev/null || true +} + # probe_app — echo " ". # # Retries only a NEGATIVE result. A 200 is believed immediately; a failure is @@ -193,6 +297,25 @@ probe_app() { [ "$try" -lt "$TRIES" ] && sleep "$SLEEP" done + # Before giving up on the health route, check whether the 3xx was merely the + # canonical host telling us where it lives. Only a path-preserving redirect + # counts — see same_path_redirect for why following an auth wall is worse + # than not following at all. + if [ "$verdict" = absent ] && is_redirect "$code"; then + local target + target=$(probe_redirect "https://$domain$health_path") + if same_path_redirect "$health_path" "$target"; then + for (( try = 1; try <= TRIES; try++ )); do + code=$(probe_code "$target") + verdict=$(health_verdict "$code") + [ "$verdict" = up ] && { echo "up health:$code (redirected to $(url_host "$target"))"; return; } + [ "$verdict" = absent ] && break + [ "$try" -lt "$TRIES" ] && sleep "$SLEEP" + done + [ "$verdict" = down ] && { echo "down health:$code (redirected to $(url_host "$target"))"; return; } + fi + fi + if [ "$verdict" = absent ]; then local rcode rverdict for (( try = 1; try <= TRIES; try++ )); do