Skip to content

fix(ops): three apps were monitored blind, and none of them was broken #424

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
github-actions merged 1 commit into from
Aug 28, 2026
Merged

fix(ops): three apps were monitored blind, and none of them was broken #424

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 64 additions & 3 deletions scripts/hetzner/test-uptime-sweep.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,70 @@ done <<<"$(extra_targets)"
|| no "now in apps.conf — delete from EXTRA_TARGETS: $dupes"

# And they must actually be covered, or the four apps apps.conf excludes stay
# exactly as unwatched as they were before this script existed.
extras="$(extra_targets | grep -c . || true)"
eq 4 "$extras" "the four pre-existing services apps.conf documents are covered"
# exactly as unwatched as they were before this script existed. Asserted BY
# NAME rather than by counting: the list now holds two categories, and a count
# would go green if one of the four were swapped for something else entirely.
for svc in bridge fleetcrown orangecat revampit; do
extra_targets | grep -q "^${svc} " \
&& ok "$svc is covered — apps.conf documents it as deliberately absent" \
|| no "$svc is in no manifest and now in no hand-list either"
done

echo
echo "url_path / url_host — reading a Location header"
eq /api/health "$(url_path https://aoz.orangecat.ch/api/health)" "a plain path"
eq /api/health "$(url_path 'https://h/api/health?x=1')" "the query is not part of the path"
eq /api/health "$(url_path https://h/api/health/)" "a trailing slash is not a different route"
eq /login "$(url_path 'https://petvity.orangecat.ch/login?returnTo=%2Fapi%2Fhealth')" \
"the auth wall's path is /login, whatever its query smuggles"
eq / "$(url_path https://h)" "no path at all reads as /"
eq aoz.orangecat.ch "$(url_host https://aoz.orangecat.ch/api/health)" "the host, for saying where we ended up"

echo
echo "is_redirect"
is_redirect 308 && ok "308 is a redirect" || no "308 should be a redirect"
is_redirect 307 && ok "307 is a redirect" || no "307 should be a redirect"
is_redirect 200 && no "200 is not a redirect" || ok "200 is not a redirect"
is_redirect 404 && no "404 is not a redirect" || ok "404 is not a redirect"

echo
echo "same_path_redirect — a host move, or an auth wall wearing the same code?"
same_path_redirect /api/health https://aoz.orangecat.ch/api/health \
&& ok "aoz-wohnen -> aoz keeps the path: follow it, the health route is real" \
|| no "a canonical-host redirect must be followed"
same_path_redirect /api/health 'https://petvity.orangecat.ch/login?returnTo=%2Fapi%2Fhealth' \
&& no "petvity's auth wall must NOT be followed — 200 from a login page is a false green" \
|| ok "an auth wall is not a health route, however inviting its 200 looks"
same_path_redirect /api/health https://h/api/health/ \
&& ok "a trailing slash is the same route" || no "trailing slash should match"
same_path_redirect /api/health https://h/en/api/health \
&& no "a locale prefix is a different route" || ok "a locale prefix is not the same route"
same_path_redirect /api/health "" \
&& no "no Location means nothing to follow" || ok "an empty Location is not a redirect target"

echo
echo "annushka — served, has a process, and can never be in apps.conf"
extra_targets | grep -q "^annushka annushka.orangecat.ch /api/health$" \
&& ok "annushka is watched: its static pages serve on while its api dies" \
|| no "annushka should be in EXTRA_TARGETS"

echo "HEALTH_PATHS — apps that answer somewhere other than /api/health"
eq /api/healthz "$(health_path_for petvity)" "petvity's real check; /api/health is its pet health-RECORDS api, behind auth"
eq /healthz "$(health_path_for bridge)" "bridge is an SSE service, not a Next app"
eq /api/health "$(health_path_for kivvi)" "an app that follows the convention needs no entry"
eq /api/health "$(health_path_for '')" "an empty name falls back to the default rather than emptying the URL"

# Both target builders must consult the table. manifest_targets passed it to awk
# as a file-argument assignment, which awk applies only when it REACHES that
# argument — long after BEGIN built the lookup. bridge (hand-listed, -v) worked;
# petvity (manifest) silently kept /api/health and stayed blind. Syntax was
# fine and the sweep still ran; only the emitted path was wrong.
manifest_targets "$MANIFEST" | grep -q "^petvity petvity.orangecat.ch /api/healthz$" \
&& ok "a MANIFEST app picks up its declared path" \
|| no "manifest_targets ignored HEALTH_PATHS — check awk gets it via -v, not a file-arg assignment"
extra_targets | grep -q "^bridge bridge.orangecat.ch /healthz$" \
&& ok "a HAND-LISTED app picks up its declared path" \
|| no "extra_targets ignored HEALTH_PATHS"

echo
if [ "$FAIL" -gt 0 ]; then
Expand Down
151 changes: 137 additions & 14 deletions scripts/hetzner/uptime-sweep.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,16 @@
# Hence: probe /api/health, and treat 5xx as DOWN. An app with no health route
# can only be asked "do you serve anything at all", which is a weaker question —
# so it is reported as LIMITED, never as a pass. That distinction is the point.
# Nine apps have a health route; the rest are monitored with one hand tied, and
# saying so is what keeps the number honest (see scripts/ci/shared-inventory.sh,
# which ratchets that health-route count).
# Twelve targets answer a health route; the rest are monitored with one hand
# tied, and saying so is what keeps the number honest (see
# scripts/ci/shared-inventory.sh, which ratchets that health-route count).
#
# THREE of those twelve were counted among "the rest" until 2026-08-28, and none
# of them was ever broken — the sweep was asking the wrong URL and believing the
# answer. aoz-wohnen and revampit answer on the canonical host they redirect to;
# petvity answers at /api/healthz, while the /api/health we asked for is its pet
# health-RECORDS api sitting behind auth. Each looked exactly like an app with
# no health route, which is the one shape this script must never get wrong.
#
# Runs off-box on GitHub's infrastructure, so it still reports when bitbaum is
# dead — which is exactly when an on-box check tells you nothing.
Expand Down Expand Up @@ -61,37 +68,73 @@ DEFAULT_HEALTH_PATH=/api/health
# test-uptime-sweep.sh fails on the duplicate and forces its removal — a
# hand-list that cannot quietly outlive its reason.
#
# Third field = health path, when the service does not use the fleet's
# /api/health convention. bridge is a ~300-line SSE fan-out service, not a Next
# app: it answers /healthz and returns 404 for `/` by design. Probing it the
# standard way reported a perfectly healthy service as DOWN, which is how a
# monitor teaches people to ignore it.
# Health paths for both lists live in HEALTH_PATHS below, so there is one place
# to look when an app does not follow the convention.
#
# annushka is here for a different reason and will NOT die the same way: it is a
# static concept site with no repo, deployed by hand, so it can never appear in
# a manifest whose other fields are a repo path and a deploy target. It does
# have a process — annushka-api on 4030 serves /api/* — which can fall over
# while the static pages carry on serving perfectly. Exactly botsmann's shape.
EXTRA_TARGETS='
bridge|bridge.orangecat.ch|/healthz
bridge|bridge.orangecat.ch
fleetcrown|fleetcrown.orangecat.ch
orangecat|orangecat.ch
revampit|revampit.orangecat.ch
annushka|annushka.orangecat.ch
'

# Apps whose health route is not at the fleet's /api/health. One table for
# manifest apps and hand-listed ones alike — an app that answers somewhere else
# is the same problem whichever list it came from.
#
# bridge is a ~300-line SSE fan-out service, not a Next app: it answers /healthz
# and 404s on `/` by design. Probed the standard way it read DOWN — a perfectly
# healthy service, which is how a monitor teaches people to ignore it.
#
# petvity is worse, because its /api/health looks like it exists. That path is
# the pet HEALTH-RECORDS api, behind auth, so it 307s to /login. The sweep read
# "no health route" and settled for asking whether the homepage renders — while
# /api/healthz was there the whole time answering {"ok":true,"db":true}, the
# database check we actually wanted. Measured 2026-08-28.
HEALTH_PATHS='
bridge|/healthz
petvity|/api/healthz
'

# ── Pure helpers (no network, no box) — exercised by test-uptime-sweep.sh ─────

# health_path_for <name> — the declared health path, or the fleet default.
health_path_for() {
local declared
declared=$(printf '%s\n' "$HEALTH_PATHS" | awk -F'|' -v n="$1" '$1 == n { print $2; exit }')
printf '%s' "${declared:-$DEFAULT_HEALTH_PATH}"
}

# manifest_targets <file> — echo "name<TAB>domain" per monitorable app.
#
# Skips comments, blanks, internal-only apps ('-' domain), and apps whose status
# says they are not ours to page on. Takes the FIRST domain of a comma list, the
# same way selfhost-deploy.yml resolves ${DOMAINS%%,*} — www aliases are the
# same app and a second probe would only double the noise.
manifest_targets() {
awk -F'|' '
awk -F'|' -v HEALTH="$DEFAULT_HEALTH_PATH" -v PATHS="$HEALTH_PATHS" '
BEGIN {
n = split(PATHS, lines, "\n")
for (i = 1; i <= n; i++) {
if (split(lines[i], f, "|") == 2 && f[1] != "") declared[f[1]] = f[2]
}
}
function health_for(name) { return (name in declared) ? declared[name] : HEALTH }
/^[[:space:]]*#/ || /^[[:space:]]*$/ { next }
{
name = $1; domains = $3; status = $9
if (domains == "-" || domains == "") next
if (status == "archived" || status == "handed-over") next
sub(/,.*/, "", domains)
print name "\t" domains "\t" HEALTH
print name "\t" domains "\t" health_for(name)
}
' HEALTH="$DEFAULT_HEALTH_PATH" "$1"
' "$1"
}

# manifest_skipped <file> — echo "name<TAB>reason" for every app NOT probed, so
Expand All @@ -110,8 +153,15 @@ manifest_skipped() {
# extra_targets — the documented-absent four, same "name<TAB>domain" shape.
extra_targets() {
printf '%s\n' "$EXTRA_TARGETS" \
| awk -F'|' -v HEALTH="$DEFAULT_HEALTH_PATH" \
'NF>=2 { print $1 "\t" $2 "\t" (NF>=3 && $3 != "" ? $3 : HEALTH) }'
| awk -F'|' -v HEALTH="$DEFAULT_HEALTH_PATH" -v PATHS="$HEALTH_PATHS" '
BEGIN {
n = split(PATHS, lines, "\n")
for (i = 1; i <= n; i++) {
if (split(lines[i], f, "|") == 2 && f[1] != "") declared[f[1]] = f[2]
}
}
function health_for(name) { return (name in declared) ? declared[name] : HEALTH }
NF>=2 { print $1 "\t" $2 "\t" health_for($1) }'
}

# health_verdict <http_code> — what /api/health told us.
Expand All @@ -129,6 +179,55 @@ health_verdict() {
esac
}

# is_redirect <http_code> — 3xx, the code that means "ask somewhere else".
is_redirect() { case "$1" in 3??) return 0 ;; *) return 1 ;; esac; }

# url_path <url> — the path, without query or fragment, trailing slash trimmed.
# `https://h/api/health?x=1` -> `/api/health`; `https://h` -> `/`.
url_path() {
local rest="${1#*://}"
case "$rest" in
*/*) rest="/${rest#*/}" ;;
*) rest="/" ;;
esac
rest="${rest%%\?*}"
rest="${rest%%#*}"
[ "$rest" = / ] || rest="${rest%/}"
printf '%s' "$rest"
}

# url_host <url> — the hostname, for saying WHERE we ended up in the report.
url_host() {
local rest="${1#*://}"
printf '%s' "${rest%%/*}"
}

# same_path_redirect <requested_path> <redirect_url> — did this redirect only
# move HOSTS, keeping the path we asked for?
#
# A 3xx on the health path is two different things wearing one status code, and
# they need opposite treatment:
#
# aoz-wohnen.orangecat.ch 308s to aoz.orangecat.ch — same path, canonical
# host. The health route exists and answers 200; we were asking the wrong
# hostname and reporting LIMITED for an app that has a working check. Worse,
# had its database died the redirect would still have been a 308 and the
# sweep would still have said "serving".
#
# petvity 307s /api/health to /login?returnTo=%2Fapi%2Fhealth — an auth wall.
# Following that returns 200 from a LOGIN PAGE. A blanket `curl -L` would
# report petvity UP on the strength of a form: a false green, which is worse
# than the blind spot it set out to fix.
#
# So: follow it only when the path survives.
same_path_redirect() {
local want="$1" got
[ -n "${2:-}" ] || return 1
got="$(url_path "$2")"
[ "$want" = / ] || want="${want%/}"
[ "$got" = "$want" ]
}

# root_verdict <http_code> — the weaker fallback question: does it serve at all?
# 3xx counts as serving: several apps redirect `/` to a locale or a canonical
# host (revampit, petvity, vitareba all do) and that is a working app.
Expand Down Expand Up @@ -178,6 +277,11 @@ probe_code() {
normalize_code "$(curl -sS -o /dev/null -w '%{http_code}' --max-time "$TIMEOUT" "$1" 2>/dev/null || true)"
}

# probe_redirect <url> — echo the Location this URL redirects to, or nothing.
probe_redirect() {
curl -sS -o /dev/null -w '%{redirect_url}' --max-time "$TIMEOUT" "$1" 2>/dev/null || true
}

# probe_app <domain> — echo "<state> <detail>".
#
# Retries only a NEGATIVE result. A 200 is believed immediately; a failure is
Expand All @@ -193,6 +297,25 @@ probe_app() {
[ "$try" -lt "$TRIES" ] && sleep "$SLEEP"
done

# Before giving up on the health route, check whether the 3xx was merely the
# canonical host telling us where it lives. Only a path-preserving redirect
# counts — see same_path_redirect for why following an auth wall is worse
# than not following at all.
if [ "$verdict" = absent ] && is_redirect "$code"; then
local target
target=$(probe_redirect "https://$domain$health_path")
if same_path_redirect "$health_path" "$target"; then
for (( try = 1; try <= TRIES; try++ )); do
code=$(probe_code "$target")
verdict=$(health_verdict "$code")
[ "$verdict" = up ] && { echo "up health:$code (redirected to $(url_host "$target"))"; return; }
[ "$verdict" = absent ] && break
[ "$try" -lt "$TRIES" ] && sleep "$SLEEP"
done
[ "$verdict" = down ] && { echo "down health:$code (redirected to $(url_host "$target"))"; return; }
fi
fi

if [ "$verdict" = absent ]; then
local rcode rverdict
for (( try = 1; try <= TRIES; try++ )); do
Expand Down
Loading