diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a417bfc5..a6dd84d9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,17 @@ jobs: - name: Run tests run: npm test -- --run + shell-lint: + name: Shell script lint + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + + - name: shellcheck deploy scripts + # shellcheck ships preinstalled on ubuntu-latest. + run: shellcheck api/deploy/backup/*.sh + test-backend: name: Backend tests runs-on: ubuntu-latest diff --git a/api/deploy/backup/cercol-db-backup.sh b/api/deploy/backup/cercol-db-backup.sh new file mode 100755 index 000000000..55baacc07 --- /dev/null +++ b/api/deploy/backup/cercol-db-backup.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Nightly two-leg backup of the cercol PostgreSQL database. +# Spec: docs/decisions/0017-database-backups-two-leg.md +# +# Leg 1: pg_dump -Fc of the cercol database only, kept on-box under +# /var/backups/cercol/ (7 most recent dumps). +# Leg 2: the same dump, gpg-encrypted (AES256, symmetric), pushed to +# Google Drive via rclone (30-day retention on the remote). +# +# Runs as root from /etc/cron.d/cercol-db-backup (see that file for the +# one-time install steps). The dump itself runs as postgres via runuser +# (peer auth); root owns the passphrase file, the backup directory and +# the rclone config. +# +# Failure contract: exits non-zero on ANY failure, appends a +# "BACKUP FAILED" line to the log (the cron line redirects output to +# /home/cercol/logs/db-backup.log) and leaves a FAILED marker file that +# the next successful run removes. No silent deaths. + +set -euo pipefail + +BACKUP_DIR="/var/backups/cercol" +PASSPHRASE_FILE="/root/.cercol-backup-passphrase" +RCLONE_REMOTE="gdrive:cercol-db-backups" +LOCAL_KEEP=7 +REMOTE_KEEP_DAYS=30 +LOG_DIR="/home/cercol/logs" +FAIL_MARKER="$LOG_DIR/db-backup.FAILED" + +ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } + +fail() { + echo "$(ts) BACKUP FAILED: $1" >&2 + mkdir -p "$LOG_DIR" + echo "$(ts) $1" > "$FAIL_MARKER" + exit 1 +} +trap 'fail "command on line $LINENO exited non-zero"' ERR + +# postgres cannot read root's cwd; avoid the noisy "could not change +# directory" warning from runuser/pg_dump. +cd / + +[ "$(id -u)" -eq 0 ] || fail "must run as root" +[ -r "$PASSPHRASE_FILE" ] || fail "passphrase file $PASSPHRASE_FILE missing or unreadable" +command -v rclone > /dev/null || fail "rclone not installed" +rclone listremotes | grep -q '^gdrive:$' || fail "rclone remote 'gdrive' not configured" + +# Backup dir: root-owned, postgres can read (needed for the restore +# test in the runbook, which runs pg_restore as postgres). +install -d -m 0750 -o root -g postgres "$BACKUP_DIR" + +stamp=$(date -u +%Y%m%dT%H%M%SZ) +dump="$BACKUP_DIR/cercol-$stamp.dump" + +# Leg 1: local dump. -Fc is already zlib-compressed; no extra gzip. +runuser -u postgres -- pg_dump -Fc --dbname=cercol > "$dump" +chown root:postgres "$dump" +chmod 0640 "$dump" +[ -s "$dump" ] || fail "dump file is empty" + +# Integrity check: a corrupt archive fails to list. +runuser -u postgres -- pg_restore --list "$dump" > /dev/null + +# Local rotation: keep the LOCAL_KEEP most recent dumps. Filenames +# embed a UTC timestamp (cercol-YYYYMMDDTHHMMSSZ.dump), so reverse +# lexical order IS reverse chronological order. +shopt -s nullglob +printf '%s\n' "$BACKUP_DIR"/cercol-*.dump | sort -r | tail -n +$((LOCAL_KEEP + 1)) | xargs -r rm -- +shopt -u nullglob + +# Leg 2: encrypt and push off-box, then drop the local ciphertext. +gpg --batch --yes --pinentry-mode loopback \ + --symmetric --cipher-algo AES256 \ + --passphrase-file "$PASSPHRASE_FILE" \ + -o "$dump.gpg" "$dump" +rclone copy "$dump.gpg" "$RCLONE_REMOTE" +rclone delete --min-age "${REMOTE_KEEP_DAYS}d" "$RCLONE_REMOTE" +rm -f "$dump.gpg" + +rm -f "$FAIL_MARKER" +echo "$(ts) BACKUP OK: $(basename "$dump") ($(du -h "$dump" | cut -f1)) local, encrypted copy pushed to $RCLONE_REMOTE" diff --git a/api/deploy/cron/cercol-db-backup b/api/deploy/cron/cercol-db-backup new file mode 100644 index 000000000..fc67226a6 --- /dev/null +++ b/api/deploy/cron/cercol-db-backup @@ -0,0 +1,24 @@ +# Cercol - nightly two-leg backup of the cercol PostgreSQL database. +# Runs api/deploy/backup/cercol-db-backup.sh as root: local pg_dump +# rotation plus gpg-encrypted rclone push to Google Drive. +# Spec: docs/decisions/0017-database-backups-two-leg.md +# +# Install: see the "Database backups" section of docs/ops/runbook.md +# for the full one-time block (passphrase file, rclone OAuth config, +# log directory, cron install, first run, restore test). +# +# Verify after install: +# ls -la /etc/cron.d/cercol-db-backup +# tail -2 /home/cercol/logs/db-backup.log # after 03:15 UTC +# ls -la /home/cercol/logs/db-backup.FAILED # must NOT exist +# +# Output is redirected to the log file on purpose: the script owns its +# failure signalling (FAILED marker + non-zero exit), unlike the +# MAILTO="" pattern of the older crons that swallows errors. + +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Daily at 03:15 UTC: offset from the 02:00 crawl parser, the 03:00 +# Sunday Bing ingest and the 04:00 token purge. +15 3 * * * root /home/cercol/api/api/deploy/backup/cercol-db-backup.sh >> /home/cercol/logs/db-backup.log 2>&1 diff --git a/docs/decisions/0017-database-backups-two-leg.md b/docs/decisions/0017-database-backups-two-leg.md new file mode 100644 index 000000000..729a6785e --- /dev/null +++ b/docs/decisions/0017-database-backups-two-leg.md @@ -0,0 +1,124 @@ +# ADR 0017: Database backups, two-leg strategy + +- **Number**: 0017 +- **Title**: Two-leg backup strategy for the cercol PostgreSQL database +- **Status**: Accepted +- **Date**: 2026-07-05 (proposed and accepted same day; design signed + off by the operator before implementation) + +## Context + +The cercol PostgreSQL database (~20 MB) holds everything the project +cannot regenerate: profiles, auth tables, results (the raw material for +the Phase 4+ team instrument), witness sessions, groups, blog posts and +funnel events. Until now it had no logical backup at all: the July 2026 +reliability audit confirmed no `pg_dump` anywhere on the server, and +`docs/ops/runbook.md` carried the TODO "Backups are the operator's +responsibility (`pg_dump`). Frequency and offsite copy procedure: TODO +document." This ADR closes that TODO. + +Hetzner Cloud machine-level backups ARE enabled on the VPS (billed as +"Backup (20% of instance price)" on the invoice): whole-VM, +crash-consistent, 7-day retention, stored with the same provider under +the same account. They are the coarse fallback, not a substitute: + +- no granular restore (whole-VM rollback also rolls back topquaranta, + which shares the box), +- no off-provider copy (a lost account or provider incident loses both + the VM and its backups), +- crash-consistent only, no application-consistent guarantee for + PostgreSQL. + +## Decision + +Two legs, proportionate to actual risk on a shared VPS: + +- **Leg 1, logical-error recovery**: nightly `pg_dump` of the cercol + database only (never topquaranta's), custom format (`-Fc`), stored + on-box under `/var/backups/cercol/` with local rotation keeping the + 7 most recent dumps. Covers accidental deletes, bad migrations and + application bugs, restorable table by table with `pg_restore`. +- **Leg 2, disk/box-loss recovery**: the same dump encrypted and pushed + off-box to Google Drive via rclone, with its own retention (30 days). + Google Drive is chosen because it is free, already in use by the + operator, and decoupled from the pending Cloudflare Pages migration + (ADR 0013). + +Implementation details (resolved during implementation, see +`api/deploy/backup/cercol-db-backup.sh`): + +- **Encryption: gpg symmetric (AES256)** with a passphrase read from a + root-only file at `/root/.cercol-backup-passphrase`. gpg 2.2 is + already installed on the server; `age` is not, and installing a new + tool for one `gpg --symmetric` call fails the proportionality test. + The passphrase file is created by the operator at install time and is + NEVER committed to the repository. +- **rclone auth: interactive OAuth token flow (user-owned quota)**, not + a service account. On a personal Google account, service-account + uploads can land in the service account's own 15 GB quota instead of + the user's Drive, and the SA identity would be one more secret to + manage. The OAuth token is created interactively by the operator + (`rclone config`) during the one-time manual install step; rclone + refreshes it automatically afterwards. +- **Schedule: daily at 03:15 UTC**, offset from the existing crons + (02:00 crawl parser, 03:00 Sunday Bing ingest, 04:00 token purge, + 05:00 daily jobs). Runs as root: the dump itself is executed via + `runuser -u postgres` (peer auth), while root owns the passphrase + file, the backup directory and the rclone config. +- **No silent deaths**: the script exits non-zero on any failure, logs + every run to `/home/cercol/logs/db-backup.log` (the cron line + redirects stdout and stderr there), and maintains a + `/home/cercol/logs/db-backup.FAILED` marker file that is written on + failure and removed on the next success, so a human or a future + digest job can detect a broken backup without reading cron mail. + This deliberately breaks with the `MAILTO=""` pattern of the other + crons, which has let jobs die silently for weeks, twice. + +## Alternatives considered + +- **Hetzner Storage Box**: rejected on cost. Hard constraint for this + decision: no new spend. +- **Cloudflare R2**: rejected to avoid coupling the backup path to an + incomplete migration (ADR 0013 is still Proposed); if that migration + is later accepted, moving Leg 2 to R2 is a one-line rclone remote + change. +- **Backblaze B2**: rejected to avoid a new account and identity to + manage for a 20 MB payload. +- **Committing encrypted dumps to Git**: rejected on security grounds + (user PII in a public repository, even encrypted, with no revocation + story) and repo hygiene (unbounded binary growth). + +## Consequences + +- One manual install step on the server, mirroring the existing cron + deployment pattern: files live in the repo + (`api/deploy/backup/cercol-db-backup.sh`, + `api/deploy/cron/cercol-db-backup`), the operator installs once. The + exact install block lives in `docs/ops/runbook.md`. +- Restore procedures (on-box, off-box, and the Hetzner whole-VM last + resort) and a quarterly restore test are documented in + `docs/ops/runbook.md`, replacing the TODO. +- Restore capability is only proven by restoring: the runbook documents + a scratch-database restore test to run once after install and then + quarterly. +- Follow-up, out of scope here: standardise output redirection and + failure alerting across the other existing cron jobs; the current + `MAILTO=""` pattern lets crons die silently for weeks (proven twice: + crawl-parser since 2026-05-28, and the 2026-04-16 Caddy outage class + of silence). + +## Related + +- `docs/ops/runbook.md` "Database backups" section: install block, + restore procedures, quarterly restore test (replaces the old TODO). +- `api/deploy/backup/cercol-db-backup.sh` and + `api/deploy/cron/cercol-db-backup`: the implementation this ADR + specifies. +- ADR 0013 (Cloudflare front hosting, Proposed): rejected R2 coupling; + if 0013 is accepted later, Leg 2 can move to R2 with a one-line + remote change. +- ADR 0006 (cron pattern for SEO ingest): the repo-ships-cron, + operator-installs-once pattern this decision reuses, minus its + silent-failure mode. +- 2026-07-05 reliability audit: confirmed no pg_dump existed anywhere + on the server and that Hetzner machine-level backups are enabled. diff --git a/docs/ops/runbook.md b/docs/ops/runbook.md index 461fd4c2d..b306fd25e 100644 --- a/docs/ops/runbook.md +++ b/docs/ops/runbook.md @@ -350,8 +350,99 @@ read files under `/home/cercol/api`. cd /home/cercol/api && sudo -u postgres psql cercol < db/migrations/-.sql ``` -Backups are the operator's responsibility (`pg_dump`). Frequency -and offsite copy procedure: TODO document. +### Database backups (ADR 0017) + +Two legs, both driven by `api/deploy/backup/cercol-db-backup.sh` +(root cron, daily 03:15 UTC, `/etc/cron.d/cercol-db-backup`): + +- **Leg 1 (on-box)**: `pg_dump -Fc` of the cercol database only, under + `/var/backups/cercol/`, 7 most recent dumps kept. +- **Leg 2 (off-box)**: the same dump gpg-encrypted (AES256 symmetric, + passphrase in root-only `/root/.cercol-backup-passphrase`) and pushed + via rclone to the `gdrive:cercol-db-backups` Drive folder, 30-day + retention. + +Health check: `tail /home/cercol/logs/db-backup.log` and confirm +`/home/cercol/logs/db-backup.FAILED` does NOT exist. The marker is +written on any failure and removed on the next success. + +#### One-time install (as root on the server) + +``` +# 1. Passphrase file (fill in a strong passphrase; ALSO store it in the +# operator's password manager: without it the off-box copies are +# unrecoverable after box loss). +[ -f /root/.cercol-backup-passphrase ] || \ + printf '%s' 'REPLACE-WITH-STRONG-PASSPHRASE' > /root/.cercol-backup-passphrase +chmod 0600 /root/.cercol-backup-passphrase + +# 2. rclone + Drive remote named exactly "gdrive" (interactive OAuth: +# pick "drive", scope "drive.file" is enough, follow the browser flow). +command -v rclone > /dev/null || apt-get install -y rclone +rclone listremotes | grep -q '^gdrive:$' || rclone config + +# 3. Log directory. +install -d -o cercol -g cercol /home/cercol/logs + +# 4. Cron. +install -m 0644 /home/cercol/api/api/deploy/cron/cercol-db-backup /etc/cron.d/ + +# 5. First run by hand + restore test (next section). +chmod +x /home/cercol/api/api/deploy/backup/cercol-db-backup.sh +/home/cercol/api/api/deploy/backup/cercol-db-backup.sh +tail -1 /home/cercol/logs/db-backup.log # expect "BACKUP OK: ..." +``` + +#### Restore: from an on-box dump (logical errors, bad migration) + +``` +# as root +latest=$(ls -1t /var/backups/cercol/cercol-*.dump | head -1) +runuser -u postgres -- pg_restore --list "$latest" | head # sanity: archive readable +# Full restore into the live database (DESTRUCTIVE, think first; +# prefer the scratch restore below to inspect data): +runuser -u postgres -- pg_restore --clean --if-exists --no-owner \ + --dbname=cercol "$latest" +``` + +For a single table, restore into a scratch database (next section) and +copy the rows across with `psql` instead of touching the live database. + +#### Restore: from the off-box copy (box loss) + +``` +# on any machine with rclone configured for the same Drive account +rclone ls gdrive:cercol-db-backups # pick the newest .dump.gpg +rclone copy gdrive:cercol-db-backups/cercol-.dump.gpg /tmp/ +gpg --batch --pinentry-mode loopback \ + --passphrase-file /root/.cercol-backup-passphrase \ + -o /tmp/cercol-restore.dump -d /tmp/cercol-.dump.gpg +# then restore as in the on-box section, pointing at /tmp/cercol-restore.dump +``` + +The passphrase is NOT in the repo and NOT in Drive. If the box is lost +the passphrase must come from the operator's password manager; storing +it there is part of the install step above. + +#### Restore test (run once after install, then quarterly) + +``` +# as root; restores the newest dump into a scratch DB, checks a core +# table, drops the scratch DB. Safe to run on the live server. +latest=$(ls -1t /var/backups/cercol/cercol-*.dump | head -1) +runuser -u postgres -- createdb cercol_restore_test +runuser -u postgres -- pg_restore --no-owner --dbname=cercol_restore_test "$latest" +runuser -u postgres -- psql -d cercol_restore_test -tc "SELECT COUNT(*) FROM results;" +# expect a plausible row count (compare: same query against cercol) +runuser -u postgres -- dropdb cercol_restore_test +``` + +#### Last resort: Hetzner machine-level backups + +Hetzner Cloud automatic backups are enabled on the VPS (whole-VM, +crash-consistent, 7-day retention, restore from the Hetzner console). +They roll back the ENTIRE machine including topquaranta, so they are +the last resort for total box loss, never for cercol-only recovery. ### Blog slug redirects (Phase 17.10)