diff --git a/.gitignore b/.gitignore index 310f733..28082a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ target/ .worktrees/ .env +# production env on the host, and its dated backups (docs/runbooks/deploy-server.md) +.env.prod* graphify-out/ .cargo-home/ diff --git a/ansible/drill-seed.yml b/ansible/drill-seed.yml index eadca10..6775a8b 100644 --- a/ansible/drill-seed.yml +++ b/ansible/drill-seed.yml @@ -118,7 +118,7 @@ - name: Start the server ansible.builtin.command: - cmd: docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml up -d server + cmd: docker compose -p voltius --env-file .env.prod -f compose.prod.yml up -d server chdir: "{{ voltius_root }}/voltius-server" changed_when: true diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index db1d92d..a3b1e60 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -1,7 +1,7 @@ voltius_root: /home/ubuntu/fourretout voltius_repo: https://github.com/VoltiusApp/server voltius_ref: main -# TRUSTED_PROXIES in .env.dockhand names this subnet. A different one silently +# TRUSTED_PROXIES in .env.prod names this subnet. A different one silently # puts every client in one rate-limit bucket. voltius_cloudflare_subnet: 172.22.0.0/16 voltius_db_volume: voltius-db_db-data diff --git a/ansible/migrate.yml b/ansible/migrate.yml index 5ede2e3..39b8641 100644 --- a/ansible/migrate.yml +++ b/ansible/migrate.yml @@ -137,7 +137,7 @@ path: "{{ voltius_root }}/{{ item }}" register: voltius_secret_state loop: - - voltius-server/.env.dockhand + - voltius-server/.env.prod - voltius-db/.env.db - name: Note that the secrets are already in place @@ -177,14 +177,14 @@ path: "{{ voltius_root }}/{{ item }}" register: voltius_restored loop: - - voltius-server/.env.dockhand + - voltius-server/.env.prod - voltius-db/.env.db failed_when: not voltius_restored.stat.exists - name: Read the target's server tag ansible.builtin.slurp: - src: "{{ voltius_root }}/voltius-server/.env.dockhand" - register: voltius_tgt_dockhand_env + src: "{{ voltius_root }}/voltius-server/.env.prod" + register: voltius_tgt_prod_env no_log: true # The bundle is packed by hand and can predate the last deploy. An older binary @@ -195,11 +195,11 @@ fail_msg: >- the secrets bundle has SERVER_TAG={{ voltius_tgt_server_tag }} but {{ voltius_source }} runs {{ voltius_src_server_tag }}. Re-pack the bundle (docs/runbooks/deploy-server.md), delete - voltius-server/.env.dockhand and voltius-db/.env.db on {{ inventory_hostname }}, and run again. + voltius-server/.env.prod and voltius-db/.env.db on {{ inventory_hostname }}, and run again. vars: voltius_src_server_tag: "{{ hostvars[voltius_source].voltius_src_image.stdout | trim | split(':') | last }}" voltius_tgt_server_tag: >- - {{ voltius_tgt_dockhand_env.content | b64decode + {{ voltius_tgt_prod_env.content | b64decode | regex_findall('(?m)^SERVER_TAG=(.*)$') | last | default('unset') | trim }} - name: Read the target's copy of .env.db @@ -338,13 +338,13 @@ tasks: - name: Pull the pinned image ansible.builtin.command: - cmd: docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml pull -q server + cmd: docker compose -p voltius --env-file .env.prod -f compose.prod.yml pull -q server chdir: "{{ voltius_root }}/voltius-server" changed_when: true - name: Start it ansible.builtin.command: - cmd: docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml up -d server + cmd: docker compose -p voltius --env-file .env.prod -f compose.prod.yml up -d server chdir: "{{ voltius_root }}/voltius-server" changed_when: true diff --git a/ansible/rehearse.yml b/ansible/rehearse.yml index 51bebd9..b942be3 100644 --- a/ansible/rehearse.yml +++ b/ansible/rehearse.yml @@ -160,7 +160,7 @@ path: "{{ voltius_root }}/{{ item }}" state: absent loop: - - voltius-server/.env.dockhand + - voltius-server/.env.prod - voltius-db/.env.db - cloudflared/.env - cloudflared-api/.env diff --git a/ansible/roles/voltius_host/tasks/main.yml b/ansible/roles/voltius_host/tasks/main.yml index fc0f507..aac8136 100644 --- a/ansible/roles/voltius_host/tasks/main.yml +++ b/ansible/roles/voltius_host/tasks/main.yml @@ -73,7 +73,7 @@ path: "{{ voltius_root }}/{{ item }}" register: voltius_host_secret_files loop: - - voltius-server/.env.dockhand + - voltius-server/.env.prod - voltius-db/.env.db - name: Say so diff --git a/docs/runbooks/bootstrap-host.md b/docs/runbooks/bootstrap-host.md index 1779755..b1e8627 100644 --- a/docs/runbooks/bootstrap-host.md +++ b/docs/runbooks/bootstrap-host.md @@ -19,7 +19,7 @@ The server image itself comes from GHCR, so nothing is compiled on the host. ## The secrets bundle -`.env.dockhand`, `.env.db`, `cloudflared/.env`, `cloudflared-api/.env`, `voltius-tofu/.env.tofu` and +`voltius-server/.env.prod`, `.env.db`, `cloudflared/.env`, `cloudflared-api/.env`, `voltius-tofu/.env.tofu` and `voltius-tofu/oci_api_key.pem` exist nowhere else. No database backup contains them, and losing `JWT_SECRET` alone signs every user out permanently. Losing `TOTP_ENC_KEY` locks every account with two-factor sign-in out of its authenticator codes: the server refuses them @@ -30,15 +30,15 @@ tars them, encrypts them to an age recipient, and uploads them to ```sh cd /home/ubuntu/fourretout/voltius-db && set -a && . ./.env.db && set +a -AGE_RECIPIENTS=age1... \ - DOCKHAND_ENV_FILE=/var/lib/docker/volumes/dockhand_dockhand_data/_data/stacks/Docker/voltius/.env.dockhand \ - /path/to/scripts/secrets-bundle.sh pack +AGE_RECIPIENTS=age1... /home/ubuntu/fourretout/voltius-server/scripts/secrets-bundle.sh pack ``` -`DOCKHAND_ENV_FILE` is needed on **this** host and only for `.env.dockhand`: the live copy sits in -the dockhand volume rather than under `$ROOT`, and is root-owned, so `pack` reads it through -passwordless sudo. Without it the run still succeeds and uploads a bundle with every production -secret missing. A rebuilt host keeps its env files under `$ROOT` and needs none of this. +Every file is read from under `$ROOT`. `pack` skips an unreadable file with a warning rather than +failing, so read its `packed N file(s)` line: anything below six means a secret is missing from the +upload. + +Bundles packed before 2026-10-18 carry the server env as `voltius-server/.env.dockhand`; `unpack` +restores it as `.env.prod`, so an old bundle still rebuilds a host. **Run it after every change to any of those files**, and keep the age private key off this machine — a key stored beside the bundle protects nothing. `verify` lists what the bucket holds. @@ -61,7 +61,7 @@ git clone --depth=1 https://github.com/VoltiusApp/server /tmp/voltius-server ``` In order, it installs Docker and age, creates the `cloudflare` network **on the fixed subnet -172.22.0.0/16** (`TRUSTED_PROXIES` in `.env.dockhand` names that subnet — a different one silently +172.22.0.0/16** (`TRUSTED_PROXIES` in `.env.prod` names that subnet — a different one silently puts every client in one rate-limit bucket), lays out `voltius-db/`, `voltius-server/`, `voltius-tofu/` and `cloudflared/` under `/home/ubuntu/fourretout`, installs the systemd path unit that copies the OpenTofu state to R2 whenever it changes, checks the three env files are present, then @@ -74,10 +74,8 @@ The generated `cloudflared/compose.yml` reads the tunnel token from `TUNNEL_TOKE `cloudflared/.env`. Do not put it back on the command line — anything that can run `docker inspect` can read a command line. -The layout it produces is not quite the one the current box grew into: today the server's compose -tree and `.env.dockhand` live inside the dockhand Docker volume (`deploy-server.md`), not in -`/home/ubuntu/fourretout/voltius-server`. A rebuilt host gets the plain directory, and -`secrets-bundle.sh` reads `.env.dockhand` from there — pack it from wherever it actually is. +This is the same layout the current box uses: since 2026-10-18 the server's tree and `.env.prod` +live in `/home/ubuntu/fourretout/voltius-server` (`deploy-server.md`), not in a Docker volume. ## What this does not cover @@ -89,7 +87,7 @@ tree and `.env.dockhand` live inside the dockhand Docker volume (`deploy-server. rebuildable from the import blocks even without that. - **The API tunnel's connector.** `api.voltius.app` is served by whichever host runs the `voltius-api` connector; `ansible/connector.yml` starts it once the server is healthy (see `ansible/README.md`). -- **Anything that is not voltius** — other services on the host, dockhand, the dev containers. The +- **Anything that is not voltius** — other services on the host, the Docker UI, the dev containers. The script starts what production needs and nothing else. ## Verify diff --git a/docs/runbooks/deploy-server.md b/docs/runbooks/deploy-server.md index 80aa83f..32d99e6 100644 --- a/docs/runbooks/deploy-server.md +++ b/docs/runbooks/deploy-server.md @@ -10,14 +10,13 @@ Merging or pushing a commit does not change production. Only a pull + recreate d | Thing | Location | |---|---| -| Deployment tree | `/var/lib/docker/volumes/dockhand_dockhand_data/_data/stacks/Docker/voltius` | +| Deployment tree | `/home/ubuntu/fourretout/voltius-server` | | Compose project | `-p voltius -f compose.prod.yml` | -| Environment | `.env.dockhand` in that tree (untracked, never in git, must survive every operation) | +| Environment | `.env.prod` in that tree (untracked, gitignored, mode 600, must survive every operation) | | Image | `ghcr.io/voltiusapp/voltius-server:sha-` | -The tree is root-owned and `/var/lib/docker` is `0710`, so every command runs as -`sudo sh -c 'cd && …'`. A plain `cd` fails. Read the path back off the container rather than -trusting this document: +The tree is owned by `ubuntu`, which is in the `docker` group, so no command below needs `sudo`. +Read the path back off the container rather than trusting this document: ```sh docker inspect voltius-server --format '{{index .Config.Labels "com.docker.compose.project.working_dir"}}' @@ -31,7 +30,7 @@ branch name, as one multi-arch manifest covering `linux/amd64` and `linux/arm64` Deploy `sha-`, never `latest`. `latest` is mutable and overlapping pushes race on it, so `latest` cannot be rolled back to and cannot be reasoned about after the fact. -`SERVER_TAG` in `.env.dockhand` selects the tag. `compose.prod.yml` refuses to start without it. +`SERVER_TAG` in `.env.prod` selects the tag. `compose.prod.yml` refuses to start without it. ## Deploy @@ -56,13 +55,14 @@ Deploy `sha-`, never `latest`. `latest` is mutable and overlapping pushes 3. Pull first, so the download is not part of the outage window: ```sh - sudo sh -c 'cd && SERVER_TAG=sha-'$SHA' docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml pull server' + cd /home/ubuntu/fourretout/voltius-server + SERVER_TAG=sha-$SHA docker compose -p voltius --env-file .env.prod -f compose.prod.yml pull server ``` -4. Record the tag you are leaving, for rollback, then set the new one in `.env.dockhand`: +4. Record the tag you are leaving, for rollback, then set the new one in `.env.prod`: ```sh - sudo sh -c 'cd && grep ^SERVER_TAG .env.dockhand' + grep ^SERVER_TAG .env.prod ``` Edit `SERVER_TAG=sha-` in place. Do not rewrite the file wholesale; it holds every @@ -71,7 +71,7 @@ Deploy `sha-`, never `latest`. `latest` is mutable and overlapping pushes 5. Recreate and wait for health: ```sh - sudo sh -c 'cd && docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml up -d server' + docker compose -p voltius --env-file .env.prod -f compose.prod.yml up -d server docker inspect voltius-server --format '{{.State.Health.Status}}' ``` @@ -80,7 +80,7 @@ Deploy `sha-`, never `latest`. `latest` is mutable and overlapping pushes at startup. 6. Once the checks under Verify pass, re-pack the secrets bundle (`bootstrap-host.md`, "The secrets - bundle"). It carries `.env.dockhand`, so until then it names the tag you just left, and + bundle"). It carries `.env.prod`, so until then it names the tag you just left, and `ansible/migrate.yml` refuses to move a host whose bundle disagrees with the image it runs. ## Verify @@ -134,13 +134,29 @@ the tunnel at `https://api.voltius.app`. The steps, for reference or for a second host: 1. Confirm the tree's tracked files are clean and at the commit whose image you are about to - deploy. `git status --porcelain` in the tree shows untracked leftovers (`.env.dockhand` and its + deploy. `git status --porcelain` in the tree shows untracked leftovers (`.env.prod` and its backups, `build.rs`, orphan `src/*.rs` files); those are expected and are not copied into a CI-built image. 2. Keep the last locally built image as a fallback: `docker tag voltius-server:latest voltius-server:pre-ghcr-`. -3. Add `SERVER_TAG=sha-` to `.env.dockhand`. +3. Add `SERVER_TAG=sha-` to `.env.prod`. 4. Update `compose.prod.yml` in the tree to this repo's version (`image:` instead of `build: .`). 5. Deploy as above. Local builds took 13–17 minutes on this host's 2 cores, competing with the running server for CPU the whole time. CI finishes sooner and the host no longer needs a Rust toolchain or a build cache. + +## Move out of the dockhand volume — done 2026-10-18 + +Until 2026-10-18 the deployment tree lived at +`/var/lib/docker/volumes/dockhand_dockhand_data/_data/stacks/Docker/voltius`, root-owned, with the +environment in `.env.dockhand`: Dockhand had created the stack and the files stayed in its volume. +Dockhand was then retired from the host, and the tree was copied to +`/home/ubuntu/fourretout/voltius-server` — the layout `bootstrap-host.sh` and `ansible/` already +produce — owned by `ubuntu`, with the environment renamed to `.env.prod` (mode 600, byte-identical). + +`docker compose config` gave the same output from both trees before the switch. The server was then +recreated with `--force-recreate --no-deps` on the same image (`sha-afe9104`), healthy after about +15 seconds, and `/health`, `/v1/billing/subscription` and `https://api.voltius.app/health/deep` +answered 200, 401 and 200. The container's compose labels now point at the new tree. + +The old copy in the dockhand volume is stale from that moment. Do not deploy from it. diff --git a/docs/runbooks/restore-database.md b/docs/runbooks/restore-database.md index 570c2ce..e36d4d5 100644 --- a/docs/runbooks/restore-database.md +++ b/docs/runbooks/restore-database.md @@ -124,5 +124,5 @@ Production was untouched: no port published, not on the `cloudflare` network, `a and `voltius-db` stayed healthy throughout. Recovery-time objective for the data is therefore about two minutes. The real outage window in a -host-loss scenario is dominated by provisioning a replacement host and restoring `.env.dockhand`, +host-loss scenario is dominated by provisioning a replacement host and restoring `.env.prod`, neither of which this drill covers. diff --git a/scripts/bootstrap-host.sh b/scripts/bootstrap-host.sh index d867f6b..a445c38 100755 --- a/scripts/bootstrap-host.sh +++ b/scripts/bootstrap-host.sh @@ -5,7 +5,7 @@ set -euo pipefail ROOT="${ROOT:-/home/ubuntu/fourretout}" REPO="${REPO:-https://github.com/VoltiusApp/server}" REF="${REF:-main}" -# TRUSTED_PROXIES in .env.dockhand names this subnet, so it is fixed, not Docker's default pick. +# TRUSTED_PROXIES in .env.prod names this subnet, so it is fixed, not Docker's default pick. CLOUDFLARE_SUBNET="${CLOUDFLARE_SUBNET:-172.22.0.0/16}" CHECK_ONLY=0 [ "${1:-}" = "--check" ] && CHECK_ONLY=1 @@ -127,11 +127,11 @@ UNIT check_secrets() { local missing=0 - for f in voltius-server/.env.dockhand voltius-db/.env.db cloudflared/.env; do + for f in voltius-server/.env.prod voltius-db/.env.db cloudflared/.env; do [ -r "$ROOT/$f" ] || { echo "[bootstrap] missing $ROOT/$f" >&2; missing=1; } done [ "$missing" -eq 0 ] || die "restore the env files first: AGE_IDENTITY= scripts/secrets-bundle.sh unpack" - grep -q '^SERVER_TAG=' "$ROOT/voltius-server/.env.dockhand" || die "SERVER_TAG is unset in .env.dockhand" + grep -q '^SERVER_TAG=' "$ROOT/voltius-server/.env.prod" || die "SERVER_TAG is unset in .env.prod" } start_db() { @@ -144,8 +144,8 @@ start_db() { } start_server() { - (cd "$ROOT/voltius-server" && docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml pull -q server \ - && docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml up -d server) + (cd "$ROOT/voltius-server" && docker compose -p voltius --env-file .env.prod -f compose.prod.yml pull -q server \ + && docker compose -p voltius --env-file .env.prod -f compose.prod.yml up -d server) } start_tunnel() { diff --git a/scripts/secrets-bundle.sh b/scripts/secrets-bundle.sh index cb84676..37783bb 100755 --- a/scripts/secrets-bundle.sh +++ b/scripts/secrets-bundle.sh @@ -6,12 +6,9 @@ BUNDLE_PREFIX="${BUNDLE_PREFIX:-voltius-prod/secrets}" AGE_RECIPIENTS="${AGE_RECIPIENTS:-}" AGE_IDENTITY="${AGE_IDENTITY:-}" ROOT="${ROOT:-/home/ubuntu/fourretout}" -# On the current production host .env.dockhand lives inside the dockhand volume, -# not under $ROOT, and is root-owned. Point this at it there. -DOCKHAND_ENV_FILE="${DOCKHAND_ENV_FILE:-}" FILES=( - "voltius-server/.env.dockhand" + "voltius-server/.env.prod" "voltius-db/.env.db" "cloudflared/.env" "cloudflared-api/.env" @@ -46,13 +43,8 @@ rclone_run() { rm -f "$envfile" } -# Where a bundle entry is read from, which is $ROOT/ unless overridden. -source_for() { - case "$1" in - voltius-server/.env.dockhand) printf '%s\n' "${DOCKHAND_ENV_FILE:-$ROOT/$1}" ;; - *) printf '%s\n' "$ROOT/$1" ;; - esac -} +# Bundles packed before 2026-10-18 carry the server env under its old name. +LEGACY_SERVER_ENV="voltius-server/.env.dockhand" pack() { need age @@ -60,7 +52,7 @@ pack() { mkwork local staged=0 for f in "${FILES[@]}"; do - local src; src=$(source_for "$f") + local src="$ROOT/$f" if [ -r "$src" ]; then install -D -m 600 "$src" "$WORK/bundle/$f" staged=$((staged + 1)) @@ -91,6 +83,10 @@ unpack() { [ -e "$ROOT/$f" ] && die "$ROOT/$f already exists; move it aside first" done mkdir -p "$WORK/out" && tar -C "$WORK/out" -xf "$WORK/secrets.tar" + if [ ! -f "$WORK/out/voltius-server/.env.prod" ] && [ -f "$WORK/out/$LEGACY_SERVER_ENV" ]; then + mv "$WORK/out/$LEGACY_SERVER_ENV" "$WORK/out/voltius-server/.env.prod" + echo "bundle predates the rename: restoring $LEGACY_SERVER_ENV as voltius-server/.env.prod" + fi for f in "${FILES[@]}"; do [ -f "$WORK/out/$f" ] || { echo "not in bundle: $f" >&2; continue; } install -D -m 600 "$WORK/out/$f" "$ROOT/$f"