chore(ops): move the production tree out of the dockhand volume, rename .env.dockhand to .env.prod

.gitignore

@@ -1,6 +1,8 @@
 target/
 .worktrees/
 .env
+# production env on the host, and its dated backups (docs/runbooks/deploy-server.md)
+.env.prod*
 graphify-out/
 .cargo-home/
 

ansible/drill-seed.yml

@@ -118,7 +118,7 @@
 
     - name: Start the server
       ansible.builtin.command:
-        cmd: docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml up -d server
+        cmd: docker compose -p voltius --env-file .env.prod -f compose.prod.yml up -d server
         chdir: "{{ voltius_root }}/voltius-server"
       changed_when: true
 

ansible/group_vars/all.yml

@@ -1,7 +1,7 @@
 voltius_root: /home/ubuntu/fourretout
 voltius_repo: https://github.com/VoltiusApp/server
 voltius_ref: main
-# TRUSTED_PROXIES in .env.dockhand names this subnet. A different one silently
+# TRUSTED_PROXIES in .env.prod names this subnet. A different one silently
 # puts every client in one rate-limit bucket.
 voltius_cloudflare_subnet: 172.22.0.0/16
 voltius_db_volume: voltius-db_db-data

ansible/migrate.yml

@@ -137,7 +137,7 @@
         path: "{{ voltius_root }}/{{ item }}"
       register: voltius_secret_state
       loop:
-        - voltius-server/.env.dockhand
+        - voltius-server/.env.prod
         - voltius-db/.env.db
 
     - name: Note that the secrets are already in place
@@ -177,14 +177,14 @@
         path: "{{ voltius_root }}/{{ item }}"
       register: voltius_restored
       loop:
-        - voltius-server/.env.dockhand
+        - voltius-server/.env.prod
         - voltius-db/.env.db
       failed_when: not voltius_restored.stat.exists
 
     - name: Read the target's server tag
       ansible.builtin.slurp:
-        src: "{{ voltius_root }}/voltius-server/.env.dockhand"
-      register: voltius_tgt_dockhand_env
+        src: "{{ voltius_root }}/voltius-server/.env.prod"
+      register: voltius_tgt_prod_env
       no_log: true
 
     # The bundle is packed by hand and can predate the last deploy. An older binary
@@ -195,11 +195,11 @@
         fail_msg: >-
           the secrets bundle has SERVER_TAG={{ voltius_tgt_server_tag }} but {{ voltius_source }} runs
           {{ voltius_src_server_tag }}. Re-pack the bundle (docs/runbooks/deploy-server.md), delete
-          voltius-server/.env.dockhand and voltius-db/.env.db on {{ inventory_hostname }}, and run again.
+          voltius-server/.env.prod and voltius-db/.env.db on {{ inventory_hostname }}, and run again.
       vars:
         voltius_src_server_tag: "{{ hostvars[voltius_source].voltius_src_image.stdout | trim | split(':') | last }}"
         voltius_tgt_server_tag: >-
-          {{ voltius_tgt_dockhand_env.content | b64decode
+          {{ voltius_tgt_prod_env.content | b64decode
              | regex_findall('(?m)^SERVER_TAG=(.*)$') | last | default('unset') | trim }}
 
     - name: Read the target's copy of .env.db
@@ -338,13 +338,13 @@
   tasks:
     - name: Pull the pinned image
       ansible.builtin.command:
-        cmd: docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml pull -q server
+        cmd: docker compose -p voltius --env-file .env.prod -f compose.prod.yml pull -q server
         chdir: "{{ voltius_root }}/voltius-server"
       changed_when: true
 
     - name: Start it
       ansible.builtin.command:
-        cmd: docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml up -d server
+        cmd: docker compose -p voltius --env-file .env.prod -f compose.prod.yml up -d server
         chdir: "{{ voltius_root }}/voltius-server"
       changed_when: true
 

ansible/rehearse.yml

@@ -160,7 +160,7 @@
         path: "{{ voltius_root }}/{{ item }}"
         state: absent
       loop:
-        - voltius-server/.env.dockhand
+        - voltius-server/.env.prod
         - voltius-db/.env.db
         - cloudflared/.env
         - cloudflared-api/.env

ansible/roles/voltius_host/tasks/main.yml

@@ -73,7 +73,7 @@
     path: "{{ voltius_root }}/{{ item }}"
   register: voltius_host_secret_files
   loop:
-    - voltius-server/.env.dockhand
+    - voltius-server/.env.prod
     - voltius-db/.env.db
 
 - name: Say so

docs/runbooks/bootstrap-host.md

@@ -19,7 +19,7 @@ The server image itself comes from GHCR, so nothing is compiled on the host.
 
 ## The secrets bundle
 
-`.env.dockhand`, `.env.db`, `cloudflared/.env`, `cloudflared-api/.env`, `voltius-tofu/.env.tofu` and
+`voltius-server/.env.prod`, `.env.db`, `cloudflared/.env`, `cloudflared-api/.env`, `voltius-tofu/.env.tofu` and
 `voltius-tofu/oci_api_key.pem` exist nowhere else. No database backup contains
 them, and losing `JWT_SECRET` alone signs every user out permanently. Losing `TOTP_ENC_KEY` locks
 every account with two-factor sign-in out of its authenticator codes: the server refuses them
@@ -30,15 +30,15 @@ tars them, encrypts them to an age recipient, and uploads them to
 
 ```sh
 cd /home/ubuntu/fourretout/voltius-db && set -a && . ./.env.db && set +a
-AGE_RECIPIENTS=age1... \
-  DOCKHAND_ENV_FILE=/var/lib/docker/volumes/dockhand_dockhand_data/_data/stacks/Docker/voltius/.env.dockhand \
-  /path/to/scripts/secrets-bundle.sh pack
+AGE_RECIPIENTS=age1... /home/ubuntu/fourretout/voltius-server/scripts/secrets-bundle.sh pack
 ```
 
-`DOCKHAND_ENV_FILE` is needed on **this** host and only for `.env.dockhand`: the live copy sits in
-the dockhand volume rather than under `$ROOT`, and is root-owned, so `pack` reads it through
-passwordless sudo. Without it the run still succeeds and uploads a bundle with every production
-secret missing. A rebuilt host keeps its env files under `$ROOT` and needs none of this.
+Every file is read from under `$ROOT`. `pack` skips an unreadable file with a warning rather than
+failing, so read its `packed N file(s)` line: anything below six means a secret is missing from the
+upload.
+
+Bundles packed before 2026-10-18 carry the server env as `voltius-server/.env.dockhand`; `unpack`
+restores it as `.env.prod`, so an old bundle still rebuilds a host.
 
 **Run it after every change to any of those files**, and keep the age private key off this
 machine — a key stored beside the bundle protects nothing. `verify` lists what the bucket holds.
@@ -61,7 +61,7 @@ git clone --depth=1 https://github.com/VoltiusApp/server /tmp/voltius-server
 ```
 
 In order, it installs Docker and age, creates the `cloudflare` network **on the fixed subnet
-172.22.0.0/16** (`TRUSTED_PROXIES` in `.env.dockhand` names that subnet — a different one silently
+172.22.0.0/16** (`TRUSTED_PROXIES` in `.env.prod` names that subnet — a different one silently
 puts every client in one rate-limit bucket), lays out `voltius-db/`, `voltius-server/`,
 `voltius-tofu/` and `cloudflared/` under `/home/ubuntu/fourretout`, installs the systemd path unit
 that copies the OpenTofu state to R2 whenever it changes, checks the three env files are present, then
@@ -74,10 +74,8 @@ The generated `cloudflared/compose.yml` reads the tunnel token from `TUNNEL_TOKE
 `cloudflared/.env`. Do not put it back on the command line — anything that can run `docker inspect`
 can read a command line.
 
-The layout it produces is not quite the one the current box grew into: today the server's compose
-tree and `.env.dockhand` live inside the dockhand Docker volume (`deploy-server.md`), not in
-`/home/ubuntu/fourretout/voltius-server`. A rebuilt host gets the plain directory, and
-`secrets-bundle.sh` reads `.env.dockhand` from there — pack it from wherever it actually is.
+This is the same layout the current box uses: since 2026-10-18 the server's tree and `.env.prod`
+live in `/home/ubuntu/fourretout/voltius-server` (`deploy-server.md`), not in a Docker volume.
 
 ## What this does not cover
 
@@ -89,7 +87,7 @@ tree and `.env.dockhand` live inside the dockhand Docker volume (`deploy-server.
   rebuildable from the import blocks even without that.
 - **The API tunnel's connector.** `api.voltius.app` is served by whichever host runs the `voltius-api`
   connector; `ansible/connector.yml` starts it once the server is healthy (see `ansible/README.md`).
-- **Anything that is not voltius** — other services on the host, dockhand, the dev containers. The
+- **Anything that is not voltius** — other services on the host, the Docker UI, the dev containers. The
   script starts what production needs and nothing else.
 
 ## Verify

docs/runbooks/deploy-server.md

@@ -10,14 +10,13 @@ Merging or pushing a commit does not change production. Only a pull + recreate d
 
 | Thing | Location |
 |---|---|
-| Deployment tree | `/var/lib/docker/volumes/dockhand_dockhand_data/_data/stacks/Docker/voltius` |
+| Deployment tree | `/home/ubuntu/fourretout/voltius-server` |
 | Compose project | `-p voltius -f compose.prod.yml` |
-| Environment | `.env.dockhand` in that tree (untracked, never in git, must survive every operation) |
+| Environment | `.env.prod` in that tree (untracked, gitignored, mode 600, must survive every operation) |
 | Image | `ghcr.io/voltiusapp/voltius-server:sha-<short>` |
 
-The tree is root-owned and `/var/lib/docker` is `0710`, so every command runs as
-`sudo sh -c 'cd <tree> && …'`. A plain `cd` fails. Read the path back off the container rather than
-trusting this document:
+The tree is owned by `ubuntu`, which is in the `docker` group, so no command below needs `sudo`.
+Read the path back off the container rather than trusting this document:
 
 ```sh
 docker inspect voltius-server --format '{{index .Config.Labels "com.docker.compose.project.working_dir"}}'
@@ -31,7 +30,7 @@ branch name, as one multi-arch manifest covering `linux/amd64` and `linux/arm64`
 Deploy `sha-<short>`, never `latest`. `latest` is mutable and overlapping pushes race on it, so
 `latest` cannot be rolled back to and cannot be reasoned about after the fact.
 
-`SERVER_TAG` in `.env.dockhand` selects the tag. `compose.prod.yml` refuses to start without it.
+`SERVER_TAG` in `.env.prod` selects the tag. `compose.prod.yml` refuses to start without it.
 
 ## Deploy
 
@@ -56,13 +55,14 @@ Deploy `sha-<short>`, never `latest`. `latest` is mutable and overlapping pushes
 3. Pull first, so the download is not part of the outage window:
 
    ```sh
-   sudo sh -c 'cd <tree> && SERVER_TAG=sha-'$SHA' docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml pull server'
+   cd /home/ubuntu/fourretout/voltius-server
+   SERVER_TAG=sha-$SHA docker compose -p voltius --env-file .env.prod -f compose.prod.yml pull server
    ```
 
-4. Record the tag you are leaving, for rollback, then set the new one in `.env.dockhand`:
+4. Record the tag you are leaving, for rollback, then set the new one in `.env.prod`:
 
    ```sh
-   sudo sh -c 'cd <tree> && grep ^SERVER_TAG .env.dockhand'
+   grep ^SERVER_TAG .env.prod
    ```
 
    Edit `SERVER_TAG=sha-<new>` in place. Do not rewrite the file wholesale; it holds every
@@ -71,7 +71,7 @@ Deploy `sha-<short>`, never `latest`. `latest` is mutable and overlapping pushes
 5. Recreate and wait for health:
 
    ```sh
-   sudo sh -c 'cd <tree> && docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml up -d server'
+   docker compose -p voltius --env-file .env.prod -f compose.prod.yml up -d server
    docker inspect voltius-server --format '{{.State.Health.Status}}'
    ```
 
@@ -80,7 +80,7 @@ Deploy `sha-<short>`, never `latest`. `latest` is mutable and overlapping pushes
    at startup.
 
 6. Once the checks under Verify pass, re-pack the secrets bundle (`bootstrap-host.md`, "The secrets
-   bundle"). It carries `.env.dockhand`, so until then it names the tag you just left, and
+   bundle"). It carries `.env.prod`, so until then it names the tag you just left, and
    `ansible/migrate.yml` refuses to move a host whose bundle disagrees with the image it runs.
 
 ## Verify
@@ -134,13 +134,29 @@ the tunnel at `https://api.voltius.app`.
 The steps, for reference or for a second host:
 
 1. Confirm the tree's tracked files are clean and at the commit whose image you are about to
-   deploy. `git status --porcelain` in the tree shows untracked leftovers (`.env.dockhand` and its
+   deploy. `git status --porcelain` in the tree shows untracked leftovers (`.env.prod` and its
    backups, `build.rs`, orphan `src/*.rs` files); those are expected and are not copied into a
    CI-built image.
 2. Keep the last locally built image as a fallback: `docker tag voltius-server:latest voltius-server:pre-ghcr-<date>`.
-3. Add `SERVER_TAG=sha-<short>` to `.env.dockhand`.
+3. Add `SERVER_TAG=sha-<short>` to `.env.prod`.
 4. Update `compose.prod.yml` in the tree to this repo's version (`image:` instead of `build: .`).
 5. Deploy as above.
 
 Local builds took 13–17 minutes on this host's 2 cores, competing with the running server for CPU
 the whole time. CI finishes sooner and the host no longer needs a Rust toolchain or a build cache.
+
+## Move out of the dockhand volume — done 2026-10-18
+
+Until 2026-10-18 the deployment tree lived at
+`/var/lib/docker/volumes/dockhand_dockhand_data/_data/stacks/Docker/voltius`, root-owned, with the
+environment in `.env.dockhand`: Dockhand had created the stack and the files stayed in its volume.
+Dockhand was then retired from the host, and the tree was copied to
+`/home/ubuntu/fourretout/voltius-server` — the layout `bootstrap-host.sh` and `ansible/` already
+produce — owned by `ubuntu`, with the environment renamed to `.env.prod` (mode 600, byte-identical).
+
+`docker compose config` gave the same output from both trees before the switch. The server was then
+recreated with `--force-recreate --no-deps` on the same image (`sha-afe9104`), healthy after about
+15 seconds, and `/health`, `/v1/billing/subscription` and `https://api.voltius.app/health/deep`
+answered 200, 401 and 200. The container's compose labels now point at the new tree.
+
+The old copy in the dockhand volume is stale from that moment. Do not deploy from it.

docs/runbooks/restore-database.md

@@ -124,5 +124,5 @@ Production was untouched: no port published, not on the `cloudflare` network, `a
 and `voltius-db` stayed healthy throughout.
 
 Recovery-time objective for the data is therefore about two minutes. The real outage window in a
-host-loss scenario is dominated by provisioning a replacement host and restoring `.env.dockhand`,
+host-loss scenario is dominated by provisioning a replacement host and restoring `.env.prod`,
 neither of which this drill covers.

scripts/bootstrap-host.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 ROOT="${ROOT:-/home/ubuntu/fourretout}"
 REPO="${REPO:-https://github.com/VoltiusApp/server}"
 REF="${REF:-main}"
-# TRUSTED_PROXIES in .env.dockhand names this subnet, so it is fixed, not Docker's default pick.
+# TRUSTED_PROXIES in .env.prod names this subnet, so it is fixed, not Docker's default pick.
 CLOUDFLARE_SUBNET="${CLOUDFLARE_SUBNET:-172.22.0.0/16}"
 CHECK_ONLY=0
 [ "${1:-}" = "--check" ] && CHECK_ONLY=1
@@ -127,11 +127,11 @@ UNIT
 
 check_secrets() {
   local missing=0
-  for f in voltius-server/.env.dockhand voltius-db/.env.db cloudflared/.env; do
+  for f in voltius-server/.env.prod voltius-db/.env.db cloudflared/.env; do
     [ -r "$ROOT/$f" ] || { echo "[bootstrap] missing $ROOT/$f" >&2; missing=1; }
   done
   [ "$missing" -eq 0 ] || die "restore the env files first: AGE_IDENTITY=<key> scripts/secrets-bundle.sh unpack"
-  grep -q '^SERVER_TAG=' "$ROOT/voltius-server/.env.dockhand" || die "SERVER_TAG is unset in .env.dockhand"
+  grep -q '^SERVER_TAG=' "$ROOT/voltius-server/.env.prod" || die "SERVER_TAG is unset in .env.prod"
 }
 
 start_db() {
@@ -144,8 +144,8 @@ start_db() {
 }
 
 start_server() {
-  (cd "$ROOT/voltius-server" && docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml pull -q server \
-    && docker compose -p voltius --env-file .env.dockhand -f compose.prod.yml up -d server)
+  (cd "$ROOT/voltius-server" && docker compose -p voltius --env-file .env.prod -f compose.prod.yml pull -q server \
+    && docker compose -p voltius --env-file .env.prod -f compose.prod.yml up -d server)
 }
 
 start_tunnel() {

scripts/secrets-bundle.sh

@@ -6,12 +6,9 @@ BUNDLE_PREFIX="${BUNDLE_PREFIX:-voltius-prod/secrets}"
 AGE_RECIPIENTS="${AGE_RECIPIENTS:-}"
 AGE_IDENTITY="${AGE_IDENTITY:-}"
 ROOT="${ROOT:-/home/ubuntu/fourretout}"
-# On the current production host .env.dockhand lives inside the dockhand volume,
-# not under $ROOT, and is root-owned. Point this at it there.
-DOCKHAND_ENV_FILE="${DOCKHAND_ENV_FILE:-}"
 
 FILES=(
-  "voltius-server/.env.dockhand"
+  "voltius-server/.env.prod"
   "voltius-db/.env.db"
   "cloudflared/.env"
   "cloudflared-api/.env"
@@ -46,21 +43,16 @@ rclone_run() {
   rm -f "$envfile"
 }
 
-# Where a bundle entry is read from, which is $ROOT/<entry> unless overridden.
-source_for() {
-  case "$1" in
-    voltius-server/.env.dockhand) printf '%s\n' "${DOCKHAND_ENV_FILE:-$ROOT/$1}" ;;
-    *) printf '%s\n' "$ROOT/$1" ;;
-  esac
-}
+# Bundles packed before 2026-10-18 carry the server env under its old name.
+LEGACY_SERVER_ENV="voltius-server/.env.dockhand"
 
 pack() {
   need age
   [ -n "$AGE_RECIPIENTS" ] || die "AGE_RECIPIENTS is unset (age1... public key, or -R file)"
   mkwork
   local staged=0
   for f in "${FILES[@]}"; do
-    local src; src=$(source_for "$f")
+    local src="$ROOT/$f"
     if [ -r "$src" ]; then
       install -D -m 600 "$src" "$WORK/bundle/$f"
       staged=$((staged + 1))
@@ -91,6 +83,10 @@ unpack() {
     [ -e "$ROOT/$f" ] && die "$ROOT/$f already exists; move it aside first"
   done
   mkdir -p "$WORK/out" && tar -C "$WORK/out" -xf "$WORK/secrets.tar"
+  if [ ! -f "$WORK/out/voltius-server/.env.prod" ] && [ -f "$WORK/out/$LEGACY_SERVER_ENV" ]; then
+    mv "$WORK/out/$LEGACY_SERVER_ENV" "$WORK/out/voltius-server/.env.prod"
+    echo "bundle predates the rename: restoring $LEGACY_SERVER_ENV as voltius-server/.env.prod"
+  fi
   for f in "${FILES[@]}"; do
     [ -f "$WORK/out/$f" ] || { echo "not in bundle: $f" >&2; continue; }
     install -D -m 600 "$WORK/out/$f" "$ROOT/$f"