From 6ac31828e0b2ef5a645338351bd3d9deae1fd0c7 Mon Sep 17 00:00:00 2001
From: Steve Munini <smunini@gmail.com>
Date: Thu, 30 Jul 2026 11:53:00 -0700
Subject: [PATCH] ci: reap testcontainer volumes, not just containers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Every CI cleanup path force-removed its containers with `docker rm -f`,
which does not remove anonymous volumes. The postgres, elasticsearch,
mongo, minio and keycloak images all declare VOLUME in their Dockerfiles,
so each testcontainer created one — and each reap orphaned it permanently.

The shared self-hosted Docker host is an LXC container on a ZFS dataset
with a 200G refquota. The orphans filled it until containerd could not
write io.containerd.metadata.v1.bolt/meta.db, at which point *every*
container start failed with "disk quota exceeded" (EDQUOT, not ENOSPC —
the dataset quota, not a full disk). `docker system df` on the host still
showed 144 volumes with only 4 active after a manual prune.

Add `-v` to all 84 call sites across 16 workflows. This removes only
anonymous volumes; named volumes are never touched by `docker rm -v`, and
no workflow creates or mounts one — every volume in play here is anonymous.

This fixes the leak at the source. The nightly janitor in #451 remains
worth having as a backstop for runs that die before reaching cleanup, but
a 05:30 UTC sweep could not have saved a run that failed at 16:03.
---
 .github/workflows/audit-events.yml         | 24 +++++++++++-----------
 .github/workflows/bulk-export-smoke.yml    | 16 +++++++--------
 .github/workflows/bulk-submit-smoke.yml    | 16 +++++++--------
 .github/workflows/ci-extended.yml          |  4 ++--
 .github/workflows/ci.yml                   | 12 +++++++++--
 .github/workflows/cluster-smoke.yml        |  8 ++++----
 .github/workflows/fhir-benchmark.yml       |  8 ++++----
 .github/workflows/hts-benchmark.yml        |  4 ++--
 .github/workflows/hts-ig-conformance.yml   |  4 ++--
 .github/workflows/hts.yml                  |  2 +-
 .github/workflows/inferno-bulk-data.yml    | 14 ++++++-------
 .github/workflows/inferno-subscription.yml | 14 ++++++-------
 .github/workflows/inferno-us-core.yml      | 14 ++++++-------
 .github/workflows/subscriptions-smoke.yml  | 16 +++++++--------
 .github/workflows/ui-tests-matrix.yml      | 16 +++++++--------
 .github/workflows/ui-tests.yml             |  4 ++--
 16 files changed, 92 insertions(+), 84 deletions(-)

diff --git a/.github/workflows/audit-events.yml b/.github/workflows/audit-events.yml
index bb49d0f35..a45083797 100644
--- a/.github/workflows/audit-events.yml
+++ b/.github/workflows/audit-events.yml
@@ -229,7 +229,7 @@ jobs:
         if: matrix.backend == 'sqlite-elasticsearch' || matrix.backend == 's3-elasticsearch'
         run: |
           ES_CONTAINER="es-audit-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           docker run -d --name "$ES_CONTAINER" -p 0:9200 \
             -e "discovery.type=single-node" \
             -e "xpack.security.enabled=false" \
@@ -260,7 +260,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           PG_CONTAINER="pg-audit-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -291,7 +291,7 @@ jobs:
         if: matrix.backend == 'mongodb'
         run: |
           MONGO_CONTAINER="mongo-audit-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
 
           echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> $GITHUB_ENV
@@ -383,7 +383,7 @@ jobs:
       - name: Start ephemeral Keycloak
         run: |
           KC_CONTAINER="kc-audit-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$KC_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$KC_CONTAINER" 2>/dev/null || true
 
           docker create --name "$KC_CONTAINER" -p 0:8080 \
             -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
@@ -760,16 +760,16 @@ jobs:
           fi
 
           echo "Stopping Keycloak..."
-          docker rm -f "${KC_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${KC_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Stopping Elasticsearch..."
-          docker rm -f "${ES_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${ES_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Stopping PostgreSQL..."
-          docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Stopping MongoDB..."
-          docker rm -f "${MONGO_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${MONGO_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Cleanup complete"
 
@@ -826,7 +826,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           PG_CONTAINER="pg-audit-db-${{ matrix.mode }}-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -862,7 +862,7 @@ jobs:
         if: matrix.backend == 'mongodb'
         run: |
           MONGO_CONTAINER="mongo-audit-db-${{ matrix.mode }}-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
 
           echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> $GITHUB_ENV
@@ -1196,5 +1196,5 @@ jobs:
       - name: Cleanup smoke resources
         if: always()
         run: |
-          docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
-          docker rm -f "${MONGO_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${MONGO_CONTAINER:-none}" 2>/dev/null || true
diff --git a/.github/workflows/bulk-export-smoke.yml b/.github/workflows/bulk-export-smoke.yml
index 76d3a54f5..5a5ef8858 100644
--- a/.github/workflows/bulk-export-smoke.yml
+++ b/.github/workflows/bulk-export-smoke.yml
@@ -136,7 +136,7 @@ jobs:
         if: contains(matrix.backend, 'elasticsearch')
         run: |
           ES_CONTAINER="es-bulk-export-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
-          docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           docker run -d --name "$ES_CONTAINER" -p 0:9200 \
             -e "discovery.type=single-node" \
             -e "xpack.security.enabled=false" \
@@ -162,7 +162,7 @@ jobs:
         if: contains(matrix.backend, 'postgres')
         run: |
           PG_CONTAINER="pg-bulk-export-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -191,7 +191,7 @@ jobs:
         if: contains(matrix.backend, 'mongodb')
         run: |
           MONGO_CONTAINER="mongo-bulk-export-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
-          docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
           echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> "$GITHUB_ENV"
 
@@ -217,7 +217,7 @@ jobs:
         if: startsWith(matrix.backend, 's3')
         run: |
           MINIO_CONTAINER="minio-bulk-export-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
-          docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MINIO_CONTAINER" -p 0:9000 -p 0:9001 \
             -e MINIO_ROOT_USER=hfs-minio \
             -e MINIO_ROOT_PASSWORD=hfs-minio-secret \
@@ -499,14 +499,14 @@ jobs:
         if: always()
         run: |
           if [ -n "${ES_CONTAINER:-}" ]; then
-            docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${PG_CONTAINER:-}" ]; then
-            docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${MONGO_CONTAINER:-}" ]; then
-            docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${MINIO_CONTAINER:-}" ]; then
-            docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
           fi
diff --git a/.github/workflows/bulk-submit-smoke.yml b/.github/workflows/bulk-submit-smoke.yml
index 8941a6c73..2a8aac707 100644
--- a/.github/workflows/bulk-submit-smoke.yml
+++ b/.github/workflows/bulk-submit-smoke.yml
@@ -123,7 +123,7 @@ jobs:
         if: contains(matrix.backend, 'elasticsearch')
         run: |
           ES_CONTAINER="es-bulk-submit-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
-          docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           docker run -d --name "$ES_CONTAINER" -p 0:9200 \
             -e "discovery.type=single-node" \
             -e "xpack.security.enabled=false" \
@@ -149,7 +149,7 @@ jobs:
         if: contains(matrix.backend, 'postgres')
         run: |
           PG_CONTAINER="pg-bulk-submit-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -178,7 +178,7 @@ jobs:
         if: contains(matrix.backend, 'mongodb')
         run: |
           MONGO_CONTAINER="mongo-bulk-submit-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
-          docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
           echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> "$GITHUB_ENV"
 
@@ -204,7 +204,7 @@ jobs:
         if: startsWith(matrix.backend, 's3')
         run: |
           MINIO_CONTAINER="minio-bulk-submit-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
-          docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MINIO_CONTAINER" -p 0:9000 -p 0:9001 \
             -e MINIO_ROOT_USER=hfs-minio \
             -e MINIO_ROOT_PASSWORD=hfs-minio-secret \
@@ -562,14 +562,14 @@ jobs:
         if: always()
         run: |
           if [ -n "${ES_CONTAINER:-}" ]; then
-            docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${PG_CONTAINER:-}" ]; then
-            docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${MONGO_CONTAINER:-}" ]; then
-            docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${MINIO_CONTAINER:-}" ]; then
-            docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
           fi
diff --git a/.github/workflows/ci-extended.yml b/.github/workflows/ci-extended.yml
index b339b0611..2115dc971 100644
--- a/.github/workflows/ci-extended.yml
+++ b/.github/workflows/ci-extended.yml
@@ -50,7 +50,7 @@ jobs:
     steps:
       - name: Remove testcontainers from this run
         run: |
-          docker ps -aq --filter "label=github.run_id=${{ github.run_id }}" | xargs -r docker rm -f || true
+          docker ps -aq --filter "label=github.run_id=${{ github.run_id }}" | xargs -r docker rm -fv || true
 
       - name: Sweep orphaned testcontainers from dead runs
         run: |
@@ -64,7 +64,7 @@ jobs:
             age_min=$(( (now - $(date -d "$started" +%s)) / 60 ))
             if [ "$age_min" -ge "$MAX_CONTAINER_AGE_MIN" ]; then
               echo "Reaping orphan $id (age ${age_min}m)"
-              docker rm -f "$id" || true
+              docker rm -fv "$id" || true
               reaped=$((reaped + 1))
             fi
           done
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 077e8b5cb..01868978b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -318,8 +318,16 @@ jobs:
           # The test helpers stamp `github.run_id`, which is exactly what makes
           # them reapable. No age guard: these are *this* run's containers, and
           # by construction they are only minutes old.
+          #
+          # `-v` is load-bearing. The postgres/elasticsearch/mongo images all
+          # declare VOLUME in their Dockerfiles, so every testcontainer gets an
+          # anonymous volume — and plain `docker rm -f` leaves it behind
+          # forever. That is what filled the shared host's 200G ZFS quota until
+          # containerd could not even write meta.db and *every* container start
+          # failed with "disk quota exceeded". `-v` removes only anonymous
+          # volumes; named volumes are never touched.
           docker ps -aq --filter "label=github.run_id=${{ github.run_id }}" \
-            | xargs -r docker rm -f || true
+            | xargs -r docker rm -fv || true
 
       - name: Sweep orphaned testcontainers from dead runs
         run: |
@@ -335,7 +343,7 @@ jobs:
             age_min=$(( (now - $(date -d "$started" +%s)) / 60 ))
             if [ "$age_min" -ge "$MAX_CONTAINER_AGE_MIN" ]; then
               echo "Reaping orphan $id (age ${age_min}m)"
-              docker rm -f "$id" || true
+              docker rm -fv "$id" || true
               reaped=$((reaped + 1))
             fi
           done
diff --git a/.github/workflows/cluster-smoke.yml b/.github/workflows/cluster-smoke.yml
index 76367f701..14d161e8e 100644
--- a/.github/workflows/cluster-smoke.yml
+++ b/.github/workflows/cluster-smoke.yml
@@ -100,7 +100,7 @@ jobs:
       - name: Start PostgreSQL
         run: |
           PG_CONTAINER="pg-cluster-smoke-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -129,7 +129,7 @@ jobs:
         run: |
           mkdir -p "$RESULTS_DIR"
           NGINX_CONTAINER="nginx-cluster-smoke-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$NGINX_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$NGINX_CONTAINER" 2>/dev/null || true
 
           # Upstreams are the two hfs processes on the runner. X-Hfs-Upstream
           # lets the smoke script prove round-robin. The Upgrade/Connection
@@ -301,8 +301,8 @@ jobs:
         if: always()
         run: |
           if [ -n "${PG_CONTAINER:-}" ]; then
-            docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${NGINX_CONTAINER:-}" ]; then
-            docker rm -f "$NGINX_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$NGINX_CONTAINER" 2>/dev/null || true
           fi
diff --git a/.github/workflows/fhir-benchmark.yml b/.github/workflows/fhir-benchmark.yml
index 4e3658011..40734ff99 100644
--- a/.github/workflows/fhir-benchmark.yml
+++ b/.github/workflows/fhir-benchmark.yml
@@ -240,7 +240,7 @@ jobs:
           echo "TGZ_CONTAINER=$TGZ_CONTAINER" >> "$GITHUB_ENV"
 
           docker build -t "$TGZ_IMAGE" fhir-benchmark/infra/tgz
-          docker rm -f "$TGZ_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$TGZ_CONTAINER" 2>/dev/null || true
           docker run -d --name "$TGZ_CONTAINER" -p 0:8080 "$TGZ_IMAGE" \
             "https://storage.googleapis.com/aidbox-public/synthea/performance/bulk_1k.tar.gz" >/dev/null
 
@@ -281,7 +281,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           set -euo pipefail
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           # Tuning approximates the benchmark's infra/postgres/postgres.conf.
           # shared_buffers/effective_cache_size are scaled DOWN from the
           # published values (10G/25G) to fit a typical self-hosted runner;
@@ -644,12 +644,12 @@ jobs:
         if: always()
         run: |
           if [ -n "${TGZ_CONTAINER:-}" ]; then
-            docker rm -f "$TGZ_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$TGZ_CONTAINER" 2>/dev/null || true
           fi
 
       - name: Stop ephemeral Postgres
         if: always() && matrix.backend == 'postgres'
         run: |
           if [ -n "${PG_CONTAINER:-}" ]; then
-            docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           fi
diff --git a/.github/workflows/hts-benchmark.yml b/.github/workflows/hts-benchmark.yml
index 4e0d39336..ac8d985fd 100644
--- a/.github/workflows/hts-benchmark.yml
+++ b/.github/workflows/hts-benchmark.yml
@@ -344,7 +344,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           set -euo pipefail
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           # `-c shared_buffers=1GB -c work_mem=64MB` give the bench a fair
           # chance against SQLite's in-process locality; the defaults are
           # tuned for tiny VPS instances and would penalise PG unfairly when
@@ -677,7 +677,7 @@ jobs:
         if: always() && matrix.backend == 'postgres'
         run: |
           if [ -n "${PG_CONTAINER:-}" ]; then
-            docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           fi
 
       # Turn the job red if any arm never started/verified — but only after the
diff --git a/.github/workflows/hts-ig-conformance.yml b/.github/workflows/hts-ig-conformance.yml
index af29b3c6a..f8118e7ee 100644
--- a/.github/workflows/hts-ig-conformance.yml
+++ b/.github/workflows/hts-ig-conformance.yml
@@ -269,7 +269,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           set -euo pipefail
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           # `-c shared_buffers=1GB -c work_mem=64MB` give PG enough cache to
           # hold the SNOMED `concept_closure` without evicting the hot
           # working set. `--shm-size=2g` ensures the larger shared-memory
@@ -809,7 +809,7 @@ jobs:
         if: always() && matrix.backend == 'postgres'
         run: |
           if [ -n "${PG_CONTAINER:-}" ]; then
-            docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           fi
 
       - name: Clean up working files
diff --git a/.github/workflows/hts.yml b/.github/workflows/hts.yml
index 8a4d645c6..18dd77474 100644
--- a/.github/workflows/hts.yml
+++ b/.github/workflows/hts.yml
@@ -704,7 +704,7 @@ jobs:
             age_min=$(( (now - $(date -d "$started" +%s)) / 60 ))
             if [ "$age_min" -ge "$MAX_CONTAINER_AGE_MIN" ]; then
               echo "Reaping orphan $id (age ${age_min}m)"
-              docker rm -f "$id" || true
+              docker rm -fv "$id" || true
               reaped=$((reaped + 1))
             fi
           done
diff --git a/.github/workflows/inferno-bulk-data.yml b/.github/workflows/inferno-bulk-data.yml
index 3430c350d..8d2860358 100644
--- a/.github/workflows/inferno-bulk-data.yml
+++ b/.github/workflows/inferno-bulk-data.yml
@@ -221,7 +221,7 @@ jobs:
       - name: Start PostgreSQL
         run: |
           PG_CONTAINER="pg-inferno-bulk-data-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -251,7 +251,7 @@ jobs:
       - name: Start MinIO
         run: |
           MINIO_CONTAINER="minio-inferno-bulk-data-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MINIO_CONTAINER" -p 0:9000 -p 0:9001 \
             -e MINIO_ROOT_USER=hfs-minio \
             -e MINIO_ROOT_PASSWORD=hfs-minio-secret \
@@ -318,7 +318,7 @@ jobs:
       - name: Start Keycloak
         run: |
           KC_CONTAINER="kc-inferno-bulk-data-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$KC_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$KC_CONTAINER" 2>/dev/null || true
 
           docker create --name "$KC_CONTAINER" -p 0:8080 \
             -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
@@ -778,13 +778,13 @@ jobs:
           fi
 
           echo "Stopping PostgreSQL..."
-          docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Stopping MinIO..."
-          docker rm -f "${MINIO_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${MINIO_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Stopping Keycloak..."
-          docker rm -f "${KC_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${KC_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Cleanup complete"
 
@@ -804,7 +804,7 @@ jobs:
 
           IDS=$(docker ps -aq --filter "label=com.docker.compose.project=$PROJECT")
           if [ -n "$IDS" ]; then
-            docker rm -f $IDS
+            docker rm -fv $IDS
           fi
 
           NETS=$(docker network ls --filter "label=com.docker.compose.project=$PROJECT" -q)
diff --git a/.github/workflows/inferno-subscription.yml b/.github/workflows/inferno-subscription.yml
index c4ed543e6..74011822e 100644
--- a/.github/workflows/inferno-subscription.yml
+++ b/.github/workflows/inferno-subscription.yml
@@ -206,7 +206,7 @@ jobs:
         if: matrix.backend == 'sqlite-elasticsearch' || matrix.backend == 's3-elasticsearch'
         run: |
           ES_CONTAINER="es-inferno-subscriptions-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           docker run -d --name "$ES_CONTAINER" -p 0:9200 \
             -e "discovery.type=single-node" \
             -e "xpack.security.enabled=false" \
@@ -237,7 +237,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           PG_CONTAINER="pg-inferno-subscriptions-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -268,7 +268,7 @@ jobs:
         if: matrix.backend == 'mongodb'
         run: |
           MONGO_CONTAINER="mongo-inferno-subscriptions-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
 
           echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> "$GITHUB_ENV"
@@ -771,9 +771,9 @@ jobs:
           fi
 
           echo "Stopping backend containers..."
-          docker rm -f "${ES_CONTAINER:-none}" 2>/dev/null || true
-          docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
-          docker rm -f "${MONGO_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${ES_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${MONGO_CONTAINER:-none}" 2>/dev/null || true
 
   inferno-stack-down:
     name: Stop Shared Inferno Subscriptions Stack
@@ -791,7 +791,7 @@ jobs:
 
           IDS=$(docker ps -aq --filter "label=com.docker.compose.project=$PROJECT")
           if [ -n "$IDS" ]; then
-            docker rm -f $IDS
+            docker rm -fv $IDS
           fi
 
           NETS=$(docker network ls --filter "label=com.docker.compose.project=$PROJECT" -q)
diff --git a/.github/workflows/inferno-us-core.yml b/.github/workflows/inferno-us-core.yml
index 8c47fc320..ab0e9deeb 100644
--- a/.github/workflows/inferno-us-core.yml
+++ b/.github/workflows/inferno-us-core.yml
@@ -281,7 +281,7 @@ jobs:
         if: matrix.backend == 'mongodb'
         run: |
           MONGO_CONTAINER="mongo-${{ matrix.suite_id }}-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
 
           echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> $GITHUB_ENV
@@ -325,7 +325,7 @@ jobs:
         if: matrix.backend == 'sqlite-elasticsearch' || matrix.backend == 's3-elasticsearch'
         run: |
           ES_CONTAINER="es-${{ matrix.suite_id }}-${{ matrix.backend }}"
-          docker rm -f $ES_CONTAINER 2>/dev/null || true
+          docker rm -fv $ES_CONTAINER 2>/dev/null || true
           docker run -d --name $ES_CONTAINER -p 0:9200 \
             -e "discovery.type=single-node" \
             -e "xpack.security.enabled=false" \
@@ -355,7 +355,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           PG_CONTAINER="pg-${{ matrix.suite_id }}"
-          docker rm -f $PG_CONTAINER 2>/dev/null || true
+          docker rm -fv $PG_CONTAINER 2>/dev/null || true
           docker run -d --name $PG_CONTAINER -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -760,13 +760,13 @@ jobs:
           fi
 
           echo "Stopping Elasticsearch..."
-          docker rm -f "${ES_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${ES_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Stopping PostgreSQL..."
-          docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Stopping MongoDB..."
-          docker rm -f "${MONGO_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${MONGO_CONTAINER:-none}" 2>/dev/null || true
 
           echo "Cleanup complete"
 
@@ -786,7 +786,7 @@ jobs:
 
           IDS=$(docker ps -aq --filter "label=com.docker.compose.project=$PROJECT")
           if [ -n "$IDS" ]; then
-            docker rm -f $IDS
+            docker rm -fv $IDS
           fi
 
           NETS=$(docker network ls --filter "label=com.docker.compose.project=$PROJECT" -q)
diff --git a/.github/workflows/subscriptions-smoke.yml b/.github/workflows/subscriptions-smoke.yml
index 8e45d3d3a..c0504c026 100644
--- a/.github/workflows/subscriptions-smoke.yml
+++ b/.github/workflows/subscriptions-smoke.yml
@@ -158,7 +158,7 @@ jobs:
         if: matrix.backend == 'sqlite-elasticsearch' || matrix.backend == 's3-elasticsearch'
         run: |
           ES_CONTAINER="es-subscriptions-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.build_key }}-${{ matrix.fhir_version }}"
-          docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           docker run -d --name "$ES_CONTAINER" -p 0:9200 \
             -e "discovery.type=single-node" \
             -e "xpack.security.enabled=false" \
@@ -189,7 +189,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           PG_CONTAINER="pg-subscriptions-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.build_key }}-${{ matrix.fhir_version }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -220,7 +220,7 @@ jobs:
         if: matrix.backend == 'mongodb'
         run: |
           MONGO_CONTAINER="mongo-subscriptions-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.build_key }}-${{ matrix.fhir_version }}"
-          docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
 
           echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> $GITHUB_ENV
@@ -284,7 +284,7 @@ jobs:
       - name: Start mailpit (email channel sink)
         run: |
           MP_CONTAINER="mailpit-subscriptions-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.build_key }}-${{ matrix.fhir_version }}-${{ matrix.backend }}"
-          docker rm -f "$MP_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MP_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MP_CONTAINER" -p 0:1025 -p 0:8025 axllent/mailpit:latest
           echo "MP_CONTAINER=$MP_CONTAINER" >> $GITHUB_ENV
 
@@ -493,14 +493,14 @@ jobs:
         if: always()
         run: |
           if [ -n "${ES_CONTAINER:-}" ]; then
-            docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${PG_CONTAINER:-}" ]; then
-            docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${MONGO_CONTAINER:-}" ]; then
-            docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           fi
           if [ -n "${MP_CONTAINER:-}" ]; then
-            docker rm -f "$MP_CONTAINER" 2>/dev/null || true
+            docker rm -fv "$MP_CONTAINER" 2>/dev/null || true
           fi
diff --git a/.github/workflows/ui-tests-matrix.yml b/.github/workflows/ui-tests-matrix.yml
index 8bcc132d4..a75c2a78d 100644
--- a/.github/workflows/ui-tests-matrix.yml
+++ b/.github/workflows/ui-tests-matrix.yml
@@ -127,7 +127,7 @@ jobs:
         if: matrix.backend == 'sqlite-elasticsearch' || matrix.backend == 's3-elasticsearch'
         run: |
           ES_CONTAINER="es-ui-matrix-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$ES_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
           docker run -d --name "$ES_CONTAINER" -p 0:9200 \
             -e "discovery.type=single-node" \
             -e "xpack.security.enabled=false" \
@@ -158,7 +158,7 @@ jobs:
         if: matrix.backend == 'postgres'
         run: |
           PG_CONTAINER="pg-ui-matrix-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$PG_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
           docker run -d --name "$PG_CONTAINER" -p 0:5432 \
             -e POSTGRES_USER=helios \
             -e POSTGRES_PASSWORD=helios \
@@ -189,7 +189,7 @@ jobs:
         if: matrix.backend == 'mongodb'
         run: |
           MONGO_CONTAINER="mongo-ui-matrix-${{ github.run_id }}-${{ github.run_attempt }}"
-          docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
           docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
 
           echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> "$GITHUB_ENV"
@@ -352,7 +352,7 @@ jobs:
           set -euo pipefail
           BASE="http://$RUNNER_IP:$HFS_PORT"
           echo "Driving browser against $BASE"
-          docker rm -f "$PW_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PW_CONTAINER" 2>/dev/null || true
           # 2g shm: Chromium needs more than docker's 64m default.
           docker run -d --name "$PW_CONTAINER" --shm-size=2g -w /work "$PLAYWRIGHT_IMAGE" sleep 3600
           docker cp crates/ui/e2e "$PW_CONTAINER":/work/e2e
@@ -377,7 +377,7 @@ jobs:
         if: always()
         run: |
           echo "Removing Playwright container..."
-          docker rm -f "$PW_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$PW_CONTAINER" 2>/dev/null || true
 
           echo "Stopping HFS..."
           if [ -f /tmp/hfs-ui-matrix-${{ matrix.backend }}.pid ]; then
@@ -395,6 +395,6 @@ jobs:
           fi
 
           echo "Stopping backend containers..."
-          docker rm -f "${ES_CONTAINER:-none}" 2>/dev/null || true
-          docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
-          docker rm -f "${MONGO_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${ES_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true
+          docker rm -fv "${MONGO_CONTAINER:-none}" 2>/dev/null || true
diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml
index abcd955d2..c757ee528 100644
--- a/.github/workflows/ui-tests.yml
+++ b/.github/workflows/ui-tests.yml
@@ -70,7 +70,7 @@ jobs:
       - name: Run browser suite (Playwright)
         run: |
           set -euo pipefail
-          docker rm -f "$RUN_CONTAINER" 2>/dev/null || true
+          docker rm -fv "$RUN_CONTAINER" 2>/dev/null || true
           # 2g shm: Chromium and the tmpfs-backed SQLite DB share /dev/shm, and
           # per-tenant conformance seeding overflows docker's 64m default.
           docker run -d --name "$RUN_CONTAINER" --shm-size=2g -w /work "$PLAYWRIGHT_IMAGE" sleep 3600
@@ -97,4 +97,4 @@ jobs:
 
       - name: Remove Playwright container
         if: always()
-        run: docker rm -f "$RUN_CONTAINER" 2>/dev/null || true
+        run: docker rm -fv "$RUN_CONTAINER" 2>/dev/null || true