Skip to content

ci: reap testcontainer volumes, not just containers #452

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
smunini merged 1 commit into from
Jul 30, 2026
Merged

ci: reap testcontainer volumes, not just containers #452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions .github/workflows/audit-events.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ jobs:
if: matrix.backend == 'sqlite-elasticsearch' || matrix.backend == 's3-elasticsearch'
run: |
ES_CONTAINER="es-audit-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$ES_CONTAINER" 2>/dev/null || true
docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
docker run -d --name "$ES_CONTAINER" -p 0:9200 \
-e "discovery.type=single-node" \
-e "xpack.security.enabled=false" \
Expand Down Expand Up @@ -260,7 +260,7 @@ jobs:
if: matrix.backend == 'postgres'
run: |
PG_CONTAINER="pg-audit-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" -p 0:5432 \
-e POSTGRES_USER=helios \
-e POSTGRES_PASSWORD=helios \
Expand Down Expand Up @@ -291,7 +291,7 @@ jobs:
if: matrix.backend == 'mongodb'
run: |
MONGO_CONTAINER="mongo-audit-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all

echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> $GITHUB_ENV
Expand Down Expand Up @@ -383,7 +383,7 @@ jobs:
- name: Start ephemeral Keycloak
run: |
KC_CONTAINER="kc-audit-${{ matrix.backend }}-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$KC_CONTAINER" 2>/dev/null || true
docker rm -fv "$KC_CONTAINER" 2>/dev/null || true

docker create --name "$KC_CONTAINER" -p 0:8080 \
-e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
Expand Down Expand Up @@ -760,16 +760,16 @@ jobs:
fi

echo "Stopping Keycloak..."
docker rm -f "${KC_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${KC_CONTAINER:-none}" 2>/dev/null || true

echo "Stopping Elasticsearch..."
docker rm -f "${ES_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${ES_CONTAINER:-none}" 2>/dev/null || true

echo "Stopping PostgreSQL..."
docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true

echo "Stopping MongoDB..."
docker rm -f "${MONGO_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${MONGO_CONTAINER:-none}" 2>/dev/null || true

echo "Cleanup complete"

Expand Down Expand Up @@ -826,7 +826,7 @@ jobs:
if: matrix.backend == 'postgres'
run: |
PG_CONTAINER="pg-audit-db-${{ matrix.mode }}-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" -p 0:5432 \
-e POSTGRES_USER=helios \
-e POSTGRES_PASSWORD=helios \
Expand Down Expand Up @@ -862,7 +862,7 @@ jobs:
if: matrix.backend == 'mongodb'
run: |
MONGO_CONTAINER="mongo-audit-db-${{ matrix.mode }}-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all

echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> $GITHUB_ENV
Expand Down Expand Up @@ -1196,5 +1196,5 @@ jobs:
- name: Cleanup smoke resources
if: always()
run: |
docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
docker rm -f "${MONGO_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${MONGO_CONTAINER:-none}" 2>/dev/null || true
16 changes: 8 additions & 8 deletions .github/workflows/bulk-export-smoke.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ jobs:
if: contains(matrix.backend, 'elasticsearch')
run: |
ES_CONTAINER="es-bulk-export-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
docker rm -f "$ES_CONTAINER" 2>/dev/null || true
docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
docker run -d --name "$ES_CONTAINER" -p 0:9200 \
-e "discovery.type=single-node" \
-e "xpack.security.enabled=false" \
Expand All @@ -162,7 +162,7 @@ jobs:
if: contains(matrix.backend, 'postgres')
run: |
PG_CONTAINER="pg-bulk-export-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" -p 0:5432 \
-e POSTGRES_USER=helios \
-e POSTGRES_PASSWORD=helios \
Expand Down Expand Up @@ -191,7 +191,7 @@ jobs:
if: contains(matrix.backend, 'mongodb')
run: |
MONGO_CONTAINER="mongo-bulk-export-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> "$GITHUB_ENV"

Expand All @@ -217,7 +217,7 @@ jobs:
if: startsWith(matrix.backend, 's3')
run: |
MINIO_CONTAINER="minio-bulk-export-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
docker run -d --name "$MINIO_CONTAINER" -p 0:9000 -p 0:9001 \
-e MINIO_ROOT_USER=hfs-minio \
-e MINIO_ROOT_PASSWORD=hfs-minio-secret \
Expand Down Expand Up @@ -499,14 +499,14 @@ jobs:
if: always()
run: |
if [ -n "${ES_CONTAINER:-}" ]; then
docker rm -f "$ES_CONTAINER" 2>/dev/null || true
docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
fi
if [ -n "${PG_CONTAINER:-}" ]; then
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
fi
if [ -n "${MONGO_CONTAINER:-}" ]; then
docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
fi
if [ -n "${MINIO_CONTAINER:-}" ]; then
docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
fi
16 changes: 8 additions & 8 deletions .github/workflows/bulk-submit-smoke.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ jobs:
if: contains(matrix.backend, 'elasticsearch')
run: |
ES_CONTAINER="es-bulk-submit-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
docker rm -f "$ES_CONTAINER" 2>/dev/null || true
docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
docker run -d --name "$ES_CONTAINER" -p 0:9200 \
-e "discovery.type=single-node" \
-e "xpack.security.enabled=false" \
Expand All @@ -149,7 +149,7 @@ jobs:
if: contains(matrix.backend, 'postgres')
run: |
PG_CONTAINER="pg-bulk-submit-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" -p 0:5432 \
-e POSTGRES_USER=helios \
-e POSTGRES_PASSWORD=helios \
Expand Down Expand Up @@ -178,7 +178,7 @@ jobs:
if: contains(matrix.backend, 'mongodb')
run: |
MONGO_CONTAINER="mongo-bulk-submit-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
docker run -d --name "$MONGO_CONTAINER" -p 0:27017 mongo:7.0 --replSet rs0 --bind_ip_all
echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> "$GITHUB_ENV"

Expand All @@ -204,7 +204,7 @@ jobs:
if: startsWith(matrix.backend, 's3')
run: |
MINIO_CONTAINER="minio-bulk-submit-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.fhir_version }}-${{ matrix.backend }}-${{ matrix.output }}"
docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
docker run -d --name "$MINIO_CONTAINER" -p 0:9000 -p 0:9001 \
-e MINIO_ROOT_USER=hfs-minio \
-e MINIO_ROOT_PASSWORD=hfs-minio-secret \
Expand Down Expand Up @@ -562,14 +562,14 @@ jobs:
if: always()
run: |
if [ -n "${ES_CONTAINER:-}" ]; then
docker rm -f "$ES_CONTAINER" 2>/dev/null || true
docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
fi
if [ -n "${PG_CONTAINER:-}" ]; then
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
fi
if [ -n "${MONGO_CONTAINER:-}" ]; then
docker rm -f "$MONGO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
fi
if [ -n "${MINIO_CONTAINER:-}" ]; then
docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
fi
4 changes: 2 additions & 2 deletions .github/workflows/ci-extended.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
steps:
- name: Remove testcontainers from this run
run: |
docker ps -aq --filter "label=github.run_id=${{ github.run_id }}" | xargs -r docker rm -f || true
docker ps -aq --filter "label=github.run_id=${{ github.run_id }}" | xargs -r docker rm -fv || true

- name: Sweep orphaned testcontainers from dead runs
run: |
Expand All @@ -64,7 +64,7 @@ jobs:
age_min=$(( (now - $(date -d "$started" +%s)) / 60 ))
if [ "$age_min" -ge "$MAX_CONTAINER_AGE_MIN" ]; then
echo "Reaping orphan $id (age ${age_min}m)"
docker rm -f "$id" || true
docker rm -fv "$id" || true
reaped=$((reaped + 1))
fi
done
Expand Down
12 changes: 10 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,8 +318,16 @@ jobs:
# The test helpers stamp `github.run_id`, which is exactly what makes
# them reapable. No age guard: these are *this* run's containers, and
# by construction they are only minutes old.
#
# `-v` is load-bearing. The postgres/elasticsearch/mongo images all
# declare VOLUME in their Dockerfiles, so every testcontainer gets an
# anonymous volume — and plain `docker rm -f` leaves it behind
# forever. That is what filled the shared host's 200G ZFS quota until
# containerd could not even write meta.db and *every* container start
# failed with "disk quota exceeded". `-v` removes only anonymous
# volumes; named volumes are never touched.
docker ps -aq --filter "label=github.run_id=${{ github.run_id }}" \
| xargs -r docker rm -f || true
| xargs -r docker rm -fv || true

- name: Sweep orphaned testcontainers from dead runs
run: |
Expand All @@ -335,7 +343,7 @@ jobs:
age_min=$(( (now - $(date -d "$started" +%s)) / 60 ))
if [ "$age_min" -ge "$MAX_CONTAINER_AGE_MIN" ]; then
echo "Reaping orphan $id (age ${age_min}m)"
docker rm -f "$id" || true
docker rm -fv "$id" || true
reaped=$((reaped + 1))
fi
done
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/cluster-smoke.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ jobs:
- name: Start PostgreSQL
run: |
PG_CONTAINER="pg-cluster-smoke-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" -p 0:5432 \
-e POSTGRES_USER=helios \
-e POSTGRES_PASSWORD=helios \
Expand Down Expand Up @@ -129,7 +129,7 @@ jobs:
run: |
mkdir -p "$RESULTS_DIR"
NGINX_CONTAINER="nginx-cluster-smoke-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$NGINX_CONTAINER" 2>/dev/null || true
docker rm -fv "$NGINX_CONTAINER" 2>/dev/null || true

# Upstreams are the two hfs processes on the runner. X-Hfs-Upstream
# lets the smoke script prove round-robin. The Upgrade/Connection
Expand Down Expand Up @@ -301,8 +301,8 @@ jobs:
if: always()
run: |
if [ -n "${PG_CONTAINER:-}" ]; then
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
fi
if [ -n "${NGINX_CONTAINER:-}" ]; then
docker rm -f "$NGINX_CONTAINER" 2>/dev/null || true
docker rm -fv "$NGINX_CONTAINER" 2>/dev/null || true
fi
8 changes: 4 additions & 4 deletions .github/workflows/fhir-benchmark.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ jobs:
echo "TGZ_CONTAINER=$TGZ_CONTAINER" >> "$GITHUB_ENV"

docker build -t "$TGZ_IMAGE" fhir-benchmark/infra/tgz
docker rm -f "$TGZ_CONTAINER" 2>/dev/null || true
docker rm -fv "$TGZ_CONTAINER" 2>/dev/null || true
docker run -d --name "$TGZ_CONTAINER" -p 0:8080 "$TGZ_IMAGE" \
"https://storage.googleapis.com/aidbox-public/synthea/performance/bulk_1k.tar.gz" >/dev/null

Expand Down Expand Up @@ -281,7 +281,7 @@ jobs:
if: matrix.backend == 'postgres'
run: |
set -euo pipefail
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
# Tuning approximates the benchmark's infra/postgres/postgres.conf.
# shared_buffers/effective_cache_size are scaled DOWN from the
# published values (10G/25G) to fit a typical self-hosted runner;
Expand Down Expand Up @@ -644,12 +644,12 @@ jobs:
if: always()
run: |
if [ -n "${TGZ_CONTAINER:-}" ]; then
docker rm -f "$TGZ_CONTAINER" 2>/dev/null || true
docker rm -fv "$TGZ_CONTAINER" 2>/dev/null || true
fi

- name: Stop ephemeral Postgres
if: always() && matrix.backend == 'postgres'
run: |
if [ -n "${PG_CONTAINER:-}" ]; then
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
fi
4 changes: 2 additions & 2 deletions .github/workflows/hts-benchmark.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ jobs:
if: matrix.backend == 'postgres'
run: |
set -euo pipefail
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
# `-c shared_buffers=1GB -c work_mem=64MB` give the bench a fair
# chance against SQLite's in-process locality; the defaults are
# tuned for tiny VPS instances and would penalise PG unfairly when
Expand Down Expand Up @@ -677,7 +677,7 @@ jobs:
if: always() && matrix.backend == 'postgres'
run: |
if [ -n "${PG_CONTAINER:-}" ]; then
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
fi

# Turn the job red if any arm never started/verified — but only after the
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/hts-ig-conformance.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ jobs:
if: matrix.backend == 'postgres'
run: |
set -euo pipefail
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
# `-c shared_buffers=1GB -c work_mem=64MB` give PG enough cache to
# hold the SNOMED `concept_closure` without evicting the hot
# working set. `--shm-size=2g` ensures the larger shared-memory
Expand Down Expand Up @@ -809,7 +809,7 @@ jobs:
if: always() && matrix.backend == 'postgres'
run: |
if [ -n "${PG_CONTAINER:-}" ]; then
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
fi

- name: Clean up working files
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/hts.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -704,7 +704,7 @@ jobs:
age_min=$(( (now - $(date -d "$started" +%s)) / 60 ))
if [ "$age_min" -ge "$MAX_CONTAINER_AGE_MIN" ]; then
echo "Reaping orphan $id (age ${age_min}m)"
docker rm -f "$id" || true
docker rm -fv "$id" || true
reaped=$((reaped + 1))
fi
done
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/inferno-bulk-data.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ jobs:
- name: Start PostgreSQL
run: |
PG_CONTAINER="pg-inferno-bulk-data-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
docker run -d --name "$PG_CONTAINER" -p 0:5432 \
-e POSTGRES_USER=helios \
-e POSTGRES_PASSWORD=helios \
Expand Down Expand Up @@ -251,7 +251,7 @@ jobs:
- name: Start MinIO
run: |
MINIO_CONTAINER="minio-inferno-bulk-data-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$MINIO_CONTAINER" 2>/dev/null || true
docker rm -fv "$MINIO_CONTAINER" 2>/dev/null || true
docker run -d --name "$MINIO_CONTAINER" -p 0:9000 -p 0:9001 \
-e MINIO_ROOT_USER=hfs-minio \
-e MINIO_ROOT_PASSWORD=hfs-minio-secret \
Expand Down Expand Up @@ -318,7 +318,7 @@ jobs:
- name: Start Keycloak
run: |
KC_CONTAINER="kc-inferno-bulk-data-${{ github.run_id }}-${{ github.run_attempt }}"
docker rm -f "$KC_CONTAINER" 2>/dev/null || true
docker rm -fv "$KC_CONTAINER" 2>/dev/null || true

docker create --name "$KC_CONTAINER" -p 0:8080 \
-e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
Expand Down Expand Up @@ -778,13 +778,13 @@ jobs:
fi

echo "Stopping PostgreSQL..."
docker rm -f "${PG_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${PG_CONTAINER:-none}" 2>/dev/null || true

echo "Stopping MinIO..."
docker rm -f "${MINIO_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${MINIO_CONTAINER:-none}" 2>/dev/null || true

echo "Stopping Keycloak..."
docker rm -f "${KC_CONTAINER:-none}" 2>/dev/null || true
docker rm -fv "${KC_CONTAINER:-none}" 2>/dev/null || true

echo "Cleanup complete"

Expand All @@ -804,7 +804,7 @@ jobs:

IDS=$(docker ps -aq --filter "label=com.docker.compose.project=$PROJECT")
if [ -n "$IDS" ]; then
docker rm -f $IDS
docker rm -fv $IDS
fi

NETS=$(docker network ls --filter "label=com.docker.compose.project=$PROJECT" -q)
Expand Down
Loading
Loading