From 022dd6ab17b19ff0ffbc400ff7d9819418d71d33 Mon Sep 17 00:00:00 2001 From: gabriel-farache Date: Wed, 12 Aug 2026 17:27:47 +0200 Subject: [PATCH 1/4] fix(black-box): capture diagnostics on cancel, pin podman-compose The blackbox job's log-collection step only ran on failure(), so a timeout/cancellation (as seen in dcm-project/control-plane#37) left no diagnostic output about which service hung. Switch it to always() so it runs (with GitHub's ~5min cancellation grace period) even when the job is cancelled, and have it dump container status, health check history, and compose logs. Also pin podman-compose to 1.6.0 instead of installing unpinned, to remove version drift as a variable. Co-authored-by: Cursor Signed-off-by: gabriel-farache --- .github/workflows/black-box.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/black-box.yaml b/.github/workflows/black-box.yaml index 05dcfad..8078fc4 100644 --- a/.github/workflows/black-box.yaml +++ b/.github/workflows/black-box.yaml @@ -38,7 +38,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y podman - pip install podman-compose + pip install podman-compose==1.6.0 - name: Pre-pull container images if: inputs.images != '' @@ -58,10 +58,24 @@ jobs: - name: Run tests run: make ${{ inputs.test-target }} - - name: Collect logs on failure - if: failure() + - name: Collect diagnostics + if: always() run: | - podman-compose logs + echo "::group::podman ps -a" + podman ps -a + echo "::endgroup::" + + echo "::group::container health" + for c in $(podman ps -aq); do + name=$(podman inspect --format '{{.Name}}' "$c") + echo "--- $name ---" + podman inspect --format '{{json .State.Health}}' "$c" || true + done + echo "::endgroup::" + + echo "::group::podman-compose logs" + podman-compose logs || true + echo "::endgroup::" - name: Stop services if: always() From f2dc056423a4713eddb33acb781201cf4458d6ec Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Thu, 13 Aug 2026 13:31:59 +0200 Subject: [PATCH 2/4] fix(black-box): harden diagnostics collection Run only on failure/cancel, never fail the job, and use podman logs per container instead of podman-compose logs. Assisted-By: Claude (Anthropic) Signed-off-by: Gloria Ciavarrini --- .github/workflows/black-box.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/black-box.yaml b/.github/workflows/black-box.yaml index 8078fc4..bf6dd7b 100644 --- a/.github/workflows/black-box.yaml +++ b/.github/workflows/black-box.yaml @@ -59,22 +59,28 @@ jobs: run: make ${{ inputs.test-target }} - name: Collect diagnostics - if: always() + if: failure() || cancelled() + continue-on-error: true run: | + set +e echo "::group::podman ps -a" podman ps -a echo "::endgroup::" echo "::group::container health" - for c in $(podman ps -aq); do - name=$(podman inspect --format '{{.Name}}' "$c") + for c in $(podman ps -aq 2>/dev/null); do + name=$(podman inspect --format '{{.Name}}' "$c" 2>/dev/null || echo "$c") echo "--- $name ---" - podman inspect --format '{{json .State.Health}}' "$c" || true + podman inspect --format '{{json .State.Health}}' "$c" done echo "::endgroup::" - echo "::group::podman-compose logs" - podman-compose logs || true + echo "::group::container logs" + for c in $(podman ps -aq 2>/dev/null); do + name=$(podman inspect --format '{{.Name}}' "$c" 2>/dev/null || echo "$c") + echo "--- $name ---" + podman logs --tail 200 "$c" + done echo "::endgroup::" - name: Stop services From 8fa2369c7b5e440c741abbe69783eed23081f4a3 Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Thu, 13 Aug 2026 15:14:50 +0200 Subject: [PATCH 3/4] enable linger for Podman healthchecks Rootless healthchecks need a user systemd session; without it compose service_healthy can hang until the job times out. Assisted-By: Claude (Anthropic) Signed-off-by: Gloria Ciavarrini --- .github/workflows/black-box.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/black-box.yaml b/.github/workflows/black-box.yaml index bf6dd7b..3bc72c1 100644 --- a/.github/workflows/black-box.yaml +++ b/.github/workflows/black-box.yaml @@ -39,6 +39,12 @@ jobs: sudo apt-get update sudo apt-get install -y podman pip install podman-compose==1.6.0 + # Rootless Podman runs healthchecks via systemd user timers. On GHA + # the user manager is often absent, so health stays "starting" and + # compose depends_on: service_healthy hangs until job timeout. + sudo loginctl enable-linger "$(id -un)" + sleep 1 + echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" - name: Pre-pull container images if: inputs.images != '' From 37d405932be94b4f2b5caf440c2de51347ae2081 Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Thu, 13 Aug 2026 15:54:03 +0200 Subject: [PATCH 4/4] wait for XDG_RUNTIME_DIR after linger Replace the fixed sleep with a short poll Assisted-By: Claude (Anthropic) Signed-off-by: Gloria Ciavarrini --- .github/workflows/black-box.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/black-box.yaml b/.github/workflows/black-box.yaml index 3bc72c1..866a462 100644 --- a/.github/workflows/black-box.yaml +++ b/.github/workflows/black-box.yaml @@ -39,11 +39,8 @@ jobs: sudo apt-get update sudo apt-get install -y podman pip install podman-compose==1.6.0 - # Rootless Podman runs healthchecks via systemd user timers. On GHA - # the user manager is often absent, so health stays "starting" and - # compose depends_on: service_healthy hangs until job timeout. sudo loginctl enable-linger "$(id -un)" - sleep 1 + timeout 30 bash -c 'until [ -d /run/user/$(id -u) ]; do sleep 0.5; done' echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" - name: Pre-pull container images