From 482299f323cd466a7c74ca00369c2e5b12de1358 Mon Sep 17 00:00:00 2001 From: Yuriy Kirillov Date: Sat, 10 Oct 2026 19:03:47 +0200 Subject: [PATCH] fix(renovate): prune unused Docker images after recreating containers renovate.py pulled a fresh image and recreated each app's container but never cleaned up the image it replaced. Over weeks of repeated runs across several apps, dangling images accumulated on the target host until the disk filled - observed in production on hawkeye.dupmachine.com, where a full root disk put every Postgres-backed app into a crash loop. deploy.py already runs this same cleanup after every deploy (`docker container prune -f && docker image prune -a -f`); renovate.py was just missing it. Add a prune_host() step, called once per host after all of that host's requested apps have been renovated, regardless of whether any of them actually changed - a previous run's now-superseded image is what needs cleaning up, not necessarily this run's own pull. Fixes #219 Co-Authored-By: Claude Sonnet 5 --- .github/actions/renovate/README.md | 2 + .github/actions/renovate/renovate.py | 17 +++++++-- .../actions/renovate/tests/test_renovate.py | 37 +++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/.github/actions/renovate/README.md b/.github/actions/renovate/README.md index 028670c..eb05d8b 100644 --- a/.github/actions/renovate/README.md +++ b/.github/actions/renovate/README.md @@ -32,6 +32,8 @@ jobs: Not every target runs every requested app - `renovate.py` decides that itself, from the target manifest's own `apps` mapping, and simply does nothing (never opening an SSH connection) if none of the requested apps are present. To tell whether a host's image actually changed (rather than the pull being a no-op), it compares the immutable image IDs used by each Compose service before the pull and after `docker compose up`, and exposes that as this action's own `updated`/`updated-hosts`/`updated-items`/`target-name` outputs. `updated-items` is the structured JSON form used to group changed services by app and host and show their version transitions in notifications; it uses the image's OCI version label when available and falls back to a short image ID. `updated-hosts` remains available as the original comma-separated compatibility output. +After every requested app has been pulled and recreated on a host, that host gets a `docker container prune -f && docker image prune -a -f` pass - the same cleanup `deploy.py` already runs after every deploy. It runs once per host per invocation regardless of whether any app's image actually changed, since a previous renovate run's now-superseded image is what's being cleaned up, not the one just pulled. Without it, repeated renovate runs across several apps leave the old image behind every time, and those dangling layers silently accumulate until the disk fills - see [#219](https://github.com/rubykatzen/flightdeck/issues/219) for the production incident that prompted this. `docker image prune` only ever removes images with no container referencing them, so an image still backing a running container is never touched. + The action prepares a host-grouped message as `telegram-message` only when something changed, but never sends it or receives Telegram credentials. Sending the message is the *caller's* job, as a separate step reading that output, same as shown above - a different caller is free to wire up a different channel, or none, without forking this action. Same checkout model as [`deploy`](../deploy): this action never checks out anything itself, it just reads `target-manifest` from whatever the caller's own preceding `actions/checkout` step already put on disk. diff --git a/.github/actions/renovate/renovate.py b/.github/actions/renovate/renovate.py index ac8df29..e9e4756 100644 --- a/.github/actions/renovate/renovate.py +++ b/.github/actions/renovate/renovate.py @@ -4,7 +4,11 @@ without touching versions - no new app bundle, no new vault-sourced env, no rebuilt release tree. Just `docker compose pull && docker compose up -d` per requested app, against its already-current release, on each -of the target's hosts. +of the target's hosts. After every app is processed on a host, prunes +that host's now-unused images (`docker image prune -a -f`, same as +deploy.py's own post-deploy cleanup) - repeated renovate runs otherwise +leave the superseded image behind every time, and those dangling layers +accumulate across weeks of runs until the disk fills (see #219). Like deploy.py, this takes a manifest *path* and parses it itself - the same shape encrypt-env's render-env.py already uses for vault manifests, @@ -104,6 +108,12 @@ def renovate_host(host, base_path, app): return image_transitions(before, after) +def prune_host(host): + connection = Connection(host) + connection.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + connection.run("docker container prune -f && docker image prune -a -f") + + def write_github_output(name, value): output_path = os.environ.get("GITHUB_OUTPUT") if output_path: @@ -133,12 +143,13 @@ def main(): base_path = target.get("path", "~/flightdeck") updated = [] - for app in matching_apps: - for host in target["hosts"]: + for host in target["hosts"]: + for app in matching_apps: print(f"Renovating {app} on {host}") changes = renovate_host(host, base_path, app) if changes: updated.append({"app": app, "host": host, "changes": changes}) + prune_host(host) write_github_output("updated", "true" if updated else "false") write_github_output("updated_hosts", ",".join(f"{item['app']}@{item['host']}" for item in updated)) diff --git a/.github/actions/renovate/tests/test_renovate.py b/.github/actions/renovate/tests/test_renovate.py index 023c0df..904ed49 100644 --- a/.github/actions/renovate/tests/test_renovate.py +++ b/.github/actions/renovate/tests/test_renovate.py @@ -189,6 +189,15 @@ def test_preserves_missing_versions_for_downstream_image_id_fallback(self): ) +class PruneHostTest(unittest.TestCase): + def test_prunes_unused_containers_and_images(self): + fake = FakeConnection("deploy@host") + with patch.object(renovate, "Connection", return_value=fake): + renovate.prune_host("deploy@host") + + self.assertIn("docker container prune -f && docker image prune -a -f", fake.commands) + + class MainTest(unittest.TestCase): def _run_main(self, directory, apps, manifest_text, manifest_name="heimdall"): manifest_path = Path(directory) / f"{manifest_name}.yml" @@ -207,10 +216,12 @@ def test_skips_a_target_that_runs_none_of_the_requested_apps(self): with ( tempfile.TemporaryDirectory() as directory, patch.object(renovate, "renovate_host") as fake_renovate_host, + patch.object(renovate, "prune_host") as fake_prune_host, ): outputs = self._run_main(directory, ["beszel"], "apps:\n traefik: {}\nhosts: [deploy@host]\n") fake_renovate_host.assert_not_called() + fake_prune_host.assert_not_called() self.assertIn("updated=false\n", outputs) self.assertIn("target_name=heimdall\n", outputs) @@ -218,27 +229,49 @@ def test_renovates_every_app_when_apps_is_empty(self): with ( tempfile.TemporaryDirectory() as directory, patch.object(renovate, "renovate_host", return_value=[]) as fake_renovate_host, + patch.object(renovate, "prune_host") as fake_prune_host, ): self._run_main(directory, [], "apps:\n traefik: {}\n gatus: {}\nhosts: [deploy@host]\n") fake_renovate_host.assert_any_call("deploy@host", "~/flightdeck", "traefik") fake_renovate_host.assert_any_call("deploy@host", "~/flightdeck", "gatus") self.assertEqual(fake_renovate_host.call_count, 2) + fake_prune_host.assert_called_once_with("deploy@host") def test_skips_a_target_with_no_apps_when_apps_is_empty(self): with ( tempfile.TemporaryDirectory() as directory, patch.object(renovate, "renovate_host") as fake_renovate_host, + patch.object(renovate, "prune_host") as fake_prune_host, ): outputs = self._run_main(directory, [], "apps: {}\nhosts: [deploy@host]\n") fake_renovate_host.assert_not_called() + fake_prune_host.assert_not_called() self.assertIn("updated=false\n", outputs) + def test_prunes_each_host_once_per_invocation_regardless_of_app_count(self): + with ( + tempfile.TemporaryDirectory() as directory, + patch.object(renovate, "renovate_host", return_value=[]), + patch.object(renovate, "prune_host") as fake_prune_host, + ): + self._run_main( + directory, + ["traefik", "gatus"], + "apps:\n traefik: {}\n gatus: {}\n" + "hosts: [deploy@app1.example.com, deploy@app2.example.com]\n", + ) + + fake_prune_host.assert_any_call("deploy@app1.example.com") + fake_prune_host.assert_any_call("deploy@app2.example.com") + self.assertEqual(fake_prune_host.call_count, 2) + def test_renovates_only_the_requested_apps_present_on_the_target(self): with ( tempfile.TemporaryDirectory() as directory, patch.object(renovate, "renovate_host", return_value=[]) as fake_renovate_host, + patch.object(renovate, "prune_host"), ): self._run_main( directory, @@ -268,6 +301,7 @@ def fake_renovate_host(host, base_path, app): with ( tempfile.TemporaryDirectory() as directory, patch.object(renovate, "renovate_host", side_effect=fake_renovate_host), + patch.object(renovate, "prune_host"), ): outputs = self._run_main(directory, ["beszel", "traefik"], manifest) @@ -285,6 +319,7 @@ def test_reports_not_updated_when_every_host_was_already_current(self): with ( tempfile.TemporaryDirectory() as directory, patch.object(renovate, "renovate_host", return_value=[]), + patch.object(renovate, "prune_host"), ): outputs = self._run_main(directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n") @@ -296,6 +331,7 @@ def test_defaults_path_when_omitted(self): with ( tempfile.TemporaryDirectory() as directory, patch.object(renovate, "renovate_host", return_value=[]) as fake_renovate_host, + patch.object(renovate, "prune_host"), ): self._run_main(directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n") @@ -305,6 +341,7 @@ def test_derives_target_name_from_manifest_filename(self): with ( tempfile.TemporaryDirectory() as directory, patch.object(renovate, "renovate_host", return_value=[]), + patch.object(renovate, "prune_host"), ): outputs = self._run_main( directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n", manifest_name="mainframe"