Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/actions/renovate/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ jobs:

Not every target runs every requested app - `renovate.py` decides that itself, from the target manifest's own `apps` mapping, and simply does nothing (never opening an SSH connection) if none of the requested apps are present. To tell whether a host's image actually changed (rather than the pull being a no-op), it compares the immutable image IDs used by each Compose service before the pull and after `docker compose up`, and exposes that as this action's own `updated`/`updated-hosts`/`updated-items`/`target-name` outputs. `updated-items` is the structured JSON form used to group changed services by app and host and show their version transitions in notifications; it uses the image's OCI version label when available and falls back to a short image ID. `updated-hosts` remains available as the original comma-separated compatibility output.

After every requested app has been pulled and recreated on a host, that host gets a `docker container prune -f && docker image prune -a -f` pass - the same cleanup `deploy.py` already runs after every deploy. It runs once per host per invocation regardless of whether any app's image actually changed, since a previous renovate run's now-superseded image is what's being cleaned up, not the one just pulled. Without it, repeated renovate runs across several apps leave the old image behind every time, and those dangling layers silently accumulate until the disk fills - see [#219](https://github.com/rubykatzen/flightdeck/issues/219) for the production incident that prompted this. `docker image prune` only ever removes images with no container referencing them, so an image still backing a running container is never touched.

The action prepares a host-grouped message as `telegram-message` only when something changed, but never sends it or receives Telegram credentials. Sending the message is the *caller's* job, as a separate step reading that output, same as shown above - a different caller is free to wire up a different channel, or none, without forking this action.

Same checkout model as [`deploy`](../deploy): this action never checks out anything itself, it just reads `target-manifest` from whatever the caller's own preceding `actions/checkout` step already put on disk.
17 changes: 14 additions & 3 deletions .github/actions/renovate/renovate.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@
without touching versions - no new app bundle, no new vault-sourced env,
no rebuilt release tree. Just `docker compose pull && docker compose
up -d` per requested app, against its already-current release, on each
of the target's hosts.
of the target's hosts. After every app is processed on a host, prunes
that host's now-unused images (`docker image prune -a -f`, same as
deploy.py's own post-deploy cleanup) - repeated renovate runs otherwise
leave the superseded image behind every time, and those dangling layers
accumulate across weeks of runs until the disk fills (see #219).

Like deploy.py, this takes a manifest *path* and parses it itself - the
same shape encrypt-env's render-env.py already uses for vault manifests,
Expand Down Expand Up @@ -104,6 +108,12 @@ def renovate_host(host, base_path, app):
return image_transitions(before, after)


def prune_host(host):
connection = Connection(host)
connection.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
connection.run("docker container prune -f && docker image prune -a -f")


def write_github_output(name, value):
output_path = os.environ.get("GITHUB_OUTPUT")
if output_path:
Expand Down Expand Up @@ -133,12 +143,13 @@ def main():

base_path = target.get("path", "~/flightdeck")
updated = []
for app in matching_apps:
for host in target["hosts"]:
for host in target["hosts"]:
for app in matching_apps:
print(f"Renovating {app} on {host}")
changes = renovate_host(host, base_path, app)
if changes:
updated.append({"app": app, "host": host, "changes": changes})
prune_host(host)

write_github_output("updated", "true" if updated else "false")
write_github_output("updated_hosts", ",".join(f"{item['app']}@{item['host']}" for item in updated))
Expand Down
37 changes: 37 additions & 0 deletions .github/actions/renovate/tests/test_renovate.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,15 @@ def test_preserves_missing_versions_for_downstream_image_id_fallback(self):
)


class PruneHostTest(unittest.TestCase):
def test_prunes_unused_containers_and_images(self):
fake = FakeConnection("deploy@host")
with patch.object(renovate, "Connection", return_value=fake):
renovate.prune_host("deploy@host")

self.assertIn("docker container prune -f && docker image prune -a -f", fake.commands)


class MainTest(unittest.TestCase):
def _run_main(self, directory, apps, manifest_text, manifest_name="heimdall"):
manifest_path = Path(directory) / f"{manifest_name}.yml"
Expand All @@ -207,38 +216,62 @@ def test_skips_a_target_that_runs_none_of_the_requested_apps(self):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host") as fake_renovate_host,
patch.object(renovate, "prune_host") as fake_prune_host,
):
outputs = self._run_main(directory, ["beszel"], "apps:\n traefik: {}\nhosts: [deploy@host]\n")

fake_renovate_host.assert_not_called()
fake_prune_host.assert_not_called()
self.assertIn("updated=false\n", outputs)
self.assertIn("target_name=heimdall\n", outputs)

def test_renovates_every_app_when_apps_is_empty(self):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host", return_value=[]) as fake_renovate_host,
patch.object(renovate, "prune_host") as fake_prune_host,
):
self._run_main(directory, [], "apps:\n traefik: {}\n gatus: {}\nhosts: [deploy@host]\n")

fake_renovate_host.assert_any_call("deploy@host", "~/flightdeck", "traefik")
fake_renovate_host.assert_any_call("deploy@host", "~/flightdeck", "gatus")
self.assertEqual(fake_renovate_host.call_count, 2)
fake_prune_host.assert_called_once_with("deploy@host")

def test_skips_a_target_with_no_apps_when_apps_is_empty(self):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host") as fake_renovate_host,
patch.object(renovate, "prune_host") as fake_prune_host,
):
outputs = self._run_main(directory, [], "apps: {}\nhosts: [deploy@host]\n")

fake_renovate_host.assert_not_called()
fake_prune_host.assert_not_called()
self.assertIn("updated=false\n", outputs)

def test_prunes_each_host_once_per_invocation_regardless_of_app_count(self):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host", return_value=[]),
patch.object(renovate, "prune_host") as fake_prune_host,
):
self._run_main(
directory,
["traefik", "gatus"],
"apps:\n traefik: {}\n gatus: {}\n"
"hosts: [deploy@app1.example.com, deploy@app2.example.com]\n",
)

fake_prune_host.assert_any_call("deploy@app1.example.com")
fake_prune_host.assert_any_call("deploy@app2.example.com")
self.assertEqual(fake_prune_host.call_count, 2)

def test_renovates_only_the_requested_apps_present_on_the_target(self):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host", return_value=[]) as fake_renovate_host,
patch.object(renovate, "prune_host"),
):
self._run_main(
directory,
Expand Down Expand Up @@ -268,6 +301,7 @@ def fake_renovate_host(host, base_path, app):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host", side_effect=fake_renovate_host),
patch.object(renovate, "prune_host"),
):
outputs = self._run_main(directory, ["beszel", "traefik"], manifest)

Expand All @@ -285,6 +319,7 @@ def test_reports_not_updated_when_every_host_was_already_current(self):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host", return_value=[]),
patch.object(renovate, "prune_host"),
):
outputs = self._run_main(directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n")

Expand All @@ -296,6 +331,7 @@ def test_defaults_path_when_omitted(self):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host", return_value=[]) as fake_renovate_host,
patch.object(renovate, "prune_host"),
):
self._run_main(directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n")

Expand All @@ -305,6 +341,7 @@ def test_derives_target_name_from_manifest_filename(self):
with (
tempfile.TemporaryDirectory() as directory,
patch.object(renovate, "renovate_host", return_value=[]),
patch.object(renovate, "prune_host"),
):
outputs = self._run_main(
directory, ["beszel"], "apps:\n beszel: {}\nhosts: [deploy@host]\n", manifest_name="mainframe"
Expand Down
Loading