From 2bef82030a57c3704d6aaf003b7b37716af6ce28 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 12:01:32 +1100 Subject: [PATCH 1/2] fix(update): remove the previous stack images after a nightly update Self-update left a full set of images from the old pin after each update. The VM disk reached 92%. - Before the re-pin it lists the images docker compose resolves to across all profiles. - After make ci and its self test pass, it runs docker image rm on each one the new stack no longer names and no container uses. - Images pulled by hand are not touched. - A failed update removes nothing. - A removal error logs a warning and the update still succeeds. --- ops/daemon-update/README.md | 2 + ops/daemon-update/self_update.py | 79 +++++++++++++ scripts/tests/test_self_update.py | 180 ++++++++++++++++++++++++++++++ 3 files changed, 261 insertions(+) diff --git a/ops/daemon-update/README.md b/ops/daemon-update/README.md index 42065afa..0d3271d0 100644 --- a/ops/daemon-update/README.md +++ b/ops/daemon-update/README.md @@ -11,6 +11,8 @@ A tick runs 10 minutes after boot, then every hour: 3. Compares it to `.dfe-stack-applied`. The same version only fast-forwards the checkout and restarts nothing. A lower version is refused, because a box installed from a cut branch can sit ahead of the newest published stack. 4. A higher version runs `git pull --ff-only`, `make init`, `make stack VERSION=` and `make ci`, then records the version. +5. After `make ci` and its self test pass, removes the images the previous pin used. Before the re-pin the tick lists every image `docker compose config --images` resolves to across all profiles. Once the update succeeds it runs `docker image rm` on each that the new stack no longer names and no container, running or stopped, still uses. Images pulled by hand are never touched, and a failed or refused update removes nothing. A removal that errors logs a `warning:` line and the update still counts as succeeded. + A failure exits non-zero, so systemd marks the unit failed and the next tick retries. `make ci` pulls the images before it stops the stack, so a failed pull leaves the running stack up. Output reaches the journal as it happens: `journalctl -u dfe-docker-update -f`. ## The checkout moves with the pins diff --git a/ops/daemon-update/self_update.py b/ops/daemon-update/self_update.py index 3d0e3cc1..8b025fff 100755 --- a/ops/daemon-update/self_update.py +++ b/ops/daemon-update/self_update.py @@ -66,6 +66,19 @@ def _apply(*, repo_dir: Path, version: str) -> None: _make(repo_dir=repo_dir, target=target) +def _docker(*, args: Sequence[str], repo_dir: Path) -> subprocess.CompletedProcess: + """Run docker in the checkout and capture what it prints.""" + return subprocess.run( + ["docker", *args], + capture_output=True, + check=False, + cwd=repo_dir, + encoding="utf-8", + errors="replace", + text=True, + ) + + def _flag_set(*, name: str) -> bool: """Return whether an environment flag is switched on.""" return os.environ.get(name, "").strip() in _TRUE_VALUES @@ -108,6 +121,51 @@ def _make(*, repo_dir: Path, target: Sequence[str]) -> None: raise UpdateError(f"{command!r} failed (exit {result.returncode})") +def _images_in_use(*, repo_dir: Path) -> set[str]: + """Return the image of every container on the host, running or stopped.""" + listed = _docker(args=("ps", "-a", "--format", "{{.Image}}"), repo_dir=repo_dir) + if listed.returncode != 0: + raise UpdateError(f"docker ps failed: {listed.stderr.strip()!r}") + return {line.strip() for line in listed.stdout.splitlines() if line.strip()} + + +def _previous_images(*, repo_dir: Path) -> set[str]: + """Return the images the current pins name, empty when they cannot be read.""" + try: + return _stack_images(repo_dir=repo_dir) + except (OSError, UpdateError) as error: + _log(message=f"warning: previous images unknown, none will be removed: {error}") + return set() + + +def _prune_images(*, previous: set[str], repo_dir: Path) -> None: + """Remove the images the previous pin used that nothing uses now. + + Only the previous stack's own references go, so an image pulled by hand stays. A failure is logged and never fails the update, which has already succeeded. + """ + try: + keep = _stack_images(repo_dir=repo_dir) | _images_in_use(repo_dir=repo_dir) + except (OSError, UpdateError) as error: + _log( + message=f"warning: image prune skipped, cannot list what is in use: {error}" + ) + return + removed = 0 + for image in sorted(previous - keep): + try: + result = _docker(args=("image", "rm", image), repo_dir=repo_dir) + except OSError as error: + _log(message=f"warning: could not remove {image!r}: {error}") + continue + if result.returncode != 0: + _log( + message=f"warning: could not remove {image!r}: {result.stderr.strip()!r}" + ) + continue + removed += 1 + _log(message=f"removed {removed} image(s) the previous pin used") + + def _refresh_checkout(*, repo_dir: Path) -> None: """Fast-forward the checkout, refusing uncommitted changes to tracked files.""" if not ((repo_dir / ".git").exists()): @@ -142,6 +200,25 @@ def _refresh_plan(*, repo_dir: Path) -> str: return "run git pull --ff-only" +def _stack_images(*, repo_dir: Path) -> set[str]: + """Return every image the checkout's compose file resolves to under the current pins, across all profiles.""" + resolved = _docker( + args=( + "compose", + "-f", + "docker-compose.yml", + "--profile", + "*", + "config", + "--images", + ), + repo_dir=repo_dir, + ) + if resolved.returncode != 0: + raise UpdateError(f"docker compose config failed: {resolved.stderr.strip()!r}") + return {line.strip() for line in resolved.stdout.splitlines() if line.strip()} + + def _target(*, allow_prerelease: bool, repo: str) -> tuple[str, str]: """Return the version this box should be on and what `make stack` pins to reach it. @@ -193,9 +270,11 @@ def _tick(*, dry_run: bool, repo_dir: Path) -> None: return _log(message=f"updating {applied!r} -> {latest!r}") + previous = _previous_images(repo_dir=repo_dir) _apply(repo_dir=repo_dir, version=pin) state_path.write_text(f"{latest}\n", encoding="utf-8", newline="\n") _log(message=f"updated to {latest!r}") + _prune_images(previous=previous, repo_dir=repo_dir) def main() -> int: diff --git a/scripts/tests/test_self_update.py b/scripts/tests/test_self_update.py index 2a5c6dbc..08efa902 100644 --- a/scripts/tests/test_self_update.py +++ b/scripts/tests/test_self_update.py @@ -10,6 +10,7 @@ import subprocess import sys +from collections.abc import Sequence from pathlib import Path from typing import TypedDict @@ -70,6 +71,17 @@ class MainCase(TypedDict): latest: str +class PruneCase(TypedDict): + """The previous and new stacks' images, the images containers use, and what is removed.""" + + expected_log: str + expected_removed: list[str] + failing: set[str] + in_use: set[str] + new_stack: set[str] + previous: set[str] + + class RefreshPlanCase(TypedDict): """A checkout's state and the words --dry-run uses for it.""" @@ -217,6 +229,48 @@ class RefreshPlanCase(TypedDict): latest="2.2.0-rc.14", ), ] +PRUNE_CASES = [ + PruneCase( + expected_log="removed 2 image(s)", + expected_removed=["a:1@sha256:a", "b:1@sha256:b"], + failing=set(), + in_use=set(), + new_stack={"a:2@sha256:c"}, + previous={"a:1@sha256:a", "b:1@sha256:b"}, + ), + PruneCase( + expected_log="removed 1 image(s)", + expected_removed=["a:1@sha256:a"], + failing=set(), + in_use=set(), + new_stack={"b:1@sha256:b"}, + previous={"a:1@sha256:a", "b:1@sha256:b"}, + ), + PruneCase( + expected_log="removed 1 image(s)", + expected_removed=["b:1@sha256:b"], + failing=set(), + in_use={"a:1@sha256:a"}, + new_stack=set(), + previous={"a:1@sha256:a", "b:1@sha256:b"}, + ), + PruneCase( + expected_log="warning: could not remove 'a:1@sha256:a': 'boom'", + expected_removed=["a:1@sha256:a", "b:1@sha256:b"], + failing={"a:1@sha256:a"}, + in_use=set(), + new_stack=set(), + previous={"a:1@sha256:a", "b:1@sha256:b"}, + ), + PruneCase( + expected_log="removed 0 image(s)", + expected_removed=[], + failing=set(), + in_use=set(), + new_stack=set(), + previous=set(), + ), +] REFRESH_PLAN_CASES = [ RefreshPlanCase(dirty=False, expected="run git pull --ff-only"), RefreshPlanCase(dirty=True, expected="refuse"), @@ -245,6 +299,7 @@ def manifest_repo() -> str: return "ghcr.io/test/manifest" applied = [] + _fake_docker(monkeypatch=monkeypatch) monkeypatch.delenv(name=stack.REPIN_ENV, raising=False) monkeypatch.delenv(name="DFE_UPDATE_ALLOW_PRERELEASE", raising=False) monkeypatch.setattr(name="_apply", target=self_update, value=apply) @@ -253,6 +308,38 @@ def manifest_repo() -> str: return applied +def _fake_docker( + *, + configured: list[set[str]] | None = None, + failing: set[str] | None = None, + in_use: set[str] | None = None, + monkeypatch: pytest.MonkeyPatch, +) -> list[str]: + """Stand in for the docker CLI, returning the images it was asked to remove. + + `configured` is what `docker compose config --images` prints on each call, the last entry repeating. `failing` names the images whose removal exits non-zero. + """ + + def docker(*, args: Sequence[str], repo_dir: Path) -> subprocess.CompletedProcess: + code = 0 + out = "" + if args[0] == "compose": + sets = configured or [set()] + out = "\n".join(sorted(sets.pop(0) if len(sets) > 1 else sets[0])) + elif args[0] == "ps": + out = "\n".join(sorted(in_use or set())) + elif args[:2] == ("image", "rm"): + removed.append(args[2]) + code = 1 if args[2] in (failing or set()) else 0 + return subprocess.CompletedProcess( + args=args, returncode=code, stdout=out, stderr="boom" if code else "" + ) + + removed = [] + monkeypatch.setattr(name="_docker", target=self_update, value=docker) + return removed + + def _git(*args: str, cwd: Path) -> None: """Run git with a throwaway identity.""" subprocess.run( @@ -392,6 +479,99 @@ def pins(*, prereleases: str) -> dict[str, str]: assert _state(repo=repo) == _OLD_IMAGE_TARGET +class TestPruneImages: + @pytest.mark.parametrize("case", PRUNE_CASES) + def test_matches_expected( + self, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + case: PruneCase, + ) -> None: + removed = _fake_docker( + configured=[case["new_stack"]], + failing=case["failing"], + in_use=case["in_use"], + monkeypatch=monkeypatch, + ) + + self_update._prune_images(previous=case["previous"], repo_dir=tmp_path) + + assert removed == case["expected_removed"] + assert case["expected_log"] in capsys.readouterr().out + + def test_an_unreadable_container_list_removes_nothing( + self, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + def broken(*, args: Sequence[str], repo_dir: Path) -> None: + raise FileNotFoundError("docker") + + monkeypatch.setattr(name="_docker", target=self_update, value=broken) + + self_update._prune_images(previous={"a:1@sha256:a"}, repo_dir=tmp_path) + + assert "image prune skipped" in capsys.readouterr().out + + def test_a_successful_update_removes_the_previous_stacks_images( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + repo = _checkout(applied="2.2.0-rc.13", tmp_path=tmp_path) + _discover(latest="2.2.0-rc.14", monkeypatch=monkeypatch) + old, new = {"a:1@sha256:a", "b:1@sha256:b"}, {"b:1@sha256:b", "a:2@sha256:c"} + removed = _fake_docker(configured=[old, new], monkeypatch=monkeypatch) + + assert _main(dry_run=False, monkeypatch=monkeypatch, repo=repo) == 0 + + assert removed == ["a:1@sha256:a"] + + def test_a_removal_error_does_not_fail_the_update( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + repo = _checkout(applied="2.2.0-rc.13", tmp_path=tmp_path) + _discover(latest="2.2.0-rc.14", monkeypatch=monkeypatch) + removed = _fake_docker( + configured=[{"a:1@sha256:a"}, {"a:2@sha256:c"}], + failing={"a:1@sha256:a"}, + monkeypatch=monkeypatch, + ) + + assert _main(dry_run=False, monkeypatch=monkeypatch, repo=repo) == 0 + + assert removed == ["a:1@sha256:a"] + assert _state(repo=repo) == "2.2.0-rc.14" + + def test_a_failed_update_removes_nothing( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + def fail(*, repo_dir: Path, version: str) -> None: + raise self_update.UpdateError("'make ci' failed (exit 2)") + + repo = _checkout(applied="2.2.0-rc.13", tmp_path=tmp_path) + _discover(latest="2.2.0-rc.14", monkeypatch=monkeypatch) + removed = _fake_docker( + configured=[{"a:1@sha256:a"}, {"a:2@sha256:c"}], monkeypatch=monkeypatch + ) + monkeypatch.setattr(name="_apply", target=self_update, value=fail) + + assert _main(dry_run=False, monkeypatch=monkeypatch, repo=repo) == 1 + + assert removed == [] + + def test_a_dry_run_removes_nothing( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + repo = _checkout(applied="2.2.0-rc.13", tmp_path=tmp_path) + _discover(latest="2.2.0-rc.14", monkeypatch=monkeypatch) + removed = _fake_docker(configured=[{"a:1@sha256:a"}], monkeypatch=monkeypatch) + + assert _main(dry_run=True, monkeypatch=monkeypatch, repo=repo) == 0 + + assert removed == [] + + class TestIsDowngrade: @pytest.mark.parametrize("case", IS_DOWNGRADE_CASES) def test_matches_expected(self, case: IsDowngradeCase) -> None: From f286c0a178aa1b5a9379b881851c1ceebe87dbff Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 12:07:47 +1100 Subject: [PATCH 2/2] fix(update): match image identities before removing images A previous pin of x:1 and a new pin of x:1@sha256:abc are one image. The string comparison treated them as different images. docker image rm x:1 could therefore untag an image the new stack uses. Each reference matches name:tag and, if it has a digest, name@digest. A previous image is kept if either form matches the new stack or a container. --- ops/daemon-update/self_update.py | 19 ++++++++++++++++++- scripts/tests/test_self_update.py | 16 ++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/ops/daemon-update/self_update.py b/ops/daemon-update/self_update.py index 8b025fff..012d7544 100755 --- a/ops/daemon-update/self_update.py +++ b/ops/daemon-update/self_update.py @@ -121,6 +121,20 @@ def _make(*, repo_dir: Path, target: Sequence[str]) -> None: raise UpdateError(f"{command!r} failed (exit {result.returncode})") +def _identities(*, image: str) -> set[str]: + """Return the names one image reference answers to, so `x:1` and `x:1@sha256:a` match. + + A reference with a digest answers to its name:tag and to its name@digest. One without answers to its name:tag. + """ + named, _, digest = image.partition("@") + name, separator, _tag = named.rpartition(":") + repository = name if separator and "/" not in _tag else named + identities = {named} + if digest: + identities.add(f"{repository}@{digest}") + return identities + + def _images_in_use(*, repo_dir: Path) -> set[str]: """Return the image of every container on the host, running or stopped.""" listed = _docker(args=("ps", "-a", "--format", "{{.Image}}"), repo_dir=repo_dir) @@ -150,8 +164,11 @@ def _prune_images(*, previous: set[str], repo_dir: Path) -> None: message=f"warning: image prune skipped, cannot list what is in use: {error}" ) return + kept = {identity for image in keep for identity in _identities(image=image)} removed = 0 - for image in sorted(previous - keep): + for image in sorted(previous): + if _identities(image=image) & kept: + continue try: result = _docker(args=("image", "rm", image), repo_dir=repo_dir) except OSError as error: diff --git a/scripts/tests/test_self_update.py b/scripts/tests/test_self_update.py index 08efa902..758b6216 100644 --- a/scripts/tests/test_self_update.py +++ b/scripts/tests/test_self_update.py @@ -262,6 +262,22 @@ class RefreshPlanCase(TypedDict): new_stack=set(), previous={"a:1@sha256:a", "b:1@sha256:b"}, ), + PruneCase( + expected_log="removed 0 image(s)", + expected_removed=[], + failing=set(), + in_use=set(), + new_stack={"x:1@sha256:abc"}, + previous={"x:1"}, + ), + PruneCase( + expected_log="removed 0 image(s)", + expected_removed=[], + failing=set(), + in_use=set(), + new_stack={"x:2@sha256:abc"}, + previous={"x:1@sha256:abc"}, + ), PruneCase( expected_log="removed 0 image(s)", expected_removed=[],