Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ops/daemon-update/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ A tick runs 10 minutes after boot, then every hour:
3. Compares it to `.dfe-stack-applied`. The same version only fast-forwards the checkout and restarts nothing. A lower version is refused, because a box installed from a cut branch can sit ahead of the newest published stack.
4. A higher version runs `git pull --ff-only`, `make init`, `make stack VERSION=<new>` and `make ci`, then records the version.

5. After `make ci` and its self test pass, removes the images the previous pin used. Before the re-pin the tick lists every image `docker compose config --images` resolves to across all profiles. Once the update succeeds it runs `docker image rm` on each that the new stack no longer names and no container, running or stopped, still uses. Images pulled by hand are never touched, and a failed or refused update removes nothing. A removal that errors logs a `warning:` line and the update still counts as succeeded.

A failure exits non-zero, so systemd marks the unit failed and the next tick retries. `make ci` pulls the images before it stops the stack, so a failed pull leaves the running stack up. Output reaches the journal as it happens: `journalctl -u dfe-docker-update -f`.

## The checkout moves with the pins
Expand Down
96 changes: 96 additions & 0 deletions ops/daemon-update/self_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,19 @@ def _apply(*, repo_dir: Path, version: str) -> None:
_make(repo_dir=repo_dir, target=target)


def _docker(*, args: Sequence[str], repo_dir: Path) -> subprocess.CompletedProcess:
"""Run docker in the checkout and capture what it prints."""
return subprocess.run(
["docker", *args],
capture_output=True,
check=False,
cwd=repo_dir,
encoding="utf-8",
errors="replace",
text=True,
)


def _flag_set(*, name: str) -> bool:
"""Return whether an environment flag is switched on."""
return os.environ.get(name, "").strip() in _TRUE_VALUES
Expand Down Expand Up @@ -108,6 +121,68 @@ def _make(*, repo_dir: Path, target: Sequence[str]) -> None:
raise UpdateError(f"{command!r} failed (exit {result.returncode})")


def _identities(*, image: str) -> set[str]:
"""Return the names one image reference answers to, so `x:1` and `x:1@sha256:a` match.

A reference with a digest answers to its name:tag and to its name@digest. One without answers to its name:tag.
"""
named, _, digest = image.partition("@")
name, separator, _tag = named.rpartition(":")
repository = name if separator and "/" not in _tag else named
identities = {named}
if digest:
identities.add(f"{repository}@{digest}")
return identities


def _images_in_use(*, repo_dir: Path) -> set[str]:
"""Return the image of every container on the host, running or stopped."""
listed = _docker(args=("ps", "-a", "--format", "{{.Image}}"), repo_dir=repo_dir)
if listed.returncode != 0:
raise UpdateError(f"docker ps failed: {listed.stderr.strip()!r}")
return {line.strip() for line in listed.stdout.splitlines() if line.strip()}


def _previous_images(*, repo_dir: Path) -> set[str]:
"""Return the images the current pins name, empty when they cannot be read."""
try:
return _stack_images(repo_dir=repo_dir)
except (OSError, UpdateError) as error:
_log(message=f"warning: previous images unknown, none will be removed: {error}")
return set()


def _prune_images(*, previous: set[str], repo_dir: Path) -> None:
"""Remove the images the previous pin used that nothing uses now.

Only the previous stack's own references go, so an image pulled by hand stays. A failure is logged and never fails the update, which has already succeeded.
"""
try:
keep = _stack_images(repo_dir=repo_dir) | _images_in_use(repo_dir=repo_dir)
except (OSError, UpdateError) as error:
_log(
message=f"warning: image prune skipped, cannot list what is in use: {error}"
)
return
kept = {identity for image in keep for identity in _identities(image=image)}
removed = 0
for image in sorted(previous):
if _identities(image=image) & kept:
continue
try:
result = _docker(args=("image", "rm", image), repo_dir=repo_dir)
except OSError as error:
_log(message=f"warning: could not remove {image!r}: {error}")
continue
if result.returncode != 0:
_log(
message=f"warning: could not remove {image!r}: {result.stderr.strip()!r}"
)
continue
removed += 1
_log(message=f"removed {removed} image(s) the previous pin used")


def _refresh_checkout(*, repo_dir: Path) -> None:
"""Fast-forward the checkout, refusing uncommitted changes to tracked files."""
if not ((repo_dir / ".git").exists()):
Expand Down Expand Up @@ -142,6 +217,25 @@ def _refresh_plan(*, repo_dir: Path) -> str:
return "run git pull --ff-only"


def _stack_images(*, repo_dir: Path) -> set[str]:
"""Return every image the checkout's compose file resolves to under the current pins, across all profiles."""
resolved = _docker(
args=(
"compose",
"-f",
"docker-compose.yml",
"--profile",
"*",
"config",
"--images",
),
repo_dir=repo_dir,
)
if resolved.returncode != 0:
raise UpdateError(f"docker compose config failed: {resolved.stderr.strip()!r}")
return {line.strip() for line in resolved.stdout.splitlines() if line.strip()}


def _target(*, allow_prerelease: bool, repo: str) -> tuple[str, str]:
"""Return the version this box should be on and what `make stack` pins to reach it.

Expand Down Expand Up @@ -193,9 +287,11 @@ def _tick(*, dry_run: bool, repo_dir: Path) -> None:
return

_log(message=f"updating {applied!r} -> {latest!r}")
previous = _previous_images(repo_dir=repo_dir)
_apply(repo_dir=repo_dir, version=pin)
state_path.write_text(f"{latest}\n", encoding="utf-8", newline="\n")
_log(message=f"updated to {latest!r}")
_prune_images(previous=previous, repo_dir=repo_dir)


def main() -> int:
Expand Down
196 changes: 196 additions & 0 deletions scripts/tests/test_self_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import subprocess
import sys
from collections.abc import Sequence
from pathlib import Path
from typing import TypedDict

Expand Down Expand Up @@ -70,6 +71,17 @@ class MainCase(TypedDict):
latest: str


class PruneCase(TypedDict):
"""The previous and new stacks' images, the images containers use, and what is removed."""

expected_log: str
expected_removed: list[str]
failing: set[str]
in_use: set[str]
new_stack: set[str]
previous: set[str]


class RefreshPlanCase(TypedDict):
"""A checkout's state and the words --dry-run uses for it."""

Expand Down Expand Up @@ -217,6 +229,64 @@ class RefreshPlanCase(TypedDict):
latest="2.2.0-rc.14",
),
]
PRUNE_CASES = [
PruneCase(
expected_log="removed 2 image(s)",
expected_removed=["a:1@sha256:a", "b:1@sha256:b"],
failing=set(),
in_use=set(),
new_stack={"a:2@sha256:c"},
previous={"a:1@sha256:a", "b:1@sha256:b"},
),
PruneCase(
expected_log="removed 1 image(s)",
expected_removed=["a:1@sha256:a"],
failing=set(),
in_use=set(),
new_stack={"b:1@sha256:b"},
previous={"a:1@sha256:a", "b:1@sha256:b"},
),
PruneCase(
expected_log="removed 1 image(s)",
expected_removed=["b:1@sha256:b"],
failing=set(),
in_use={"a:1@sha256:a"},
new_stack=set(),
previous={"a:1@sha256:a", "b:1@sha256:b"},
),
PruneCase(
expected_log="warning: could not remove 'a:1@sha256:a': 'boom'",
expected_removed=["a:1@sha256:a", "b:1@sha256:b"],
failing={"a:1@sha256:a"},
in_use=set(),
new_stack=set(),
previous={"a:1@sha256:a", "b:1@sha256:b"},
),
PruneCase(
expected_log="removed 0 image(s)",
expected_removed=[],
failing=set(),
in_use=set(),
new_stack={"x:1@sha256:abc"},
previous={"x:1"},
),
PruneCase(
expected_log="removed 0 image(s)",
expected_removed=[],
failing=set(),
in_use=set(),
new_stack={"x:2@sha256:abc"},
previous={"x:1@sha256:abc"},
),
PruneCase(
expected_log="removed 0 image(s)",
expected_removed=[],
failing=set(),
in_use=set(),
new_stack=set(),
previous=set(),
),
]
REFRESH_PLAN_CASES = [
RefreshPlanCase(dirty=False, expected="run git pull --ff-only"),
RefreshPlanCase(dirty=True, expected="refuse"),
Expand Down Expand Up @@ -245,6 +315,7 @@ def manifest_repo() -> str:
return "ghcr.io/test/manifest"

applied = []
_fake_docker(monkeypatch=monkeypatch)
monkeypatch.delenv(name=stack.REPIN_ENV, raising=False)
monkeypatch.delenv(name="DFE_UPDATE_ALLOW_PRERELEASE", raising=False)
monkeypatch.setattr(name="_apply", target=self_update, value=apply)
Expand All @@ -253,6 +324,38 @@ def manifest_repo() -> str:
return applied


def _fake_docker(
*,
configured: list[set[str]] | None = None,
failing: set[str] | None = None,
in_use: set[str] | None = None,
monkeypatch: pytest.MonkeyPatch,
) -> list[str]:
"""Stand in for the docker CLI, returning the images it was asked to remove.

`configured` is what `docker compose config --images` prints on each call, the last entry repeating. `failing` names the images whose removal exits non-zero.
"""

def docker(*, args: Sequence[str], repo_dir: Path) -> subprocess.CompletedProcess:
code = 0
out = ""
if args[0] == "compose":
sets = configured or [set()]
out = "\n".join(sorted(sets.pop(0) if len(sets) > 1 else sets[0]))
elif args[0] == "ps":
out = "\n".join(sorted(in_use or set()))
elif args[:2] == ("image", "rm"):
removed.append(args[2])
code = 1 if args[2] in (failing or set()) else 0
return subprocess.CompletedProcess(
args=args, returncode=code, stdout=out, stderr="boom" if code else ""
)

removed = []
monkeypatch.setattr(name="_docker", target=self_update, value=docker)
return removed


def _git(*args: str, cwd: Path) -> None:
"""Run git with a throwaway identity."""
subprocess.run(
Expand Down Expand Up @@ -392,6 +495,99 @@ def pins(*, prereleases: str) -> dict[str, str]:
assert _state(repo=repo) == _OLD_IMAGE_TARGET


class TestPruneImages:
@pytest.mark.parametrize("case", PRUNE_CASES)
def test_matches_expected(
self,
capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
case: PruneCase,
) -> None:
removed = _fake_docker(
configured=[case["new_stack"]],
failing=case["failing"],
in_use=case["in_use"],
monkeypatch=monkeypatch,
)

self_update._prune_images(previous=case["previous"], repo_dir=tmp_path)

assert removed == case["expected_removed"]
assert case["expected_log"] in capsys.readouterr().out

def test_an_unreadable_container_list_removes_nothing(
self,
capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
def broken(*, args: Sequence[str], repo_dir: Path) -> None:
raise FileNotFoundError("docker")

monkeypatch.setattr(name="_docker", target=self_update, value=broken)

self_update._prune_images(previous={"a:1@sha256:a"}, repo_dir=tmp_path)

assert "image prune skipped" in capsys.readouterr().out

def test_a_successful_update_removes_the_previous_stacks_images(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
repo = _checkout(applied="2.2.0-rc.13", tmp_path=tmp_path)
_discover(latest="2.2.0-rc.14", monkeypatch=monkeypatch)
old, new = {"a:1@sha256:a", "b:1@sha256:b"}, {"b:1@sha256:b", "a:2@sha256:c"}
removed = _fake_docker(configured=[old, new], monkeypatch=monkeypatch)

assert _main(dry_run=False, monkeypatch=monkeypatch, repo=repo) == 0

assert removed == ["a:1@sha256:a"]

def test_a_removal_error_does_not_fail_the_update(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
repo = _checkout(applied="2.2.0-rc.13", tmp_path=tmp_path)
_discover(latest="2.2.0-rc.14", monkeypatch=monkeypatch)
removed = _fake_docker(
configured=[{"a:1@sha256:a"}, {"a:2@sha256:c"}],
failing={"a:1@sha256:a"},
monkeypatch=monkeypatch,
)

assert _main(dry_run=False, monkeypatch=monkeypatch, repo=repo) == 0

assert removed == ["a:1@sha256:a"]
assert _state(repo=repo) == "2.2.0-rc.14"

def test_a_failed_update_removes_nothing(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
def fail(*, repo_dir: Path, version: str) -> None:
raise self_update.UpdateError("'make ci' failed (exit 2)")

repo = _checkout(applied="2.2.0-rc.13", tmp_path=tmp_path)
_discover(latest="2.2.0-rc.14", monkeypatch=monkeypatch)
removed = _fake_docker(
configured=[{"a:1@sha256:a"}, {"a:2@sha256:c"}], monkeypatch=monkeypatch
)
monkeypatch.setattr(name="_apply", target=self_update, value=fail)

assert _main(dry_run=False, monkeypatch=monkeypatch, repo=repo) == 1

assert removed == []

def test_a_dry_run_removes_nothing(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
repo = _checkout(applied="2.2.0-rc.13", tmp_path=tmp_path)
_discover(latest="2.2.0-rc.14", monkeypatch=monkeypatch)
removed = _fake_docker(configured=[{"a:1@sha256:a"}], monkeypatch=monkeypatch)

assert _main(dry_run=True, monkeypatch=monkeypatch, repo=repo) == 0

assert removed == []


class TestIsDowngrade:
@pytest.mark.parametrize("case", IS_DOWNGRADE_CASES)
def test_matches_expected(self, case: IsDowngradeCase) -> None:
Expand Down
Loading