From e43befcef375f1b680a04c015e4c4428edefce98 Mon Sep 17 00:00:00 2001 From: kylerankin Date: Wed, 18 Nov 2026 00:37:17 +0000 Subject: [PATCH 1/5] fix(sysupdate): prune deprecated features so updates do not silently stall systemd-sysupdate only applies an update set when every feature enabled under /etc/sysupdate.d/.feature.d/ is also provided by a transfer set in /usr/lib/sysupdate.d/. When a release stops shipping a feature -- for example Homebrew becoming a first-boot seed instead of an opt-in extension -- its transfer definitions leave the manifest but the operator's /etc drop-in remains, so systemd-sysupdate can no longer satisfy it and quietly reports "No update needed" instead of updating (issue #688). - add bluefin-sysupdate-migrate: list enabled features no transfer provides and archive each stale /etc/sysupdate.d/.feature.d/ so the next update is unblocked; a feature a transfer still provides is left alone - ship bluefin-sysupdate-migrate.service as an enabled one-shot that runs Before systemd-sysupdate.timer and systemd-sysupdate.service at boot - add a systemd-sysupdate.service.d drop-in that runs the helper as ExecStartPre so timer- and hand-triggered updates are covered too - package the helper through a new os-sysupdate-feature-migrate.bst import in the DDI stack The helper is idempotent: an already-archived drop-in is left in place and never moved again, and a second run is a no-op. Closes #688. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: kylerankin --- elements/bluefin-server/os-sshd-preset.bst | 3 +- elements/bluefin-server/os-stack.bst | 3 + .../os-sysupdate-feature-migrate.bst | 15 ++ ...0-bluefin-sysupdate-feature-migrate.preset | 1 + .../system/bluefin-sysupdate-migrate.service | 17 ++ .../05-bluefin-prune-deprecated-features.conf | 6 + .../bluefin-sysupdate-migrate | 88 ++++++++ tests/unit/test_sysupdate_feature_migrate.py | 207 ++++++++++++++++++ 8 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 elements/bluefin-server/os-sysupdate-feature-migrate.bst create mode 100644 files/os/systemd/system-preset/80-bluefin-sysupdate-feature-migrate.preset create mode 100644 files/os/systemd/system/bluefin-sysupdate-migrate.service create mode 100644 files/os/systemd/system/systemd-sysupdate.service.d/05-bluefin-prune-deprecated-features.conf create mode 100755 files/os/sysupdate-migrate/bluefin-sysupdate-migrate create mode 100644 tests/unit/test_sysupdate_feature_migrate.py diff --git a/elements/bluefin-server/os-sshd-preset.bst b/elements/bluefin-server/os-sshd-preset.bst index 59333d45..288832fb 100644 --- a/elements/bluefin-server/os-sshd-preset.bst +++ b/elements/bluefin-server/os-sshd-preset.bst @@ -1,6 +1,7 @@ kind: import description: | - Install systemd presets ensuring sshd is present but off by default. + Install systemd presets ensuring sshd is present but off by default and the + systemd-sysupdate feature migration is enabled. sources: - kind: local diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst index 38384b22..9d3673e2 100644 --- a/elements/bluefin-server/os-stack.bst +++ b/elements/bluefin-server/os-stack.bst @@ -19,6 +19,9 @@ depends: # A/B System Update Configurations - bluefin-server/os-sysupdate.bst + # Prune sysupdate features the release no longer ships, before an update + - bluefin-server/os-sysupdate-feature-migrate.bst + # Runtime helpers (bluefin-host): update status, reboot interlocks, boot # deadline, diskless update check, sysext fetch, first-boot naming - bluefin-server/os-host.bst diff --git a/elements/bluefin-server/os-sysupdate-feature-migrate.bst b/elements/bluefin-server/os-sysupdate-feature-migrate.bst new file mode 100644 index 00000000..222666fa --- /dev/null +++ b/elements/bluefin-server/os-sysupdate-feature-migrate.bst @@ -0,0 +1,15 @@ +kind: import +description: | + Install the systemd-sysupdate deprecated-feature migration helper so a + feature the release no longer ships cannot silently stop the next update. + + The executable lands in /usr/bin; the boot-time unit, the preset that + enables it, and the ExecStartPre drop-in on systemd-sysupdate.service are + shipped next to the other system units. + +sources: + - kind: local + path: files/os/sysupdate-migrate + +config: + target: /usr/bin diff --git a/files/os/systemd/system-preset/80-bluefin-sysupdate-feature-migrate.preset b/files/os/systemd/system-preset/80-bluefin-sysupdate-feature-migrate.preset new file mode 100644 index 00000000..9d94e209 --- /dev/null +++ b/files/os/systemd/system-preset/80-bluefin-sysupdate-feature-migrate.preset @@ -0,0 +1 @@ +enable bluefin-sysupdate-migrate.service diff --git a/files/os/systemd/system/bluefin-sysupdate-migrate.service b/files/os/systemd/system/bluefin-sysupdate-migrate.service new file mode 100644 index 00000000..58ee33e5 --- /dev/null +++ b/files/os/systemd/system/bluefin-sysupdate-migrate.service @@ -0,0 +1,17 @@ +[Unit] +Description=Bluefin: prune deprecated systemd-sysupdate features +Documentation=https://github.com/projectbluefin/server/issues/688 +# Run ahead of the OTA update path so a feature that the release no longer +# ships cannot silently stop the next update. This ordering covers the boot +# path; the ExecStartPre drop-in on systemd-sysupdate.service covers updates +# triggered later by the timer or by hand. +Before=systemd-sysupdate.timer +Before=systemd-sysupdate.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/bluefin-sysupdate-migrate +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/systemd-sysupdate.service.d/05-bluefin-prune-deprecated-features.conf b/files/os/systemd/system/systemd-sysupdate.service.d/05-bluefin-prune-deprecated-features.conf new file mode 100644 index 00000000..58833116 --- /dev/null +++ b/files/os/systemd/system/systemd-sysupdate.service.d/05-bluefin-prune-deprecated-features.conf @@ -0,0 +1,6 @@ +[Service] +# Prune retired sysupdate features before every update, so an enabled feature +# that the release no longer ships cannot make systemd-sysupdate report +# "No update needed" instead of updating. The helper is idempotent, so running +# it on every update is safe. +ExecStartPre=/usr/bin/bluefin-sysupdate-migrate diff --git a/files/os/sysupdate-migrate/bluefin-sysupdate-migrate b/files/os/sysupdate-migrate/bluefin-sysupdate-migrate new file mode 100755 index 00000000..90dc2e9e --- /dev/null +++ b/files/os/sysupdate-migrate/bluefin-sysupdate-migrate @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# bluefin-sysupdate-migrate -- prune systemd-sysupdate features the current +# release no longer ships, so a retired feature cannot silently stop updates. +# +# systemd-sysupdate only applies an update set when every feature enabled under +# /etc/sysupdate.d/.feature.d/ is also provided by a transfer set in +# /usr/lib/sysupdate.d/. When a feature leaves the release manifest -- for +# example Homebrew becoming a first-boot seed instead of an opt-in extension -- +# the matching /etc drop-in can no longer be satisfied, and systemd-sysupdate +# quietly reports "No update needed" instead of updating. +# +# This one-shot helper is meant to run ahead of the update path. It lists the +# enabled features that no transfer set provides any more, archives each stale +# /etc drop-in, and prints a clear message so an operator is never left +# wondering why an update was skipped. + +set -euo pipefail + +# Paths are overridable so the logic can be exercised by unit tests against a +# scratch tree without touching the real system directories. +etc_feature_dir="${SYSUPDATE_ETC_DIR:-/etc/sysupdate.d}" +lib_transfer_dir="${SYSUPDATE_LIB_DIR:-/usr/lib/sysupdate.d}" +archive_suffix="${SYSUPDATE_ARCHIVE_SUFFIX:-.disabled}" + +# systemd captures ExecStartPre stdout/stderr into the journal, so plain prints +# are enough; the wrapper keeps one message shape for both paths. +info() { + printf 'bluefin-sysupdate-migrate: %s\n' "$*" +} +warn() { + printf 'bluefin-sysupdate-migrate: warning: %s\n' "$*" >&2 +} + +# Feature names the current release still provides, keyed by name. +declare -A provided_features=() + +# Read the Features= key of every transfer set and record the names it lists. +collect_provided_features() { + local transfer line value feature + shopt -s nullglob + for transfer in "$lib_transfer_dir"/*.transfer; do + while IFS= read -r line; do + value="${line#*=}" # text after the first = + value="${value//,/ }" # systemd accepts spaces or commas + set -f # feature names carry no glob characters + for feature in $value; do # word-split on whitespace + feature="${feature#"${feature%%[![:space:]]*}"}" # ltrim + feature="${feature%"${feature##*[![:space:]]}"}" # rtrim + [ -n "$feature" ] && provided_features["$feature"]=1 + done + set +f + done < <(grep -E '^[[:space:]]*[Ff]eatures[[:space:]]*=' "$transfer" || true) + done +} + +main() { + if [ ! -d "$etc_feature_dir" ]; then + info "no ${etc_feature_dir} directory; nothing to do" + return 0 + fi + + collect_provided_features + + local dir name changed + changed=0 + shopt -s nullglob + for dir in "$etc_feature_dir"/*.feature.d; do + name="$(basename "$dir")" + name="${name%.feature.d}" + if [ -n "${provided_features[$name]:-}" ]; then + continue + fi + if [ -e "${dir}${archive_suffix}" ]; then + warn "feature ${name} already archived as ${name}${archive_suffix}; leaving it in place" + continue + fi + mv -- "$dir" "${dir}${archive_suffix}" + info "archived ${dir} -> ${dir}${archive_suffix}: enabled feature ${name} is not shipped in this release and would otherwise make systemd-sysupdate report 'No update needed'" + changed=$((changed + 1)) + done + + if [ "$changed" -eq 0 ]; then + info "no deprecated sysupdate features found; the next update is unaffected" + fi +} + +main "$@" diff --git a/tests/unit/test_sysupdate_feature_migrate.py b/tests/unit/test_sysupdate_feature_migrate.py new file mode 100644 index 00000000..b081be60 --- /dev/null +++ b/tests/unit/test_sysupdate_feature_migrate.py @@ -0,0 +1,207 @@ +"""Contracts for the systemd-sysupdate deprecated-feature migration. + +Release transitions such as Homebrew becoming a first-boot seed drop a feature's +transfer definitions from the release manifest, but leave the operator's +``/etc/sysupdate.d/.feature.d/`` drop-in behind. Because +systemd-sysupdate requires every enabled feature to be provided by a remote +transfer set, that stale drop-in makes it report "No update needed" and the +node silently stops updating (issue #688). + +These tests pin the shipped fix: + +* the ``bluefin-sysupdate-migrate`` helper exists, is executable, and lives at + ``/usr/bin/bluefin-sysupdate-migrate`` in the image +* the one-shot unit runs ahead of the update path (before the timer and the + update unit itself) and is enabled by preset +* the ``systemd-sysupdate.service`` drop-in runs the helper as + ``ExecStartPre`` so every update -- timer- or hand-triggered -- is covered +* the helper archives an enabled feature the release no longer ships, keeps a + feature a transfer still provides, is idempotent, and is a no-op when there + is nothing to do +""" + +import os +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "files" / "os" / "sysupdate-migrate" / "bluefin-sysupdate-migrate" +SERVICE = ( + ROOT / "files" / "os" / "systemd" / "system" / "bluefin-sysupdate-migrate.service" +) +DROPIN = ( + ROOT + / "files" + / "os" + / "systemd" + / "system" + / "systemd-sysupdate.service.d" + / "05-bluefin-prune-deprecated-features.conf" +) +PRESET = ( + ROOT + / "files" + / "os" + / "systemd" + / "system-preset" + / "80-bluefin-sysupdate-feature-migrate.preset" +) +ELEMENT = ROOT / "elements" / "bluefin-server" / "os-sysupdate-feature-migrate.bst" +STACK = ROOT / "elements" / "bluefin-server" / "os-stack.bst" + + +def run_helper(etc_dir: Path, lib_dir: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [str(SCRIPT)], + env={ + **os.environ, + "SYSUPDATE_ETC_DIR": str(etc_dir), + "SYSUPDATE_LIB_DIR": str(lib_dir), + }, + capture_output=True, + text=True, + check=False, + ) + + +# ── Packaging contracts ----------------------------------------------------- + + +def test_helper_is_packaged_and_executable() -> None: + assert SCRIPT.is_file(), "the migration helper is missing" + assert os.access(SCRIPT, os.X_OK), ( + "the helper must be executable so the systemd unit can run it" + ) + element = ELEMENT.read_text(encoding="utf-8") + assert "path: files/os/sysupdate-migrate" in element, ( + "the helper must ship from files/os/sysupdate-migrate" + ) + assert "target: /usr/bin" in element, ( + "the helper must land at /usr/bin/bluefin-sysupdate-migrate" + ) + assert ( + "bluefin-server/os-sysupdate-feature-migrate.bst" in STACK.read_text("utf-8") + ), "the element must be part of the DDI stack" + + +def test_migration_unit_runs_before_the_update_path() -> None: + service = SERVICE.read_text(encoding="utf-8") + assert "[Unit]" in service and "[Service]" in service and "[Install]" in service + assert "Description=Bluefin: prune deprecated systemd-sysupdate features" in service + assert "Type=oneshot" in service, "the migration is a one-shot helper" + assert "ExecStart=/usr/bin/bluefin-sysupdate-migrate" in service + assert "Before=systemd-sysupdate.timer" in service, ( + "the migration must run ahead of the update timer" + ) + assert "Before=systemd-sysupdate.service" in service + assert "WantedBy=multi-user.target" in service, ( + "the unit must be enabled so it runs at boot" + ) + + +def test_update_unit_prunes_features_before_every_update() -> None: + dropin = DROPIN.read_text(encoding="utf-8") + assert "[Service]" in dropin + assert "ExecStartPre=/usr/bin/bluefin-sysupdate-migrate" in dropin, ( + "the prune must run before systemd-sysupdate itself, so timer- and " + "hand-triggered updates are both covered" + ) + + +def test_migration_unit_is_enabled_by_preset() -> None: + preset = PRESET.read_text(encoding="utf-8") + assert preset == "enable bluefin-sysupdate-migrate.service\n" + + +# ── Behavioural contracts --------------------------------------------------- + + +def _make_tree(tmp_path: Path, provided: list[str], enabled: list[str]) -> tuple[Path, Path]: + etc = tmp_path / "etc" / "sysupdate.d" + lib = tmp_path / "usr" / "lib" / "sysupdate.d" + etc.mkdir(parents=True) + lib.mkdir(parents=True) + if provided: + transfers = "\n".join(f"Features={name}" for name in provided) + (lib / "10-example.transfer").write_text( + "[Transfer]\n\n" + transfers + "\n\n[Source]\nType=url-file\n", + encoding="utf-8", + ) + for name in enabled: + (etc / f"{name}.feature.d").mkdir() + return etc, lib + + +def test_orphaned_feature_is_archived_and_provided_one_kept(tmp_path: Path) -> None: + etc, lib = _make_tree(tmp_path, provided=["k0s"], enabled=["k0s", "homebrew"]) + result = run_helper(etc, lib) + assert result.returncode == 0, result.stderr + assert (etc / "homebrew.feature.d.disabled").is_dir(), ( + "a feature no transfer provides must be archived" + ) + assert (etc / "k0s.feature.d").is_dir(), ( + "a feature a transfer still provides must be left in place" + ) + assert "not shipped in this release" in result.stdout + + +def test_helper_is_idempotent_on_rerun(tmp_path: Path) -> None: + etc, lib = _make_tree(tmp_path, provided=[], enabled=["homebrew"]) + first = run_helper(etc, lib) + assert first.returncode == 0 + assert (etc / "homebrew.feature.d.disabled").is_dir() + second = run_helper(etc, lib) + assert second.returncode == 0, second.stderr + assert (etc / "homebrew.feature.d.disabled").is_dir(), ( + "the archived drop-in must survive a second run untouched" + ) + assert not (etc / "homebrew.feature.d.disabled.disabled").exists(), ( + "a second run must not archive the drop-in again" + ) + assert "no deprecated sysupdate features found" in second.stdout + + +def test_no_orphaned_feature_is_a_noop(tmp_path: Path) -> None: + etc, lib = _make_tree(tmp_path, provided=["k0s", "homebrew"], enabled=["k0s", "homebrew"]) + result = run_helper(etc, lib) + assert result.returncode == 0, result.stderr + assert (etc / "k0s.feature.d").is_dir() + assert (etc / "homebrew.feature.d").is_dir() + assert "no deprecated sysupdate features found" in result.stdout + + +def test_missing_etc_directory_is_a_noop(tmp_path: Path) -> None: + lib = tmp_path / "usr" / "lib" / "sysupdate.d" + lib.mkdir(parents=True) + etc = tmp_path / "does-not-exist" + result = run_helper(etc, lib) + assert result.returncode == 0, result.stderr + assert "nothing to do" in result.stdout + + +def test_existing_archive_is_not_clobbered(tmp_path: Path) -> None: + # If an operator re-enabled a feature while the archived drop-in is still + # present, the helper must not move the new drop-in over the archive. + etc, lib = _make_tree(tmp_path, provided=[], enabled=["homebrew"]) + (etc / "homebrew.feature.d.disabled").mkdir() + result = run_helper(etc, lib) + assert result.returncode == 0, result.stderr + assert "already archived" in result.stderr + assert (etc / "homebrew.feature.d.disabled").is_dir() + assert not (etc / "homebrew.feature.d.disabled" / "homebrew.feature.d").exists() + + +def test_features_list_accepts_spaces_and_commas(tmp_path: Path) -> None: + etc, lib = _make_tree(tmp_path, provided=[], enabled=["k0s", "homebrew"]) + # A single transfer listing both features, mixed separators. + (lib / "10-example.transfer").write_text( + "[Transfer]\nFeatures=k0s, homebrew\n[Source]\nType=url-file\n", + encoding="utf-8", + ) + result = run_helper(etc, lib) + assert result.returncode == 0, result.stderr + assert (etc / "k0s.feature.d").is_dir() + assert (etc / "homebrew.feature.d").is_dir() + From 7da34ea84b0857d786626c293ab65787aae93a33 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:02:29 -0500 Subject: [PATCH 2/5] fix(sysupdate): decide feature retirement from the remote manifest The helper read Features= from local transfers, but the stalled node still ships the retired transfers locally; only the remote release dropped them. Decide from the remote SHA256SUMS at each transfer's [Source] Path and MatchPattern instead, and leave features alone when the manifest is unreachable. Enable the unit via /usr so updated nodes run it, make the ExecStartPre non-fatal, and list the new units in os-k0s-first-boot.bst. Hive-Fix-Attempt: 1/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- elements/bluefin-server/os-k0s-first-boot.bst | 5 +- elements/bluefin-server/os-sshd-preset.bst | 3 +- elements/oci/bluefin-server-usr.bst | 4 + ...0-bluefin-sysupdate-feature-migrate.preset | 1 - .../05-bluefin-prune-deprecated-features.conf | 4 +- .../bluefin-sysupdate-migrate | 117 ++++++++++++++---- tests/unit/test_sysupdate_feature_migrate.py | 115 ++++++++++------- 7 files changed, 170 insertions(+), 79 deletions(-) delete mode 100644 files/os/systemd/system-preset/80-bluefin-sysupdate-feature-migrate.preset diff --git a/elements/bluefin-server/os-k0s-first-boot.bst b/elements/bluefin-server/os-k0s-first-boot.bst index ef2c22fd..0e20ac2a 100644 --- a/elements/bluefin-server/os-k0s-first-boot.bst +++ b/elements/bluefin-server/os-k0s-first-boot.bst @@ -13,7 +13,10 @@ description: | Kubernetes; the boot deadline; the diskless update check; bluefin-update-status.service for the login banners and bluefin-update-status-migrate.service, which enables it on nodes that - updated from an image without it). + updated from an image without it), and bluefin-sysupdate-migrate.service + plus the systemd-sysupdate.service drop-in + (05-bluefin-prune-deprecated-features.conf), which archive enabled + sysupdate features the remote release no longer ships. sources: - kind: local diff --git a/elements/bluefin-server/os-sshd-preset.bst b/elements/bluefin-server/os-sshd-preset.bst index 288832fb..59333d45 100644 --- a/elements/bluefin-server/os-sshd-preset.bst +++ b/elements/bluefin-server/os-sshd-preset.bst @@ -1,7 +1,6 @@ kind: import description: | - Install systemd presets ensuring sshd is present but off by default and the - systemd-sysupdate feature migration is enabled. + Install systemd presets ensuring sshd is present but off by default. sources: - kind: local diff --git a/elements/oci/bluefin-server-usr.bst b/elements/oci/bluefin-server-usr.bst index d580c7ef..18603d00 100644 --- a/elements/oci/bluefin-server-usr.bst +++ b/elements/oci/bluefin-server-usr.bst @@ -168,6 +168,10 @@ config: # does, once per installed node (projectbluefin/server#367). ln -sf ../bluefin-update-status-migrate.service \ /sysroot/usr/lib/systemd/system/multi-user.target.wants/bluefin-update-status-migrate.service + # Presets apply on first boot only; enable the sysupdate feature prune in + # /usr so nodes updating into this image run it too (#688). + ln -sf ../bluefin-sysupdate-migrate.service \ + /sysroot/usr/lib/systemd/system/multi-user.target.wants/bluefin-sysupdate-migrate.service # Enabled in /usr, not by preset: a node that updated into this image # must still give its FleetLock slot back once an operator configures a # server, or its group stops rebooting. Inert without a server. diff --git a/files/os/systemd/system-preset/80-bluefin-sysupdate-feature-migrate.preset b/files/os/systemd/system-preset/80-bluefin-sysupdate-feature-migrate.preset deleted file mode 100644 index 9d94e209..00000000 --- a/files/os/systemd/system-preset/80-bluefin-sysupdate-feature-migrate.preset +++ /dev/null @@ -1 +0,0 @@ -enable bluefin-sysupdate-migrate.service diff --git a/files/os/systemd/system/systemd-sysupdate.service.d/05-bluefin-prune-deprecated-features.conf b/files/os/systemd/system/systemd-sysupdate.service.d/05-bluefin-prune-deprecated-features.conf index 58833116..a533c112 100644 --- a/files/os/systemd/system/systemd-sysupdate.service.d/05-bluefin-prune-deprecated-features.conf +++ b/files/os/systemd/system/systemd-sysupdate.service.d/05-bluefin-prune-deprecated-features.conf @@ -2,5 +2,5 @@ # Prune retired sysupdate features before every update, so an enabled feature # that the release no longer ships cannot make systemd-sysupdate report # "No update needed" instead of updating. The helper is idempotent, so running -# it on every update is safe. -ExecStartPre=/usr/bin/bluefin-sysupdate-migrate +# it on every update is safe. The "-" keeps a helper failure from blocking updates. +ExecStartPre=-/usr/bin/bluefin-sysupdate-migrate diff --git a/files/os/sysupdate-migrate/bluefin-sysupdate-migrate b/files/os/sysupdate-migrate/bluefin-sysupdate-migrate index 90dc2e9e..c683539e 100755 --- a/files/os/sysupdate-migrate/bluefin-sysupdate-migrate +++ b/files/os/sysupdate-migrate/bluefin-sysupdate-migrate @@ -4,14 +4,20 @@ # release no longer ships, so a retired feature cannot silently stop updates. # # systemd-sysupdate only applies an update set when every feature enabled under -# /etc/sysupdate.d/.feature.d/ is also provided by a transfer set in -# /usr/lib/sysupdate.d/. When a feature leaves the release manifest -- for -# example Homebrew becoming a first-boot seed instead of an opt-in extension -- -# the matching /etc drop-in can no longer be satisfied, and systemd-sysupdate -# quietly reports "No update needed" instead of updating. +# /etc/sysupdate.d/.feature.d/ is also provided by the release. When a +# feature leaves the release manifest -- for example Homebrew becoming a +# first-boot seed instead of an opt-in extension -- the matching /etc drop-in +# can no longer be satisfied, and systemd-sysupdate quietly reports "No update +# needed" instead of updating. # -# This one-shot helper is meant to run ahead of the update path. It lists the -# enabled features that no transfer set provides any more, archives each stale +# The node that stalls still ships the retired feature's transfers locally; only +# the remote release dropped them. So a feature is retired when none of the +# local transfers that list it find a file matching their [Source] MatchPattern +# in the remote SHA256SUMS at the newest version. A feature with no local +# transfer at all is also retired. If a manifest cannot be fetched, the feature +# is left alone: an offline node must never lose a working feature. +# +# This one-shot helper runs ahead of the update path, archives each retired # /etc drop-in, and prints a clear message so an operator is never left # wondering why an update was skipped. @@ -22,6 +28,7 @@ set -euo pipefail etc_feature_dir="${SYSUPDATE_ETC_DIR:-/etc/sysupdate.d}" lib_transfer_dir="${SYSUPDATE_LIB_DIR:-/usr/lib/sysupdate.d}" archive_suffix="${SYSUPDATE_ARCHIVE_SUFFIX:-.disabled}" +fetch_timeout="${SYSUPDATE_FETCH_TIMEOUT:-20}" # systemd captures ExecStartPre stdout/stderr into the journal, so plain prints # are enough; the wrapper keeps one message shape for both paths. @@ -32,25 +39,85 @@ warn() { printf 'bluefin-sysupdate-migrate: warning: %s\n' "$*" >&2 } -# Feature names the current release still provides, keyed by name. -declare -A provided_features=() +# Per feature: "yes" once a remote manifest lists a match for one of its +# transfers, "unknown" when a manifest could not be read, else absent. +declare -A feature_state=() +declare -A feature_seen=() +declare -A manifest_cache=() + +trim() { + local v="$1" + v="${v#"${v%%[![:space:]]*}"}" + printf '%s' "${v%"${v##*[![:space:]]}"}" +} -# Read the Features= key of every transfer set and record the names it lists. -collect_provided_features() { - local transfer line value feature +# Fetch (once per URL) the SHA256SUMS next to a transfer's Source Path. +# Prints the manifest; returns 1 when it cannot be read. +fetch_manifest() { + local url="${1%/}/SHA256SUMS" + if [ -z "${manifest_cache[$url]+x}" ]; then + if manifest_cache[$url]="$(curl -fsSL --max-time "$fetch_timeout" -- "$url" 2>/dev/null)"; then + : + else + manifest_cache[$url]="__FETCH_FAILED__" + fi + fi + [ "${manifest_cache[$url]}" != "__FETCH_FAILED__" ] || return 1 + printf '%s\n' "${manifest_cache[$url]}" +} + +# Turn a MatchPattern into an anchored regex; @v is the version wildcard. +pattern_regex() { + local pat="$1" + pat="$(printf '%s' "$pat" | sed -e 's/[][\.^$*+?(){}|\/]/\\&/g')" + printf '^%s$' "${pat//@v/[^[:space:]]+}" +} + +# Record, for every transfer, whether the remote release still ships the files +# its features need. +collect_feature_state() { + local transfer line section key value feature features path pattern manifest regex name found shopt -s nullglob for transfer in "$lib_transfer_dir"/*.transfer; do + features="" path="" pattern="" section="" while IFS= read -r line; do - value="${line#*=}" # text after the first = - value="${value//,/ }" # systemd accepts spaces or commas - set -f # feature names carry no glob characters - for feature in $value; do # word-split on whitespace - feature="${feature#"${feature%%[![:space:]]*}"}" # ltrim - feature="${feature%"${feature##*[![:space:]]}"}" # rtrim - [ -n "$feature" ] && provided_features["$feature"]=1 - done - set +f - done < <(grep -E '^[[:space:]]*[Ff]eatures[[:space:]]*=' "$transfer" || true) + line="$(trim "$line")" + case "$line" in + "["*"]") section="$line"; continue ;; + ""|"#"*|";"*) continue ;; + esac + key="$(trim "${line%%=*}")" + value="$(trim "${line#*=}")" + case "$section:$key" in + "[Transfer]:Features") features="$features ${value//,/ }" ;; + "[Source]:Path") path="$value" ;; + "[Source]:MatchPattern") pattern="${value%% *}" ;; + esac + done < "$transfer" + + [ -n "$features" ] || continue + found=unknown + if [ -n "$path" ] && [ -n "$pattern" ] && manifest="$(fetch_manifest "$path")"; then + found=no + regex="$(pattern_regex "$pattern")" + while IFS= read -r line; do + name="${line##*[[:space:]]}" + name="${name#\*}" + if [[ "$name" =~ $regex ]]; then found=yes; break; fi + done <<< "$manifest" + fi + set -f + for feature in $features; do + feature_seen["$feature"]=1 + case "$found:${feature_state[$feature]:-}" in + yes:*) feature_state["$feature"]=yes ;; + unknown:yes) ;; + unknown:*) feature_state["$feature"]=unknown ;; + no:yes|no:unknown) ;; + no:*) feature_state["$feature"]=no ;; + esac + done + set +f done } @@ -60,7 +127,7 @@ main() { return 0 fi - collect_provided_features + collect_feature_state local dir name changed changed=0 @@ -68,7 +135,7 @@ main() { for dir in "$etc_feature_dir"/*.feature.d; do name="$(basename "$dir")" name="${name%.feature.d}" - if [ -n "${provided_features[$name]:-}" ]; then + if [ -n "${feature_seen[$name]:-}" ] && [ "${feature_state[$name]:-}" != "no" ]; then continue fi if [ -e "${dir}${archive_suffix}" ]; then @@ -76,7 +143,7 @@ main() { continue fi mv -- "$dir" "${dir}${archive_suffix}" - info "archived ${dir} -> ${dir}${archive_suffix}: enabled feature ${name} is not shipped in this release and would otherwise make systemd-sysupdate report 'No update needed'" + info "archived ${dir} -> ${dir}${archive_suffix}: enabled feature ${name} is not in the release manifest and would otherwise make systemd-sysupdate report 'No update needed'" changed=$((changed + 1)) done diff --git a/tests/unit/test_sysupdate_feature_migrate.py b/tests/unit/test_sysupdate_feature_migrate.py index b081be60..5db69679 100644 --- a/tests/unit/test_sysupdate_feature_migrate.py +++ b/tests/unit/test_sysupdate_feature_migrate.py @@ -40,14 +40,7 @@ / "systemd-sysupdate.service.d" / "05-bluefin-prune-deprecated-features.conf" ) -PRESET = ( - ROOT - / "files" - / "os" - / "systemd" - / "system-preset" - / "80-bluefin-sysupdate-feature-migrate.preset" -) +USR = ROOT / "elements" / "oci" / "bluefin-server-usr.bst" ELEMENT = ROOT / "elements" / "bluefin-server" / "os-sysupdate-feature-migrate.bst" STACK = ROOT / "elements" / "bluefin-server" / "os-stack.bst" @@ -104,29 +97,37 @@ def test_migration_unit_runs_before_the_update_path() -> None: def test_update_unit_prunes_features_before_every_update() -> None: dropin = DROPIN.read_text(encoding="utf-8") assert "[Service]" in dropin - assert "ExecStartPre=/usr/bin/bluefin-sysupdate-migrate" in dropin, ( + assert "ExecStartPre=-/usr/bin/bluefin-sysupdate-migrate" in dropin, ( "the prune must run before systemd-sysupdate itself, so timer- and " "hand-triggered updates are both covered" ) -def test_migration_unit_is_enabled_by_preset() -> None: - preset = PRESET.read_text(encoding="utf-8") - assert preset == "enable bluefin-sysupdate-migrate.service\n" +def test_migration_unit_is_enabled_in_usr() -> None: + # Presets only apply on first boot; updated nodes need the /usr link. + usr = USR.read_text(encoding="utf-8") + assert "multi-user.target.wants/bluefin-sysupdate-migrate.service" in usr # ── Behavioural contracts --------------------------------------------------- -def _make_tree(tmp_path: Path, provided: list[str], enabled: list[str]) -> tuple[Path, Path]: +def _make_tree( + tmp_path: Path, remote: list[str], local: dict[str, str], enabled: list[str] +) -> tuple[Path, Path]: + """Local transfers (feature -> file pattern) against a file:// remote.""" etc = tmp_path / "etc" / "sysupdate.d" lib = tmp_path / "usr" / "lib" / "sysupdate.d" - etc.mkdir(parents=True) - lib.mkdir(parents=True) - if provided: - transfers = "\n".join(f"Features={name}" for name in provided) - (lib / "10-example.transfer").write_text( - "[Transfer]\n\n" + transfers + "\n\n[Source]\nType=url-file\n", + rel = tmp_path / "release" + for d in (etc, lib, rel): + d.mkdir(parents=True) + (rel / "SHA256SUMS").write_text( + "".join(f"{'0' * 64} *{name}\n" for name in remote), encoding="utf-8" + ) + for i, (feature, pattern) in enumerate(local.items()): + (lib / f"{10 + i}-{feature}.transfer").write_text( + f"[Transfer]\nFeatures={feature}\n\n[Source]\nType=url-file\n" + f"Path=file://{rel}/\nMatchPattern={pattern}\n", encoding="utf-8", ) for name in enabled: @@ -134,37 +135,58 @@ def _make_tree(tmp_path: Path, provided: list[str], enabled: list[str]) -> tuple return etc, lib -def test_orphaned_feature_is_archived_and_provided_one_kept(tmp_path: Path) -> None: - etc, lib = _make_tree(tmp_path, provided=["k0s"], enabled=["k0s", "homebrew"]) +def test_feature_dropped_from_remote_is_archived_though_local_transfer_exists( + tmp_path: Path, +) -> None: + # The stalled node still ships the homebrew transfer locally; only the + # remote release dropped it. + etc, lib = _make_tree( + tmp_path, + remote=["k0s_1.2.raw.zst", "bluefin_1.2.raw"], + local={"k0s": "k0s_@v.raw.zst", "homebrew": "homebrew_@v.raw.zst"}, + enabled=["k0s", "homebrew"], + ) result = run_helper(etc, lib) assert result.returncode == 0, result.stderr - assert (etc / "homebrew.feature.d.disabled").is_dir(), ( - "a feature no transfer provides must be archived" - ) - assert (etc / "k0s.feature.d").is_dir(), ( - "a feature a transfer still provides must be left in place" + assert (etc / "homebrew.feature.d.disabled").is_dir() + assert (etc / "k0s.feature.d").is_dir() + assert "not in the release manifest" in result.stdout + + +def test_feature_without_local_transfer_is_archived(tmp_path: Path) -> None: + etc, lib = _make_tree(tmp_path, remote=[], local={}, enabled=["homebrew"]) + assert run_helper(etc, lib).returncode == 0 + assert (etc / "homebrew.feature.d.disabled").is_dir() + + +def test_unreachable_manifest_leaves_features_alone(tmp_path: Path) -> None: + etc, lib = _make_tree( + tmp_path, remote=[], local={"homebrew": "homebrew_@v.raw.zst"}, enabled=["homebrew"] ) - assert "not shipped in this release" in result.stdout + for f in (tmp_path / "release").iterdir(): + f.unlink() + result = run_helper(etc, lib) + assert result.returncode == 0, result.stderr + assert (etc / "homebrew.feature.d").is_dir(), "offline must never archive" def test_helper_is_idempotent_on_rerun(tmp_path: Path) -> None: - etc, lib = _make_tree(tmp_path, provided=[], enabled=["homebrew"]) - first = run_helper(etc, lib) - assert first.returncode == 0 - assert (etc / "homebrew.feature.d.disabled").is_dir() + etc, lib = _make_tree(tmp_path, remote=[], local={}, enabled=["homebrew"]) + assert run_helper(etc, lib).returncode == 0 second = run_helper(etc, lib) assert second.returncode == 0, second.stderr - assert (etc / "homebrew.feature.d.disabled").is_dir(), ( - "the archived drop-in must survive a second run untouched" - ) - assert not (etc / "homebrew.feature.d.disabled.disabled").exists(), ( - "a second run must not archive the drop-in again" - ) + assert (etc / "homebrew.feature.d.disabled").is_dir() + assert not (etc / "homebrew.feature.d.disabled.disabled").exists() assert "no deprecated sysupdate features found" in second.stdout def test_no_orphaned_feature_is_a_noop(tmp_path: Path) -> None: - etc, lib = _make_tree(tmp_path, provided=["k0s", "homebrew"], enabled=["k0s", "homebrew"]) + etc, lib = _make_tree( + tmp_path, + remote=["k0s_1.raw.zst", "homebrew_1.raw.zst"], + local={"k0s": "k0s_@v.raw.zst", "homebrew": "homebrew_@v.raw.zst"}, + enabled=["k0s", "homebrew"], + ) result = run_helper(etc, lib) assert result.returncode == 0, result.stderr assert (etc / "k0s.feature.d").is_dir() @@ -175,33 +197,30 @@ def test_no_orphaned_feature_is_a_noop(tmp_path: Path) -> None: def test_missing_etc_directory_is_a_noop(tmp_path: Path) -> None: lib = tmp_path / "usr" / "lib" / "sysupdate.d" lib.mkdir(parents=True) - etc = tmp_path / "does-not-exist" - result = run_helper(etc, lib) + result = run_helper(tmp_path / "does-not-exist", lib) assert result.returncode == 0, result.stderr assert "nothing to do" in result.stdout def test_existing_archive_is_not_clobbered(tmp_path: Path) -> None: - # If an operator re-enabled a feature while the archived drop-in is still - # present, the helper must not move the new drop-in over the archive. - etc, lib = _make_tree(tmp_path, provided=[], enabled=["homebrew"]) + etc, lib = _make_tree(tmp_path, remote=[], local={}, enabled=["homebrew"]) (etc / "homebrew.feature.d.disabled").mkdir() result = run_helper(etc, lib) assert result.returncode == 0, result.stderr assert "already archived" in result.stderr - assert (etc / "homebrew.feature.d.disabled").is_dir() assert not (etc / "homebrew.feature.d.disabled" / "homebrew.feature.d").exists() def test_features_list_accepts_spaces_and_commas(tmp_path: Path) -> None: - etc, lib = _make_tree(tmp_path, provided=[], enabled=["k0s", "homebrew"]) - # A single transfer listing both features, mixed separators. + etc, lib = _make_tree( + tmp_path, remote=["x_1.raw"], local={}, enabled=["k0s", "homebrew"] + ) (lib / "10-example.transfer").write_text( - "[Transfer]\nFeatures=k0s, homebrew\n[Source]\nType=url-file\n", + f"[Transfer]\nFeatures=k0s, homebrew\n[Source]\nType=url-file\n" + f"Path=file://{tmp_path}/release/\nMatchPattern=x_@v.raw\n", encoding="utf-8", ) result = run_helper(etc, lib) assert result.returncode == 0, result.stderr assert (etc / "k0s.feature.d").is_dir() assert (etc / "homebrew.feature.d").is_dir() - From db69895794d11776ee0ebbab233ea558bcd060c3 Mon Sep 17 00:00:00 2001 From: kylerankin Date: Wed, 18 Nov 2026 04:28:58 +0000 Subject: [PATCH 3/5] fix(sysupdate): stop track-binaries.py raising at import time The unit check aborted pytest collection for the whole suite: the Tree class referenced itself in the copy() return annotation before the name was bound, raising NameError at class-definition time. Add `from __future__ import annotations` so the forward reference stays a string and the module imports cleanly. The remaining unit failures (ci_image_artifacts on PurePosixPath.full_match, and shellcheck SC2015/SC2317) are environment-only: the local runner has Python 3.11 and shellcheck 0.9.0, while CI uses Python 3.15 and shellcheck 0.11.0-2, which no longer flags a test command as the middle of A && B || C and is less spammy about unreachable commands nested in indirectly called functions. Closes #688. Signed-off-by: kylerankin --- .github/scripts/track-binaries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/track-binaries.py b/.github/scripts/track-binaries.py index 63f8227a..4637376a 100644 --- a/.github/scripts/track-binaries.py +++ b/.github/scripts/track-binaries.py @@ -93,7 +93,7 @@ Stdlib only. GITHUB_TOKEN or GH_TOKEN, if set, is sent to api.github.com only. """ - +from __future__ import annotations import argparse import hashlib From 1a83131468f7882cf27b91e220f9758ea25d2880 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:54:05 -0500 Subject: [PATCH 4/5] fix(sysupdate): restore archived features and cache manifest fetches Restore .feature.d.disabled when the release ships the feature again, fetch each manifest once per run, and correct element/doc wording. Hive-Fix-Attempt: 3/6 Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- docs/skills/systemd-sysupdate-verification.md | 3 ++ .../os-sysupdate-feature-migrate.bst | 4 +- .../bluefin-sysupdate-migrate | 21 ++++++-- tests/unit/test_sysupdate_feature_migrate.py | 54 ++++++++++++++++++- 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/docs/skills/systemd-sysupdate-verification.md b/docs/skills/systemd-sysupdate-verification.md index 5ad86f44..0e98cbb7 100644 --- a/docs/skills/systemd-sysupdate-verification.md +++ b/docs/skills/systemd-sysupdate-verification.md @@ -67,6 +67,9 @@ the same names (`files/os/sysupdate.d/.feature` and `3N-.transfer`), enabled per node with `updatectl enable zfs` or a drop-in such as `/etc/sysupdate.d/zfs.feature.d/enable.conf` containing `[Feature] Enabled=true`. +`bluefin-sysupdate-migrate` renames that drop-in directory to +`zfs.feature.d.disabled` when the release stops shipping the feature, and +restores it when a later release ships the feature again. The k0s and NVIDIA Container Toolkit sysexts stay separate components (`files/os/sysupdate.k0s.d/`, `files/os/sysupdate.nvidia-container-toolkit.d/`; `systemd-sysupdate --component= update`) with their own version axes. diff --git a/elements/bluefin-server/os-sysupdate-feature-migrate.bst b/elements/bluefin-server/os-sysupdate-feature-migrate.bst index 222666fa..95b70305 100644 --- a/elements/bluefin-server/os-sysupdate-feature-migrate.bst +++ b/elements/bluefin-server/os-sysupdate-feature-migrate.bst @@ -3,8 +3,8 @@ description: | Install the systemd-sysupdate deprecated-feature migration helper so a feature the release no longer ships cannot silently stop the next update. - The executable lands in /usr/bin; the boot-time unit, the preset that - enables it, and the ExecStartPre drop-in on systemd-sysupdate.service are + The executable lands in /usr/bin; the boot-time unit (enabled by a wants + symlink in bluefin-server-usr.bst) and the ExecStartPre drop-in on systemd-sysupdate.service are shipped next to the other system units. sources: diff --git a/files/os/sysupdate-migrate/bluefin-sysupdate-migrate b/files/os/sysupdate-migrate/bluefin-sysupdate-migrate index c683539e..c03d291a 100755 --- a/files/os/sysupdate-migrate/bluefin-sysupdate-migrate +++ b/files/os/sysupdate-migrate/bluefin-sysupdate-migrate @@ -52,7 +52,8 @@ trim() { } # Fetch (once per URL) the SHA256SUMS next to a transfer's Source Path. -# Prints the manifest; returns 1 when it cannot be read. +# Sets the caller's "manifest" variable (no subshell, so the cache persists); +# returns 1 when it cannot be read. fetch_manifest() { local url="${1%/}/SHA256SUMS" if [ -z "${manifest_cache[$url]+x}" ]; then @@ -63,7 +64,7 @@ fetch_manifest() { fi fi [ "${manifest_cache[$url]}" != "__FETCH_FAILED__" ] || return 1 - printf '%s\n' "${manifest_cache[$url]}" + manifest="${manifest_cache[$url]}" } # Turn a MatchPattern into an anchored regex; @v is the version wildcard. @@ -97,7 +98,7 @@ collect_feature_state() { [ -n "$features" ] || continue found=unknown - if [ -n "$path" ] && [ -n "$pattern" ] && manifest="$(fetch_manifest "$path")"; then + if [ -n "$path" ] && [ -n "$pattern" ] && fetch_manifest "$path"; then found=no regex="$(pattern_regex "$pattern")" while IFS= read -r line; do @@ -147,6 +148,20 @@ main() { changed=$((changed + 1)) done + # A release that ships the feature again restores the operator's opt-in. + for dir in "$etc_feature_dir"/*.feature.d"${archive_suffix}"; do + name="$(basename "$dir")" + name="${name%.feature.d"${archive_suffix}"}" + [ "${feature_state[$name]:-}" = "yes" ] || continue + if [ -e "${dir%"${archive_suffix}"}" ]; then + warn "feature ${name} has both an active and an archived drop-in; leaving the archive in place" + continue + fi + mv -- "$dir" "${dir%"${archive_suffix}"}" + info "restored ${dir%"${archive_suffix}"} from ${dir}: feature ${name} is in the release manifest again" + changed=$((changed + 1)) + done + if [ "$changed" -eq 0 ]; then info "no deprecated sysupdate features found; the next update is unaffected" fi diff --git a/tests/unit/test_sysupdate_feature_migrate.py b/tests/unit/test_sysupdate_feature_migrate.py index 5db69679..e2fb68cc 100644 --- a/tests/unit/test_sysupdate_feature_migrate.py +++ b/tests/unit/test_sysupdate_feature_migrate.py @@ -12,7 +12,7 @@ * the ``bluefin-sysupdate-migrate`` helper exists, is executable, and lives at ``/usr/bin/bluefin-sysupdate-migrate`` in the image * the one-shot unit runs ahead of the update path (before the timer and the - update unit itself) and is enabled by preset + update unit itself) and is enabled by a /usr wants symlink * the ``systemd-sysupdate.service`` drop-in runs the helper as ``ExecStartPre`` so every update -- timer- or hand-triggered -- is covered * the helper archives an enabled feature the release no longer ships, keeps a @@ -224,3 +224,55 @@ def test_features_list_accepts_spaces_and_commas(tmp_path: Path) -> None: assert result.returncode == 0, result.stderr assert (etc / "k0s.feature.d").is_dir() assert (etc / "homebrew.feature.d").is_dir() + + +def test_archived_feature_is_restored_when_release_ships_it_again( + tmp_path: Path, +) -> None: + etc, lib = _make_tree( + tmp_path, + remote=["zfs_1.3.raw.zst"], + local={"zfs": "zfs_@v.raw.zst"}, + enabled=[], + ) + (etc / "zfs.feature.d.disabled").mkdir() + result = run_helper(etc, lib) + assert result.returncode == 0, result.stderr + assert (etc / "zfs.feature.d").is_dir() + assert not (etc / "zfs.feature.d.disabled").exists() + assert "restored" in result.stdout + + +def test_archived_feature_stays_archived_while_release_lacks_it( + tmp_path: Path, +) -> None: + etc, lib = _make_tree( + tmp_path, remote=[], local={"zfs": "zfs_@v.raw.zst"}, enabled=[] + ) + (etc / "zfs.feature.d.disabled").mkdir() + assert run_helper(etc, lib).returncode == 0 + assert (etc / "zfs.feature.d.disabled").is_dir() + assert not (etc / "zfs.feature.d").exists() + + +def test_manifest_is_fetched_once_per_url(tmp_path: Path) -> None: + etc, lib = _make_tree( + tmp_path, + remote=["a_1.raw"], + local={"a": "a_@v.raw", "b": "b_@v.raw", "c": "c_@v.raw"}, + enabled=["a"], + ) + bindir = tmp_path / "bin" + bindir.mkdir() + log = tmp_path / "curl.log" + curl = bindir / "curl" + curl.write_text(f'#!/bin/sh\necho x >> {log}\nexec /usr/bin/curl "$@"\n') + curl.chmod(0o755) + env = { + **os.environ, + "PATH": f"{bindir}:{os.environ['PATH']}", + "SYSUPDATE_ETC_DIR": str(etc), + "SYSUPDATE_LIB_DIR": str(lib), + } + subprocess.run([str(SCRIPT)], env=env, check=True, capture_output=True) + assert len(log.read_text().splitlines()) == 1 From 40e42f3d81bfca855a2113aeb17aa7670efdecf6 Mon Sep 17 00:00:00 2001 From: kylerankin Date: Wed, 18 Nov 2026 10:32:53 +0000 Subject: [PATCH 5/5] sysupdate-migrate: reject non-manifest 200 responses and drop unused pytest import - fetch_manifest now validates the fetched SHA256SUMS body against a "<64 hex> " pattern; a 200 response that is not a real manifest (captive portal, proxy page) is treated as unreachable so an offline-style node never archives every enabled feature. - Remove the unused `import pytest` that made the required `unit` ruff check (F401) fail. - Add a test pinning the non-manifest 200 body leaving features in place. --- .../bluefin-sysupdate-migrate | 15 ++++++++++++--- tests/unit/test_sysupdate_feature_migrate.py | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/files/os/sysupdate-migrate/bluefin-sysupdate-migrate b/files/os/sysupdate-migrate/bluefin-sysupdate-migrate index c03d291a..1ceed215 100755 --- a/files/os/sysupdate-migrate/bluefin-sysupdate-migrate +++ b/files/os/sysupdate-migrate/bluefin-sysupdate-migrate @@ -55,10 +55,19 @@ trim() { # Sets the caller's "manifest" variable (no subshell, so the cache persists); # returns 1 when it cannot be read. fetch_manifest() { - local url="${1%/}/SHA256SUMS" + local url="${1%/}/SHA256SUMS" body if [ -z "${manifest_cache[$url]+x}" ]; then - if manifest_cache[$url]="$(curl -fsSL --max-time "$fetch_timeout" -- "$url" 2>/dev/null)"; then - : + if body="$(curl -fsSL --max-time "$fetch_timeout" -- "$url" 2>/dev/null)"; then + # `curl -f` only fails on HTTP >= 400, so a captive portal or proxy + # error page can come back 200 with an HTML body and be trusted as a + # manifest. Only trust a real SHA256SUMS: at least one "<64 hex> + # " line. Anything else is treated as unreachable, so a + # non-release response never archives a working feature. + if printf '%s\n' "$body" | grep -qE '^[0-9a-f]{64}([[:space:]]|\*)'; then + manifest_cache[$url]="$body" + else + manifest_cache[$url]="__FETCH_FAILED__" + fi else manifest_cache[$url]="__FETCH_FAILED__" fi diff --git a/tests/unit/test_sysupdate_feature_migrate.py b/tests/unit/test_sysupdate_feature_migrate.py index e2fb68cc..ca79cf16 100644 --- a/tests/unit/test_sysupdate_feature_migrate.py +++ b/tests/unit/test_sysupdate_feature_migrate.py @@ -24,8 +24,6 @@ import subprocess from pathlib import Path -import pytest - ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "files" / "os" / "sysupdate-migrate" / "bluefin-sysupdate-migrate" SERVICE = ( @@ -170,6 +168,22 @@ def test_unreachable_manifest_leaves_features_alone(tmp_path: Path) -> None: assert (etc / "homebrew.feature.d").is_dir(), "offline must never archive" +def test_non_manifest_manifest_leaves_features_alone(tmp_path: Path) -> None: + # A 200 response that is not a SHA256SUMS (a captive portal or proxy error + # page) must never archive a working feature: it is treated as + # unreachable. `curl -f` only fails on HTTP >= 400, so a valid-looking + # status with an HTML body would otherwise set found=no for every transfer. + etc, lib = _make_tree( + tmp_path, remote=[], local={"homebrew": "homebrew_@v.raw.zst"}, enabled=["homebrew"] + ) + (tmp_path / "release" / "SHA256SUMS").write_text( + "Captive Login\n", encoding="utf-8" + ) + result = run_helper(etc, lib) + assert result.returncode == 0, result.stderr + assert (etc / "homebrew.feature.d").is_dir(), "a non-manifest 200 must never archive" + + def test_helper_is_idempotent_on_rerun(tmp_path: Path) -> None: etc, lib = _make_tree(tmp_path, remote=[], local={}, enabled=["homebrew"]) assert run_helper(etc, lib).returncode == 0