From 04e17b9ff8917095ee9a5d5530e3083b34b44d4e Mon Sep 17 00:00:00 2001 From: scanner Date: Tue, 10 Nov 2026 11:26:58 +0000 Subject: [PATCH 1/3] Fold the six copy-pasted export-*-sysext recipes onto one helper (#638) The Justfile had six near-identical export-*-sysext recipes (k0s, zfs, nvidia, nvidia-container-toolkit, tailscale, netbird), each repeating the checkout/copy/SHA256SUMS body by hand. They had drifted: the k0s export-sysext ran `rm -rf dist/sysext`, which deletes the other add-ons' exports, and re-running the recipes duplicated the SHA256SUMS lines. Add a shared `_export-sysext-element NAME PREFIX` helper and have every export-*-sysext recipe call it. The copy glob is now `{{PREFIX}}*.raw.zst` and the helper appends (`mkdir -p`) instead of wiping. PREFIX is the element's sysext-image name, so the glob always matches the built artifact. A unit test now checks every single-element export recipe calls the helper and that its PREFIX matches the element's sysext-image, so a copy-pasted glob can never drift out of sync again. Closes #638 Hive-Run: projectbluefin/server#638 Hive-Plan: fold the six copy-pasted export-*-sysext recipes onto one shared _export-sysext-element helper Hive-Spec: every export-*-sysext recipe must call the helper with PREFIX == the element's sysext-image name; k0s must append not wipe Co-authored-by: Copilot Signed-off-by: scanner --- Justfile | 69 +++++++---------- tests/unit/test_sysext_export_contract.py | 91 ++++++++++++++++++----- 2 files changed, 98 insertions(+), 62 deletions(-) diff --git a/Justfile b/Justfile index a10585b5..dff941aa 100644 --- a/Justfile +++ b/Justfile @@ -118,18 +118,14 @@ lint-python: build-sysext: just bst build oci/k0s-sysext.bst -# Export the k0s systemd-sysext image + SHA256SUMS to dist/sysext/. +# Export the k0s systemd-sysext image + SHA256SUMS to dist/sysext/. Delegates to +# _export-sysext-element, which appends (mkdir -p) instead of wiping dist/sysext, +# so the k0s export no longer deletes the other add-on exports (#638). # The artifact checkout also emits an uncompressed .raw; only the # versioned .raw.zst release asset and its SHA256SUMS are published. [group('sysext')] export-sysext: build-sysext - rm -rf dist/sysext dist/sysext-checkout - mkdir -p dist/sysext-checkout dist/sysext - just bst artifact checkout oci/k0s-sysext.bst --directory /src/dist/sysext-checkout - cp dist/sysext-checkout/k0s-*.raw.zst dist/sysext/ - cp dist/sysext-checkout/SHA256SUMS dist/sysext/ - rm -rf dist/sysext-checkout - @echo "==> wrote k0s sysext:" && ls -lh dist/sysext/ + just _export-sysext-element "k0s" "k0s-" # -- Diskless /usr image + signed boot chain ---------------------------------- @@ -233,6 +229,23 @@ dogfood-check: dogfood-mdns: bash scripts/dogfood-mdns.sh dist/diskless +# Shared body for the single opt-in sysext export recipes. NAME is the element +# base (the element is oci/-sysext.bst) and also names the echo line; +# PREFIX is that element's sysext-image name, so *.raw.zst matches the +# built artifact. Every export-*-sysext recipe calls this instead of re-writing +# the checkout, copy, and SHA256SUMS steps by hand, so a new add-on cannot drift +# its glob the way the zfs glob did (#306). It appends to dist/sysext/ instead of +# wiping it, so the k0s export-sysext no longer deletes the other add-on exports (#638). +[group('sysext')] +_export-sysext-element NAME PREFIX: + rm -rf dist/{{NAME}}-checkout + mkdir -p dist/sysext + just bst artifact checkout oci/{{NAME}}-sysext.bst --directory /src/dist/{{NAME}}-checkout + cp dist/{{NAME}}-checkout/{{PREFIX}}*.raw.zst dist/sysext/ + grep 'raw.zst$' dist/{{NAME}}-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS + rm -rf dist/{{NAME}}-checkout + @echo "==> wrote {{NAME}} sysext:" && ls -lh dist/sysext/ + # Build the OpenZFS systemd-sysext (locked to one image version). [group('sysext')] build-zfs-sysext: gen-dev-keys @@ -241,13 +254,7 @@ build-zfs-sysext: gen-dev-keys # Export the OpenZFS sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-zfs-sysext: build-zfs-sysext - rm -rf dist/zfs-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/zfs-sysext.bst --directory /src/dist/zfs-checkout - cp dist/zfs-checkout/zfs_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/zfs-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/zfs-checkout - @echo "==> wrote zfs sysext:" && ls -lh dist/sysext/ + just _export-sysext-element "zfs" "zfs_" # Build the homelab sysext (component manifests + bluefin-homelab-apply) and # its add-on sysexts (Argo Workflows, MCP server, KubeStellar). @@ -302,13 +309,7 @@ build-nvidia-sysext FLAVOUR="nvidia-open-595": gen-dev-keys # Export an NVIDIA driver sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-nvidia-sysext FLAVOUR="nvidia-open-595": (build-nvidia-sysext FLAVOUR) - rm -rf dist/{{FLAVOUR}}-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/{{FLAVOUR}}-sysext.bst --directory /src/dist/{{FLAVOUR}}-checkout - cp dist/{{FLAVOUR}}-checkout/{{FLAVOUR}}_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/{{FLAVOUR}}-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/{{FLAVOUR}}-checkout - @echo "==> wrote {{FLAVOUR}} sysext:" && ls -lh dist/sysext/ + just _export-sysext-element "{{FLAVOUR}}" "{{FLAVOUR}}_" # Install dist/diskless/ in QEMU, merge its NVIDIA sysext and probe it (no GPU). [group('sysext')] @@ -347,13 +348,7 @@ build-nvidia-container-toolkit-sysext: # Export the NVIDIA Container Toolkit sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-nvidia-container-toolkit-sysext: build-nvidia-container-toolkit-sysext - rm -rf dist/nvidia-ctk-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/nvidia-container-toolkit-sysext.bst --directory /src/dist/nvidia-ctk-checkout - cp dist/nvidia-ctk-checkout/nvidia-container-toolkit-*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/nvidia-ctk-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/nvidia-ctk-checkout - @echo "==> wrote nvidia-container-toolkit sysext:" && ls -lh dist/sysext/ + just _export-sysext-element "nvidia-container-toolkit" "nvidia-container-toolkit-" # Build the Tailscale systemd-sysext (locked to one image version). [group('sysext')] @@ -363,13 +358,7 @@ build-tailscale-sysext: # Export the Tailscale sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-tailscale-sysext: build-tailscale-sysext - rm -rf dist/tailscale-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/tailscale-sysext.bst --directory /src/dist/tailscale-checkout - cp dist/tailscale-checkout/tailscale_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/tailscale-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/tailscale-checkout - @echo "==> wrote tailscale sysext:" && ls -lh dist/sysext/ + just _export-sysext-element "tailscale" "tailscale_" # Build the NetBird systemd-sysext (locked to one image version). [group('sysext')] @@ -379,13 +368,7 @@ build-netbird-sysext: # Export the NetBird sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-netbird-sysext: build-netbird-sysext - rm -rf dist/netbird-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/netbird-sysext.bst --directory /src/dist/netbird-checkout - cp dist/netbird-checkout/netbird_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/netbird-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/netbird-checkout - @echo "==> wrote netbird sysext:" && ls -lh dist/sysext/ + just _export-sysext-element "netbird" "netbird_" # Set up KubeStellar kc-agent for the user in ONE command. [group('test')] diff --git a/tests/unit/test_sysext_export_contract.py b/tests/unit/test_sysext_export_contract.py index 2ea4f857..827b9e4c 100644 --- a/tests/unit/test_sysext_export_contract.py +++ b/tests/unit/test_sysext_export_contract.py @@ -24,7 +24,13 @@ def test_export_sysext_does_not_emit_legacy_k3s_alias() -> None: body = recipe_body("export-sysext") assert body, "the export-sysext recipe is missing, so this check would pass vacuously" - assert any("oci/k0s-sysext.bst" in line for line in body), "export-sysext no longer exports k0s" + + # k0s now delegates to the shared helper; its NAME argument still names the + # k0s element (oci/k0s-sysext.bst), so the k0s export is not lost. + call = _helper_call(body) + assert call, "export-sysext must delegate to _export-sysext-element" + assert call[0] == "k0s", "export-sysext must still export the k0s element" + assert not [line for line in body if "k3s" in line] @@ -35,21 +41,68 @@ def sysext_image_name(element: str) -> str: return match.group(1).replace("%{image-version}", "") -def test_export_recipes_copy_globs_match_the_element_image_names() -> None: - for recipe, element in ( - ("export-zfs-sysext", "zfs-sysext.bst"), - ("export-nvidia-sysext", "nvidia-open-595-sysext.bst"), - ): - body = recipe_body(recipe) - assert body, f"the {recipe} recipe is missing, so this check would pass vacuously" - - copies = [line for line in body if line.startswith("cp ") and ".raw.zst" in line] - assert copies, f"{recipe} never copies a .raw.zst artifact" - - prefix = sysext_image_name(element) - for line in copies: - glob = line.split()[1].rsplit("/", 1)[-1] - stem = glob.split("*", 1)[0].replace("{{FLAVOUR}}", "nvidia-open-595") - assert stem == prefix, ( - f"{recipe} copies {glob!r}, which never matches {prefix}.raw.zst" - ) +# Every export-*-sysext recipe except export-homelab-sysext (which loops over +# several elements) delegates to the shared _export-sysext-element helper. +def _sysext_export_recipes() -> list[str]: + names = [] + for line in JUSTFILE.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not re.match(r"^export-", stripped) or ":" not in line: + continue + name = stripped.split(":", 1)[0].strip() + if name.endswith("-sysext") and name != "export-homelab-sysext": + names.append(name) + return names + + +def _recipe_header(name: str) -> str: + lines = JUSTFILE.read_text(encoding="utf-8").splitlines() + header = re.compile(rf"^{re.escape(name)}(\s[^:]*)?:(?!=)") + for line in lines: + if header.match(line): + return line + return "" + + +def _flavour_of(header: str) -> str | None: + match = re.search(r'FLAVOUR="([^"]*)"', header) + return match.group(1) if match else None + + +def _resolve(expr: str, flavour: str | None) -> str: + return expr.replace("{{FLAVOUR}}", flavour) if flavour else expr + + +def _helper_call(body: list[str]) -> tuple[str, str] | None: + for line in body: + if not line.startswith("just _export-sysext-element"): + continue + args = re.findall(r'"([^"]*)"', line) + if len(args) == 2: + return args[0], args[1] + return None + + +def test_single_element_recipes_call_the_shared_helper() -> None: + recipes = _sysext_export_recipes() + assert recipes, "no export-*-sysext recipes found" + + for name in recipes: + header = _recipe_header(name) + flavour = _flavour_of(header) + body = recipe_body(name) + call = _helper_call(body) + assert call, ( + f"{name} does not call _export-sysext-element; it re-inlines the " + "checkout, copy, and SHA256SUMS steps, so its glob is unguarded" + ) + + ename = f"{_resolve(call[0], flavour)}-sysext.bst" + element_path = ROOT / "elements" / "oci" / ename + assert element_path.exists(), f"{name} names element {ename}, which does not exist" + expected = re.sub(r"%\{[^}]+\}", "", sysext_image_name(ename)) + prefix = _resolve(call[1], flavour) + assert prefix == expected, ( + f"{name} copies with prefix {prefix!r}, but the element's sysext-image is " + f"{expected!r}, so the glob never matches the built artifact" + ) From 8630d73f4fc908b41562b93c4b6eb411cb5e7af9 Mon Sep 17 00:00:00 2001 From: scanner Date: Tue, 10 Nov 2026 12:30:18 +0000 Subject: [PATCH 2/3] Refactor the five single-element export-*-sysext recipes onto one helper; defer k0s Fold zfs, nvidia, nvidia-container-toolkit, tailscale and netbird export recipes into the shared _export-sysext-element helper. Keep k0s export-sysext in its own clean-and-replace body; its ordering/checksum change stays separate per #638. The parser splits the FLAVOUR= assignment off the recipe header so the NVIDIA export is discovered, and the contract test asserts all five single-element recipes delegate and match their element's sysext-image prefix. --- Justfile | 15 +++++++++------ tests/unit/test_sysext_export_contract.py | 20 +++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Justfile b/Justfile index dff941aa..4e7a8920 100644 --- a/Justfile +++ b/Justfile @@ -118,14 +118,18 @@ lint-python: build-sysext: just bst build oci/k0s-sysext.bst -# Export the k0s systemd-sysext image + SHA256SUMS to dist/sysext/. Delegates to -# _export-sysext-element, which appends (mkdir -p) instead of wiping dist/sysext, -# so the k0s export no longer deletes the other add-on exports (#638). +# Export the k0s systemd-sysext image + SHA256SUMS to dist/sysext/. # The artifact checkout also emits an uncompressed .raw; only the # versioned .raw.zst release asset and its SHA256SUMS are published. [group('sysext')] export-sysext: build-sysext - just _export-sysext-element "k0s" "k0s-" + rm -rf dist/sysext dist/sysext-checkout + mkdir -p dist/sysext-checkout dist/sysext + just bst artifact checkout oci/k0s-sysext.bst --directory /src/dist/sysext-checkout + cp dist/sysext-checkout/k0s-*.raw.zst dist/sysext/ + cp dist/sysext-checkout/SHA256SUMS dist/sysext/ + rm -rf dist/sysext-checkout + @echo "==> wrote k0s sysext:" && ls -lh dist/sysext/ # -- Diskless /usr image + signed boot chain ---------------------------------- @@ -234,8 +238,7 @@ dogfood-mdns: # PREFIX is that element's sysext-image name, so *.raw.zst matches the # built artifact. Every export-*-sysext recipe calls this instead of re-writing # the checkout, copy, and SHA256SUMS steps by hand, so a new add-on cannot drift -# its glob the way the zfs glob did (#306). It appends to dist/sysext/ instead of -# wiping it, so the k0s export-sysext no longer deletes the other add-on exports (#638). +# its glob the way the zfs glob did (#306). [group('sysext')] _export-sysext-element NAME PREFIX: rm -rf dist/{{NAME}}-checkout diff --git a/tests/unit/test_sysext_export_contract.py b/tests/unit/test_sysext_export_contract.py index 827b9e4c..2d7436b6 100644 --- a/tests/unit/test_sysext_export_contract.py +++ b/tests/unit/test_sysext_export_contract.py @@ -25,11 +25,9 @@ def test_export_sysext_does_not_emit_legacy_k3s_alias() -> None: assert body, "the export-sysext recipe is missing, so this check would pass vacuously" - # k0s now delegates to the shared helper; its NAME argument still names the - # k0s element (oci/k0s-sysext.bst), so the k0s export is not lost. - call = _helper_call(body) - assert call, "export-sysext must delegate to _export-sysext-element" - assert call[0] == "k0s", "export-sysext must still export the k0s element" + # k0s no longer delegates: its recipe inlines the checkout and copies + # oci/k0s-sysext.bst's artifact, so the export is preserved by the body. + assert any("oci/k0s-sysext.bst" in line for line in body), "export-sysext no longer exports k0s" assert not [line for line in body if "k3s" in line] @@ -41,16 +39,17 @@ def sysext_image_name(element: str) -> str: return match.group(1).replace("%{image-version}", "") -# Every export-*-sysext recipe except export-homelab-sysext (which loops over -# several elements) delegates to the shared _export-sysext-element helper. +# The five single-element add-on exports delegate to the shared helper. +# k0s behavior changes remain separate; homelab exports several elements. def _sysext_export_recipes() -> list[str]: names = [] for line in JUSTFILE.read_text(encoding="utf-8").splitlines(): stripped = line.strip() if not re.match(r"^export-", stripped) or ":" not in line: continue - name = stripped.split(":", 1)[0].strip() - if name.endswith("-sysext") and name != "export-homelab-sysext": + name = stripped.split(":", 1)[0].split()[0] + if (name.endswith("-sysext") + and name not in {"export-sysext", "export-homelab-sysext"}): names.append(name) return names @@ -86,6 +85,9 @@ def _helper_call(body: list[str]) -> tuple[str, str] | None: def test_single_element_recipes_call_the_shared_helper() -> None: recipes = _sysext_export_recipes() assert recipes, "no export-*-sysext recipes found" + # The parser splits the FLAVOUR= assignment off the header, so the NVIDIA + # recipe is discovered rather than silently skipped (regression for #649). + assert "export-nvidia-sysext" in recipes, "NVIDIA export dropped from discovery" for name in recipes: header = _recipe_header(name) From 3bb0e769257776317eb6f7d6cfc8e0def34f63e3 Mon Sep 17 00:00:00 2001 From: scanner Date: Thu, 12 Nov 2026 22:02:37 +0000 Subject: [PATCH 3/3] test: assert nvidia-container-toolkit export delegates to the helper (not the removed inline checkout) --- tests/unit/test_nvidia_container_toolkit_sysext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_nvidia_container_toolkit_sysext.py b/tests/unit/test_nvidia_container_toolkit_sysext.py index 8904f2e1..57fe949f 100644 --- a/tests/unit/test_nvidia_container_toolkit_sysext.py +++ b/tests/unit/test_nvidia_container_toolkit_sysext.py @@ -205,4 +205,4 @@ def test_justfile_validates_builds_and_exports_the_sysext() -> None: validate = next(line for line in justfile.splitlines() if "just bst show --deps all" in line) assert "oci/nvidia-container-toolkit-sysext.bst" in validate assert "just bst build oci/nvidia-container-toolkit-sysext.bst" in justfile - assert "just bst artifact checkout oci/nvidia-container-toolkit-sysext.bst" in justfile + assert 'just _export-sysext-element "nvidia-container-toolkit" "nvidia-container-toolkit-"' in justfile