diff --git a/Justfile b/Justfile index 4195da27..fe68643f 100644 --- a/Justfile +++ b/Justfile @@ -118,18 +118,31 @@ lint-python: build-sysext: just bst build oci/k0s-sysext.bst +# Export one opt-in sysext element's .raw.zst + SHA256SUMS line to dist/sysext/. +# Every export-*sysext recipe goes through here so they stay order-independent: +# dist/sysext/ is never wiped, and a re-export replaces its own SHA256SUMS line +# instead of appending a duplicate. PREFIX is the element's sysext-image name +# without the version; the artifact checkout also emits an uncompressed .raw, +# but only the versioned .raw.zst release asset is published. +_export-sysext-element ELEMENT PREFIX: + #!/usr/bin/env bash + set -euo pipefail + scratch="dist/{{ELEMENT}}-checkout" + rm -rf "${scratch}" + mkdir -p dist/sysext + just bst artifact checkout "oci/{{ELEMENT}}.bst" --directory "/src/${scratch}" + cp "${scratch}"/{{PREFIX}}*.raw.zst dist/sysext/ + grep 'raw.zst$' "${scratch}/SHA256SUMS" > "${scratch}/new-sums" + touch dist/sysext/SHA256SUMS + awk 'NR == FNR { drop[$NF]; next } !($NF in drop)' "${scratch}/new-sums" dist/sysext/SHA256SUMS > "${scratch}/kept-sums" + cat "${scratch}/kept-sums" "${scratch}/new-sums" > dist/sysext/SHA256SUMS + rm -rf "${scratch}" + echo "==> wrote {{PREFIX}} sysext:" && ls -lh dist/sysext/ + # Export the k0s systemd-sysext image + SHA256SUMS to dist/sysext/. -# The artifact checkout also emits an uncompressed .raw; only the -# versioned .raw.zst release asset and its SHA256SUMS are published. [group('sysext')] export-sysext: build-sysext - rm -rf dist/sysext dist/sysext-checkout - mkdir -p dist/sysext-checkout dist/sysext - just bst artifact checkout oci/k0s-sysext.bst --directory /src/dist/sysext-checkout - cp dist/sysext-checkout/k0s-*.raw.zst dist/sysext/ - cp dist/sysext-checkout/SHA256SUMS dist/sysext/ - rm -rf dist/sysext-checkout - @echo "==> wrote k0s sysext:" && ls -lh dist/sysext/ + just _export-sysext-element k0s-sysext k0s- # -- Diskless /usr image + signed boot chain ---------------------------------- @@ -241,13 +254,7 @@ build-zfs-sysext: gen-dev-keys # Export the OpenZFS sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-zfs-sysext: build-zfs-sysext - rm -rf dist/zfs-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/zfs-sysext.bst --directory /src/dist/zfs-checkout - cp dist/zfs-checkout/zfs_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/zfs-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/zfs-checkout - @echo "==> wrote zfs sysext:" && ls -lh dist/sysext/ + just _export-sysext-element zfs-sysext zfs_ # Build the homelab sysext (component manifests + bluefin-homelab-apply) and # its add-on sysexts (Argo Workflows, MCP server, KubeStellar). @@ -258,17 +265,10 @@ build-homelab-sysext: # Export the homelab sysext and its add-ons + SHA256SUMS to dist/sysext/. [group('sysext')] export-homelab-sysext: build-homelab-sysext - #!/usr/bin/env bash - set -euo pipefail - mkdir -p dist/sysext - for name in homelab argo-workflows mcp kubestellar; do - rm -rf dist/homelab-checkout - just bst artifact checkout "oci/${name}-sysext.bst" --directory /src/dist/homelab-checkout - cp dist/homelab-checkout/${name}_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/homelab-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - done - rm -rf dist/homelab-checkout - echo "==> wrote homelab sysexts:" && ls -lh dist/sysext/ + just _export-sysext-element homelab-sysext homelab_ + just _export-sysext-element argo-workflows-sysext argo-workflows_ + just _export-sysext-element mcp-sysext mcp_ + just _export-sysext-element kubestellar-sysext kubestellar_ # Re-render files/homelab/manifests/ and files/homelab/addons/ from the pins in the render script (network, podman). [group('sysext')] @@ -302,13 +302,7 @@ build-nvidia-sysext FLAVOUR="nvidia-open-595": gen-dev-keys # Export an NVIDIA driver sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-nvidia-sysext FLAVOUR="nvidia-open-595": (build-nvidia-sysext FLAVOUR) - rm -rf dist/{{FLAVOUR}}-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/{{FLAVOUR}}-sysext.bst --directory /src/dist/{{FLAVOUR}}-checkout - cp dist/{{FLAVOUR}}-checkout/{{FLAVOUR}}_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/{{FLAVOUR}}-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/{{FLAVOUR}}-checkout - @echo "==> wrote {{FLAVOUR}} sysext:" && ls -lh dist/sysext/ + just _export-sysext-element {{FLAVOUR}}-sysext {{FLAVOUR}}_ # Install dist/diskless/ in QEMU, merge its NVIDIA sysext and probe it (no GPU). [group('sysext')] @@ -347,13 +341,7 @@ build-nvidia-container-toolkit-sysext: # Export the NVIDIA Container Toolkit sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-nvidia-container-toolkit-sysext: build-nvidia-container-toolkit-sysext - rm -rf dist/nvidia-ctk-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/nvidia-container-toolkit-sysext.bst --directory /src/dist/nvidia-ctk-checkout - cp dist/nvidia-ctk-checkout/nvidia-container-toolkit-*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/nvidia-ctk-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/nvidia-ctk-checkout - @echo "==> wrote nvidia-container-toolkit sysext:" && ls -lh dist/sysext/ + just _export-sysext-element nvidia-container-toolkit-sysext nvidia-container-toolkit- # Build the Tailscale systemd-sysext (locked to one image version). [group('sysext')] @@ -363,13 +351,7 @@ build-tailscale-sysext: # Export the Tailscale sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-tailscale-sysext: build-tailscale-sysext - rm -rf dist/tailscale-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/tailscale-sysext.bst --directory /src/dist/tailscale-checkout - cp dist/tailscale-checkout/tailscale_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/tailscale-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/tailscale-checkout - @echo "==> wrote tailscale sysext:" && ls -lh dist/sysext/ + just _export-sysext-element tailscale-sysext tailscale_ # Build the NetBird systemd-sysext (locked to one image version). [group('sysext')] @@ -379,13 +361,7 @@ build-netbird-sysext: # Export the NetBird sysext + SHA256SUMS to dist/sysext/. [group('sysext')] export-netbird-sysext: build-netbird-sysext - rm -rf dist/netbird-checkout - mkdir -p dist/sysext - just bst artifact checkout oci/netbird-sysext.bst --directory /src/dist/netbird-checkout - cp dist/netbird-checkout/netbird_*.raw.zst dist/sysext/ - grep 'raw.zst$' dist/netbird-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS - rm -rf dist/netbird-checkout - @echo "==> wrote netbird sysext:" && ls -lh dist/sysext/ + just _export-sysext-element netbird-sysext netbird_ # Set up KubeStellar kc-agent for the user in ONE command. [group('test')] diff --git a/tests/unit/test_homelab_sysext.py b/tests/unit/test_homelab_sysext.py index b27c8024..c916d695 100644 --- a/tests/unit/test_homelab_sysext.py +++ b/tests/unit/test_homelab_sysext.py @@ -459,8 +459,8 @@ def test_homelab_sysext_ships_in_the_signed_release_set() -> None: assert '"homelab_${v}\\\\.raw\\\\.zst"' in (ROOT / "scripts" / "publish-release.sh").read_text() justfile = (ROOT / "Justfile").read_text() assert "oci/homelab-sysext.bst" in justfile - assert "for name in homelab argo-workflows mcp kubestellar; do" in justfile - assert "cp dist/homelab-checkout/${name}_*.raw.zst dist/sysext/" in justfile + for name in ("homelab", "argo-workflows", "mcp", "kubestellar"): + assert f"just _export-sysext-element {name}-sysext {name}_" in justfile def test_applier_never_deletes() -> None: diff --git a/tests/unit/test_nvidia_container_toolkit_sysext.py b/tests/unit/test_nvidia_container_toolkit_sysext.py index 8904f2e1..ae954cff 100644 --- a/tests/unit/test_nvidia_container_toolkit_sysext.py +++ b/tests/unit/test_nvidia_container_toolkit_sysext.py @@ -205,4 +205,4 @@ def test_justfile_validates_builds_and_exports_the_sysext() -> None: validate = next(line for line in justfile.splitlines() if "just bst show --deps all" in line) assert "oci/nvidia-container-toolkit-sysext.bst" in validate assert "just bst build oci/nvidia-container-toolkit-sysext.bst" in justfile - assert "just bst artifact checkout oci/nvidia-container-toolkit-sysext.bst" in justfile + assert "just _export-sysext-element nvidia-container-toolkit-sysext nvidia-container-toolkit-" in justfile diff --git a/tests/unit/test_sysext_export_contract.py b/tests/unit/test_sysext_export_contract.py index 2ea4f857..348ea45d 100644 --- a/tests/unit/test_sysext_export_contract.py +++ b/tests/unit/test_sysext_export_contract.py @@ -3,6 +3,9 @@ ROOT = Path(__file__).resolve().parents[2] JUSTFILE = ROOT / "Justfile" +HELPER = "_export-sysext-element" +EXPORT_RECIPE = re.compile(r"^(export-(?:[\w-]+-)?sysext)\b[^:]*:(?!=)", re.MULTILINE) +HELPER_CALL = re.compile(rf"^just {re.escape(HELPER)} (\S+) (\S+)$") def recipe_body(name: str) -> list[str]: @@ -20,36 +23,57 @@ def recipe_body(name: str) -> list[str]: return [] -def test_export_sysext_does_not_emit_legacy_k3s_alias() -> None: - body = recipe_body("export-sysext") - - assert body, "the export-sysext recipe is missing, so this check would pass vacuously" - assert any("oci/k0s-sysext.bst" in line for line in body), "export-sysext no longer exports k0s" - assert not [line for line in body if "k3s" in line] +def export_recipes() -> list[str]: + return EXPORT_RECIPE.findall(JUSTFILE.read_text(encoding="utf-8")) def sysext_image_name(element: str) -> str: text = (ROOT / "elements" / "oci" / element).read_text(encoding="utf-8") match = re.search(r'^\s*sysext-image:\s*"([^"]+)"', text, re.MULTILINE) assert match, f"{element} does not declare a sysext-image name" - return match.group(1).replace("%{image-version}", "") + return re.sub(r"%\{[\w-]+-version\}$", "", match.group(1)) -def test_export_recipes_copy_globs_match_the_element_image_names() -> None: - for recipe, element in ( - ("export-zfs-sysext", "zfs-sysext.bst"), - ("export-nvidia-sysext", "nvidia-open-595-sysext.bst"), - ): +def test_export_sysext_does_not_emit_legacy_k3s_alias() -> None: + body = recipe_body("export-sysext") + + assert body, "the export-sysext recipe is missing, so this check would pass vacuously" + assert f"just {HELPER} k0s-sysext k0s-" in body, "export-sysext no longer exports k0s" + assert not [line for line in body if "k3s" in line] + + +def test_every_export_recipe_only_calls_the_shared_helper() -> None: + recipes = export_recipes() + assert len(recipes) >= 7, f"found only {recipes}; the recipe pattern no longer matches" + + for recipe in recipes: body = recipe_body(recipe) - assert body, f"the {recipe} recipe is missing, so this check would pass vacuously" + assert body, f"{recipe} has no body" + inlined = [line for line in body if not HELPER_CALL.match(line)] + assert not inlined, f"{recipe} re-inlines the export body instead of calling {HELPER}: {inlined}" - copies = [line for line in body if line.startswith("cp ") and ".raw.zst" in line] - assert copies, f"{recipe} never copies a .raw.zst artifact" - prefix = sysext_image_name(element) - for line in copies: - glob = line.split()[1].rsplit("/", 1)[-1] - stem = glob.split("*", 1)[0].replace("{{FLAVOUR}}", "nvidia-open-595") - assert stem == prefix, ( - f"{recipe} copies {glob!r}, which never matches {prefix}.raw.zst" +def test_export_recipes_copy_globs_match_the_element_image_names() -> None: + for recipe in export_recipes(): + for line in recipe_body(recipe): + match = HELPER_CALL.match(line) + assert match, f"{recipe}: {line!r} does not call {HELPER}" + element, prefix = ( + part.replace("{{FLAVOUR}}", "nvidia-open-595") for part in match.groups() ) + assert prefix == sysext_image_name(f"{element}.bst"), ( + f"{recipe} copies {prefix}*.raw.zst, which never matches {element}'s image name" + ) + + +def test_helper_keeps_other_exports_and_replaces_its_own_checksum_line() -> None: + body = recipe_body(HELPER) + + assert body, f"the {HELPER} recipe is missing, so this check would pass vacuously" + assert not [line for line in body if re.search(r"rm -rf\s+\"?dist/sysext\"?(\s|$)", line)], ( + "the helper wipes dist/sysext/, so one export erases the others" + ) + assert not [line for line in body if re.search(r">>\s*dist/sysext/SHA256SUMS", line)], ( + "the helper appends to SHA256SUMS blindly, so a re-export duplicates its line" + ) + assert any("drop[$NF]" in line for line in body), "the helper no longer drops stale SHA256SUMS lines"