Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 31 additions & 55 deletions Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -118,18 +118,31 @@ lint-python:
build-sysext:
just bst build oci/k0s-sysext.bst

# Export one opt-in sysext element's .raw.zst + SHA256SUMS line to dist/sysext/.
# Every export-*sysext recipe goes through here so they stay order-independent:
# dist/sysext/ is never wiped, and a re-export replaces its own SHA256SUMS line
# instead of appending a duplicate. PREFIX is the element's sysext-image name
# without the version; the artifact checkout also emits an uncompressed .raw,
# but only the versioned .raw.zst release asset is published.
_export-sysext-element ELEMENT PREFIX:
#!/usr/bin/env bash
set -euo pipefail
scratch="dist/{{ELEMENT}}-checkout"
rm -rf "${scratch}"
mkdir -p dist/sysext
just bst artifact checkout "oci/{{ELEMENT}}.bst" --directory "/src/${scratch}"
cp "${scratch}"/{{PREFIX}}*.raw.zst dist/sysext/
grep 'raw.zst$' "${scratch}/SHA256SUMS" > "${scratch}/new-sums"
touch dist/sysext/SHA256SUMS
awk 'NR == FNR { drop[$NF]; next } !($NF in drop)' "${scratch}/new-sums" dist/sysext/SHA256SUMS > "${scratch}/kept-sums"
cat "${scratch}/kept-sums" "${scratch}/new-sums" > dist/sysext/SHA256SUMS
rm -rf "${scratch}"
echo "==> wrote {{PREFIX}} sysext:" && ls -lh dist/sysext/

# Export the k0s systemd-sysext image + SHA256SUMS to dist/sysext/.
# The artifact checkout also emits an uncompressed .raw; only the
# versioned .raw.zst release asset and its SHA256SUMS are published.
[group('sysext')]
export-sysext: build-sysext
rm -rf dist/sysext dist/sysext-checkout
mkdir -p dist/sysext-checkout dist/sysext
just bst artifact checkout oci/k0s-sysext.bst --directory /src/dist/sysext-checkout
cp dist/sysext-checkout/k0s-*.raw.zst dist/sysext/
cp dist/sysext-checkout/SHA256SUMS dist/sysext/
rm -rf dist/sysext-checkout
@echo "==> wrote k0s sysext:" && ls -lh dist/sysext/
just _export-sysext-element k0s-sysext k0s-

# -- Diskless /usr image + signed boot chain ----------------------------------

Expand Down Expand Up @@ -241,13 +254,7 @@ build-zfs-sysext: gen-dev-keys
# Export the OpenZFS sysext + SHA256SUMS to dist/sysext/.
[group('sysext')]
export-zfs-sysext: build-zfs-sysext
rm -rf dist/zfs-checkout
mkdir -p dist/sysext
just bst artifact checkout oci/zfs-sysext.bst --directory /src/dist/zfs-checkout
cp dist/zfs-checkout/zfs_*.raw.zst dist/sysext/
grep 'raw.zst$' dist/zfs-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS
rm -rf dist/zfs-checkout
@echo "==> wrote zfs sysext:" && ls -lh dist/sysext/
just _export-sysext-element zfs-sysext zfs_

# Build the homelab sysext (component manifests + bluefin-homelab-apply) and
# its add-on sysexts (Argo Workflows, MCP server, KubeStellar).
Expand All @@ -258,17 +265,10 @@ build-homelab-sysext:
# Export the homelab sysext and its add-ons + SHA256SUMS to dist/sysext/.
[group('sysext')]
export-homelab-sysext: build-homelab-sysext
#!/usr/bin/env bash
set -euo pipefail
mkdir -p dist/sysext
for name in homelab argo-workflows mcp kubestellar; do
rm -rf dist/homelab-checkout
just bst artifact checkout "oci/${name}-sysext.bst" --directory /src/dist/homelab-checkout
cp dist/homelab-checkout/${name}_*.raw.zst dist/sysext/
grep 'raw.zst$' dist/homelab-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS
done
rm -rf dist/homelab-checkout
echo "==> wrote homelab sysexts:" && ls -lh dist/sysext/
just _export-sysext-element homelab-sysext homelab_
just _export-sysext-element argo-workflows-sysext argo-workflows_
just _export-sysext-element mcp-sysext mcp_
just _export-sysext-element kubestellar-sysext kubestellar_

# Re-render files/homelab/manifests/ and files/homelab/addons/ from the pins in the render script (network, podman).
[group('sysext')]
Expand Down Expand Up @@ -302,13 +302,7 @@ build-nvidia-sysext FLAVOUR="nvidia-open-595": gen-dev-keys
# Export an NVIDIA driver sysext + SHA256SUMS to dist/sysext/.
[group('sysext')]
export-nvidia-sysext FLAVOUR="nvidia-open-595": (build-nvidia-sysext FLAVOUR)
rm -rf dist/{{FLAVOUR}}-checkout
mkdir -p dist/sysext
just bst artifact checkout oci/{{FLAVOUR}}-sysext.bst --directory /src/dist/{{FLAVOUR}}-checkout
cp dist/{{FLAVOUR}}-checkout/{{FLAVOUR}}_*.raw.zst dist/sysext/
grep 'raw.zst$' dist/{{FLAVOUR}}-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS
rm -rf dist/{{FLAVOUR}}-checkout
@echo "==> wrote {{FLAVOUR}} sysext:" && ls -lh dist/sysext/
just _export-sysext-element {{FLAVOUR}}-sysext {{FLAVOUR}}_

# Install dist/diskless/ in QEMU, merge its NVIDIA sysext and probe it (no GPU).
[group('sysext')]
Expand Down Expand Up @@ -347,13 +341,7 @@ build-nvidia-container-toolkit-sysext:
# Export the NVIDIA Container Toolkit sysext + SHA256SUMS to dist/sysext/.
[group('sysext')]
export-nvidia-container-toolkit-sysext: build-nvidia-container-toolkit-sysext
rm -rf dist/nvidia-ctk-checkout
mkdir -p dist/sysext
just bst artifact checkout oci/nvidia-container-toolkit-sysext.bst --directory /src/dist/nvidia-ctk-checkout
cp dist/nvidia-ctk-checkout/nvidia-container-toolkit-*.raw.zst dist/sysext/
grep 'raw.zst$' dist/nvidia-ctk-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS
rm -rf dist/nvidia-ctk-checkout
@echo "==> wrote nvidia-container-toolkit sysext:" && ls -lh dist/sysext/
just _export-sysext-element nvidia-container-toolkit-sysext nvidia-container-toolkit-

# Build the Tailscale systemd-sysext (locked to one image version).
[group('sysext')]
Expand All @@ -363,13 +351,7 @@ build-tailscale-sysext:
# Export the Tailscale sysext + SHA256SUMS to dist/sysext/.
[group('sysext')]
export-tailscale-sysext: build-tailscale-sysext
rm -rf dist/tailscale-checkout
mkdir -p dist/sysext
just bst artifact checkout oci/tailscale-sysext.bst --directory /src/dist/tailscale-checkout
cp dist/tailscale-checkout/tailscale_*.raw.zst dist/sysext/
grep 'raw.zst$' dist/tailscale-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS
rm -rf dist/tailscale-checkout
@echo "==> wrote tailscale sysext:" && ls -lh dist/sysext/
just _export-sysext-element tailscale-sysext tailscale_

# Build the NetBird systemd-sysext (locked to one image version).
[group('sysext')]
Expand All @@ -379,13 +361,7 @@ build-netbird-sysext:
# Export the NetBird sysext + SHA256SUMS to dist/sysext/.
[group('sysext')]
export-netbird-sysext: build-netbird-sysext
rm -rf dist/netbird-checkout
mkdir -p dist/sysext
just bst artifact checkout oci/netbird-sysext.bst --directory /src/dist/netbird-checkout
cp dist/netbird-checkout/netbird_*.raw.zst dist/sysext/
grep 'raw.zst$' dist/netbird-checkout/SHA256SUMS >> dist/sysext/SHA256SUMS
rm -rf dist/netbird-checkout
@echo "==> wrote netbird sysext:" && ls -lh dist/sysext/
just _export-sysext-element netbird-sysext netbird_

# Set up KubeStellar kc-agent for the user in ONE command.
[group('test')]
Expand Down
4 changes: 2 additions & 2 deletions tests/unit/test_homelab_sysext.py
Original file line number Diff line number Diff line change
Expand Up @@ -459,8 +459,8 @@ def test_homelab_sysext_ships_in_the_signed_release_set() -> None:
assert '"homelab_${v}\\\\.raw\\\\.zst"' in (ROOT / "scripts" / "publish-release.sh").read_text()
justfile = (ROOT / "Justfile").read_text()
assert "oci/homelab-sysext.bst" in justfile
assert "for name in homelab argo-workflows mcp kubestellar; do" in justfile
assert "cp dist/homelab-checkout/${name}_*.raw.zst dist/sysext/" in justfile
for name in ("homelab", "argo-workflows", "mcp", "kubestellar"):
assert f"just _export-sysext-element {name}-sysext {name}_" in justfile


def test_applier_never_deletes() -> None:
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/test_nvidia_container_toolkit_sysext.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,4 +205,4 @@ def test_justfile_validates_builds_and_exports_the_sysext() -> None:
validate = next(line for line in justfile.splitlines() if "just bst show --deps all" in line)
assert "oci/nvidia-container-toolkit-sysext.bst" in validate
assert "just bst build oci/nvidia-container-toolkit-sysext.bst" in justfile
assert "just bst artifact checkout oci/nvidia-container-toolkit-sysext.bst" in justfile
assert "just _export-sysext-element nvidia-container-toolkit-sysext nvidia-container-toolkit-" in justfile
66 changes: 45 additions & 21 deletions tests/unit/test_sysext_export_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@

ROOT = Path(__file__).resolve().parents[2]
JUSTFILE = ROOT / "Justfile"
HELPER = "_export-sysext-element"
EXPORT_RECIPE = re.compile(r"^(export-(?:[\w-]+-)?sysext)\b[^:]*:(?!=)", re.MULTILINE)
HELPER_CALL = re.compile(rf"^just {re.escape(HELPER)} (\S+) (\S+)$")


def recipe_body(name: str) -> list[str]:
Expand All @@ -20,36 +23,57 @@ def recipe_body(name: str) -> list[str]:
return []


def test_export_sysext_does_not_emit_legacy_k3s_alias() -> None:
body = recipe_body("export-sysext")

assert body, "the export-sysext recipe is missing, so this check would pass vacuously"
assert any("oci/k0s-sysext.bst" in line for line in body), "export-sysext no longer exports k0s"
assert not [line for line in body if "k3s" in line]
def export_recipes() -> list[str]:
return EXPORT_RECIPE.findall(JUSTFILE.read_text(encoding="utf-8"))


def sysext_image_name(element: str) -> str:
text = (ROOT / "elements" / "oci" / element).read_text(encoding="utf-8")
match = re.search(r'^\s*sysext-image:\s*"([^"]+)"', text, re.MULTILINE)
assert match, f"{element} does not declare a sysext-image name"
return match.group(1).replace("%{image-version}", "")
return re.sub(r"%\{[\w-]+-version\}$", "", match.group(1))


def test_export_recipes_copy_globs_match_the_element_image_names() -> None:
for recipe, element in (
("export-zfs-sysext", "zfs-sysext.bst"),
("export-nvidia-sysext", "nvidia-open-595-sysext.bst"),
):
def test_export_sysext_does_not_emit_legacy_k3s_alias() -> None:
body = recipe_body("export-sysext")

assert body, "the export-sysext recipe is missing, so this check would pass vacuously"
assert f"just {HELPER} k0s-sysext k0s-" in body, "export-sysext no longer exports k0s"
assert not [line for line in body if "k3s" in line]


def test_every_export_recipe_only_calls_the_shared_helper() -> None:
recipes = export_recipes()
assert len(recipes) >= 7, f"found only {recipes}; the recipe pattern no longer matches"

for recipe in recipes:
body = recipe_body(recipe)
assert body, f"the {recipe} recipe is missing, so this check would pass vacuously"
assert body, f"{recipe} has no body"
inlined = [line for line in body if not HELPER_CALL.match(line)]
assert not inlined, f"{recipe} re-inlines the export body instead of calling {HELPER}: {inlined}"

copies = [line for line in body if line.startswith("cp ") and ".raw.zst" in line]
assert copies, f"{recipe} never copies a .raw.zst artifact"

prefix = sysext_image_name(element)
for line in copies:
glob = line.split()[1].rsplit("/", 1)[-1]
stem = glob.split("*", 1)[0].replace("{{FLAVOUR}}", "nvidia-open-595")
assert stem == prefix, (
f"{recipe} copies {glob!r}, which never matches {prefix}<version>.raw.zst"
def test_export_recipes_copy_globs_match_the_element_image_names() -> None:
for recipe in export_recipes():
for line in recipe_body(recipe):
match = HELPER_CALL.match(line)
assert match, f"{recipe}: {line!r} does not call {HELPER}"
element, prefix = (
part.replace("{{FLAVOUR}}", "nvidia-open-595") for part in match.groups()
)
assert prefix == sysext_image_name(f"{element}.bst"), (
f"{recipe} copies {prefix}*.raw.zst, which never matches {element}'s image name"
)


def test_helper_keeps_other_exports_and_replaces_its_own_checksum_line() -> None:
body = recipe_body(HELPER)

assert body, f"the {HELPER} recipe is missing, so this check would pass vacuously"
assert not [line for line in body if re.search(r"rm -rf\s+\"?dist/sysext\"?(\s|$)", line)], (
"the helper wipes dist/sysext/, so one export erases the others"
)
assert not [line for line in body if re.search(r">>\s*dist/sysext/SHA256SUMS", line)], (
"the helper appends to SHA256SUMS blindly, so a re-export duplicates its line"
)
assert any("drop[$NF]" in line for line in body), "the helper no longer drops stale SHA256SUMS lines"
Loading