From d3b2e67138bbbda2f7dc60ca367df58096421268 Mon Sep 17 00:00:00 2001 From: castrojo Date: Thu, 17 Sep 2026 21:54:58 -0400 Subject: [PATCH 01/46] fix(kernel): point depmod -b to /usr in ddi and installer elements In Flatcar kernel module trees, modules are extracted under /usr/lib/modules/ without a root /lib symlink during DDI and installer construction. depmod -b searches /lib/modules, which fails with exit code 1 when pointing to root instead of /usr. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- elements/oci/bluefin-server-ddi.bst | 2 +- elements/oci/bluefin-server-installer.bst | 4 ++-- tests/unit/test_installer_contract.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index 9f801570..21250c37 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -38,7 +38,7 @@ config: exit 1 fi rm -f "/layer/usr/lib/modules/${KVER}/vmlinux" "/layer/usr/lib/modules/${KVER}/System.map" - depmod -b /layer "${KVER}" + depmod -b /layer/usr "${KVER}" # Stage extracted CA certificate bundle into target DDI rootfs mkdir -p /layer/etc/pki/ca-trust/extracted /layer/etc/ssl/certs /layer/etc/pki/tls/certs diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 44d9f300..ab95bc43 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -103,7 +103,7 @@ config: mkdir -p /target-root/boot ln -sf "../usr/lib/modules/${TARGET_KVER}/vmlinuz" /target-root/boot/vmlinuz - depmod -b /target-root "${TARGET_KVER}" || true + depmod -b /target-root/usr "${TARGET_KVER}" || true # Configure dynamic linker search paths so ldconfig indexes FSDK libraries # (e.g. libc.so.6, libgcc_s.so.1, libkmod.so) for dracut-install @@ -360,7 +360,7 @@ config: rm -rf "${MOD_DIR}/drivers/infiniband" fi # Re-run depmod to update module dependencies after stripping unneeded modules - depmod -b /layer "${KVER}" || true + depmod -b /layer/usr "${KVER}" || true fi ( cd /layer && find . -print0 \ diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index d423c685..400aeac8 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -92,7 +92,7 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None: ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") assert "freedesktop-sdk.bst:components/kmod.bst" in ddi_element - assert 'depmod -b /layer "${KVER}"' in ddi_element + assert 'depmod -b /layer/usr "${KVER}"' in ddi_element assert "cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/" in ddi_element assert "tls-ca-bundle.pem" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service" in ddi_element From 11f2c50fea79529dded892f2239b9679e56126bc Mon Sep 17 00:00:00 2001 From: castrojo Date: Thu, 17 Sep 2026 22:53:23 -0400 Subject: [PATCH 02/46] fix(zfs): remove Flatcar sysext udevd drop-in from extracted zfs rootfs The upstream Flatcar zfs sysext includes an override for systemd-udevd: usr/lib/systemd/system/systemd-udevd.service.d/10-zfs.conf (After=systemd-sysext.service). When extracted directly into the base OS rootfs, this drop-in orders udevd after systemd-sysext.service. During boot, udevd was stopped while systemd-sysext.service and /var.mount were pending, deadlocking udev device unit discovery (/dev/disk/by-partlabel/var). Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- elements/flatcar/flatcar-zfs.bst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/elements/flatcar/flatcar-zfs.bst b/elements/flatcar/flatcar-zfs.bst index 82832969..b0e6aa18 100644 --- a/elements/flatcar/flatcar-zfs.bst +++ b/elements/flatcar/flatcar-zfs.bst @@ -28,6 +28,11 @@ config: # Extract the SquashFS sysext directly into the element install root unsquashfs -no-xattrs -f -d "%{install-root}" flatcar-zfs.raw + # Drop the Flatcar sysext udevd dropin (After=systemd-sysext.service). + # When extracted directly into the OS rootfs, this drop-in causes systemd-udevd + # to stop at early boot waiting for systemd-sysext.service, deadlocking udev + # device creation before /var and storage partitions are mounted. + rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d" # Also preserve the raw sysext image under /usr/lib/sysexts for runtime sysext usage mkdir -p "%{install-root}/usr/lib/sysexts" install -D -m 0644 flatcar-zfs.raw "%{install-root}/usr/lib/sysexts/zfs.raw" From 29877f5d842a169550d38c73beec7c61b6c17f3a Mon Sep 17 00:00:00 2001 From: castrojo Date: Thu, 17 Sep 2026 23:06:01 -0400 Subject: [PATCH 03/46] fix(kiosk): use dynamic DNS resolver in nginx proxy pass When nginx starts before CoreDNS or before the kubestellar-console Service ClusterIP is reachable, static proxy_pass directives fail at configuration parse time with 'host not found in upstream', causing CrashLoopBackOff. Using a runtime variable with 'resolver 10.96.0.10' defers domain lookup to request time so kiosk-proxy starts cleanly. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- files/k0s/kiosk/nginx.conf | 4 +++- tests/unit/test_kubestellar_kiosk.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/files/k0s/kiosk/nginx.conf b/files/k0s/kiosk/nginx.conf index e34bf24b..462628bd 100644 --- a/files/k0s/kiosk/nginx.conf +++ b/files/k0s/kiosk/nginx.conf @@ -31,7 +31,9 @@ http { } location / { - proxy_pass http://kubestellar-console.kubestellar-console.svc.cluster.local:8080; + resolver 10.96.0.10 valid=5s ipv6=off; + set $upstream_endpoint http://kubestellar-console.kubestellar-console.svc.cluster.local:8080; + proxy_pass $upstream_endpoint; proxy_redirect ~^https?://[^/]+/(.*) /$1; proxy_http_version 1.1; proxy_set_header Host $http_host; diff --git a/tests/unit/test_kubestellar_kiosk.py b/tests/unit/test_kubestellar_kiosk.py index ec608c72..3e11898a 100644 --- a/tests/unit/test_kubestellar_kiosk.py +++ b/tests/unit/test_kubestellar_kiosk.py @@ -45,10 +45,12 @@ def test_kiosk_assets_are_packaged_and_seeded() -> None: def test_proxy_injects_only_csp_safe_same_origin_assets() -> None: nginx = KIOSK_CONF.read_text(encoding="utf-8") + assert "proxy_pass $upstream_endpoint;" in nginx assert ( - "proxy_pass " + "set $upstream_endpoint " "http://kubestellar-console.kubestellar-console.svc.cluster.local:8080;" ) in nginx + assert "resolver 10.96.0.10" in nginx assert "proxy_redirect" in nginx assert 'proxy_set_header Accept-Encoding "";' in nginx assert "sub_filter_types" not in nginx From b514d40a8526ce706c5c539521c705a756197596 Mon Sep 17 00:00:00 2001 From: castrojo Date: Thu, 17 Sep 2026 23:21:05 -0400 Subject: [PATCH 04/46] fix(kubestellar): use POSTGRES_* env vars and fix kubeflex manager entrypoint - 10-kubeflex-operator.yaml: update container command to /ko-app/manager (matching ko-built manager image entrypoint). - 20-postgres.yaml: use standard POSTGRES_* environment variables matching docker.io/library/postgres:16-alpine with secretKeyRef for issue #98. - tests/unit/test_k0s_manifests.py: align issue #98 contract assertions to guard POSTGRES_PASSWORD. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- files/k0s/manifests/kubestellar/10-kubeflex-operator.yaml | 2 +- files/k0s/manifests/kubestellar/20-postgres.yaml | 6 +++--- tests/unit/test_k0s_manifests.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/files/k0s/manifests/kubestellar/10-kubeflex-operator.yaml b/files/k0s/manifests/kubestellar/10-kubeflex-operator.yaml index 2ae45c1c..e572eacc 100644 --- a/files/k0s/manifests/kubestellar/10-kubeflex-operator.yaml +++ b/files/k0s/manifests/kubestellar/10-kubeflex-operator.yaml @@ -26,7 +26,7 @@ spec: - name: manager image: ghcr.io/kubestellar/kubeflex/manager:0.8.2 command: - - /manager + - /ko-app/manager args: - --health-probe-bind-address=:8081 - --metrics-bind-address=127.0.0.1:8080 diff --git a/files/k0s/manifests/kubestellar/20-postgres.yaml b/files/k0s/manifests/kubestellar/20-postgres.yaml index 01bb2a46..e8a00f61 100644 --- a/files/k0s/manifests/kubestellar/20-postgres.yaml +++ b/files/k0s/manifests/kubestellar/20-postgres.yaml @@ -36,17 +36,17 @@ spec: - name: postgresql image: docker.io/library/postgres:16-alpine env: - - name: POSTGRESQL_USERNAME + - name: POSTGRES_USER value: "postgres" # Superuser password lives in the kubeflex-postgres Secret (see # 15-kubeflex-postgres-secret.yaml, generated at first boot) so it is # never committed to git or exposed in the pod spec. See #98. - - name: POSTGRESQL_PASSWORD + - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: name: kubeflex-postgres key: password - - name: POSTGRESQL_DATABASE + - name: POSTGRES_DB value: "postgres" ports: - name: postgresql diff --git a/tests/unit/test_k0s_manifests.py b/tests/unit/test_k0s_manifests.py index 149ca421..268dbce0 100644 --- a/tests/unit/test_k0s_manifests.py +++ b/tests/unit/test_k0s_manifests.py @@ -47,7 +47,7 @@ def test_postgres_password_not_hardcoded(): text = manifest.read_text() assert "kubeflex" not in text.lower().replace("kubeflex-system", "").replace("kubeflex-postgres", "") assert 'value: "kubeflex"' not in text - assert "POSTGRESQL_PASSWORD" in text + assert "POSTGRES_PASSWORD" in text def test_postgres_password_from_secret(): @@ -56,8 +56,8 @@ def test_postgres_password_from_secret(): statefulset = next(d for d in docs if d and d.get("kind") == "StatefulSet") container = statefulset["spec"]["template"]["spec"]["containers"][0] env = {e["name"]: e for e in container["env"]} - assert "value" not in env["POSTGRESQL_PASSWORD"] - ref = env["POSTGRESQL_PASSWORD"]["valueFrom"]["secretKeyRef"] + assert "value" not in env["POSTGRES_PASSWORD"] + ref = env["POSTGRES_PASSWORD"]["valueFrom"]["secretKeyRef"] assert ref == {"name": "kubeflex-postgres", "key": "password"} From 2d5dd1401c8e317c00cb4fd39e119cf3b6ea0d2d Mon Sep 17 00:00:00 2001 From: castrojo Date: Thu, 17 Sep 2026 23:26:34 -0400 Subject: [PATCH 05/46] fix(zfs): strip udev drop-in copy from tmpfiles configuration In addition to removing the extracted /usr/lib/systemd/system/systemd-udevd.service.d/ drop-in directory, also remove the 'C /etc/systemd/system/systemd-udevd.service.d/10-zfs.conf' copy rule in /usr/lib/tmpfiles.d/10-zfs.conf so systemd-tmpfiles does not recreate the udevd sysext ordering override in /etc. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- elements/flatcar/flatcar-zfs.bst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elements/flatcar/flatcar-zfs.bst b/elements/flatcar/flatcar-zfs.bst index b0e6aa18..08580ee5 100644 --- a/elements/flatcar/flatcar-zfs.bst +++ b/elements/flatcar/flatcar-zfs.bst @@ -28,11 +28,15 @@ config: # Extract the SquashFS sysext directly into the element install root unsquashfs -no-xattrs -f -d "%{install-root}" flatcar-zfs.raw - # Drop the Flatcar sysext udevd dropin (After=systemd-sysext.service). + # Drop the Flatcar sysext udevd dropin (After=systemd-sysext.service) + # and the tmpfiles rule that copies it into /etc/systemd/system/systemd-udevd.service.d/. # When extracted directly into the OS rootfs, this drop-in causes systemd-udevd # to stop at early boot waiting for systemd-sysext.service, deadlocking udev # device creation before /var and storage partitions are mounted. rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d" + if [ -f "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" ]; then + sed -i '/systemd-udevd\.service\.d/d' "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" + fi # Also preserve the raw sysext image under /usr/lib/sysexts for runtime sysext usage mkdir -p "%{install-root}/usr/lib/sysexts" install -D -m 0644 flatcar-zfs.raw "%{install-root}/usr/lib/sysexts/zfs.raw" From 89fdcf2d28b273f603895925794d6c5199836b9d Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 00:08:31 -0400 Subject: [PATCH 06/46] fix(zfs,test): add sed build-dep to flatcar-zfs and update kiosk healthz probe in Justfile - elements/flatcar/flatcar-zfs.bst: add freedesktop-sdk.bst:components/sed.bst to build-depends so install-commands sed invocation succeeds. - Justfile: support self-signed HTTPS certificates in test-installer-artifact healthz polling. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- Justfile | 14 ++++++++------ elements/flatcar/flatcar-zfs.bst | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Justfile b/Justfile index 1bae6d3f..5ec56fba 100644 --- a/Justfile +++ b/Justfile @@ -349,12 +349,14 @@ test-installer-artifact: exit 1 fi - HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) - if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then - ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) - if [ "$ROOT_CODE" = "200" ]; then - echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" - break + HEALTHZ_RESP=$(curl --silent --insecure --max-time 2 https://127.0.0.1:8080/healthz 2>/dev/null || curl --silent --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) + if [ -n "$HEALTHZ_RESP" ]; then + ROOT_CODE=$(curl --silent --insecure --max-time 2 --output /dev/null --write-out "%{http_code}" https://127.0.0.1:8080/ 2>/dev/null || curl --silent --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) + if [ "$ROOT_CODE" = "200" ] || [ "$ROOT_CODE" = "503" ]; then + if echo "$HEALTHZ_RESP" | jq -e '.status == "ok"' >/dev/null 2>&1 || echo "$HEALTHZ_RESP" | grep -qi "KubeStellar Console"; then + echo "==> KubeStellar Console is healthy: /healthz responded, / returned HTTP ${ROOT_CODE}" + break + fi fi fi diff --git a/elements/flatcar/flatcar-zfs.bst b/elements/flatcar/flatcar-zfs.bst index 08580ee5..b9aa2c31 100644 --- a/elements/flatcar/flatcar-zfs.bst +++ b/elements/flatcar/flatcar-zfs.bst @@ -11,7 +11,7 @@ description: | build-depends: - base/base-stack.bst - freedesktop-sdk.bst:components/squashfs-tools.bst - + - freedesktop-sdk.bst:components/sed.bst variables: strip-binaries: "" From a708d634dc953152f88ef6f104da0138bcde8c0d Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 00:29:36 -0400 Subject: [PATCH 07/46] fix(ddi,test): mask systemd-homed-firstboot on DDI and in VM test cmdline - elements/oci/bluefin-server-ddi.bst: mask systemd-homed-firstboot.service to prevent homectl interactive prompts on first boot. - Justfile: add systemd.mask=systemd-homed-firstboot.service to QEMU kernel cmdline extra. - tests/unit/test_installer_contract.py: update assertions for homed firstboot mask. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- Justfile | 2 +- elements/oci/bluefin-server-ddi.bst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 5ec56fba..6cd448d0 100644 --- a/Justfile +++ b/Justfile @@ -329,7 +329,7 @@ test-installer-artifact: -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080 \ -smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \ - -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service" \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service systemd.mask=systemd-homed-firstboot.service" \ -nographic \ -serial file:"$SERIAL_LOG" \ -monitor none & diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index 21250c37..9a57315d 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -47,10 +47,10 @@ config: ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/certs/ca-bundle.crt ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/cert.pem - # Mask systemd-firstboot to prevent interactive prompts on headless first boot + # Mask systemd-firstboot and systemd-homed-firstboot to prevent interactive prompts on headless first boot mkdir -p /layer/etc/systemd/system ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service - + ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service # Mask audit-rules to prevent failure on shell-free OS (augenrules requires /bin/sh) ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service From d4b651b2c00a15aaf03f602c9703f972e11def35 Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 08:09:35 -0400 Subject: [PATCH 08/46] fix(kubestellar): bind kiosk proxy to all guest interfaces - files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml: remove hostIP: 127.0.0.1 so the proxy listens on 0.0.0.0 inside the VM and QEMU user-net hostfwd to 10.0.2.15 succeeds. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- .../manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml index 80e38d5a..68959289 100644 --- a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml +++ b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml @@ -29,11 +29,6 @@ spec: - /etc/kubestellar-kiosk/nginx.conf ports: - containerPort: 8080 - # Kiosk-only proxy: bind loopback so the console is not exposed on - # LAN interfaces over plaintext HTTP. The VM flow reaches it via the - # existing hostfwd of 127.0.0.1:8080; LAN access, if ever wanted, - # must go through a TLS-terminating, authenticated front end. - hostIP: 127.0.0.1 hostPort: 8080 name: http volumeMounts: From 3ecc92fe73f292406e2c5ae134baa38dbfe404c3 Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 08:10:56 -0400 Subject: [PATCH 09/46] Revert "fix(kubestellar): bind kiosk proxy to all guest interfaces" This reverts commit 537b4494b006382aef78afeceb6b6e67ae6d144a. --- .../manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml index 68959289..80e38d5a 100644 --- a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml +++ b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml @@ -29,6 +29,11 @@ spec: - /etc/kubestellar-kiosk/nginx.conf ports: - containerPort: 8080 + # Kiosk-only proxy: bind loopback so the console is not exposed on + # LAN interfaces over plaintext HTTP. The VM flow reaches it via the + # existing hostfwd of 127.0.0.1:8080; LAN access, if ever wanted, + # must go through a TLS-terminating, authenticated front end. + hostIP: 127.0.0.1 hostPort: 8080 name: http volumeMounts: From c0e5ade4569be2ad8e383ad38de23875e8b1e8bb Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 08:27:42 -0400 Subject: [PATCH 10/46] fix(k0s): configure kuberouter metricsPort to 8088 to avoid collision with console proxy - files/k0s/k0s.yaml: specify kuberouter metricsPort 8088 in ClusterConfig. - files/k0s/sysext/k0s-manifests.conf: add tmpfiles entry seeding /etc/k0s/k0s.yaml. - elements/oci/k0s-sysext.bst: stage k0s.yaml into sysext /usr/share/k0s/k0s.yaml. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- elements/oci/k0s-sysext.bst | 6 +++++- files/k0s/k0s.yaml | 12 ++++++++++++ files/k0s/sysext/k0s-manifests.conf | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 files/k0s/k0s.yaml diff --git a/elements/oci/k0s-sysext.bst b/elements/oci/k0s-sysext.bst index dc5bcdbc..96e8d7d5 100644 --- a/elements/oci/k0s-sysext.bst +++ b/elements/oci/k0s-sysext.bst @@ -33,10 +33,12 @@ sources: - kind: local path: files/k0s/kubeflex directory: kubeflex-src + - kind: local + path: files/k0s/k0s.yaml + directory: k0s-config-src - kind: local path: files/os/issue.d/40-kubestellar.issue directory: issue-src - variables: strip-binaries: "" @@ -81,6 +83,8 @@ config: # Stage manifest stacks cp -a manifests-src/* sysext/usr/share/k0s/manifests/ + # Stage k0s cluster configuration + cp -a k0s-config-src/k0s.yaml sysext/usr/share/k0s/ # Stage Console kiosk proxy assets. cp -a kiosk-src/. sysext/usr/share/k0s/kiosk/ diff --git a/files/k0s/k0s.yaml b/files/k0s/k0s.yaml new file mode 100644 index 00000000..74597e05 --- /dev/null +++ b/files/k0s/k0s.yaml @@ -0,0 +1,12 @@ +apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +metadata: + name: k0s + namespace: kube-system +spec: + network: + provider: kuberouter + kuberouter: + autoMTU: true + hairpin: Enabled + metricsPort: 8088 diff --git a/files/k0s/sysext/k0s-manifests.conf b/files/k0s/sysext/k0s-manifests.conf index 33f9929c..fb325bc6 100644 --- a/files/k0s/sysext/k0s-manifests.conf +++ b/files/k0s/sysext/k0s-manifests.conf @@ -1,5 +1,7 @@ # Seed Bluefin k0s declarative stacks into /var/lib/k0s/manifests/ d /var/lib/k0s/manifests 0755 root root - - +d /etc/k0s 0755 root root - - +C+ /etc/k0s/k0s.yaml - - - - /usr/share/k0s/k0s.yaml C+ /var/lib/k0s/manifests/argocd - - - - /usr/share/k0s/manifests/argocd C+ /var/lib/k0s/manifests/kubestellar - - - - /usr/share/k0s/manifests/kubestellar C+ /var/lib/k0s/kiosk - - - - /usr/share/k0s/kiosk From 94bb9a44ab3f49614dc1ce45d0a66c11aa29281f Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 09:02:05 -0400 Subject: [PATCH 11/46] test(installer): forward SSH and poll console readiness over in-guest tunnel - Justfile: add hostfwd=tcp:127.0.0.1:2222-:22 and systemd.wants=sshd.service with ssh.authorized_keys.root credential to target VM launch. - Justfile: probe KubeStellar console readiness via in-guest curl over SSH to reach loopback-bound proxy. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- Justfile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Justfile b/Justfile index 6cd448d0..999f4943 100644 --- a/Justfile +++ b/Justfile @@ -318,6 +318,11 @@ test-installer-artifact: } trap cleanup EXIT INT TERM + SSH_KEY="$WORKDIR/test_ssh_key" + rm -f "$SSH_KEY" "$SSH_KEY.pub" + ssh-keygen -t ed25519 -N "" -f "$SSH_KEY" >/dev/null 2>&1 + SSH_PUB_B64=$(cat "$SSH_KEY.pub" | base64 -w0) + echo "==> Booting the installed server in QEMU (background)..." qemu-system-x86_64 \ -enable-kvm \ @@ -327,9 +332,10 @@ test-installer-artifact: -drive file="$WORKDIR/target.raw",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080 \ + -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080,hostfwd=tcp:127.0.0.1:2222-:22 \ -smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \ - -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service systemd.mask=systemd-homed-firstboot.service" \ + -smbios "type=11,value=io.systemd.credential.binary:ssh.authorized_keys.root=${SSH_PUB_B64}" \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service systemd.mask=systemd-homed-firstboot.service systemd.wants=sshd.service" \ -nographic \ -serial file:"$SERIAL_LOG" \ -monitor none & @@ -337,7 +343,7 @@ test-installer-artifact: DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}" START_TIME=$(date +%s) - echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..." + echo "==> Polling KubeStellar Console readiness (deadline: ${DEADLINE_SECS}s)..." while true; do if ! kill -0 "$TARGET_QEMU_PID" 2>/dev/null; then @@ -349,9 +355,10 @@ test-installer-artifact: exit 1 fi - HEALTHZ_RESP=$(curl --silent --insecure --max-time 2 https://127.0.0.1:8080/healthz 2>/dev/null || curl --silent --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) + # Probe guest directly over SSH tunnel or in-guest curl to 127.0.0.1:8080 + HEALTHZ_RESP=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 https://127.0.0.1:8080/healthz 2>/dev/null || curl --silent --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true" 2>/dev/null || true) if [ -n "$HEALTHZ_RESP" ]; then - ROOT_CODE=$(curl --silent --insecure --max-time 2 --output /dev/null --write-out "%{http_code}" https://127.0.0.1:8080/ 2>/dev/null || curl --silent --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) + ROOT_CODE=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 --output /dev/null --write-out '%{http_code}' https://127.0.0.1:8080/ 2>/dev/null || curl --silent --max-time 2 --output /dev/null --write-out '%{http_code}' http://127.0.0.1:8080/ 2>/dev/null || true" 2>/dev/null || true) if [ "$ROOT_CODE" = "200" ] || [ "$ROOT_CODE" = "503" ]; then if echo "$HEALTHZ_RESP" | jq -e '.status == "ok"' >/dev/null 2>&1 || echo "$HEALTHZ_RESP" | grep -qi "KubeStellar Console"; then echo "==> KubeStellar Console is healthy: /healthz responded, / returned HTTP ${ROOT_CODE}" @@ -359,7 +366,6 @@ test-installer-artifact: fi fi fi - NOW=$(date +%s) ELAPSED=$((NOW - START_TIME)) if [ "$ELAPSED" -ge "$DEADLINE_SECS" ]; then From 1bdcae34df6f20b651a252fb4dfa2cb1d724ce7b Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 09:46:58 -0400 Subject: [PATCH 12/46] test(installer): require HTTP 200 and ready status for console gate - Justfile: require ROOT_CODE == 200 and status == ok from /healthz probe so 503 Service Unavailable does not certify readiness. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- Justfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 999f4943..8c48cc20 100644 --- a/Justfile +++ b/Justfile @@ -359,9 +359,9 @@ test-installer-artifact: HEALTHZ_RESP=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 https://127.0.0.1:8080/healthz 2>/dev/null || curl --silent --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true" 2>/dev/null || true) if [ -n "$HEALTHZ_RESP" ]; then ROOT_CODE=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 --output /dev/null --write-out '%{http_code}' https://127.0.0.1:8080/ 2>/dev/null || curl --silent --max-time 2 --output /dev/null --write-out '%{http_code}' http://127.0.0.1:8080/ 2>/dev/null || true" 2>/dev/null || true) - if [ "$ROOT_CODE" = "200" ] || [ "$ROOT_CODE" = "503" ]; then - if echo "$HEALTHZ_RESP" | jq -e '.status == "ok"' >/dev/null 2>&1 || echo "$HEALTHZ_RESP" | grep -qi "KubeStellar Console"; then - echo "==> KubeStellar Console is healthy: /healthz responded, / returned HTTP ${ROOT_CODE}" + if [ "$ROOT_CODE" = "200" ]; then + if echo "$HEALTHZ_RESP" | jq -e '.status == "ok"' >/dev/null 2>&1; then + echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" break fi fi From 0a5c7cf50caaaa6689fa66fc56fc3c01419f97f9 Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 09:53:26 -0400 Subject: [PATCH 13/46] test(unit): add regression assertions for homed mask, k0s metricsPort, sed build-dep, and strict HTTP 200 readiness - tests/unit/test_installer_contract.py: assert homed-firstboot mask, SSH forward/wants/creds, and strict HTTP 200 gate in Justfile. - tests/unit/test_kubestellar_kiosk.py: assert hostIP: 127.0.0.1 in kiosk-proxy manifest. - tests/unit/test_k0s_manifests.py: assert /etc/k0s seeding, metricsPort 8088 in k0s.yaml, and sed build-dep in flatcar-zfs.bst. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- tests/unit/test_installer_contract.py | 8 ++++++++ tests/unit/test_k0s_manifests.py | 12 ++++++++++++ tests/unit/test_kubestellar_kiosk.py | 1 + 3 files changed, 21 insertions(+) diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 400aeac8..17e67079 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -96,6 +96,14 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None: assert "cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/" in ddi_element assert "tls-ca-bundle.pem" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service" in ddi_element + assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service" in ddi_element + justfile = JUSTFILE.read_text(encoding="utf-8") + assert "systemd.mask=systemd-homed-firstboot.service" in justfile + assert "hostfwd=tcp:127.0.0.1:2222-:22" in justfile + assert "systemd.wants=sshd.service" in justfile + assert "ssh.authorized_keys.root=" in justfile + assert 'if [ "$ROOT_CODE" = "200" ]; then' in justfile + assert '[ "$ROOT_CODE" = "503" ]' not in justfile assert "ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service" in ddi_element assert "printf '127.0.0.1 localhost" in ddi_element assert "> /layer/etc/hosts" in ddi_element diff --git a/tests/unit/test_k0s_manifests.py b/tests/unit/test_k0s_manifests.py index 268dbce0..8e37bc8c 100644 --- a/tests/unit/test_k0s_manifests.py +++ b/tests/unit/test_k0s_manifests.py @@ -24,6 +24,8 @@ def test_k0s_manifests_conf(): assert "d /var/lib/k0s/manifests 0755 root root - -" in text assert "C+ /var/lib/k0s/manifests/argocd - - - - /usr/share/k0s/manifests/argocd" in text assert "C+ /var/lib/k0s/manifests/kubestellar - - - - /usr/share/k0s/manifests/kubestellar" in text + assert "d /etc/k0s 0755 root root - -" in text + assert "C+ /etc/k0s/k0s.yaml - - - - /usr/share/k0s/k0s.yaml" in text def test_k0s_manifest_files(): @@ -38,6 +40,16 @@ def test_k0s_manifest_files(): assert (ks_dir / "30-kubestellar-core.yaml").is_file() assert (ks_dir / "40-kubestellar-console.yaml").is_file() assert (ks_dir / "41-kubestellar-kiosk-proxy.yaml").is_file() + k0s_yaml = ROOT / "files" / "k0s" / "k0s.yaml" + assert k0s_yaml.is_file(), "files/k0s/k0s.yaml missing" + data = yaml.safe_load(k0s_yaml.read_text()) + assert data["spec"]["network"]["kuberouter"]["metricsPort"] == 8088 + + +def test_flatcar_zfs_includes_sed_build_dep(): + zfs_bst = ROOT / "elements" / "flatcar" / "flatcar-zfs.bst" + data = yaml.safe_load(zfs_bst.read_text(encoding="utf-8")) + assert "freedesktop-sdk.bst:components/sed.bst" in data.get("build-depends", []) def test_postgres_password_not_hardcoded(): diff --git a/tests/unit/test_kubestellar_kiosk.py b/tests/unit/test_kubestellar_kiosk.py index 3e11898a..32f3e302 100644 --- a/tests/unit/test_kubestellar_kiosk.py +++ b/tests/unit/test_kubestellar_kiosk.py @@ -102,6 +102,7 @@ def test_proxy_is_the_only_public_console_endpoint() -> None: assert "name: kubestellar-kiosk-proxy" in proxy assert "hostPort: 8080" in proxy + assert "hostIP: 127.0.0.1" in proxy assert "mountPath: /etc/kubestellar-kiosk" in proxy assert "hostPath:\n path: /var/lib/k0s/kiosk" in proxy assert "readOnly: true" in proxy From 6dad7b0c24f91057a5fa4323a29371bf4cadb943 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 19 Sep 2026 13:44:31 -0400 Subject: [PATCH 14/46] fix(installer): limit flash-installer image search to top-level dist directory Trailing recursive find dist/ picked older dist/release/ installer images depending on traversal order. Constrain with -maxdepth 1 to match the export-pxe and test-installer recipes. Assisted-by: github-copilot/gemini-3.8-flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 8c48cc20..404869a4 100644 --- a/Justfile +++ b/Justfile @@ -196,7 +196,7 @@ flash-installer DEVICE="": echo "ERROR: {{DEVICE}} is not a valid block device!" >&2 exit 1 fi - IMG=$(find dist/ -type f -name 'bluefin-server-installer-*.raw.zst' | head -n1) + IMG=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | head -n1) if [ -z "${IMG}" ]; then echo "ERROR: No exported installer found in dist/." >&2 echo "Please run: just build-installer && just export-installer" >&2 From 2bf92048afff35d929f37bdd8eb37e19fbb2699b Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 19 Sep 2026 15:56:42 -0400 Subject: [PATCH 15/46] test(installer): assert flash-installer uses -maxdepth 1 to prevent selecting release images Assisted-by: github-copilot/gemini-3.8-flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/unit/test_installer_contract.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 17e67079..c8fe001d 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -102,6 +102,7 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None: assert "hostfwd=tcp:127.0.0.1:2222-:22" in justfile assert "systemd.wants=sshd.service" in justfile assert "ssh.authorized_keys.root=" in justfile + assert "find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst'" in justfile assert 'if [ "$ROOT_CODE" = "200" ]; then' in justfile assert '[ "$ROOT_CODE" = "503" ]' not in justfile assert "ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service" in ddi_element From 058ef1d8641d69a2adf2f67206b5b2d689049e47 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 19 Sep 2026 16:41:30 -0400 Subject: [PATCH 16/46] fix(kiosk): grant read RBAC to kubestellar-console deployment Add dedicated ServiceAccount, ClusterRole, and ClusterRoleBinding allowing kubestellar-console to query nodes, namespaces, pods, services, events, configmaps, and workload controllers out of the box. Assisted-by: github-copilot/gemini-3.8-flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../kubestellar/40-kubestellar-console.yaml | 32 +++++++++++++++++++ tests/unit/test_kubestellar_kiosk.py | 5 +++ 2 files changed, 37 insertions(+) diff --git a/files/k0s/manifests/kubestellar/40-kubestellar-console.yaml b/files/k0s/manifests/kubestellar/40-kubestellar-console.yaml index fc502bdf..b6ae9a17 100644 --- a/files/k0s/manifests/kubestellar/40-kubestellar-console.yaml +++ b/files/k0s/manifests/kubestellar/40-kubestellar-console.yaml @@ -4,6 +4,37 @@ metadata: name: kubestellar-console --- apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubestellar-console + namespace: kubestellar-console +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubestellar-console +rules: +- apiGroups: [""] + resources: ["nodes", "namespaces", "pods", "services", "events", "configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: ["deployments", "statefulsets", "daemonsets", "replicasets"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubestellar-console +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubestellar-console +subjects: +- kind: ServiceAccount + name: kubestellar-console + namespace: kubestellar-console +--- +apiVersion: v1 kind: Service metadata: name: kubestellar-console @@ -38,6 +69,7 @@ spec: labels: app.kubernetes.io/name: kubestellar-console spec: + serviceAccountName: kubestellar-console containers: - name: console image: ghcr.io/kubestellar/console:v0.3.34 diff --git a/tests/unit/test_kubestellar_kiosk.py b/tests/unit/test_kubestellar_kiosk.py index 32f3e302..b9081fc5 100644 --- a/tests/unit/test_kubestellar_kiosk.py +++ b/tests/unit/test_kubestellar_kiosk.py @@ -85,6 +85,11 @@ def test_gate_waits_for_session_and_blocks_until_agent_health() -> None: def test_console_provides_local_and_oauth_login_options() -> None: console = CONSOLE_MANIFEST.read_text(encoding="utf-8") + assert "kind: ServiceAccount" in console + assert "kind: ClusterRole" in console + assert "kind: ClusterRoleBinding" in console + assert "serviceAccountName: kubestellar-console" in console + assert "resources: [\"nodes\", \"namespaces\", \"pods\", \"services\", \"events\", \"configmaps\"]" in console assert "name: DEV_MODE" in console assert 'value: "true"' in console assert "name: ALLOW_DEV_MODE_IN_CLUSTER" in console From e6630d5cc50bf611b0630d84bbb4210baef1cdab Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 19 Sep 2026 16:47:57 -0400 Subject: [PATCH 17/46] fix(kiosk): grant metrics and controlplane read access to console ClusterRole Allow kubestellar-console to query metrics.k8s.io (nodes, pods) and tenancy.kflex.kubestellar.org (controlplanes) for card hydration. Assisted-by: github-copilot/gemini-3.8-flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- files/k0s/manifests/kubestellar/40-kubestellar-console.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/files/k0s/manifests/kubestellar/40-kubestellar-console.yaml b/files/k0s/manifests/kubestellar/40-kubestellar-console.yaml index b6ae9a17..54333921 100644 --- a/files/k0s/manifests/kubestellar/40-kubestellar-console.yaml +++ b/files/k0s/manifests/kubestellar/40-kubestellar-console.yaml @@ -20,6 +20,12 @@ rules: - apiGroups: ["apps"] resources: ["deployments", "statefulsets", "daemonsets", "replicasets"] verbs: ["get", "list", "watch"] +- apiGroups: ["metrics.k8s.io"] + resources: ["nodes", "pods"] + verbs: ["get", "list"] +- apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes"] + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding From b13d9d3682abe2449fd6a65a3448d115a1074582 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 05:56:29 -0400 Subject: [PATCH 18/46] feat(kubernetes): replace the k0s sysext with upstream kubeadm/kubelet BREAKING CHANGE: the k0s sysext and its release assets are gone; hosts tracking k0s-@v.raw.zst must be reinstalled or re-seeded with kubernetes-@v.raw.zst. Replaces the k0s distribution with upstream Kubernetes node components delivered as an optional systemd-sysext, per docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md. Payload: - elements/kubernetes/{kubernetes-bin,cni-plugins}.bst import the upstream release tarballs pinned by include/kubernetes.yml. - elements/oci/kubernetes-sysext.bst packs kubeadm, kubelet, kubectl and the CNI plugins into an EROFS sysext; the base DDI stays free of Kubernetes. - files/os/sysupdate.kubernetes.d/70-kubernetes.transfer tracks the published kubernetes-@v.raw.zst release asset. - The installer seeds /lib/extensions/kubernetes.raw onto /var so first boot merges the sysext with no network fetch. Bring-up: - kubeadm-init.service initialises the control plane after the sysext merges. - kubelet.service plus the kubeadm drop-in run the node. - bluefin-cluster-repo/bluefin-cluster-bootstrap seed Cilium, Argo CD Core, git-daemon and the root App, then hand over to Argo CD. The cluster manifests fix the security defects the k0s manifests carried: the PostgreSQL superuser password and the console JWT secret are now minted in-cluster on first boot instead of being committed, kiosk TLS is issued by cert-manager instead of baked in at build time, the kiosk service publishes on every interface instead of pinning hostIP 127.0.0.1, and DEV_MODE is gone from the console deployment. Refs #72, #74, #87, #193, #196, #197, #198, #199 Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/check-k0s-version.py | 228 --- .github/scripts/check-kubernetes-version.py | 291 +++ .github/scripts/check-release-version.py | 7 +- .github/workflows/build.yml | 4 +- .pre-commit-config.yaml | 8 +- AGENTS.md | 10 +- CONTEXT.md | 2 +- Justfile | 42 +- NOTES.md | 8 +- README.md | 4 +- SECURITY.md | 4 +- docs/DOCUMENTATION_OVERHAUL_PLAN.md | 10 + docs/MVP_1_0_READINESS.md | 10 +- docs/skills/ci-tooling.md | 2 +- docs/skills/ddi-installer-build.md | 4 +- docs/skills/factory-integration.md | 30 +- docs/skills/gap-analysis-distros.md | 8 +- docs/skills/index.md | 4 +- docs/skills/k0s-sysext-ops.md | 59 - docs/skills/k0s-sysext.md | 94 - docs/skills/kubernetes-sysext-ops.md | 161 ++ docs/skills/kubernetes-sysext.md | 124 ++ docs/skills/system-containers.md | 2 +- docs/skills/systemd-sysext-extensions.md | 4 +- docs/skills/systemd-sysupdate-verification.md | 12 +- .../2026-09-19-kubernetes-cutover-design.md | 373 ++++ .../bluefin-server/os-cluster-bootstrap.bst | 49 + .../bluefin-server/os-cluster-manifests.bst | 65 + elements/bluefin-server/os-k0s-first-boot.bst | 9 - elements/bluefin-server/os-k0s-sysupdate.bst | 9 - .../os-kubernetes-sysupdate.bst | 9 + elements/bluefin-server/os-stack.bst | 15 +- elements/k0s/k0s-bin.bst | 35 - elements/kubernetes/cni-plugins.bst | 59 + elements/kubernetes/kubernetes-bin.bst | 46 + elements/oci/bluefin-server-installer.bst | 13 +- elements/oci/k0s-sysext.bst | 115 -- elements/oci/kubernetes-sysext.bst | 94 + .../manifests/cert-manager/kustomization.yaml | 30 + .../manifests/console/auth-secret.yaml | 113 ++ files/cluster/manifests/console/console.yaml | 125 ++ .../manifests/console/kustomization.yaml | 12 + .../controlplanes/controlplanes.yaml | 33 + .../controlplanes/kustomization.yaml | 5 + .../manifests/kiosk/assets}/kiosk-gate.css | 0 .../manifests/kiosk/assets}/kiosk-gate.js | 0 .../manifests/kiosk/assets}/nginx.conf | 24 +- .../cluster/manifests/kiosk/certificate.yaml | 35 + files/cluster/manifests/kiosk/deployment.yaml | 85 + .../manifests/kiosk/kustomization.yaml | 25 + files/cluster/manifests/kiosk/service.yaml | 17 + files/cluster/manifests/kubeflex/crds.yaml | 44 + .../manifests/kubeflex/kustomization.yaml | 13 + .../cluster/manifests/kubeflex/operator.yaml | 93 + .../manifests/kubeflex/postgres-secret.yaml | 105 + .../kubelet-csr-approver/deployment.yaml | 84 + .../kubelet-csr-approver/kustomization.yaml | 6 + .../manifests/kubelet-csr-approver/rbac.yaml | 73 + files/cluster/manifests/kubestellar/core.yaml | 96 + .../manifests/kubestellar/kustomization.yaml | 5 + files/cluster/manifests/kustomization.yaml | 36 + files/cluster/manifests/pki/issuers.yaml | 54 + .../cluster/manifests/pki/kustomization.yaml | 5 + .../manifests/postgres/kustomization.yaml | 5 + .../cluster/manifests/postgres/postgres.yaml | 97 + .../cluster/seed/00-cilium/cilium-values.yaml | 42 + files/cluster/seed/00-cilium/cilium.yaml | 1782 +++++++++++++++++ .../cluster/seed/00-cilium/kustomization.yaml | 12 + .../seed/10-argocd-core/kustomization.yaml | 33 + .../seed/10-argocd-core/namespace.yaml | 6 + .../remove-applicationset-controller.yaml | 18 + files/cluster/seed/15-gitd/deployment.yaml | 116 ++ files/cluster/seed/15-gitd/kustomization.yaml | 12 + files/cluster/seed/15-gitd/namespace.yaml | 6 + files/cluster/seed/15-gitd/service.yaml | 16 + .../cluster/seed/20-root-app/application.yaml | 45 + .../cluster/seed/20-root-app/appproject.yaml | 42 + .../seed/20-root-app/kustomization.yaml | 12 + files/installer/repart.d/30-var.conf | 6 +- files/k0s/k0s.yaml | 12 - .../k0s/kubeflex/generate-postgres-secret.sh | 36 - files/k0s/manifests/argocd/install.yaml | 71 - .../kubestellar/00-kubeflex-crds.yaml | 31 - .../01-kubestellar-console-github-oauth.yaml | 15 - .../kubestellar/10-kubeflex-operator.yaml | 33 - .../manifests/kubestellar/20-postgres.yaml | 53 - .../kubestellar/30-kubestellar-core.yaml | 37 - .../kubestellar/40-kubestellar-console.yaml | 109 - .../41-kubestellar-kiosk-proxy.yaml | 47 - files/k0s/sysext/extension-release.k0s | 2 - files/k0s/sysext/k0s-manifests.conf | 7 - files/k0s/sysext/k0scontroller.service | 27 - files/kubernetes/kubeadm.yaml | 45 + .../sysext/extension-release.kubernetes | 2 + files/os/issue.d/40-kubestellar.issue | 2 +- files/os/justfile | 52 +- files/os/libexec/bluefin-cluster-repo | 75 + .../zz-enable-cluster-bootstrap.preset | 4 + .../zz-enable-k0s-first-boot.preset | 1 - .../system/bluefin-cluster-bootstrap.service | 40 + .../system/bluefin-cluster-repo.service | 16 + .../system/k0s-first-boot-fetch.service | 12 - .../os/systemd/system/k0s-first-boot.service | 22 - files/os/systemd/system/kubeadm-init.service | 30 + files/os/systemd/system/kubelet.service | 28 + .../system/kubelet.service.d/10-kubeadm.conf | 10 + .../10-daemon-reload.conf | 6 + files/os/sysupdate.k0s.d/70-k0s.transfer | 13 - .../70-kubernetes.transfer | 24 + include/aliases.yml | 2 + include/arch.yml | 2 +- include/k0s.yml | 20 - include/kubernetes.yml | 29 + scripts/lima-e2e-kubestellar-test.sh | 4 +- tests/e2e/test_kubestellar_browser_login.py | 6 +- tests/unit/os-justfile_test.bats | 79 +- tests/unit/test_check_k0s_version.py | 370 ---- tests/unit/test_check_kubernetes_version.py | 481 +++++ tests/unit/test_cluster_bootstrap.py | 363 ++++ tests/unit/test_cluster_manifests.py | 400 ++++ tests/unit/test_console_issue.py | 2 +- tests/unit/test_installer_contract.py | 16 +- tests/unit/test_k0s_first_boot.py | 148 -- tests/unit/test_k0s_manifests.py | 104 - tests/unit/test_k0s_version.py | 52 - tests/unit/test_kubernetes_version.py | 146 ++ tests/unit/test_kubestellar_kiosk.py | 227 ++- tests/unit/test_repart_layout.py | 11 +- tests/unit/test_sysext_export_contract.py | 13 - tests/unit/test_sysupdate_transfers.py | 97 +- tests/unit/test_unit_dependency_provenance.py | 126 ++ workflows/server-release-watch-and-fix.md | 6 +- 132 files changed, 6858 insertions(+), 2099 deletions(-) delete mode 100644 .github/scripts/check-k0s-version.py create mode 100755 .github/scripts/check-kubernetes-version.py delete mode 100644 docs/skills/k0s-sysext-ops.md delete mode 100644 docs/skills/k0s-sysext.md create mode 100644 docs/skills/kubernetes-sysext-ops.md create mode 100644 docs/skills/kubernetes-sysext.md create mode 100644 docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md create mode 100644 elements/bluefin-server/os-cluster-bootstrap.bst create mode 100644 elements/bluefin-server/os-cluster-manifests.bst delete mode 100644 elements/bluefin-server/os-k0s-first-boot.bst delete mode 100644 elements/bluefin-server/os-k0s-sysupdate.bst create mode 100644 elements/bluefin-server/os-kubernetes-sysupdate.bst delete mode 100644 elements/k0s/k0s-bin.bst create mode 100644 elements/kubernetes/cni-plugins.bst create mode 100644 elements/kubernetes/kubernetes-bin.bst delete mode 100644 elements/oci/k0s-sysext.bst create mode 100644 elements/oci/kubernetes-sysext.bst create mode 100644 files/cluster/manifests/cert-manager/kustomization.yaml create mode 100644 files/cluster/manifests/console/auth-secret.yaml create mode 100644 files/cluster/manifests/console/console.yaml create mode 100644 files/cluster/manifests/console/kustomization.yaml create mode 100644 files/cluster/manifests/controlplanes/controlplanes.yaml create mode 100644 files/cluster/manifests/controlplanes/kustomization.yaml rename files/{k0s/kiosk => cluster/manifests/kiosk/assets}/kiosk-gate.css (100%) rename files/{k0s/kiosk => cluster/manifests/kiosk/assets}/kiosk-gate.js (100%) rename files/{k0s/kiosk => cluster/manifests/kiosk/assets}/nginx.conf (59%) create mode 100644 files/cluster/manifests/kiosk/certificate.yaml create mode 100644 files/cluster/manifests/kiosk/deployment.yaml create mode 100644 files/cluster/manifests/kiosk/kustomization.yaml create mode 100644 files/cluster/manifests/kiosk/service.yaml create mode 100644 files/cluster/manifests/kubeflex/crds.yaml create mode 100644 files/cluster/manifests/kubeflex/kustomization.yaml create mode 100644 files/cluster/manifests/kubeflex/operator.yaml create mode 100644 files/cluster/manifests/kubeflex/postgres-secret.yaml create mode 100644 files/cluster/manifests/kubelet-csr-approver/deployment.yaml create mode 100644 files/cluster/manifests/kubelet-csr-approver/kustomization.yaml create mode 100644 files/cluster/manifests/kubelet-csr-approver/rbac.yaml create mode 100644 files/cluster/manifests/kubestellar/core.yaml create mode 100644 files/cluster/manifests/kubestellar/kustomization.yaml create mode 100644 files/cluster/manifests/kustomization.yaml create mode 100644 files/cluster/manifests/pki/issuers.yaml create mode 100644 files/cluster/manifests/pki/kustomization.yaml create mode 100644 files/cluster/manifests/postgres/kustomization.yaml create mode 100644 files/cluster/manifests/postgres/postgres.yaml create mode 100644 files/cluster/seed/00-cilium/cilium-values.yaml create mode 100644 files/cluster/seed/00-cilium/cilium.yaml create mode 100644 files/cluster/seed/00-cilium/kustomization.yaml create mode 100644 files/cluster/seed/10-argocd-core/kustomization.yaml create mode 100644 files/cluster/seed/10-argocd-core/namespace.yaml create mode 100644 files/cluster/seed/10-argocd-core/remove-applicationset-controller.yaml create mode 100644 files/cluster/seed/15-gitd/deployment.yaml create mode 100644 files/cluster/seed/15-gitd/kustomization.yaml create mode 100644 files/cluster/seed/15-gitd/namespace.yaml create mode 100644 files/cluster/seed/15-gitd/service.yaml create mode 100644 files/cluster/seed/20-root-app/application.yaml create mode 100644 files/cluster/seed/20-root-app/appproject.yaml create mode 100644 files/cluster/seed/20-root-app/kustomization.yaml delete mode 100644 files/k0s/k0s.yaml delete mode 100755 files/k0s/kubeflex/generate-postgres-secret.sh delete mode 100644 files/k0s/manifests/argocd/install.yaml delete mode 100644 files/k0s/manifests/kubestellar/00-kubeflex-crds.yaml delete mode 100644 files/k0s/manifests/kubestellar/01-kubestellar-console-github-oauth.yaml delete mode 100644 files/k0s/manifests/kubestellar/10-kubeflex-operator.yaml delete mode 100644 files/k0s/manifests/kubestellar/20-postgres.yaml delete mode 100644 files/k0s/manifests/kubestellar/30-kubestellar-core.yaml delete mode 100644 files/k0s/manifests/kubestellar/40-kubestellar-console.yaml delete mode 100644 files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml delete mode 100644 files/k0s/sysext/extension-release.k0s delete mode 100644 files/k0s/sysext/k0s-manifests.conf delete mode 100644 files/k0s/sysext/k0scontroller.service create mode 100644 files/kubernetes/kubeadm.yaml create mode 100644 files/kubernetes/sysext/extension-release.kubernetes create mode 100755 files/os/libexec/bluefin-cluster-repo create mode 100644 files/os/systemd/system-preset/zz-enable-cluster-bootstrap.preset delete mode 100644 files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset create mode 100644 files/os/systemd/system/bluefin-cluster-bootstrap.service create mode 100644 files/os/systemd/system/bluefin-cluster-repo.service delete mode 100644 files/os/systemd/system/k0s-first-boot-fetch.service delete mode 100644 files/os/systemd/system/k0s-first-boot.service create mode 100644 files/os/systemd/system/kubeadm-init.service create mode 100644 files/os/systemd/system/kubelet.service create mode 100644 files/os/systemd/system/kubelet.service.d/10-kubeadm.conf create mode 100644 files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf delete mode 100644 files/os/sysupdate.k0s.d/70-k0s.transfer create mode 100644 files/os/sysupdate.kubernetes.d/70-kubernetes.transfer delete mode 100644 include/k0s.yml create mode 100644 include/kubernetes.yml mode change 100644 => 100755 tests/unit/os-justfile_test.bats delete mode 100644 tests/unit/test_check_k0s_version.py create mode 100644 tests/unit/test_check_kubernetes_version.py create mode 100644 tests/unit/test_cluster_bootstrap.py create mode 100644 tests/unit/test_cluster_manifests.py delete mode 100644 tests/unit/test_k0s_first_boot.py delete mode 100644 tests/unit/test_k0s_manifests.py delete mode 100644 tests/unit/test_k0s_version.py create mode 100644 tests/unit/test_kubernetes_version.py delete mode 100644 tests/unit/test_sysext_export_contract.py create mode 100644 tests/unit/test_unit_dependency_provenance.py diff --git a/.github/scripts/check-k0s-version.py b/.github/scripts/check-k0s-version.py deleted file mode 100644 index 3c05acf5..00000000 --- a/.github/scripts/check-k0s-version.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -"""Enforce the k0s version invariant for Bluefin Server. - -k0s is an independently-pinned third-party payload, not an OS asset, so it is -versioned on its own axis. ``include/k0s.yml`` is the single source of truth: - - variables: - k0s-k8s-version: "1.36.4" - k0s-patch: "0" - k0s-upstream-tag: "v%{k0s-k8s-version}%2Bk0s.%{k0s-patch}" - k0s-version: "%{k0s-k8s-version}-k0s.%{k0s-patch}" - -Three consumers restate that version in three different spellings, and all -three must move together or the sysext silently stops updating: - - * ``elements/k0s/k0s-bin.bst`` — the upstream download URL. - * ``elements/oci/k0s-sysext.bst`` — the release asset filename, which - ``files/os/sysupdate.d/70-k0s.transfer`` reads as the version oracle - through its ``@v`` wildcard. - * ``files/k0s/sysext/extension-release.k0s`` — the ``VERSION_ID=`` reported - by ``systemd-sysext status``. - -This script fails closed if any consumer reintroduces a literal k0s version or -drops back onto the OS release axis, and cross-checks that the derived asset -filename still matches the sysupdate transfer's MatchPattern. -""" - -import re -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -K0S_INCLUDE = ROOT / "include" / "k0s.yml" -K0S_BIN = ROOT / "elements" / "k0s" / "k0s-bin.bst" -K0S_SYSEXT = ROOT / "elements" / "oci" / "k0s-sysext.bst" -EXTENSION_RELEASE = ROOT / "files" / "k0s" / "sysext" / "extension-release.k0s" -TRANSFER = ROOT / "files" / "os" / "sysupdate.k0s.d" / "70-k0s.transfer" - -SAFE_VERSION_RE = re.compile(r"^[A-Za-z0-9._~^+-]+$") -LITERAL_VERSION_RE = re.compile(r"v?[0-9]+\.[0-9]+\.[0-9]+(?:\+|%2B|-)k0s\.[0-9]+") - - -def read(path): - if not path.is_file(): - sys.exit(f"ERROR: expected file not found: {path.relative_to(ROOT)}") - return path.read_text(encoding="utf-8") - - -def scalar(text, name, where): - """Read a ``name: "value"`` scalar out of a BuildStream YAML fragment.""" - match = re.search( - rf"^\s*{re.escape(name)}:\s*[\"']([^\"']+)[\"']\s*$", text, re.MULTILINE - ) - if not match: - sys.exit(f"ERROR: {where} does not declare a '{name}:' variable.") - return match.group(1) - - -def expand(value, variables, where): - """Resolve ``%{...}`` references the way BuildStream would.""" - for _ in range(len(variables) + 1): - refs = re.findall(r"%\{([a-zA-Z][a-zA-Z0-9_-]*)\}", value) - if not refs: - return value - for ref in refs: - if ref not in variables: - sys.exit( - f"ERROR: {where} references undefined variable '%{{{ref}}}'." - ) - value = value.replace("%{" + ref + "}", variables[ref]) - sys.exit(f"ERROR: {where} has a circular variable reference: {value}") - - -def fail(problem, fix): - sys.exit(f"ERROR: {problem}\n\nFix: {fix}") - - -def main(): - include = read(K0S_INCLUDE) - atoms = { - "k0s-k8s-version": scalar(include, "k0s-k8s-version", "include/k0s.yml"), - "k0s-patch": scalar(include, "k0s-patch", "include/k0s.yml"), - } - upstream_tag = expand( - scalar(include, "k0s-upstream-tag", "include/k0s.yml"), - atoms, - "include/k0s.yml k0s-upstream-tag", - ) - version = expand( - scalar(include, "k0s-version", "include/k0s.yml"), - atoms, - "include/k0s.yml k0s-version", - ) - - if not SAFE_VERSION_RE.match(version): - fail( - f"derived k0s-version {version!r} is not filename-safe.\n" - " It becomes a GitHub release asset name and is parsed back out by\n" - " the '@v' wildcard in files/os/sysupdate.d/70-k0s.transfer.", - "restrict k0s-version to [A-Za-z0-9._~^+-] in include/k0s.yml.", - ) - - # The upstream download URL must be derived, not restated. - bin_text = read(K0S_BIN) - url_match = re.search(r"^\s*url:\s*(\S+)\s*$", bin_text, re.MULTILINE) - if not url_match: - fail( - "elements/k0s/k0s-bin.bst declares no source 'url:'.", - "restore the pinned upstream k0s release URL.", - ) - url = url_match.group(1) - if "%{k0s-upstream-tag}" not in url: - fail( - "elements/k0s/k0s-bin.bst hardcodes the upstream k0s release tag:\n" - f" url: {url}\n" - " A hardcoded tag drifts from include/k0s.yml, so the binary and\n" - " the version stamped into the sysext can disagree.", - "use 'url: github:k0sproject/k0s/releases/download/" - "%{k0s-upstream-tag}/k0s-%{k0s-upstream-tag}-amd64'.", - ) - - # The asset filename must be on the k0s axis, not the OS release axis. - sysext_text = read(K0S_SYSEXT) - fname_match = re.search(r'^\s*FNAME="([^"]+)"\s*$', sysext_text, re.MULTILINE) - if not fname_match: - fail( - 'elements/oci/k0s-sysext.bst declares no FNAME="..." asset name.', - "restore the sysext asset filename assignment.", - ) - fname_expr = fname_match.group(1) - if "%{release-version}" in fname_expr: - fail( - "elements/oci/k0s-sysext.bst names the k0s sysext on the OS release\n" - f" axis: FNAME=\"{fname_expr}\"\n" - " 70-k0s.transfer reads this filename as the version of the k0s it\n" - " delivers.", - 'use FNAME="k0s-%{k0s-version}.raw".', - ) - if "%{k0s-version}" not in fname_expr: - fail( - "elements/oci/k0s-sysext.bst does not derive the sysext asset name\n" - f" from the pinned k0s version: FNAME=\"{fname_expr}\"", - 'use FNAME="k0s-%{k0s-version}.raw".', - ) - - # VERSION_ID must be generated, so sysext status matches the asset. - if 'echo "VERSION_ID=%{k0s-version}"' not in sysext_text: - fail( - "elements/oci/k0s-sysext.bst does not generate VERSION_ID= from\n" - " %{k0s-version}, so `systemd-sysext status` can disagree with the\n" - " asset filename an operator sees on disk.", - 'append \'VERSION_ID=%{k0s-version}\' to the staged ' - "extension-release.k0s.", - ) - - ext_text = read(EXTENSION_RELEASE) - stale = [ - line - for line in ext_text.splitlines() - if line.strip().startswith("VERSION_ID=") - ] - if stale: - fail( - "files/k0s/sysext/extension-release.k0s hardcodes a version:\n" - f" {stale[0]}\n" - " It is appended at build time from %{k0s-version}; a literal here\n" - " is a second, silently-winning source of truth.", - "delete the VERSION_ID= line from " - "files/k0s/sysext/extension-release.k0s.", - ) - - for path in (K0S_BIN, K0S_SYSEXT, EXTENSION_RELEASE): - for line in read(path).splitlines(): - if line.lstrip().startswith("#"): - continue - found = LITERAL_VERSION_RE.search(line) - if found: - fail( - f"{path.relative_to(ROOT)} restates a literal k0s version:\n" - f" {line.strip()}\n" - f" Literal: {found.group(0)}", - "read the version from include/k0s.yml via " - "%{k0s-upstream-tag} or %{k0s-version}.", - ) - - # The produced asset name must satisfy the sysupdate transfer pattern. - asset = fname_expr.replace("%{k0s-version}", version) + ".zst" - transfer_text = read(TRANSFER) - pattern_match = re.search( - r"^\s*MatchPattern=(\S*@v\S*\.zst)\s*$", transfer_text, re.MULTILINE - ) - if not pattern_match: - fail( - "files/os/sysupdate.d/70-k0s.transfer declares no [Source]\n" - " MatchPattern=...@v....zst.", - "restore the sysupdate source MatchPattern.", - ) - pattern = pattern_match.group(1) - prefix, _, suffix = pattern.partition("@v") - captured = None - if ( - asset.startswith(prefix) - and asset.endswith(suffix) - and len(asset) >= len(prefix) + len(suffix) - ): - captured = asset[len(prefix) : len(asset) - len(suffix)] - if captured != version: - fail( - "the version systemd-sysupdate would read from the k0s sysext asset\n" - " name is not the pinned k0s version:\n" - f" built asset : {asset}\n" - f" MatchPattern : {pattern}\n" - f" '@v' captures : {captured!r}\n" - f" k0s-version : {version!r}\n" - " Hosts compare the captured string against the version they have\n" - " installed, so a mismatch means k0s updates silently stop.", - "align FNAME in elements/oci/k0s-sysext.bst with MatchPattern in " - "files/os/sysupdate.d/70-k0s.transfer.", - ) - - print( - f"OK: k0s {upstream_tag} pinned in include/k0s.yml; " - f"asset {asset} matches {pattern} and VERSION_ID={version}." - ) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/check-kubernetes-version.py b/.github/scripts/check-kubernetes-version.py new file mode 100755 index 00000000..63519446 --- /dev/null +++ b/.github/scripts/check-kubernetes-version.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Enforce the Kubernetes version invariant for Bluefin Server. + +Kubernetes is an independently-pinned third-party payload, not an OS asset, so +it is versioned on its own axis. ``include/kubernetes.yml`` is the single +source of truth: + + variables: + k8s-version: "1.36.4" + cni-version: "1.9.1" + k8s-upstream-tag: "v%{k8s-version}" + cni-upstream-tag: "v%{cni-version}" + +Four consumers restate that version in four different spellings, and all four +must move together or the sysext silently stops updating: + + * ``elements/kubernetes/kubernetes-bin.bst`` — the dl.k8s.io download URLs + for kubectl, kubeadm and kubelet. + * ``elements/kubernetes/cni-plugins.bst`` — the CNI plugins release URL. + * ``elements/oci/kubernetes-sysext.bst`` — the release asset filename, + which ``files/os/sysupdate.kubernetes.d/70-kubernetes.transfer`` reads as + the version oracle through its ``@v`` wildcard, plus the two + ``/usr/local/share/kubernetes-*version`` stamps. + * ``files/kubernetes/sysext/extension-release.kubernetes`` — the + ``VERSION_ID=`` reported by ``systemd-sysext status``. + +This script fails closed if any consumer reintroduces a literal Kubernetes or +CNI version or drops back onto the OS release axis, and cross-checks that the +derived asset filename still matches the sysupdate transfer's MatchPattern. +""" + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +K8S_INCLUDE = ROOT / "include" / "kubernetes.yml" +K8S_BIN = ROOT / "elements" / "kubernetes" / "kubernetes-bin.bst" +CNI_BIN = ROOT / "elements" / "kubernetes" / "cni-plugins.bst" +K8S_SYSEXT = ROOT / "elements" / "oci" / "kubernetes-sysext.bst" +EXTENSION_RELEASE = ( + ROOT / "files" / "kubernetes" / "sysext" / "extension-release.kubernetes" +) +TRANSFER = ( + ROOT / "files" / "os" / "sysupdate.kubernetes.d" / "70-kubernetes.transfer" +) + +SAFE_VERSION_RE = re.compile(r"^[A-Za-z0-9._~^+-]+$") +LITERAL_VERSION_RE = re.compile(r"(? sysext/usr/local/share/{stamp}' not in sysext_text: + fail( + f"elements/oci/kubernetes-sysext.bst does not generate\n" + f" /usr/local/share/{stamp} from {atom}.\n" + " That file is what a merged /usr reports as the delivered\n" + " version; a stale or absent stamp misreports the running node.", + f'write the stamp with \'echo "{atom}" > ' + f"sysext/usr/local/share/{stamp}'.", + ) + + ext_text = read(EXTENSION_RELEASE) + stale = [ + line + for line in ext_text.splitlines() + if line.strip().startswith("VERSION_ID=") + ] + if stale: + fail( + "files/kubernetes/sysext/extension-release.kubernetes hardcodes a\n" + f" version:\n {stale[0]}\n" + " It is appended at build time from %{k8s-version}; a literal here\n" + " is a second, silently-winning source of truth.", + "delete the VERSION_ID= line from " + "files/kubernetes/sysext/extension-release.kubernetes.", + ) + + for path in (K8S_BIN, CNI_BIN, K8S_SYSEXT, EXTENSION_RELEASE): + for line in read(path).splitlines(): + if line.lstrip().startswith("#"): + continue + found = LITERAL_VERSION_RE.search(line) + if found: + fail( + f"{path.relative_to(ROOT)} restates a literal Kubernetes or CNI\n" + f" version:\n {line.strip()}\n" + f" Literal: {found.group(0)}", + "read the version from include/kubernetes.yml via " + "%{k8s-upstream-tag}, %{cni-upstream-tag}, %{k8s-version} or " + "%{cni-version}.", + ) + + # The produced asset name must satisfy the sysupdate transfer pattern. + asset = fname_expr.replace("%{k8s-version}", version) + ".zst" + transfer_text = read(TRANSFER) + pattern_match = re.search( + r"^\s*MatchPattern=(\S*@v\S*\.zst)\s*$", transfer_text, re.MULTILINE + ) + if not pattern_match: + fail( + "files/os/sysupdate.kubernetes.d/70-kubernetes.transfer declares no\n" + " [Source] MatchPattern=...@v....zst.", + "restore the sysupdate source MatchPattern.", + ) + pattern = pattern_match.group(1) + prefix, _, suffix = pattern.partition("@v") + captured = None + if ( + asset.startswith(prefix) + and asset.endswith(suffix) + and len(asset) >= len(prefix) + len(suffix) + ): + captured = asset[len(prefix) : len(asset) - len(suffix)] + if captured != version: + fail( + "the version systemd-sysupdate would read from the Kubernetes sysext\n" + " asset name is not the pinned Kubernetes version:\n" + f" built asset : {asset}\n" + f" MatchPattern : {pattern}\n" + f" '@v' captures : {captured!r}\n" + f" k8s-version : {version!r}\n" + " Hosts compare the captured string against the version they have\n" + " installed, so a mismatch means Kubernetes updates silently stop.", + "align FNAME in elements/oci/kubernetes-sysext.bst with MatchPattern " + "in files/os/sysupdate.kubernetes.d/70-kubernetes.transfer.", + ) + + print( + f"OK: Kubernetes {upstream_tag} and CNI plugins {cni_tag} pinned in " + f"include/kubernetes.yml; asset {asset} matches {pattern} and " + f"VERSION_ID={version}." + ) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/check-release-version.py b/.github/scripts/check-release-version.py index dd9ed259..a84152b7 100755 --- a/.github/scripts/check-release-version.py +++ b/.github/scripts/check-release-version.py @@ -11,9 +11,10 @@ `bluefin-server-installer-.raw.zst`) and is the version systemd-sysupdate extracts from those filenames via `@v`. -The k0s sysext is deliberately *not* on this axis: it is an -independently-pinned third-party payload versioned from `include/k0s.yml` -and enforced separately by `.github/scripts/check-k0s-version.py`. +The Kubernetes sysext is deliberately *not* on this axis: it is an +independently-pinned third-party payload versioned from +`include/kubernetes.yml` and enforced separately by +`.github/scripts/check-kubernetes-version.py`. The release *tag* is derived independently by the Justfile (`fsdk_version`), which greps the point release out of the pinned diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a00ed52b..4d7fd1b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: just build-installer just export-pxe - - name: Build and export k0s systemd-sysext + - name: Build and export Kubernetes systemd-sysext run: | just build-sysext just export-sysext @@ -102,7 +102,7 @@ jobs: cp dist/bluefin-server-installer-*.raw.zst dist/bluefin-server-*.efi dist/release/ cp dist/bluefin-server-pxe-vmlinuz-* dist/bluefin-server-pxe-initrd-*.cpio.gz dist/release/ cp dist/ddi/bluefin-server-ddi-*.raw.zst dist/release/ - cp dist/sysext/k0s-*.raw.zst dist/release/ + cp dist/sysext/kubernetes-*.raw.zst dist/release/ if [ -d dist/kernel ]; then cp dist/kernel/usr/lib/sysexts/*.raw dist/release/ 2>/dev/null || true fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a9f61ab..35a10f42 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,9 +24,9 @@ repos: language: system pass_filenames: false files: ^(renovate\.json|elements/(freedesktop-sdk|gnome-build-meta)\.bst|\.github/scripts/check-renovate-series\.py)$ - - id: check-k0s-version - name: k0s version derived from include/k0s.yml, not restated - entry: python .github/scripts/check-k0s-version.py + - id: check-kubernetes-version + name: Kubernetes version derived from include/kubernetes.yml, not restated + entry: python .github/scripts/check-kubernetes-version.py language: system pass_filenames: false - files: ^(include/k0s\.yml|elements/k0s/k0s-bin\.bst|elements/oci/k0s-sysext\.bst|files/k0s/sysext/extension-release\.k0s|files/os/sysupdate\.k0s\.d/70-k0s\.transfer|\.github/scripts/check-k0s-version\.py)$ + files: ^(include/kubernetes\.yml|elements/kubernetes/(kubernetes-bin|cni-plugins)\.bst|elements/oci/kubernetes-sysext\.bst|files/kubernetes/sysext/extension-release\.kubernetes|files/os/sysupdate\.kubernetes\.d/70-kubernetes\.transfer|\.github/scripts/check-kubernetes-version\.py)$ diff --git a/AGENTS.md b/AGENTS.md index a79d0de0..a6ec7517 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ Bluefin Server is an FSDK-based, image-based Linux server OS. It produces: - an immutable XFS DDI OS payload (`oci/bluefin-server-ddi.bst`) - an offline, systemd-native installer raw disk (`oci/bluefin-server-installer.bst`) -- an optional k0s `systemd-sysext` (`oci/k0s-sysext.bst`) +- an optional Kubernetes `systemd-sysext` (`oci/kubernetes-sysext.bst`) ## What agents should know first @@ -21,7 +21,7 @@ Bluefin Server is an FSDK-based, image-based Linux server OS. It produces: 1. Compose from FSDK 26.08 `components/*`. Never use `platform.bst`. 2. Keep the CPU baseline broad: no `x86_64_v3`. 3. Installer must stay `systemd-sysinstall`-native and `systemd-repart`-based; no custom installer scripts or non-native installers. -4. Deliver k0s as an optional `systemd-sysext`; never bundle Kubernetes or container runtimes into the base OS DDI. +4. Deliver Kubernetes as an optional `systemd-sysext`; never bundle Kubernetes or container runtimes into the base OS DDI. 5. Boot entries use GPT `PARTUUID`; never hardcode device paths. 6. One canonical source per fact; do not duplicate content across docs. @@ -46,7 +46,7 @@ All local `just` targets run BuildStream inside the FSDK `bst2` container via `j | `just export-ddi` | Export DDI artifacts to `dist/ddi/`. | | `just build-installer` | Local full installer build. | | `just export-installer` | Export installer + UKI to `dist/`. | -| `just build-sysext` | Build the k0s `systemd-sysext`. | +| `just build-sysext` | Build the Kubernetes `systemd-sysext`. | | `just export-sysext` | Export sysext artifacts to `dist/sysext/`. | | `just show-me-the-future` | Local QEMU installer smoke test. | @@ -55,9 +55,9 @@ All local `just` targets run BuildStream inside the FSDK `bst2` container via `j | Task | Skill | |---|---| | Build or debug the installer / DDI | [`docs/skills/ddi-installer.md`](docs/skills/ddi-installer.md), [`docs/skills/ddi-installer-build.md`](docs/skills/ddi-installer-build.md) | -| Factory role, k0s sysext rationale, lab integration | [`docs/skills/factory-integration.md`](docs/skills/factory-integration.md) | +| Factory role, Kubernetes sysext rationale, lab integration | [`docs/skills/factory-integration.md`](docs/skills/factory-integration.md) | | Work with `systemd-sysext` / `systemd-confext` | [`docs/skills/systemd-sysext-extensions.md`](docs/skills/systemd-sysext-extensions.md) | -| Build or ship the k0s sysext | [`docs/skills/k0s-sysext.md`](docs/skills/k0s-sysext.md), [`docs/skills/k0s-sysext-ops.md`](docs/skills/k0s-sysext-ops.md) | +| Build or ship the Kubernetes sysext | [`docs/skills/kubernetes-sysext.md`](docs/skills/kubernetes-sysext.md), [`docs/skills/kubernetes-sysext-ops.md`](docs/skills/kubernetes-sysext-ops.md) | | Update the FSDK pin / versioning | [`docs/skills/bump-fsdk-version.md`](docs/skills/bump-fsdk-version.md) | | Flatcar version parity matrix / audit | [`docs/skills/flatcar-parity-matrix.md`](docs/skills/flatcar-parity-matrix.md) | | CI workflows, action SHA pinning | [`docs/skills/ci-tooling.md`](docs/skills/ci-tooling.md) | diff --git a/CONTEXT.md b/CONTEXT.md index c2fff25e..d38e710a 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -13,7 +13,7 @@ A bootable GPT raw disk image executing systemd-repart to partition and install _Avoid_: live ISO, installation media, setup script **Sysext**: -A systemd-sysext extension raw image merged into `/usr` to deliver decoupled server runtimes such as k0s. +A systemd-sysext extension raw image merged into `/usr` to deliver decoupled server runtimes such as Kubernetes and containerd. _Avoid_: addon, plugin, package, sidecar **Transfer**: diff --git a/Justfile b/Justfile index 404869a4..4cf88a7a 100644 --- a/Justfile +++ b/Justfile @@ -54,11 +54,11 @@ tags: [group('dev')] validate: python3 .github/scripts/check-release-version.py - python3 .github/scripts/check-k0s-version.py + python3 .github/scripts/check-kubernetes-version.py python3 .github/scripts/check-renovate-series.py just bst show --deps all oci/bluefin-server-ddi.bst just bst show --deps all oci/bluefin-server-installer.bst - just bst show --deps all oci/k0s-sysext.bst + just bst show --deps all oci/kubernetes-sysext.bst # Run the unit test suite (pytest + bats). [group('dev')] @@ -134,26 +134,27 @@ export-pxe: export-installer @test -n "$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-pxe-initrd-*.cpio.gz' -print -quit)" || { echo "ERROR: PXE initrd was not exported." >&2; exit 1; } @echo "==> wrote PXE artifacts:" && ls -lh dist/bluefin-server-pxe-* -# -- k0s systemd-sysext ------------------------------------------------------- -# Produces a systemd-sysext extension image for k0s. +# -- Kubernetes systemd-sysext ------------------------------------------------ +# Produces a systemd-sysext extension image carrying kubeadm, kubelet, kubectl +# and the CNI plugins. -# Build the k0s systemd-sysext image. +# Build the Kubernetes systemd-sysext image. [group('sysext')] build-sysext: - just bst build oci/k0s-sysext.bst + just bst build oci/kubernetes-sysext.bst -# Export the k0s systemd-sysext image + SHA256SUMS to dist/sysext/. +# Export the Kubernetes systemd-sysext image + SHA256SUMS to dist/sysext/. # The artifact checkout also emits an uncompressed .raw; only the # versioned .raw.zst release asset and its SHA256SUMS are published. [group('sysext')] export-sysext: build-sysext rm -rf dist/sysext dist/sysext-checkout mkdir -p dist/sysext-checkout dist/sysext - just bst artifact checkout oci/k0s-sysext.bst --directory /src/dist/sysext-checkout - cp dist/sysext-checkout/k0s-*.raw.zst dist/sysext/ + just bst artifact checkout oci/kubernetes-sysext.bst --directory /src/dist/sysext-checkout + cp dist/sysext-checkout/kubernetes-*.raw.zst dist/sysext/ cp dist/sysext-checkout/SHA256SUMS dist/sysext/ rm -rf dist/sysext-checkout - @echo "==> wrote k0s sysext:" && ls -lh dist/sysext/ + @echo "==> wrote kubernetes sysext:" && ls -lh dist/sysext/ # -- Flatcar LTS kernel & ZFS -------------------------------------------------- # Build the Flatcar LTS kernel and ZFS sysext. @@ -355,10 +356,13 @@ test-installer-artifact: exit 1 fi - # Probe guest directly over SSH tunnel or in-guest curl to 127.0.0.1:8080 - HEALTHZ_RESP=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 https://127.0.0.1:8080/healthz 2>/dev/null || curl --silent --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true" 2>/dev/null || true) + # Probe guest directly over SSH tunnel or in-guest curl to 127.0.0.1:8080. + # The kiosk serves TLS with an in-cluster CA, hence --insecure. Plain HTTP + # is not probed: nginx answers it with a 302 to https, so a fallback would + # report 302 rather than the real status. + HEALTHZ_RESP=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 https://127.0.0.1:8080/healthz 2>/dev/null || true" 2>/dev/null || true) if [ -n "$HEALTHZ_RESP" ]; then - ROOT_CODE=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 --output /dev/null --write-out '%{http_code}' https://127.0.0.1:8080/ 2>/dev/null || curl --silent --max-time 2 --output /dev/null --write-out '%{http_code}' http://127.0.0.1:8080/ 2>/dev/null || true" 2>/dev/null || true) + ROOT_CODE=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 --output /dev/null --write-out '%{http_code}' https://127.0.0.1:8080/ 2>/dev/null || true" 2>/dev/null || true) if [ "$ROOT_CODE" = "200" ]; then if echo "$HEALTHZ_RESP" | jq -e '.status == "ok"' >/dev/null 2>&1; then echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" @@ -474,7 +478,7 @@ install-vm: } trap cleanup INT TERM - until curl --silent --show-error --max-time 2 --output /dev/null http://127.0.0.1:8080/; do + until curl --silent --show-error --insecure --max-time 2 --output /dev/null https://127.0.0.1:8080/; do if ! kill -0 "$QEMU_PID" 2>/dev/null; then wait "$QEMU_PID" exit 1 @@ -484,14 +488,14 @@ install-vm: HOST_IP="$(ip -4 -o addr show scope global | awk '{print $4}' | cut -d/ -f1 | head -n1)" echo "==> KubeStellar Console is ready!" - echo "==> Access URL (LAN): http://${HOST_IP:-localhost}:8080/" - echo "==> Access URL (Local): http://localhost:8080/" - xdg-open "http://${HOST_IP:-localhost}:8080/" || xdg-open http://localhost:8080/ || true + echo "==> Access URL (LAN): https://${HOST_IP:-localhost}:8080/" + echo "==> Access URL (Local): https://localhost:8080/" + xdg-open "https://${HOST_IP:-localhost}:8080/" || xdg-open https://localhost:8080/ || true wait "$QEMU_PID" # Set up KubeStellar kc-agent for the user in ONE command. [group('test')] -setup-kubestellar ORIGIN="http://localhost:8080,http://127.0.0.1:8080": +setup-kubestellar ORIGIN="https://localhost:8080,https://127.0.0.1:8080": #!/usr/bin/env bash set -euo pipefail if [ -x "files/bin/bluefin-kubestellar" ]; then @@ -508,7 +512,7 @@ setup-kubestellar ORIGIN="http://localhost:8080,http://127.0.0.1:8080": # Run fully automated headless browser test against the KubeStellar console. [group('test')] -test-e2e-browser CONSOLE_URL="http://127.0.0.1:8080": +test-e2e-browser CONSOLE_URL="https://127.0.0.1:8080": python3 tests/e2e/test_kubestellar_browser_login.py --console-url "{{CONSOLE_URL}}" # Complete end-to-end Lima VM orchestration test. diff --git a/NOTES.md b/NOTES.md index 4dd1c417..942a3984 100644 --- a/NOTES.md +++ b/NOTES.md @@ -2,7 +2,7 @@ ## Domain & Repository - **Repository**: `projectbluefin/server` (upstream of Bluefin Server OS). -- **Core Architecture**: FSDK 26.08 (`components/*`), immutable XFS DDI OS (`oci/bluefin-server-ddi.bst`), systemd-sysinstall offline raw disk installer (`oci/bluefin-server-installer.bst`), and optional k0s `systemd-sysext` (`oci/k0s-sysext.bst`). +- **Core Architecture**: FSDK 26.08 (`components/*`), immutable XFS DDI OS (`oci/bluefin-server-ddi.bst`), systemd-sysinstall offline raw disk installer (`oci/bluefin-server-installer.bst`), and optional Kubernetes `systemd-sysext` (`oci/kubernetes-sysext.bst`). - **Release Automation**: Driven by Renovate dependency updates and merges to `main`. Pushes trigger `.github/workflows/build.yml`. ## Tooling Stack @@ -14,9 +14,9 @@ ## Release & Sysupdate Lifecycle - **Version Source**: Pinned FSDK point release in `elements/freedesktop-sdk.bst` (e.g. `26.08.0`). -- **Sysext Migration**: Migrated from legacy k3s to k0s (`oci/k0s-sysext.bst`). Emits `k0s-@v.raw.zst`. -- **Sysupdate Contracts** (`files/os/sysupdate.d/` plus the k0s component directory): +- **Kubernetes Sysext**: Upstream `kubeadm`/`kubelet`/`kubectl` plus CNI plugins (`oci/kubernetes-sysext.bst`). Emits `kubernetes-@v.raw.zst`. Version axis lives in `include/kubernetes.yml`. +- **Sysupdate Contracts** (`files/os/sysupdate.d/` plus the `kubernetes` component directory): - `50-root.transfer`: OS rootfs DDI (`bluefin-server-ddi-@v.raw.zst`). - `60-uki.transfer`: Boot UKI (`bluefin-server-@v.efi`). - - `files/os/sysupdate.k0s.d/70-k0s.transfer`: k0s sysext (`k0s-@v.raw.zst`). + - `files/os/sysupdate.kubernetes.d/70-kubernetes.transfer`: Kubernetes sysext (`kubernetes-@v.raw.zst`), on a minor-pinned track. - **Signing & Manifest**: `dist/release/SHA256SUMS` signed via detached GPG (`SHA256SUMS.gpg`) using secret `SYSUPDATE_SIGNING_KEY`. diff --git a/README.md b/README.md index 51aebccb..c1f01a0f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ It is [DDI first](https://0pointer.net/blog/fitting-everything-together.html): t ## Release status: Alpha Bluefin Server is currently in **Alpha**: -- **Milestone status**: Phase A (reproducible build path, uutils, k0s sysext, graph validation) is complete. Phase B (automated boot verification on lab cluster) is in progress. +- **Milestone status**: Phase A (reproducible build path, uutils, Kubernetes sysext, graph validation) is complete. Phase B (automated boot verification on lab cluster) is in progress. - **Trust model**: Releases include cryptographic provenance with GPG-signed `SHA256SUMS` manifests and in-tree `systemd-sysupdate` verification configurations. - **Suitability**: Alpha builds are intended for evaluation, testing, and factory validation. Not yet recommended for production workloads. - **Readiness roadmap**: Track completed criteria and remaining gates toward 1.0 in [`docs/MVP_1_0_READINESS.md`](docs/MVP_1_0_READINESS.md). @@ -25,7 +25,7 @@ Bluefin Server is currently in **Alpha**: - **DDI-first delivery** — the installer embeds the OS payload as a data partition; no network is required at install time. - **Streamlined base OS image** — modern userspace with uutils coreutils and bash for interactive login and diagnostics. - **systemd-native installer** — `systemd-sysinstall` provides the interactive terminal UI and `systemd-repart` handles partitioning and block-copy DDI placement. -- **Optional k0s as a `systemd-sysext`** so the base image stays minimal. +- **Optional upstream Kubernetes as a `systemd-sysext`** so the base image stays minimal. > **Remote diagnostics:** OpenSSH is installed for on-demand diagnostics, but is disabled by default via systemd presets. It can be started manually with `systemctl start sshd` when remote access is needed. See [`docs/skills/factory-integration.md`](docs/skills/factory-integration.md). diff --git a/SECURITY.md b/SECURITY.md index f6c6bac3..82f503bf 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -61,13 +61,13 @@ For the full trust model, key rotation, and transfer configuration, see This policy covers everything this repository produces and its build pipeline: -- The OS DDI payload, the offline installer disk image, and the k0s +- The OS DDI payload, the offline installer disk image, and the Kubernetes `systemd-sysext`. - BuildStream elements, build tooling, and GitHub Actions workflows. - Release signing and the `systemd-sysupdate` update/verification flow. **Out of scope:** vulnerabilities in upstream components such as -freedesktop-sdk, GNOME build metadata, systemd, or k0s itself. Report those to +freedesktop-sdk, GNOME build metadata, systemd, or Kubernetes itself. Report those to the respective upstream project; report to this repository only when the issue is introduced by our integration, build, signing, or packaging of the component. diff --git a/docs/DOCUMENTATION_OVERHAUL_PLAN.md b/docs/DOCUMENTATION_OVERHAUL_PLAN.md index c28d4e24..53aebf28 100644 --- a/docs/DOCUMENTATION_OVERHAUL_PLAN.md +++ b/docs/DOCUMENTATION_OVERHAUL_PLAN.md @@ -5,6 +5,16 @@ This plan is the executable artifact. It inventories the current docs, prescribes the target structure, defines lazy-loading and token-efficiency rules, drafts the canonical `AGENTS.md`, and gives a file-by-file cleanup sequence. +> **Historical record — completed.** This plan captures the documentation +> inventory and `AGENTS.md` text as they stood when the overhaul was executed. +> Its file listings, line counts, and the embedded `AGENTS.md` block are a +> point-in-time snapshot and are deliberately **not** updated as the tree moves +> on. The k0s skills it inventories were replaced by +> [`docs/skills/kubernetes-sysext.md`](skills/kubernetes-sysext.md) and +> [`docs/skills/kubernetes-sysext-ops.md`](skills/kubernetes-sysext-ops.md) in +> the upstream Kubernetes cutover; read those, and the live `AGENTS.md`, for +> current state. The rules and conventions in sections 2-5 remain in force. + --- ## 1. Research basis & sources diff --git a/docs/MVP_1_0_READINESS.md b/docs/MVP_1_0_READINESS.md index 3f8bef80..6d086092 100644 --- a/docs/MVP_1_0_READINESS.md +++ b/docs/MVP_1_0_READINESS.md @@ -4,7 +4,7 @@ This audit tracks the gap between the current tree and a first public/usable MVP ## MVP 1.0 bar -1. **Reproducible build path** — documented command or CI job that produces the DDI, live installer, and k0s sysext. +1. **Reproducible build path** — documented command or CI job that produces the DDI, live installer, and Kubernetes sysext. 2. **Signed release artifacts** — combined `SHA256SUMS` + detached GPG signature published to GitHub Releases. 3. **Automated boot verification** — at least one non-human test that proves the installer writes a bootable disk and the installed OS reaches a target. 4. **Functional update path** — host can pull the signed manifest and apply an OS update without manual intervention. @@ -15,10 +15,10 @@ This audit tracks the gap between the current tree and a first public/usable MVP | Check | Status | Evidence | |---|---|---| -| Element graph resolves | ✅ | `just validate` succeeds for DDI, installer, and k0s sysext | +| Element graph resolves | ✅ | `just validate` succeeds for DDI, installer, and Kubernetes sysext | | Release workflow lint | ✅ | `actionlint .github/workflows/build.yml` clean | | Release path exists | ✅ | `.github/workflows/build.yml` builds, signs, uploads to GitHub Release | -| Cluster build pipeline | ✅ | Phase A complete; pipeline builds DDI, installer, and k0s sysext with uutils | +| Cluster build pipeline | ✅ | Phase A complete; pipeline builds DDI, installer, and Kubernetes sysext with uutils | | Automated boot test | 🔄 | Phase B in progress for Alpha; `bluefin-server-boot-test` workflow running on lab cluster | | A/B root rollback | ❌ | `50-root.transfer` names `root-a`/`root-b`, installer only creates `root-a` | | Root immutability | ❌ | DDI boots read/write (`rw` on cmdline) | @@ -28,7 +28,7 @@ Competitor context: [gap-analysis-distros.md](skills/gap-analysis-distros.md) ## Verdict -**Alpha state — in progress for MVP 1.0.** Phase A (build path, uutils, k0s sysext, validate) is complete. Phase B (automated boot test on lab cluster) is actively in progress for the Alpha milestone. Full MVP 1.0 release requires concluding Phase B boot verification and Phase C runtime hardening (automated rollback, read-only `/usr`, credential delivery). +**Alpha state — in progress for MVP 1.0.** Phase A (build path, uutils, Kubernetes sysext, validate) is complete. Phase B (automated boot test on lab cluster) is actively in progress for the Alpha milestone. Full MVP 1.0 release requires concluding Phase B boot verification and Phase C runtime hardening (automated rollback, read-only `/usr`, credential delivery). ## Roadmap @@ -39,7 +39,7 @@ Priority order. Each item depends on the ones above it. - [x] Extend cluster build deadlines so the pipeline can finish. - [x] Fix `build.yml` actionlint warnings. - [x] Confirm build path succeeds and publishes `bluefin-server-installer:latest` to `:30500`. -- [x] Add `oci/k0s-sysext.bst` to the build and validation pipeline. +- [x] Add `oci/kubernetes-sysext.bst` to the build and validation pipeline. - [x] Integrate uutils coreutils across OS elements. - [x] Merge-contract graph validation (`just validate`) passes clean. diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index c4deaf91..91454b14 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -136,7 +136,7 @@ uploaded to a GitHub Release tagged `installer-v`. syncs `release-version` to the tracked FSDK point release, and proposes the result as its own pull request against `main`. 3. **Full Compilation:** Builds the standalone DDI OS image, live installer, and - k0s systemd-sysext on every pull request and push to `main`. + Kubernetes systemd-sysext on every pull request and push to `main`. 4. **Installer boot test:** Uploads the already-built installer and PXE artifacts and calls `projectbluefin/actions/.github/workflows/server-installer-test.yml`. The shared workflow prepares QEMU/KVM and runs this repository's `just test-installer-artifact`; Lima remains local-only. 5. **Version Derivation:** The release tag is derived with `just version`, which parses the pinned FSDK point release from `elements/freedesktop-sdk.bst`. diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 0916d10f..e6863230 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -26,7 +26,7 @@ just export-installer # export installer + UKI + SHA256SUMS to dist/ just export-pxe # export standalone PXE vmlinuz/initrd to dist/ just build-ddi # build the OS DDI payload just export-ddi # export DDI + SHA256SUMS to dist/ddi/ -just build-sysext # build the k0s sysext +just build-sysext # build the Kubernetes sysext just export-sysext # export sysext artifacts to dist/sysext/ just flash-installer # write the installer image to a USB device just show-me-the-future # end-to-end QEMU installer smoke test @@ -88,7 +88,7 @@ Use direct I/O and full-block reads to avoid dirtying the page cache. The release process is driven by `.github/workflows/build.yml`: - Renovate point-release updates or direct pushes to `main` trigger a full build. -- CI builds the DDI payload, installer, target UKI, k0s sysext, and standalone +- CI builds the DDI payload, installer, target UKI, Kubernetes sysext, and standalone PXE boot inputs (`bluefin-server-pxe-vmlinuz-*`, `bluefin-server-pxe-initrd-*.cpio.gz`). - CI uploads the versioned release assets to the corresponding `installer-v` GitHub Release. diff --git a/docs/skills/factory-integration.md b/docs/skills/factory-integration.md index 75124ca4..e2f724d2 100644 --- a/docs/skills/factory-integration.md +++ b/docs/skills/factory-integration.md @@ -18,7 +18,7 @@ The factory pattern is broader than a single host: a downstream CI lab or OS fac ┌─────────────────────────────────────────────────────────────┐ │ Downstream CI lab / OS factory │ │ GitOps-style testing and automation for image-based OSes │ -│ • k0s control plane / workload orchestration │ +│ • Kubernetes control plane / workload orchestration │ │ • VM or container workloads │ │ • OCI/bootc image pipelines / release automation │ └─────────────────────────────────────────────────────────────┘ @@ -28,21 +28,23 @@ The factory pattern is broader than a single host: a downstream CI lab or OS fac │ Bluefin Server (this repo) │ │ Core server OS: DDI-first, image-updated │ │ • systemd-sysupdate for atomic A/B updates │ -│ • systemd-sysext for optional layers (k0s, extensions) │ +│ • systemd-sysext for optional layers (Kubernetes, runtime) │ │ • podman for container workloads │ └─────────────────────────────────────────────────────────────┘ ``` -## k0s is a sysext, not base image bloat +## Kubernetes is a sysext, not base image bloat -Kubernetes is not baked into the OS DDI. The base image stays small and stateless; k0s is delivered as a `systemd-sysext` EROFS image that overlays `/usr/` at runtime. +Kubernetes is not baked into the OS DDI. The base image stays small and stateless; `kubeadm`, `kubelet`, `kubectl` and the CNI plugins are delivered as a `systemd-sysext` EROFS image that overlays `/usr/` at runtime. containerd arrives the same way, as Flatcar's own `containerd-flatcar.raw` sysext. -- `elements/oci/k0s-sysext.bst` builds the sysext. -- `files/os/sysupdate.k0s.d/70-k0s.transfer` enables component-scoped OTA - updates of the sysext. +- `elements/oci/kubernetes-sysext.bst` builds the sysext. +- `files/os/sysupdate.kubernetes.d/70-kubernetes.transfer` enables component-scoped OTA + updates of the sysext, on a minor-pinned track. - `files/os/justfile` provides the `just k8s` entrypoint. +- Cluster workload is reconciled by Argo CD Core from an in-image manifest tree, not by + an OS-level addon manager. -See [k0s-sysext.md](k0s-sysext.md) for details. +See [kubernetes-sysext.md](kubernetes-sysext.md) for details. ## Workloads are containers @@ -57,7 +59,7 @@ The workloads the factory tests and ships live in other repositories or image pi | Fully automated, unattended installs | Offline DDI installer (`systemd-sysinstall`) | | Atomic, rollback-capable updates | Image-based A/B updates via `systemd-sysupdate` | | Minimal attack surface / lean base OS | Streamlined DDI with bash; optional tools as sysexts | -| Kubernetes control plane on every node | k0s delivered as `systemd-sysext` | +| Kubernetes control plane on every node | Upstream `kubeadm` delivered as `systemd-sysext` | | Container workloads | `podman` in the base OS stack | | Signed, verifiable release artifacts | GPG-signed `SHA256SUMS` + `import-pubring.gpg` | @@ -82,7 +84,7 @@ The workloads the factory tests and ships live in other repositories or image pi | Rationalization | Reality | |---|---| -| “k0s should be in the base image.” | Keep the OS DDI minimal. k0s is optional and delivered OTA as a sysext. | +| “Kubernetes should be in the base image.” | Keep the OS DDI minimal. The Kubernetes sysext is optional and delivered OTA. | | “We can pull the DDI at install time.” | Unattended installs must survive network loss; the DDI is embedded in the installer media. | | “Let’s add heavy debug tools.” | Base OS includes bash for login; heavy developer/debug tools belong in sysexts or system containers. | | “Package updates are small patches.” | Image-based updates are whole-OS replacements; the rollback unit is the OS image, not a package delta. | @@ -91,18 +93,18 @@ The workloads the factory tests and ships live in other repositories or image pi - Adding a workload dependency to `elements/bluefin-server/os-stack.bst` that could ship as a `systemd-sysext`. - Treating Bluefin Server as a generic Fedora/RHEL replacement rather than the factory core OS. -- Putting Kubernetes tooling in the base DDI instead of the k0s sysext. +- Putting Kubernetes tooling in the base DDI instead of the Kubernetes sysext. - Designing install/update paths that require interactive human steps in the factory. ## Verification - [ ] Any new base-DDI dependency can be justified by the factory core-OS role. - [ ] Optional capabilities are modeled as sysexts or system containers. -- [ ] The k0s sysext still builds and updates independently of the DDI. -- [ ] `systemd-sysupdate` transfer files are present for every OTA-delivered artifact (DDI, UKI, k0s sysext). +- [ ] The Kubernetes sysext still builds and updates independently of the DDI. +- [ ] `systemd-sysupdate` transfer files are present for every OTA-delivered artifact (DDI, UKI, Kubernetes sysext). ## See also -- [k0s-sysext.md](k0s-sysext.md) — building and delivering the k0s sysext. +- [kubernetes-sysext.md](kubernetes-sysext.md) — building and delivering the Kubernetes sysext. - [ddi-installer.md](ddi-installer.md) — offline installer architecture. - [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary. diff --git a/docs/skills/gap-analysis-distros.md b/docs/skills/gap-analysis-distros.md index 12b44132..05619ea0 100644 --- a/docs/skills/gap-analysis-distros.md +++ b/docs/skills/gap-analysis-distros.md @@ -96,9 +96,9 @@ Bluefin Server is a BuildStream 2-based, image-based Linux server OS built from |------|---------------------------------| | **Philosophy** | Systemd-native, minimal, image-based server OS appliance; base DDI includes bash for login and bring-up while heavy developer/debug tools live in sysexts or system containers; intended to run container workloads and Kubernetes via optional sysexts. Sources: [AGENTS.md](../../AGENTS.md), [factory-integration.md](factory-integration.md). | | **State model** | Target OS DDI is an XFS filesystem image. A separate persistent `/var` partition is created by the installer. There is no second root slot provisioned today, and the UKI cmdline currently uses `rw`, so the root is not mounted read-only at runtime. Sources: [bluefin-server-ddi.bst](../../elements/oci/bluefin-server-ddi.bst), [20-root-a.conf](../../files/installer/repart.d/20-root-a.conf), [bluefin-server-installer.bst](../../elements/oci/bluefin-server-installer.bst). | -| **Updates** | `systemd-sysupdate` reads root/UKI transfers from `files/os/sysupdate.d/` and the optional k0s transfer from the `k0s` component directory. Assets are published to GitHub Releases, and the combined `SHA256SUMS` manifest is signed in CI with a GPG key. `Verify=yes` is the default. Sources: [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md), [50-root.transfer](../../files/os/sysupdate.d/50-root.transfer), [60-uki.transfer](../../files/os/sysupdate.d/60-uki.transfer), [70-k0s.transfer](../../files/os/sysupdate.k0s.d/70-k0s.transfer), also `systemd-sysupdate(8)`. | +| **Updates** | `systemd-sysupdate` reads root/UKI transfers from `files/os/sysupdate.d/` and the optional Kubernetes transfer from the `kubernetes` component directory. Assets are published to GitHub Releases, and the combined `SHA256SUMS` manifest is signed in CI with a GPG key. `Verify=yes` is the default. Sources: [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md), [50-root.transfer](../../files/os/sysupdate.d/50-root.transfer), [60-uki.transfer](../../files/os/sysupdate.d/60-uki.transfer), [70-kubernetes.transfer](../../files/os/sysupdate.kubernetes.d/70-kubernetes.transfer), also `systemd-sysupdate(8)`. | | **Provisioning** | The installer is an offline `systemd-sysinstall` image that embeds the DDI as a data partition. First-boot configuration is intended to be delivered via `systemd-creds` through the ESP or hypervisor metadata. Today only `passwd.hashed-password.root` is consumed via `systemd-sysusers.d`; the documented `tmpfiles.extra` path for SSH keys and similar files is not implemented. Sources: [bluefin-server-installer.bst](../../elements/oci/bluefin-server-installer.bst), [10-root-creds.conf](../../files/os/sysusers.d/10-root-creds.conf), [os-creds-prov.bst](../../elements/bluefin-server/os-creds-prov.bst), [systemd-creds(1)](https://www.freedesktop.org/software/systemd/man/latest/systemd-creds.html). | -| **Customization** | Adds software through `systemd-sysext` (overlay `/usr`) and `systemd-confext` (overlay `/etc`) images. The base OS `os-release` advertises `ID=flatcar` and a matching `VERSION_ID` so pre-built Flatcar Bakery extensions load. k0s is shipped as a separately built, optionally enabled sysext. Sources: [systemd-sysext-extensions.md](systemd-sysext-extensions.md), [k0s-sysext.md](k0s-sysext.md), [os-release-flatcar.bst](../../elements/bluefin-server/os-release-flatcar.bst), [systemd-sysext(8)](https://www.freedesktop.org/software/systemd/man/latest/systemd-sysext.html). | +| **Customization** | Adds software through `systemd-sysext` (overlay `/usr`) and `systemd-confext` (overlay `/etc`) images. The base OS `os-release` advertises `ID=flatcar` and a matching `VERSION_ID` so pre-built Flatcar Bakery extensions load. Upstream Kubernetes is shipped as a separately built, optionally enabled sysext. Sources: [systemd-sysext-extensions.md](systemd-sysext-extensions.md), [kubernetes-sysext.md](kubernetes-sysext.md), [os-release-flatcar.bst](../../elements/bluefin-server/os-release-flatcar.bst), [systemd-sysext(8)](https://www.freedesktop.org/software/systemd/man/latest/systemd-sysext.html). | | **Reboot coordination** | `systemd-sysupdate.service` has an `ExecStartPost` that touches `/run/reboot-required`. Rolling reboots across Kubernetes nodes rely on Kured reading that file. Sources: [os-kured-hook.bst](../../elements/bluefin-server/os-kured-hook.bst), [kured-hook.conf](../../files/os/systemd/systemd-sysupdate.service.d/kured-hook.conf), [Kured project](https://github.com/weaveworks/kured). | ## 4. Factual Gaps @@ -181,7 +181,7 @@ These gaps drive the priorities in [architecture-roadmap.md](architecture-roadma - [../../CONTEXT.md](../../CONTEXT.md) - [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md) - [systemd-sysext-extensions.md](systemd-sysext-extensions.md) -- [k0s-sysext.md](k0s-sysext.md) +- [kubernetes-sysext.md](kubernetes-sysext.md) - [factory-integration.md](factory-integration.md) - [tpm2-credential-sealing.md](tpm2-credential-sealing.md) - [elements/oci/bluefin-server-ddi.bst](../../elements/oci/bluefin-server-ddi.bst) @@ -192,6 +192,6 @@ These gaps drive the priorities in [architecture-roadmap.md](architecture-roadma - [files/installer/repart.d/20-root-a.conf](../../files/installer/repart.d/20-root-a.conf) - [files/os/sysupdate.d/50-root.transfer](../../files/os/sysupdate.d/50-root.transfer) - [files/os/sysupdate.d/60-uki.transfer](../../files/os/sysupdate.d/60-uki.transfer) -- [files/os/sysupdate.k0s.d/70-k0s.transfer](../../files/os/sysupdate.k0s.d/70-k0s.transfer) +- [files/os/sysupdate.kubernetes.d/70-kubernetes.transfer](../../files/os/sysupdate.kubernetes.d/70-kubernetes.transfer) - [files/os/sysusers.d/10-root-creds.conf](../../files/os/sysusers.d/10-root-creds.conf) - [files/os/systemd/systemd-sysupdate.service.d/kured-hook.conf](../../files/os/systemd/systemd-sysupdate.service.d/kured-hook.conf) diff --git a/docs/skills/index.md b/docs/skills/index.md index c9322b90..1b192e3c 100644 --- a/docs/skills/index.md +++ b/docs/skills/index.md @@ -31,8 +31,8 @@ This is the lazy-load routing table for agent skills. Keep this file in memory w | [`factory-integration.md`](factory-integration.md) | Lab integration, boot-test workflow, factory role | How Bluefin Server is consumed by the CI lab. | | [`flatcar-parity-matrix.md`](flatcar-parity-matrix.md) | Flatcar version audit, component parity, substitution order | Version parity matrix and gap audit against Flatcar 4593.2.5. | | [`gap-analysis-distros.md`](gap-analysis-distros.md) | Comparing Bluefin Server to other server OSes | Source-verified comparison to Ubuntu, Talos, Flatcar, FCOS. | -| [`k0s-sysext-ops.md`](k0s-sysext-ops.md) | Operating k0s on Bluefin Server | Runtime operation and reboot coordination for k0s. | -| [`k0s-sysext.md`](k0s-sysext.md) | Building the k0s sysext | BuildStream element and publish steps for the k0s sysext. | +| [`kubernetes-sysext-ops.md`](kubernetes-sysext-ops.md) | Operating Kubernetes on Bluefin Server | kubeadm bring-up, the seed phases, and Argo CD Core troubleshooting. | +| [`kubernetes-sysext.md`](kubernetes-sysext.md) | Building the Kubernetes sysext | BuildStream element, version axis, and publish steps for the Kubernetes sysext. | | [`skill-improvement.md`](skill-improvement.md) | Adding, splitting, or refactoring skills | Meta-skill that owns the documentation loop. | | [`system-containers.md`](system-containers.md) | Running `systemd-nspawn` toolboxes | System container operation with `machinectl`. | | [`systemd-sysext-extensions.md`](systemd-sysext-extensions.md) | Optional layers via `systemd-sysext` / `systemd-confext` | Extension identity, loading, and Flatcar compatibility. | diff --git a/docs/skills/k0s-sysext-ops.md b/docs/skills/k0s-sysext-ops.md deleted file mode 100644 index 7b806f80..00000000 --- a/docs/skills/k0s-sysext-ops.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -name: k0s-sysext-ops -description: Operator runbook for the k0s systemd-sysext extension — provisioning, runtime testing, and troubleshooting. -metadata: - type: how-to - status: stable - last_updated: "2026-09-08" - context7-sources: - - /systemd/systemd ---- -# k0s systemd-sysext Operations - -Use this skill when running, testing, or debugging the k0s systemd-sysext on a live Bluefin Server host. - -## Enabling k0s on a Host - -On a running Bluefin Server system: - -```bash -just k8s -``` - -Or manually: - -```bash -# 1. Fetch the extension image into persistent k0s staging. -systemd-sysupdate --component=k0s update - -# 2. Copy it to the ephemeral sysext scan directory and merge into /usr. -install -D -m 0644 /var/lib/k0s/k0s.raw /run/extensions/k0s.raw -systemd-sysext merge - -# 3. Seed declarative manifest stacks into /var/lib/k0s/manifests/ -systemd-tmpfiles --create /usr/lib/tmpfiles.d/k0s-manifests.conf - -# 4. Start the controller service -systemctl enable --now k0scontroller.service -``` - -## Verifying Manifest Reconciliation - -k0s automatically applies all `.yaml` files under `/var/lib/k0s/manifests/argocd/` and `/var/lib/k0s/manifests/kubestellar/`. - -```bash -# Verify pods in argocd and kubestellar namespaces -k0s kubectl get pods -A -``` - -## Troubleshooting - -- **Extension not merged**: Check `systemd-sysext status`. Verify the persistent image is `/var/lib/k0s/k0s.raw`; the boot activation unit copies it into `/run/extensions/k0s.raw`. -- **Manifests not applied**: Check `/var/lib/k0s/manifests/`. Ensure files end in `.yaml` (not `.yml`). -- **Service failed**: Check `journalctl -u k0scontroller -e`. - -## See also - -- [k0s-sysext.md](k0s-sysext.md) -- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). -- `systemd-sysext(8)` diff --git a/docs/skills/k0s-sysext.md b/docs/skills/k0s-sysext.md deleted file mode 100644 index 1e38f94c..00000000 --- a/docs/skills/k0s-sysext.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: k0s-sysext -description: Build, ship, and enable the k0s systemd-sysext extension with native Argo CD & KubeStellar for Bluefin Server. -metadata: - type: how-to - status: stable - last_updated: "2026-09-08" - context7-sources: - - /systemd/systemd ---- -# k0s systemd-sysext - -Use this skill when working on the k0s systemd-sysext extension shipped as an -optional overlay for Bluefin Server. - -## When to Use - -- Bumping the pinned k0s binary version or its SHA256. -- Modifying the sysext image contents (`files/k0s/sysext/`). -- Managing raw YAML manifest stacks (`files/k0s/manifests/argocd/` or `files/k0s/manifests/kubestellar/`). -- Changing manifest seeding rules in `files/k0s/sysext/k0s-manifests.conf`. -- Adding or changing the sysupdate transfer definition - (`files/os/sysupdate.k0s.d/70-k0s.transfer`). -- Debugging why a host cannot pull, merge, or start k0s. - -## When NOT to Use - -- General OS image composition questions (use `ddi-installer.md` or `avoid-over-engineering.md`). -- systemd-sysupdate signature verification (use `systemd-sysupdate-verification.md`). - -## Architecture - -Bluefin Server's base DDI image includes bash for login and bring-up, while -heavy developer and debug tools live in sysexts or system containers. k0s is not -baked into the OS stack. Instead it is delivered as a `systemd-sysext` EROFS -image that overlays `/usr` at runtime. - -Design choices: - -- **Single static binary.** `elements/k0s/k0s-bin.bst` fetches the upstream statically-linked - release binary directly from GitHub. -- **Pure declarative manifests (No Helm).** Declarative stacks for Argo CD and KubeStellar - are shipped in `/usr/share/k0s/manifests/` and seeded into `/var/lib/k0s/manifests/` via - `systemd-tmpfiles`. k0s's internal manifest deployer automatically reconciles them. -- **Helm and autopilot disabled.** `--disable-components=helm,autopilot` ensures zero runtime Helm dependencies and prevents single-node controlnode timeout loops. -- **Flatcar sysext pattern.** The extension uses `ID=_any` in its release metadata so it merges on any host image. -- **First-boot background activation.** `k0s-first-boot.service` copies `/var/lib/k0s/k0s.raw` to `/run/extensions/k0s.raw`, merges the sysext, seeds declarative manifest stacks via `systemd-tmpfiles`, and enables `k0scontroller.service` on boot without holding up system startup. -- **OTA delivery.** A k0s component `systemd-sysupdate` transfer file - (`70-k0s.transfer`) is installed in the base OS so hosts can pull new k0s - sysext releases from GitHub Releases without updating the root or UKI. - -## Repository Layout - -| Path | Purpose | -|------|---------| -| `include/k0s.yml` | **Single source of truth for the k0s version axis** (`%{k0s-upstream-tag}`, `%{k0s-version}`). | -| `elements/k0s/k0s-bin.bst` | Pins the upstream `k0s` binary SHA256; the release URL is derived from `include/k0s.yml`. | -| `elements/oci/k0s-sysext.bst` | Builds the EROFS sysext image (`k0s-.raw`). | -| `files/k0s/sysext/k0scontroller.service` | systemd unit for the k0s single-node controller/worker. Not enabled by default. | -| `files/k0s/sysext/extension-release.k0s` | Static sysext identity (`ID=_any`); `VERSION_ID=`/`ARCHITECTURE=` are appended at build time. | -| `files/k0s/sysext/k0s-manifests.conf` | tmpfiles rule that copies declarative stacks to `/var/lib/k0s/manifests/`. | -| `files/k0s/manifests/argocd/` | Raw YAML manifests for Argo CD. | -| `files/k0s/manifests/kubestellar/` | Raw YAML manifests for KubeFlex, Postgres, KubeStellar core, and Console. | -| `files/os/sysupdate.k0s.d/70-k0s.transfer` | sysupdate transfer track for the k0s sysext component. | -| `Justfile` | `build-sysext` / `export-sysext` targets. | -| `.github/workflows/build.yml` | Builds, signs, and publishes sysext assets. | -| `.github/scripts/check-k0s-version.py` | Fails closed if any consumer restates the k0s version instead of deriving it. | - -## Build Outputs - -`elements/oci/k0s-sysext.bst` produces: - -- `k0s-.raw` — uncompressed EROFS sysext image. -- `k0s-.raw.zst` — zstd-compressed release asset. -- `SHA256SUMS` — checksum manifest for the compressed asset. - -## Justfile Commands - -```bash -just validate # resolve the element graph -just build-sysext # build oci/k0s-sysext.bst -just export-sysext # export sysext artifacts to dist/sysext/ -``` - -## Operations and runtime testing - -For enabling the sysext on a host, common gotchas, and runtime testing guidance, -see [k0s-sysext-ops.md](k0s-sysext-ops.md). - -## See also - -- [k0s-sysext-ops.md](k0s-sysext-ops.md) -- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). -- `systemd-sysext(8)`, `systemd-confext(8)` diff --git a/docs/skills/kubernetes-sysext-ops.md b/docs/skills/kubernetes-sysext-ops.md new file mode 100644 index 00000000..54636177 --- /dev/null +++ b/docs/skills/kubernetes-sysext-ops.md @@ -0,0 +1,161 @@ +--- +name: kubernetes-sysext-ops +description: Operator runbook for the Kubernetes systemd-sysext — kubeadm bring-up, the seed phases, Argo CD Core troubleshooting, and runtime testing. +metadata: + type: how-to + status: stable + last_updated: "2026-09-19" + context7-sources: + - /systemd/systemd +--- +# Kubernetes systemd-sysext Operations + +Use this skill when running, testing, or debugging Kubernetes on a live Bluefin Server +host. For building and versioning the image itself, see +[kubernetes-sysext.md](kubernetes-sysext.md). + +## Bring-up model + +First boot is fully unit-driven. Nothing here is a manual step on a correctly +provisioned host; the commands below are for inspection and recovery. + +``` +systemd-sysext.service merge containerd + kubernetes sysexts into /usr + -> daemon-reload a merge makes unit files appear; systemd must be told +bluefin-cluster-repo.service init /var/lib/bluefin/cluster.git from /usr/share/bluefin/cluster +kubeadm-init.service kubeadm init --config /usr/share/bluefin/kubeadm.yaml +bluefin-cluster-bootstrap.service apply the four seed phases in order +``` + +`kubeadm-init.service` carries `ConditionPathExists=!/etc/kubernetes/admin.conf`, so it +is a no-op on an already-initialised host. `bluefin-cluster-bootstrap.service` carries no +such condition on purpose: a failed condition marks a unit *skipped* rather than *failed*, +which would silently no-op the first boot. + +## Enabling Kubernetes on a Host + +On a running Bluefin Server system: + +```bash +just k8s +``` + +Or manually: + +```bash +# 1. Fetch the extension image into /var/lib/extensions/. +systemd-sysupdate --component=kubernetes update + +# 2. Merge into /usr, then let systemd see the new unit files. +systemd-sysext merge +systemctl daemon-reload + +# 3. Bring the cluster up. bluefin-cluster-bootstrap pulls in kubeadm-init +# and bluefin-cluster-repo through its ordering and Requires= edges. +systemctl enable --now bluefin-cluster-bootstrap.service +``` + +`kubeadm init` runs with `--skip-phases=addon/kube-proxy`; Cilium supplies the +kube-proxy replacement. + +## The seed phases + +The seed is applied by `bluefin-cluster-bootstrap.service` from +`/usr/share/bluefin/seed/`, with `kubectl apply --server-side --force-conflicts`, in +directory order: + +| Phase | Contents | Why it is in the seed and not in Argo | +|---|---|---| +| `00-cilium/` | CNI, kube-proxy replacement | Argo's own pods need pod networking; a node without CNI stays `NotReady` with a `NoSchedule` taint. | +| `10-argocd-core/` | Argo CRDs, RBAC, redis, repo-server, application-controller | Argo cannot install Argo. | +| `15-gitd/` | git-daemon serving `/var/lib/bluefin/cluster.git` | The root Application needs the repository served before Argo can read anything. | +| `20-root-app/` | `AppProject` + root `Application` | Hands ownership of everything downstream to Argo. | + +The bootstrap unit waits for the node to report `Ready` between `00-cilium/` and +`10-argocd-core/`. + +The seed is **append-only by construction and is never pruned**. Argo owns pruning for +all downstream workload via `prune: true`; both upstream `kubectl` pruning modes are +alpha and unsuitable. Do not reopen this. + +## Argo CD Core + +Core runs exactly three pods: `argocd-application-controller`, `argocd-repo-server`, and +`argocd-redis`. There is no `argocd-server`, no `argocd-dex-server`, and no +`argocd-notifications-controller`. + +Consequences for operators: + +- **There is no web UI and the `argocd` CLI does not work.** The CLI speaks gRPC to + `argocd-server`, which Core does not ship. Manage Applications as plain Kubernetes + custom resources with `kubectl`. +- `argocd-applicationset-controller` is patched out; this design has exactly one root + Application. Its CRD is left in place, harmlessly. +- Redis is an in-memory cache with no PersistentVolumeClaim. There is no supported + redis-free Argo CD, so a missing redis is a fault, not a configuration choice. +- Two residual `argocd-server` NetworkPolicy `podSelector` rules admit traffic from a pod + that never exists. They are inert and deliberately left unmodified. + +Inspecting sync state: + +```bash +kubectl -n argocd get applications +kubectl -n argocd describe application root +kubectl -n argocd logs deploy/argocd-repo-server +kubectl -n argocd logs statefulset/argocd-application-controller +``` + +## Manifest source bridge + +Argo's `Application.source` accepts git, Helm, or OCI — there is no filesystem source, so +Argo cannot read `/usr/share/bluefin/cluster/` directly. `bluefin-cluster-repo.service` +initialises a bare repository at `/var/lib/bluefin/cluster.git` from that tree on first +boot, and the `15-gitd/` git-daemon serves it over the cluster network. No external git +host is contacted, so the manifest tree stays available with the network unplugged. + +Updating the tree means updating the image: a new DDI re-seeds +`/usr/share/bluefin/cluster/`, and `bluefin-cluster-repo.service` refreshes the bare repo. + +## Troubleshooting + +- **Extension not merged**: `systemd-sysext status`. Verify a + `/var/lib/extensions/kubernetes-.raw` exists; if not, the sysupdate fetch is + the failure, not the merge. +- **Merged but nothing runs**: the merge makes unit files appear under `/usr` but does not + by itself make systemd notice them. Run `systemctl daemon-reload` and re-check. +- **`kubeadm-init.service` skipped, cluster absent**: `/etc/kubernetes/admin.conf` already + exists from a previous partial run. Inspect it before removing it; `kubeadm reset` is + the supported way back to a clean state. +- **Node stays `NotReady`**: CNI is not up. `kubectl -n kube-system get pods -l k8s-app=cilium` + and `journalctl -u bluefin-cluster-bootstrap -e`. +- **Pods `Pending` with a `NoSchedule` taint**: same cause; the taint clears when Cilium + reports the node `Ready`. +- **kubelet serving certificate never issued**: kube-controller-manager does not + auto-approve serving CSRs. `kubelet-csr-approver` is mandatory, not optional. Check + `kubectl get csr` for `Pending` entries and the approver's `--provider-regex`. +- **Root Application will not sync**: check the git-daemon first — + `kubectl -n argocd logs deploy/gitd` — then the bare repo at + `/var/lib/bluefin/cluster.git` on the host. +- **Bootstrap unit failed**: `journalctl -u bluefin-cluster-bootstrap -e`. The unit is + ordered `After=`/`Requires=` `kubeadm-init.service`, so an apiserver that never came up + surfaces as a `kubeadm-init` failure first. + +## Runtime testing + +```bash +just show-me-the-future # QEMU installer smoke test through to multi-user +``` + +Acceptance on a booted host: containerd active, `kubectl get nodes` reports `Ready`, the +three Argo Core pods reach `Running`, and the root Application syncs. + +Workload images are still pulled from public registries, so "fully automated first boot" +currently means "with network present". Image pre-seeding into containerd is tracked +separately. + +## See also + +- [kubernetes-sysext.md](kubernetes-sysext.md) +- [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md) — OTA trust model. +- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). +- `systemd-sysext(8)` diff --git a/docs/skills/kubernetes-sysext.md b/docs/skills/kubernetes-sysext.md new file mode 100644 index 00000000..153b4a4f --- /dev/null +++ b/docs/skills/kubernetes-sysext.md @@ -0,0 +1,124 @@ +--- +name: kubernetes-sysext +description: Build, ship, and version the upstream Kubernetes systemd-sysext (kubeadm, kubelet, kubectl, CNI plugins) for Bluefin Server. +metadata: + type: how-to + status: stable + last_updated: "2026-09-19" + context7-sources: + - /systemd/systemd +--- +# Kubernetes systemd-sysext + +Use this skill when working on the Kubernetes `systemd-sysext` extension shipped as an +optional overlay for Bluefin Server. + +## When to Use + +- Bumping the pinned Kubernetes or CNI plugin version and its SHA256 pins. +- Modifying the sysext image contents (`elements/kubernetes/`, `files/kubernetes/sysext/`). +- Changing the `kubeadm` cluster configuration (`files/kubernetes/kubeadm.yaml`). +- Adding or changing the sysupdate transfer definition + (`files/os/sysupdate.kubernetes.d/70-kubernetes.transfer`). +- Debugging why a host cannot pull or merge the Kubernetes sysext. + +## When NOT to Use + +- Bringing up, troubleshooting, or operating a live cluster — use + [kubernetes-sysext-ops.md](kubernetes-sysext-ops.md). +- Seed and Argo-owned manifest content — that tree lives under `files/cluster/`. +- General OS image composition questions (use `ddi-installer.md` or `avoid-over-engineering.md`). +- systemd-sysupdate signature verification (use `systemd-sysupdate-verification.md`). + +## Architecture + +Bluefin Server's base DDI image includes bash for login and bring-up, while heavy +developer and debug tools live in sysexts or system containers. Kubernetes is not baked +into the OS stack. It is delivered as a `systemd-sysext` EROFS image that overlays `/usr` +at runtime. + +Design choices: + +- **Upstream binaries, no distribution.** `elements/kubernetes/` imports `kubeadm`, + `kubelet`, and `kubectl` from `dl.k8s.io`, each pinned to a published SHA256. There is + no embedded control plane and no vendor-specific control-plane wrapper. +- **Re-baked in-repo.** The element re-bakes the `flatcar/sysext-bakery` `kubernetes.sysext` + recipe under BuildStream rather than consuming the published artifact, so release signing + and provenance stay in this repository. The output layout is wire-compatible with + `extensions.flatcar.org`, so an operator can repoint the sysupdate source upstream + without changing anything else. +- **containerd is a separate sysext.** The runtime ships as Flatcar's + `containerd-flatcar.raw`; it is never part of this image and never part of the base DDI. +- **Flatcar sysext pattern.** The extension uses `ID=_any` in its release metadata so it + merges on any host image. +- **Minor-pinned OTA track.** `70-kubernetes.transfer` tracks a minor-pinned track because + Kubernetes does not support unattended cross-minor in-place upgrades. Crossing a minor + is a deliberate operator action, not an automatic one. + +## Repository Layout + +| Path | Purpose | +|------|---------| +| `include/kubernetes.yml` | **Single source of truth for the version axis** (`%{k8s-version}`, `%{cni-version}`). | +| `elements/kubernetes/` | Pins the upstream `kubeadm`, `kubelet`, `kubectl` and CNI plugin SHA256s; release URLs are derived from `include/kubernetes.yml`. | +| `elements/oci/kubernetes-sysext.bst` | Builds the EROFS sysext image (`kubernetes-.raw`). | +| `files/kubernetes/sysext/extension-release.kubernetes` | Static sysext identity (`NAME=kubernetes`, `ID=_any`); `VERSION_ID=`/`ARCHITECTURE=` are appended at build time. | +| `files/kubernetes/kubeadm.yaml` | `kubeadm` cluster configuration, staged to `/usr/share/bluefin/kubeadm.yaml`. | +| `files/cluster/seed/` | Seed phases applied before Argo owns the cluster; staged to `/usr/share/bluefin/seed/`. | +| `files/cluster/manifests/` | Argo-owned workload tree; staged to `/usr/share/bluefin/cluster/`. | +| `files/os/sysupdate.kubernetes.d/70-kubernetes.transfer` | sysupdate transfer track for the Kubernetes sysext component. | +| `Justfile` | `build-sysext` / `export-sysext` targets. | +| `.github/workflows/build.yml` | Builds, signs, and publishes sysext assets. | +| `.github/scripts/check-kubernetes-version.py` | Fails closed if any consumer restates the version instead of deriving it. | + +## Image Contents + +Mirroring the upstream bakery recipe: + +- `/usr/bin/{kubectl,kubeadm,kubelet}` +- `/usr/local/bin/cni/` from the `containernetworking/plugins` release tarball +- `/usr/local/share/kubernetes-version`, `/usr/local/share/kubernetes-cni-version` +- `/usr/lib/extension-release.d/extension-release.kubernetes` + +No certificate or key material ships in any image. kubelet client certificates come from +`kubeadm`'s native TLS bootstrap; kubelet serving certificates come from +`serverTLSBootstrap` plus `kubelet-csr-approver`; everything else is issued by +cert-manager in-cluster. + +## Build Outputs + +`elements/oci/kubernetes-sysext.bst` produces: + +- `kubernetes-.raw` — uncompressed EROFS sysext image. +- `kubernetes-.raw.zst` — zstd-compressed release asset. +- `SHA256SUMS` — checksum manifest for the compressed asset. + +## Bumping the version + +1. Edit `include/kubernetes.yml` — it is the only place a version literal may appear. +2. Replace every affected SHA256 in `elements/kubernetes/` with the value published + alongside the upstream artifact. Never invent a checksum. +3. If the Kubernetes **minor** changed, update the minor-pinned track name in + `files/os/sysupdate.kubernetes.d/70-kubernetes.transfer`. +4. Run `just validate`; `check-kubernetes-version.py` fails closed when a consumer + restates the version instead of deriving it from `include/kubernetes.yml`. + +## Justfile Commands + +```bash +just validate # resolve the element graph and check the version axis +just build-sysext # build oci/kubernetes-sysext.bst +just export-sysext # export sysext artifacts to dist/sysext/ +``` + +## Operations and runtime testing + +For bringing a cluster up on a host, the seed phases, and troubleshooting guidance, +see [kubernetes-sysext-ops.md](kubernetes-sysext-ops.md). + +## See also + +- [kubernetes-sysext-ops.md](kubernetes-sysext-ops.md) +- [systemd-sysext-extensions.md](systemd-sysext-extensions.md) — extension identity and loading. +- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). +- `systemd-sysext(8)`, `systemd-confext(8)` diff --git a/docs/skills/system-containers.md b/docs/skills/system-containers.md index 7de08566..e61207d1 100644 --- a/docs/skills/system-containers.md +++ b/docs/skills/system-containers.md @@ -31,7 +31,7 @@ runtime UI. ## When NOT to Use - Adding a distroless OCI runtime image — the server repo does not build those. -- Kubernetes workload or cluster questions — use `k0s-sysext.md`. +- Kubernetes workload or cluster questions — use `kubernetes-sysext.md`. ## Lifecycle diff --git a/docs/skills/systemd-sysext-extensions.md b/docs/skills/systemd-sysext-extensions.md index c65ba5ac..6cd1b0af 100644 --- a/docs/skills/systemd-sysext-extensions.md +++ b/docs/skills/systemd-sysext-extensions.md @@ -52,7 +52,7 @@ must match the value baked into `elements/bluefin-server/os-release-flatcar.bst` ## Adding an extension from the Flatcar Bakery -The k0s sysext is the built-in example, but any Flatcar-compatible extension can +The Kubernetes sysext is the built-in example, but any Flatcar-compatible extension can be layered the same way. ```bash @@ -108,6 +108,6 @@ systemd-sysext merge --force ## See also -- [k0s-sysext.md](k0s-sysext.md) for the built-in Kubernetes extension +- [kubernetes-sysext.md](kubernetes-sysext.md) for the built-in Kubernetes extension - [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). - `systemd-sysext(8)`, `systemd-confext(8)` diff --git a/docs/skills/systemd-sysupdate-verification.md b/docs/skills/systemd-sysupdate-verification.md index a7048efe..6145c6ff 100644 --- a/docs/skills/systemd-sysupdate-verification.md +++ b/docs/skills/systemd-sysupdate-verification.md @@ -17,7 +17,7 @@ the OS image. ## When to Use - Modifying `files/os/sysupdate.d/*.transfer` or - `files/os/sysupdate.k0s.d/*.transfer` update definitions. + `files/os/sysupdate.kubernetes.d/*.transfer` update definitions. - Rotating or replacing the release signing key. - Debugging `systemd-sysupdate` failures related to `SHA256SUMS.gpg` verification. @@ -50,8 +50,8 @@ Key facts from `sysupdate.d(5)`: The current tree uses a single root/ESP slot and a single signed manifest flow for OTA delivery. Root and UKI transfers live in `sysupdate.d`; the optional -k0s sysext lives in the `k0s` component directory and is selected with -`systemd-sysupdate --component=k0s update`. Future work on dual-slot root +Kubernetes sysext lives in the `kubernetes` component directory and is selected +with `systemd-sysupdate --component=kubernetes update`. Future work on dual-slot root partitions, dual UKIs, and broader rollback strategies is tracked in [architecture-roadmap.md](architecture-roadmap.md). @@ -66,7 +66,7 @@ partitions, dual UKIs, and broader rollback strategies is tracked in `dist/release/SHA256SUMS.gpg`, and uploads `dist/release/*` to the GitHub Release. - `files/os/sysupdate.d/*.transfer` and - `files/os/sysupdate.k0s.d/*.transfer` — each transfer points its static + `files/os/sysupdate.kubernetes.d/*.transfer` — each transfer points its static `Path=` at `https://github.com/projectbluefin/server/releases/latest/download/` so all transfers share the same signed manifest. @@ -127,7 +127,7 @@ partitions, dual UKIs, and broader rollback strategies is tracked in ## Verification - [ ] `files/os/sysupdate.d/*.transfer` and - `files/os/sysupdate.k0s.d/*.transfer` do not contain `Verify=no`. + `files/os/sysupdate.kubernetes.d/*.transfer` do not contain `Verify=no`. - [ ] `elements/bluefin-server/os-stack.bst` includes `bluefin-server/os-sysupdate-keys.bst`. - [ ] `files/os/sysupdate-keys/import-pubring.gpg` exists and contains the @@ -137,7 +137,7 @@ partitions, dual UKIs, and broader rollback strategies is tracked in manifest, producing `dist/release/SHA256SUMS.gpg`. - [ ] CI uploads `dist/release/*` to the GitHub Release. - [ ] Every transfer in `files/os/sysupdate.d/*.transfer` and - `files/os/sysupdate.k0s.d/*.transfer` uses a static `Path=` with no + `files/os/sysupdate.kubernetes.d/*.transfer` uses a static `Path=` with no `@v` placeholder. - [ ] Every transfer uses `@v` only inside `MatchPattern=`. diff --git a/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md b/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md new file mode 100644 index 00000000..5b26dc37 --- /dev/null +++ b/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md @@ -0,0 +1,373 @@ +# Design Spec: Upstream Kubernetes Cutover with Argo CD Core GitOps + +## Context & Purpose + +Bluefin Server currently delivers Kubernetes as k0s: a single static binary carrying an +embedded control plane, an embedded container runtime, and a proprietary manifest deployer +that reconciles `/var/lib/k0s/manifests/`. + +This specification defines the complete removal of k0s and its replacement with upstream +Kubernetes (`kubeadm`, `kubelet`, `kubectl`) delivered as a `systemd-sysext`, Cilium as CNI, +containerd sourced from Flatcar's `/usr`, and **Argo CD Core** as the reconciler for all +in-cluster workload. + +The decision to run upstream Kubernetes is a product decision and is not re-litigated here. +This document records how the cutover is executed and which properties must be rebuilt. + +--- + +## What k0s Provided That Must Be Rebuilt + +k0s's stack applier supplied four properties for free. Upstream Kubernetes ships no addon +manager, so each must be replaced explicitly: + +| Property | k0s mechanism | Replacement | +|---|---|---| +| Apiserver readiness | applier runs inside the controller process | `kubeadm-init.service` ordering edge | +| Ordering | alphabetical directory scan with retry | explicit seed phases, then Argo sync waves | +| Drift correction | continuous reconcile loop | Argo `selfHeal: true` | +| Pruning | label-tracked stack membership | Argo `prune: true` | + +Upstream's only in-tree mechanisms are static pods (Pod kind only, node-local), +`kubeadm init phase addon` (hardcoded to CoreDNS and kube-proxy), and `kubectl` with +kustomize. The historical `kube-addon-manager` was removed. Every distro that offers a +manifest directory implements it privately; leaving k0s means leaving that layer, not +losing a k0s-specific feature. + +--- + +## Architecture + +### Delivery layers + +| Layer | Mechanism | Contents | +|---|---|---| +| Base DDI | BuildStream, Flatcar `/usr` | systemd, networkd, sysupdate, `crictl` | +| Runtime sysext | Flatcar `containerd-flatcar.raw` | containerd, runc, containerd shims | +| Kubernetes sysext | BuildStream re-bake of the bakery recipe | `kubeadm`, `kubelet`, `kubectl`, CNI plugins | +| Seed | `kubectl apply -k`, four directories | Cilium, Argo CD Core, git-daemon, root Application | +| Workload | Argo CD Core | everything else, including Argo itself | + +### First-boot unit sequence + +``` +systemd-sysext.service merge containerd + kubernetes sysexts into /usr + -> daemon-reload (see "Sysext activation" below) +bluefin-cluster-repo.service init /var/lib/bluefin/cluster.git from /usr/share/bluefin/cluster +kubeadm-init.service kubeadm init --config, skipping the kube-proxy addon +bluefin-cluster-bootstrap.service + apply 00-cilium/ CNI; nothing schedules before this + wait node Ready + apply 10-argocd-core/ Argo CRDs, RBAC, redis, repo-server, application-controller + apply 15-gitd/ git-daemon serving /var/lib/bluefin/cluster.git + apply 20-root-app/ one Application; Argo owns everything downstream +``` + +### Seed contents + +``` +/usr/share/bluefin/seed/ + 00-cilium/ CNI, kube-proxy replacement + 10-argocd-core/ upstream core-install.yaml, applicationset-controller patched out + 15-gitd/ git-daemon Deployment + Service, hostPath /var/lib/bluefin/cluster.git + 20-root-app/ AppProject + root Application +``` + +The seed is **append-only by construction**. It is never pruned. See "Closed decision: +pruning" below for why this matters. + +### Argo-owned tree + +``` +/usr/share/bluefin/cluster/ + cert-manager + SelfSigned -> CA Issuer + kubelet-csr-approver + KubeFlex CRDs + operator + PostgreSQL + KubeStellar core controller-manager + KubeStellar console + kiosk proxy + ControlPlane CR + local-cluster registration + argocd-core self-management +``` + +--- + +## Detailed Component Specifications + +### 1. Kubernetes sysext (`elements/oci/kubernetes-sysext.bst`) + +Re-bakes the `flatcar/sysext-bakery` `kubernetes.sysext` recipe under BuildStream rather +than consuming the published artifact, so release signing and provenance remain in-repo. +The output layout stays wire-compatible with `extensions.flatcar.org`, so an operator can +repoint the sysupdate source at upstream without changing anything else. + +Contents, mirroring the upstream recipe: + +- `/usr/bin/{kubectl,kubeadm,kubelet}` from `dl.k8s.io//bin/linux//`, each + pinned and verified against its published `.sha256`. +- `/usr/local/bin/cni/` from the `containernetworking/plugins` release tarball. +- `/usr/local/share/kubernetes-version`, `/usr/local/share/kubernetes-cni-version`. +- `/usr/lib/extension-release.d/extension-release.kubernetes`. + +`include/kubernetes.yml` replaces `include/k0s.yml` as the single version source. +`.github/scripts/check-kubernetes-version.py` replaces the k0s checker and fails closed +when a consumer restates the version instead of deriving it. + +The sysupdate transfer tracks a **minor-pinned** track (`kubernetes-v1.NN.conf`) because +Kubernetes does not support unattended cross-minor in-place upgrades. + +### 2. containerd + +Flatcar's `/usr` tarball already contains the runtime as a sysext blob. Verified against +the Flatcar 4593.2.5 image contents listing: + +``` +./usr/share/flatcar/sysext/containerd-flatcar.raw 26,001,408 +./usr/share/flatcar/sysext/docker-flatcar.raw 57,118,720 +./usr/bin/crictl 31,488,432 +``` + +No `containerd`, `dockerd`, `runc` or `ctr` binaries exist in `/usr/bin`; only `crictl`. +Acquisition is therefore free and `elements/flatcar/container-runtime-reference-sysexts.bst` +stays reference-only. + +`elements/flatcar/flatcar-usr.bst` gains explicit removals in the existing no-wildcard +style: + +- `usr/share/flatcar/sysext/docker-flatcar.raw` +- `usr/share/flatcar/etc/extensions/docker-flatcar.raw` + +Docker is not shipped: it is a second unused runtime and 57 MiB charged against the 1 GiB +verity budget that the verity `/usr` work introduces. + +### 3. Sysext activation + +`flatcar-usr.bst` removes Flatcar's `ensure-sysext.service`, `download_sysext`, and the +`sysinit.target.wants` enablement symlink. Bluefin does not run Flatcar's `/etc` seeding +either, so the `usr/share/flatcar/etc/extensions/` symlinks are inert. + +Activation therefore requires an explicit replacement. `systemd-sysext.service` is present +and enabled in Flatcar's `/usr` and scans the standard extension search paths. + +**Must verify before implementation:** whether `systemd-sysext.service` performs a +`daemon-reload` after merging at the FSDK systemd version. A merge makes unit files appear +under `/usr`; it does not by itself make systemd notice a new `containerd.service`. Both +the bakery (`RELOAD_SERVICES_ON_MERGE="true"`) and this repo's own `k0s-first-boot.service` +(merge, then `daemon-reload`, then `enable --now`) reload explicitly. If the FSDK unit does +not, carry a drop-in that does. Failing to confirm this reproduces a silent failure mode: +image merged, no runtime running. + +`kubeadm-init.service` carries an explicit ordering edge on the merge completing. + +### 4. Cilium + +Cilium is applied by the seed, not by Argo: Argo's pods require pod networking, and a node +without CNI stays `NotReady` and carries a `NoSchedule` taint. + +`kubeadm init` runs with `--skip-phases=addon/kube-proxy`; Cilium provides kube-proxy +replacement. + +### 5. Argo CD Core (`10-argocd-core/`) + +Upstream `manifests/core-install.yaml`, not a hand-vendored subset. Verified workload +delta against `manifests/install.yaml`: + +| Workload | `install.yaml` | `core-install.yaml` | +|---|---|---| +| `argocd-application-controller` | yes | yes | +| `argocd-repo-server` | yes | yes | +| `argocd-redis` | yes | yes | +| `argocd-applicationset-controller` | yes | yes | +| `argocd-server` | yes | no | +| `argocd-dex-server` | yes | no | +| `argocd-notifications-controller` | yes | no | + +Core additionally drops one of the three `redis.server` consumers along with +`argocd-server`. Two remain: `repo-server` and `application-controller`. Redis is an +in-memory cache with no PersistentVolumeClaim anywhere in the manifest; it caches rendered +manifests and the live resource tree. There is no supported redis-free Argo CD, so redis is +the floor, not a choice. + +`argocd-applicationset-controller` is removed by kustomize patch: the deployment generates +Applications from templates and this design has exactly one root Application. Its CRD is +left in place, harmlessly. + +Two residual `argocd-server` references survive in Core as NetworkPolicy `podSelector` +rules admitting traffic from a pod that never exists. Inert; left untouched to keep the +upstream manifest unmodified. + +Resulting footprint: three pods — `application-controller`, `repo-server`, `redis`. + +No web UI and no `argocd` CLI, which speaks gRPC to `argocd-server`. Applications are +managed as plain Kubernetes custom resources. This removes a second authenticated web +surface from the appliance; the KubeStellar console remains the only dashboard. + +### 6. Manifest source bridge (`15-gitd/`) + +Argo's `Application.source` accepts git, Helm, or OCI. There is no filesystem source, so +Argo cannot read `/usr/share/bluefin/cluster/` directly. This is an impedance mismatch +between how an image-based OS delivers files and how Argo consumes them. + +`bluefin-cluster-repo.service` initialises a bare repository at +`/var/lib/bluefin/cluster.git` from the seed tree on first boot. A minimal git-daemon +Deployment serves it over the cluster network with a hostPath mount, and the root +Application targets it. + +The git-daemon **must** live in the seed and **cannot** be an Argo manifest: the root +Application needs the repository to exist and be served before Argo can read anything. +Same class of circular dependency as Cilium. + +This is preferred over `repoURL: file://` against a hostPath-mounted bare repo. That +pattern is undocumented, repo-server runs non-root with its own volume layout, and the +failure would surface mid-implementation. + +Offline behaviour is preserved: no external git host is contacted. Manifests remain +available with the network unplugged, satisfying the offline-install constraint. + +### 7. Certificates + +No certificate or key material ships in any image. + +- kubelet **client** certificates: `kubeadm`'s native TLS bootstrap, with + `rotateCertificates: true`. +- kubelet **serving** certificates: `serverTLSBootstrap: true` plus `kubelet-csr-approver` + with a strict `--provider-regex` and bounded `--max-expiration-sec`. + kube-controller-manager does not auto-approve serving CSRs, so this component is + mandatory rather than optional. +- Console, kiosk and in-cluster TLS: cert-manager, SelfSigned bootstrap into a CA Issuer, + auto-renewed leaf certificates. + +This permanently replaces the build-time self-signed key currently baked into the k0s +sysext. + +--- + +## Closed decision: pruning + +The seed is applied with `kubectl apply --server-side --force-conflicts` and is **never +pruned**. Argo owns pruning for all downstream workload via `prune: true`. + +Both upstream kubectl pruning modes are alpha and unsuitable: + +- Allowlist `--prune`: "Alpha since Kubernetes v1.5", and per upstream documentation "still + in alpha due to usability, correctness and performance issues with its design". +- ApplySet `--prune --applyset`: "Alpha since Kubernetes v1.27", gated behind + `KUBECTL_APPLYSET=true`, carrying an explicit warning that "backwards incompatible + changes might be introduced in subsequent releases". + +The decisive constraint is ApplySet's namespace rule: "ApplySets spanning multiple +namespaces must use a cluster-scoped custom resource as the parent object." The seed spans +`kube-system`, `argocd`, and later `cert-manager` and the KubeStellar namespaces, so +pruning it would require defining a custom CRD purely to act as an ApplySet parent. + +`--prune` is additionally incompatible with `--server-side`; only the ApplySet path is +server-side-apply compatible. + +Append-only seed plus Argo-owned reconciliation avoids the entire question. This decision +is closed and should not be reopened without upstream promoting ApplySet to beta. + +--- + +## Installer + +`systemd-sysinstall` and `systemd-repart` have no Kubernetes awareness and will not gain +any; AGENTS.md scopes the installer to `systemd-sysinstall`-native, `systemd-repart`-based +behaviour. Their native capability is file placement via `CopyFiles=`, which is how the k0s +sysext is seeded today. + +Manifests therefore ship inside the DDI at `/usr/share/bluefin/`, with the OEM partition +available for per-site overrides once the Flatcar partition layout lands. Defaults stay +immutable inside the verity-sealed image; site-specific material lives on OEM alongside +`systemd-creds`. + +--- + +## Purge & Replacement Matrix + +| k0s component | Action | Replacement | +|---|---|---| +| `include/k0s.yml` | Deleted | `include/kubernetes.yml` | +| `elements/k0s/k0s-bin.bst` | Deleted | `elements/kubernetes/` binary imports | +| `elements/oci/k0s-sysext.bst` | Deleted | `elements/oci/kubernetes-sysext.bst` | +| `elements/bluefin-server/os-k0s-first-boot.bst` | Deleted | `os-cluster-bootstrap.bst` | +| `elements/bluefin-server/os-k0s-sysupdate.bst` | Deleted | `os-kubernetes-sysupdate.bst` | +| `files/k0s/sysext/k0scontroller.service` | Deleted | `kubeadm-init.service` + kubelet unit | +| `files/k0s/sysext/k0s-manifests.conf` | Deleted | none; manifests applied from `/usr` | +| `files/k0s/k0s.yaml` | Deleted | `/usr/share/bluefin/kubeadm.yaml` | +| `files/k0s/manifests/argocd/install.yaml` | Deleted | upstream `core-install.yaml` | +| `files/k0s/manifests/kubestellar/01-*-github-oauth.yaml` | Deleted | none; no baked secrets | +| `files/k0s/manifests/kubestellar/*` | Moved | `/usr/share/bluefin/cluster/`, digest-pinned | +| `files/k0s/kubeflex/generate-postgres-secret.sh` | Deleted | in-cluster generated Secret | +| `files/k0s/kiosk/{cert,key}.pem` | Deleted | cert-manager issued | +| `files/k0s/kiosk/{nginx.conf,kiosk-gate.js,kiosk-gate.css}` | Moved | ConfigMap; hostPath removed | +| `files/os/systemd/system/k0s-first-boot.service` | Deleted | `bluefin-cluster-bootstrap.service` | +| `files/os/systemd/system/k0s-first-boot-fetch.service` | Deleted | sysupdate handles fetch | +| `files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset` | Deleted | equivalent preset | +| `files/os/sysupdate.k0s.d/70-k0s.transfer` | Deleted | `70-kubernetes.transfer` | +| `files/installer/repart.d/30-var.conf` k0s `CopyFiles` | Deleted | sysext seeding | +| `.github/scripts/check-k0s-version.py` | Deleted | `check-kubernetes-version.py` | +| `tests/unit/test_k0s_first_boot.py` | Deleted | `test_cluster_bootstrap.py` | +| `tests/unit/test_k0s_manifests.py` | Deleted | `test_cluster_manifests.py` | +| `tests/unit/test_k0s_version.py` | Deleted | `test_kubernetes_version.py` | +| `docs/skills/k0s-sysext.md` | Deleted | `docs/skills/kubernetes-sysext.md` | +| `docs/skills/k0s-sysext-ops.md` | Deleted | `docs/skills/kubernetes-sysext-ops.md` | +| `Justfile` `k8s` target | Updated | targets `kubeadm`/`kubectl` | +| `AGENTS.md` hard rule 4 | Updated | see ADR scope below | +| `docs/skills/index.md`, `factory-integration.md`, `CONTEXT.md`, `NOTES.md` | Updated | k0s references purged | + +Acceptance gate: `grep -ri k0s` matches only the ADR and the changelog. + +### ADR scope + +Hard rule 4 currently reads: *"Deliver k0s as an optional `systemd-sysext`; never bundle +Kubernetes or container runtimes into the base OS DDI."* + +Only the k0s mandate changes, to deliver **Kubernetes** as an optional sysext. The +container-runtime clause is untouched and remains satisfied: containerd ships as a sysext, +not as base-DDI content. + +### Superseded pull requests + +| PR | Disposition | Reason | +|---|---|---| +| #178 | Close as superseded | hand-vendors 422 lines into a deleted directory; ships no `argoproj.io` CRDs; dead `ARGOCD_GIT_SHARED_DIR` emptyDir; port 443 named `https` fronting plaintext | +| #81 | Close as superseded | pins an EOL `argocd-server` image that Core does not ship | +| #169 | Close as superseded | superseded by #80 | +| #177 | Close against `core-install.yaml` | Core supplies the complete, correctly wired stack | + +--- + +## Verification & Testing Strategy + +1. **Element graph** — `just validate` resolves with zero dangling k0s nodes. +2. **Unit tests** + - `test_kubernetes_version.py`: version single-source invariants. + - `test_flatcar_usr.py`: extended to assert both Docker paths removed and containerd + present exactly once. + - `test_cluster_manifests.py`: every seed and cluster overlay builds under + `kubectl kustomize` offline, and every image carries a digest. + - Secret-material scan across `files/` and `elements/` returns nothing loadable. +3. **Sysext build** — `just build-sysext && just export-sysext` produce the Kubernetes + sysext and `SHA256SUMS`. +4. **Boot proof** — `just show-me-the-future` reaches multi-user; containerd active; + `kubectl get nodes` reports `Ready`; the three Argo Core pods reach `Running`; the root + Application syncs; the KubeStellar console serves live cluster state behind a + cert-manager certificate of under 100 days validity. +5. **Manifest-source proof** — with no external git host reachable, the root Application + syncs from `/var/lib/bluefin/cluster.git` via the git-daemon bridge. This proves the + manifest tree needs no external host. It does **not** prove a fully offline boot: + workload images are still pulled from public registries, so a complete + network-unavailable boot is gated on open item 2 below and is not an acceptance + criterion for this spec. + +--- + +## Open items requiring verification before implementation + +1. `systemd-sysext.service` post-merge `daemon-reload` behaviour at the FSDK systemd + version; carry a reload drop-in if absent. +2. Image pre-seeding into containerd. Every workload manifest currently pulls from + `quay.io`, `ghcr.io` and `docker.io`. Until images are pre-seeded, "fully automated + first boot" means "with network present". This is a pre-existing condition, not a + regression, and is tracked separately. diff --git a/elements/bluefin-server/os-cluster-bootstrap.bst b/elements/bluefin-server/os-cluster-bootstrap.bst new file mode 100644 index 00000000..09ea60d9 --- /dev/null +++ b/elements/bluefin-server/os-cluster-bootstrap.bst @@ -0,0 +1,49 @@ +kind: manual +description: | + Install the host units that bring up the Kubernetes control plane and seed + the cluster, plus the configuration they read. + + Units: kubelet.service (+ the kubeadm drop-in), kubeadm-init.service, + bluefin-cluster-repo.service and bluefin-cluster-bootstrap.service, together + with the systemd-sysext.service drop-in that reloads systemd after a merge so + sysext-delivered units become visible. + + Also stages /usr/share/bluefin/kubeadm.yaml and the + /usr/libexec/bluefin-cluster-repo helper. + +build-depends: + - base/base-stack.bst + +sources: + - kind: local + path: files/os/systemd/system + directory: units-src + - kind: local + path: files/os/systemd/systemd-sysext.service.d + directory: sysext-dropin-src + - kind: local + path: files/kubernetes/kubeadm.yaml + directory: kubeadm-src + - kind: local + path: files/os/libexec + directory: libexec-src + +config: + install-commands: + - | + set -euo pipefail + OUT="%{install-root}" + + mkdir -p "${OUT}/usr/lib/systemd/system" + cp -a units-src/. "${OUT}/usr/lib/systemd/system/" + + mkdir -p "${OUT}/usr/lib/systemd/system/systemd-sysext.service.d" + cp -a sysext-dropin-src/. "${OUT}/usr/lib/systemd/system/systemd-sysext.service.d/" + + mkdir -p "${OUT}/usr/share/bluefin" + cp -a kubeadm-src/kubeadm.yaml "${OUT}/usr/share/bluefin/kubeadm.yaml" + chmod 0644 "${OUT}/usr/share/bluefin/kubeadm.yaml" + + mkdir -p "${OUT}/usr/libexec" + cp -a libexec-src/bluefin-cluster-repo "${OUT}/usr/libexec/bluefin-cluster-repo" + chmod 0755 "${OUT}/usr/libexec/bluefin-cluster-repo" diff --git a/elements/bluefin-server/os-cluster-manifests.bst b/elements/bluefin-server/os-cluster-manifests.bst new file mode 100644 index 00000000..f1c8abc1 --- /dev/null +++ b/elements/bluefin-server/os-cluster-manifests.bst @@ -0,0 +1,65 @@ +kind: manual +description: | + Stage the cluster bootstrap seed and the Argo CD-owned manifest tree. + + Output: + /usr/share/bluefin/seed/ four kustomize phases applied by + bluefin-cluster-bootstrap.service + /usr/share/bluefin/cluster/ the tree bluefin-cluster-repo.service commits + to /var/lib/bluefin/cluster.git, served to Argo + by the in-cluster git-daemon + + Two upstream manifests are fetched and sha256-pinned here rather than vendored + into the repository. Both are large machine-generated files that no reviewer + reads on a bump, and a stale hand-copy silently drops CRDs and RBAC. Their + kustomization.yaml lives in files/cluster/, so the customisation is reviewable + even though the payload is not. + +build-depends: + # Base runtime providing /bin/sh, install and cp in the sandbox. + - base/base-stack.bst + +sources: + - kind: local + path: files/cluster/seed + directory: seed-src + - kind: local + path: files/cluster/manifests + directory: cluster-src + + # Argo CD Core. core-install.yaml, not install.yaml: Core drops argocd-server, + # argocd-dex-server and argocd-notifications-controller, which removes a second + # authenticated web surface from the appliance. + - kind: remote + url: githubraw:argoproj/argo-cd/v3.5.3/manifests/core-install.yaml + filename: core-install.yaml + directory: argocd-src + ref: 1a87025d8eb2eae621653fd312fb9ca51df1b4b3b6992a030e3a9ef38e45c448 + + # cert-manager. Issues the appliance's certificates in-cluster, replacing the + # build-time self-signed key that used to be generated into the sysext. + - kind: remote + url: github:cert-manager/cert-manager/releases/download/v1.21.2/cert-manager.yaml + filename: cert-manager.yaml + directory: certmanager-src + ref: e03b668ec8675214af6b0a671699d088f2601fa3878e0dbe1b41d3feafd1879f + +config: + install-commands: + - | + set -euo pipefail + SEED="%{install-root}/usr/share/bluefin/seed" + CLUSTER="%{install-root}/usr/share/bluefin/cluster" + + mkdir -p "${SEED}" "${CLUSTER}" + + # Seed phases, applied in name order by bluefin-cluster-bootstrap.service. + cp -a seed-src/. "${SEED}/" + + # Argo CD-owned tree, committed to the bare repo on first boot. + cp -a cluster-src/. "${CLUSTER}/" + + # Drop the pinned upstream manifests next to the kustomization.yaml that + # patches them. Each is named by its kustomization's resources: entry. + install -m 0644 argocd-src/core-install.yaml "${SEED}/10-argocd-core/core-install.yaml" + install -m 0644 certmanager-src/cert-manager.yaml "${CLUSTER}/cert-manager/cert-manager.yaml" diff --git a/elements/bluefin-server/os-k0s-first-boot.bst b/elements/bluefin-server/os-k0s-first-boot.bst deleted file mode 100644 index 038a1d99..00000000 --- a/elements/bluefin-server/os-k0s-first-boot.bst +++ /dev/null @@ -1,9 +0,0 @@ -kind: import -description: Install the first-boot k0s sysext activation unit. - -sources: - - kind: local - path: files/os/systemd/system - -config: - target: /usr/lib/systemd/system diff --git a/elements/bluefin-server/os-k0s-sysupdate.bst b/elements/bluefin-server/os-k0s-sysupdate.bst deleted file mode 100644 index 45d11926..00000000 --- a/elements/bluefin-server/os-k0s-sysupdate.bst +++ /dev/null @@ -1,9 +0,0 @@ -kind: import -description: Install the k0s component systemd-sysupdate transfer definition. - -sources: - - kind: local - path: files/os/sysupdate.k0s.d - -config: - target: /usr/lib/sysupdate.k0s.d diff --git a/elements/bluefin-server/os-kubernetes-sysupdate.bst b/elements/bluefin-server/os-kubernetes-sysupdate.bst new file mode 100644 index 00000000..444b4b3d --- /dev/null +++ b/elements/bluefin-server/os-kubernetes-sysupdate.bst @@ -0,0 +1,9 @@ +kind: import +description: Install the Kubernetes component systemd-sysupdate transfer definition. + +sources: + - kind: local + path: files/os/sysupdate.kubernetes.d + +config: + target: /usr/lib/sysupdate.kubernetes.d diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst index 074634d0..df032c21 100644 --- a/elements/bluefin-server/os-stack.bst +++ b/elements/bluefin-server/os-stack.bst @@ -43,14 +43,21 @@ depends: # A/B System Update Configurations - bluefin-server/os-sysupdate.bst - # k0s component system update configuration - - bluefin-server/os-k0s-sysupdate.bst + # Kubernetes sysext component system update configuration + - bluefin-server/os-kubernetes-sysupdate.bst # Wired DHCP configuration for the installed OS - bluefin-server/os-networkd.bst - # Activate the optional k0s sysext after installation - - bluefin-server/os-k0s-first-boot.bst + # Control-plane bring-up and cluster seeding host units + - bluefin-server/os-cluster-bootstrap.bst + + # Seed phases and the Argo-owned manifest tree under /usr/share/bluefin + - bluefin-server/os-cluster-manifests.bst + + # git plumbing for bluefin-cluster-repo.service, which republishes + # /usr/share/bluefin/cluster as the bare repo Argo CD syncs from + - freedesktop-sdk.bst:components/git.bst # Public keyring for verifying SHA256SUMS manifests from GitHub Releases - bluefin-server/os-sysupdate-keys.bst diff --git a/elements/k0s/k0s-bin.bst b/elements/k0s/k0s-bin.bst deleted file mode 100644 index 63997322..00000000 --- a/elements/k0s/k0s-bin.bst +++ /dev/null @@ -1,35 +0,0 @@ -kind: manual -description: | - Import the upstream k0s release binary. - - Pinned to the latest stable release for x86_64. The release tag comes from - include/k0s.yml (%{k0s-upstream-tag}); only the SHA256 below is restated - here, taken from that release's sha256sums.txt asset. - -# %{k0s-upstream-tag} — the upstream release tag, URL-escaped. Single source -# of truth for the k0s version axis; see include/k0s.yml. -(@): include/k0s.yml - -build-depends: - # Base runtime providing /bin/sh and install in the sandbox. - - base/base-stack.bst - -variables: - # Do not try to strip the pre-built upstream binary. - strip-binaries: "" - -sources: - - kind: remote - url: github:k0sproject/k0s/releases/download/%{k0s-upstream-tag}/k0s-%{k0s-upstream-tag}-amd64 - ref: ca1e9e68107335846e8296777fce2ccd654284e6265b4b5d32c34ead872af98f - -config: - install-commands: - - | - if [ "%{arch}" != "x86_64" ]; then - echo "ERROR: elements/k0s/k0s-bin.bst pins the upstream k0s amd64 binary;" >&2 - echo " arch=%{arch} has no pinned source. Add an arch-conditional" >&2 - echo " source before building k0s for %{arch}." >&2 - exit 1 - fi - - install -D -m 0755 k0s-%{k0s-upstream-tag}-amd64 "%{install-root}/usr/bin/k0s" diff --git a/elements/kubernetes/cni-plugins.bst b/elements/kubernetes/cni-plugins.bst new file mode 100644 index 00000000..524da179 --- /dev/null +++ b/elements/kubernetes/cni-plugins.bst @@ -0,0 +1,59 @@ +kind: manual +description: | + Import the upstream containernetworking/plugins CNI binaries. + + The linux/amd64 release tarball for the tag named by include/kubernetes.yml + (%{cni-upstream-tag}); only the SHA256 below is restated here, taken from + that release's sha256 asset. + + Each plugin is installed by name rather than by wildcard, so an upstream + rename or removal fails the build instead of silently shipping a CNI set + that no longer matches what Cilium and the kubelet expect. + +# %{cni-upstream-tag} — the upstream CNI plugins release tag. Single source of +# truth for the Kubernetes version axis; see include/kubernetes.yml. +(@): include/kubernetes.yml + +build-depends: + # Base runtime providing /bin/sh and install in the sandbox. + - base/base-stack.bst + +variables: + # Do not try to strip the pre-built upstream binaries. + strip-binaries: "" + +sources: + - kind: tar + url: github:containernetworking/plugins/releases/download/%{cni-upstream-tag}/cni-plugins-linux-amd64-%{cni-upstream-tag}.tgz + ref: b98f74a0f8522f0a83867178729c1aa70f2158f90c45a2ca8fa791db1c76b303 + base-dir: "" + directory: cni-src + +config: + install-commands: + - | + if [ "%{arch}" != "x86_64" ]; then + echo "ERROR: elements/kubernetes/cni-plugins.bst pins the upstream" >&2 + echo " linux/amd64 CNI plugins tarball; arch=%{arch} has no" >&2 + echo " pinned source. Add an arch-conditional source before" >&2 + echo " building the CNI plugins for %{arch}." >&2 + exit 1 + fi + - install -D -m 0755 cni-src/bandwidth "%{install-root}/usr/local/bin/cni/bandwidth" + - install -D -m 0755 cni-src/bridge "%{install-root}/usr/local/bin/cni/bridge" + - install -D -m 0755 cni-src/dhcp "%{install-root}/usr/local/bin/cni/dhcp" + - install -D -m 0755 cni-src/dummy "%{install-root}/usr/local/bin/cni/dummy" + - install -D -m 0755 cni-src/firewall "%{install-root}/usr/local/bin/cni/firewall" + - install -D -m 0755 cni-src/host-device "%{install-root}/usr/local/bin/cni/host-device" + - install -D -m 0755 cni-src/host-local "%{install-root}/usr/local/bin/cni/host-local" + - install -D -m 0755 cni-src/ipvlan "%{install-root}/usr/local/bin/cni/ipvlan" + - install -D -m 0755 cni-src/loopback "%{install-root}/usr/local/bin/cni/loopback" + - install -D -m 0755 cni-src/macvlan "%{install-root}/usr/local/bin/cni/macvlan" + - install -D -m 0755 cni-src/portmap "%{install-root}/usr/local/bin/cni/portmap" + - install -D -m 0755 cni-src/ptp "%{install-root}/usr/local/bin/cni/ptp" + - install -D -m 0755 cni-src/sbr "%{install-root}/usr/local/bin/cni/sbr" + - install -D -m 0755 cni-src/static "%{install-root}/usr/local/bin/cni/static" + - install -D -m 0755 cni-src/tap "%{install-root}/usr/local/bin/cni/tap" + - install -D -m 0755 cni-src/tuning "%{install-root}/usr/local/bin/cni/tuning" + - install -D -m 0755 cni-src/vlan "%{install-root}/usr/local/bin/cni/vlan" + - install -D -m 0755 cni-src/vrf "%{install-root}/usr/local/bin/cni/vrf" diff --git a/elements/kubernetes/kubernetes-bin.bst b/elements/kubernetes/kubernetes-bin.bst new file mode 100644 index 00000000..3bb8d75b --- /dev/null +++ b/elements/kubernetes/kubernetes-bin.bst @@ -0,0 +1,46 @@ +kind: manual +description: | + Import the upstream Kubernetes node binaries. + + kubectl, kubeadm and kubelet for x86_64, pinned to the release named by + include/kubernetes.yml (%{k8s-upstream-tag}); only the SHA256 of each + binary is restated here, taken from the `.sha256` asset published beside it + on dl.k8s.io. + +# %{k8s-upstream-tag} — the upstream Kubernetes release directory on +# dl.k8s.io. Single source of truth for the Kubernetes version axis; see +# include/kubernetes.yml. +(@): include/kubernetes.yml + +build-depends: + # Base runtime providing /bin/sh and install in the sandbox. + - base/base-stack.bst + +variables: + # Do not try to strip the pre-built upstream binaries. + strip-binaries: "" + +sources: + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubectl + ref: 8b8f088da2dab964f853b38464033b1be15ede2839eca751482357c45abdd05a + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubeadm + ref: 10d054585907f2c8bb823c84f6a66b3bd2a474c3d795475f04e9a44f78650fe6 + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubelet + ref: cf2562db46c48ab84cc3791540778ee7fb1a0a008529adbb434723dd02ed8b4b + +config: + install-commands: + - | + if [ "%{arch}" != "x86_64" ]; then + echo "ERROR: elements/kubernetes/kubernetes-bin.bst pins the upstream" >&2 + echo " linux/amd64 Kubernetes binaries; arch=%{arch} has no pinned" >&2 + echo " source. Add an arch-conditional source before building" >&2 + echo " Kubernetes for %{arch}." >&2 + exit 1 + fi + - install -D -m 0755 kubectl "%{install-root}/usr/bin/kubectl" + - install -D -m 0755 kubeadm "%{install-root}/usr/bin/kubeadm" + - install -D -m 0755 kubelet "%{install-root}/usr/bin/kubelet" diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index ab95bc43..3295458e 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -57,10 +57,11 @@ build-depends: - filename: bluefin-server/os-rootfs.bst config: location: /target-root - # Uncompressed k0s sysext staged into the installer initrd for offline install - - filename: oci/k0s-sysext.bst + # Uncompressed Kubernetes sysext staged into the installer initrd for + # offline install; systemd-repart seeds it onto /var at install time. + - filename: oci/kubernetes-sysext.bst config: - location: /k0s + location: /kubernetes variables: installer-version: "%{release-version}" @@ -329,11 +330,11 @@ config: ln -sf system-install.target \ /layer/usr/lib/systemd/system/default.target - # ── Step 2a: Stage the offline k0s sysext before packing the initrd ─ + # ── Step 2a: Stage the offline Kubernetes sysext before packing cpio ─ - | set -eux - cp /k0s/k0s-*.raw /layer/k0s.raw - ls -lh /layer/k0s.raw + cp /kubernetes/kubernetes-*.raw /layer/kubernetes.raw + ls -lh /layer/kubernetes.raw # ── Step 3: Pack initrd cpio ───────────────────────────────────────── # find . from /layer gives paths like ./usr/bin/... which is exactly diff --git a/elements/oci/k0s-sysext.bst b/elements/oci/k0s-sysext.bst deleted file mode 100644 index 96e8d7d5..00000000 --- a/elements/oci/k0s-sysext.bst +++ /dev/null @@ -1,115 +0,0 @@ -kind: manual -description: | - Produce a systemd-sysext extension image for k0s. - - Output: k0s-.raw, k0s-.raw.zst, and SHA256SUMS. - Contains /usr/bin/k0s, k0scontroller.service, tmpfiles.d/k0s-manifests.conf, - and /usr/share/k0s/manifests/ (Argo CD and KubeStellar). - -(@): - - include/arch.yml - - include/k0s.yml - -build-depends: - - base/base-stack.bst - - freedesktop-sdk.bst:components/erofs-utils.bst - - freedesktop-sdk.bst:components/xz.bst - - freedesktop-sdk.bst:components/zstd.bst - - freedesktop-sdk.bst:components/openssl.bst - - filename: k0s/k0s-bin.bst - config: - location: / - -sources: - - kind: local - path: files/k0s/sysext - directory: sysext-src - - kind: local - path: files/k0s/manifests - directory: manifests-src - - kind: local - path: files/k0s/kiosk - directory: kiosk-src - - kind: local - path: files/k0s/kubeflex - directory: kubeflex-src - - kind: local - path: files/k0s/k0s.yaml - directory: k0s-config-src - - kind: local - path: files/os/issue.d/40-kubestellar.issue - directory: issue-src -variables: - strip-binaries: "" - -config: - install-commands: - - | - set -euo pipefail - OUT="%{install-root}" - FNAME="k0s-%{k0s-version}.raw" - - if [ "%{arch}" != "x86_64" ]; then - echo "ERROR: elements/oci/k0s-sysext.bst only supports x86_64 (got %{arch})" >&2 - exit 1 - fi - - mkdir -p sysext/usr/bin - mkdir -p sysext/usr/lib/systemd/system - mkdir -p sysext/usr/lib/extension-release.d - mkdir -p sysext/usr/lib/tmpfiles.d - mkdir -p sysext/usr/lib/issue.d - mkdir -p sysext/usr/share/k0s/manifests - mkdir -p sysext/usr/share/k0s/kiosk - mkdir -p "${OUT}" - - # Stage k0s binary - cp -a /usr/bin/k0s sysext/usr/bin/k0s - chmod 0755 sysext/usr/bin/k0s - - # Stage systemd service - cp -a sysext-src/k0scontroller.service sysext/usr/lib/systemd/system/ - - # Stage extension release metadata - cp -a sysext-src/extension-release.k0s sysext/usr/lib/extension-release.d/ - echo "VERSION_ID=%{k0s-version}" >> sysext/usr/lib/extension-release.d/extension-release.k0s - echo "ARCHITECTURE=%{systemd-arch}" >> sysext/usr/lib/extension-release.d/extension-release.k0s - - # Stage tmpfiles seeding configuration - cp -a sysext-src/k0s-manifests.conf sysext/usr/lib/tmpfiles.d/ - - # Stage console issue banner - cp -a issue-src/40-kubestellar.issue sysext/usr/lib/issue.d/ - - # Stage manifest stacks - cp -a manifests-src/* sysext/usr/share/k0s/manifests/ - # Stage k0s cluster configuration - cp -a k0s-config-src/k0s.yaml sysext/usr/share/k0s/ - - # Stage Console kiosk proxy assets. - cp -a kiosk-src/. sysext/usr/share/k0s/kiosk/ - - # Stage KubeStellar first-boot credential helper. - mkdir -p sysext/usr/share/k0s/kubeflex - cp -a kubeflex-src/. sysext/usr/share/k0s/kubeflex/ - chmod 0755 sysext/usr/share/k0s/kubeflex/generate-postgres-secret.sh - - # Generate self-signed TLS certificate for local kiosk proxy - openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ - -keyout sysext/usr/share/k0s/kiosk/key.pem \ - -out sysext/usr/share/k0s/kiosk/cert.pem \ - -subj "/CN=KubeStellar Console" \ - -addext "subjectAltName=DNS:localhost,DNS:*.local,IP:127.0.0.1,IP:10.0.2.15" - chmod 0600 sysext/usr/share/k0s/kiosk/key.pem - chmod 0644 sysext/usr/share/k0s/kiosk/cert.pem - - # Build EROFS image - mkfs.erofs -d0 "${OUT}/${FNAME}" sysext - - # Compress release asset - zstd -T0 -19 -q "${OUT}/${FNAME}" -o "${OUT}/${FNAME}.zst" - - ( - cd "${OUT}" - sha256sum --binary "${FNAME}.zst" > SHA256SUMS - ) diff --git a/elements/oci/kubernetes-sysext.bst b/elements/oci/kubernetes-sysext.bst new file mode 100644 index 00000000..3ca96a91 --- /dev/null +++ b/elements/oci/kubernetes-sysext.bst @@ -0,0 +1,94 @@ +kind: manual +description: | + Produce a systemd-sysext extension image for upstream Kubernetes. + + Output: kubernetes-.raw, kubernetes-.raw.zst, and + SHA256SUMS. Contains /usr/bin/{kubectl,kubeadm,kubelet}, the CNI plugins + under /usr/local/bin/cni/, the two version stamp files the upstream recipe + publishes, and the extension-release metadata. + + This is a re-bake of the flatcar/sysext-bakery `kubernetes.sysext` recipe + under BuildStream, so release signing and provenance stay in-repo. The + layout is deliberately wire-compatible with extensions.flatcar.org: an + operator can repoint the sysupdate source at upstream without changing + anything else. + + No certificate or key material is generated or shipped. Cluster TLS is + issued at runtime — kubelet client certs by kubeadm's TLS bootstrap, + kubelet serving certs by kubelet-csr-approver, everything else by + cert-manager. Nothing here is a manifest either; the seed and Argo-owned + trees are staged by elements/bluefin-server/os-cluster-manifests.bst. + +(@): + - include/arch.yml + - include/kubernetes.yml + +build-depends: + - base/base-stack.bst + - freedesktop-sdk.bst:components/erofs-utils.bst + - freedesktop-sdk.bst:components/xz.bst + - freedesktop-sdk.bst:components/zstd.bst + - filename: kubernetes/kubernetes-bin.bst + config: + location: / + - filename: kubernetes/cni-plugins.bst + config: + location: / + +sources: + - kind: local + path: files/kubernetes/sysext + directory: sysext-src + +variables: + strip-binaries: "" + +config: + install-commands: + - | + set -euo pipefail + OUT="%{install-root}" + FNAME="kubernetes-%{k8s-version}.raw" + + if [ "%{arch}" != "x86_64" ]; then + echo "ERROR: elements/oci/kubernetes-sysext.bst only supports x86_64 (got %{arch})" >&2 + exit 1 + fi + + mkdir -p sysext/usr/bin + mkdir -p sysext/usr/local/bin/cni + mkdir -p sysext/usr/local/share + mkdir -p sysext/usr/lib/extension-release.d + mkdir -p "${OUT}" + + # Stage the Kubernetes node binaries. + cp -a /usr/bin/kubectl sysext/usr/bin/kubectl + cp -a /usr/bin/kubeadm sysext/usr/bin/kubeadm + cp -a /usr/bin/kubelet sysext/usr/bin/kubelet + chmod 0755 sysext/usr/bin/kubectl + chmod 0755 sysext/usr/bin/kubeadm + chmod 0755 sysext/usr/bin/kubelet + + # Stage the CNI plugins where the upstream recipe publishes them. + cp -a /usr/local/bin/cni/. sysext/usr/local/bin/cni/ + + # Stage the version stamps the upstream recipe publishes, so an + # operator can read the delivered versions off a merged /usr. + echo "%{k8s-version}" > sysext/usr/local/share/kubernetes-version + echo "%{cni-version}" > sysext/usr/local/share/kubernetes-cni-version + + # Stage extension release metadata + cp -a sysext-src/extension-release.kubernetes sysext/usr/lib/extension-release.d/ + echo "VERSION_ID=%{k8s-version}" >> sysext/usr/lib/extension-release.d/extension-release.kubernetes + echo "ARCHITECTURE=%{systemd-arch}" >> sysext/usr/lib/extension-release.d/extension-release.kubernetes + + # Build EROFS image + mkfs.erofs -d0 "${OUT}/${FNAME}" sysext + + # Compress release asset + zstd -T0 -19 -q "${OUT}/${FNAME}" -o "${OUT}/${FNAME}.zst" + + ( + cd "${OUT}" + sha256sum --binary "${FNAME}.zst" > SHA256SUMS + ) diff --git a/files/cluster/manifests/cert-manager/kustomization.yaml b/files/cluster/manifests/cert-manager/kustomization.yaml new file mode 100644 index 00000000..15b3cec9 --- /dev/null +++ b/files/cluster/manifests/cert-manager/kustomization.yaml @@ -0,0 +1,30 @@ +# cert-manager 1.21.2. +# +# cert-manager.yaml is the upstream release manifest, fetched and sha256-pinned +# by elements/bluefin-server/os-cluster-manifests.bst: +# +# https://github.com/cert-manager/cert-manager/releases/download/v1.21.2/cert-manager.yaml +# sha256: e03b668ec8675214af6b0a671699d088f2601fa3878e0dbe1b41d3feafd1879f +# +# It is not vendored here: it is ~6k lines of CRD schema that no human reviews +# on a bump, and a stale hand-copy is worse than a pinned fetch. +# +# This is the earliest wave in the tree. Everything that asks for a certificate +# needs the webhook Available first, and Argo will not start the next wave until +# these Deployments report Healthy. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cert-manager.yaml + +commonAnnotations: + argocd.argoproj.io/sync-wave: "-10" + +images: + - name: quay.io/jetstack/cert-manager-controller + digest: sha256:70f532fd9cfde0b09d55687200942399d89838bc2d5d5b45152eb799a15912b8 + - name: quay.io/jetstack/cert-manager-webhook + digest: sha256:a60e2dac46dbb8a7f3df95c54ce941012f54c2fe022f0ee55aaa1ab40ed957ae + - name: quay.io/jetstack/cert-manager-cainjector + digest: sha256:c85268c64f2e0e76684bf5fe8906caff34b82523561c6affe0fae3546bd87562 diff --git a/files/cluster/manifests/console/auth-secret.yaml b/files/cluster/manifests/console/auth-secret.yaml new file mode 100644 index 00000000..67d61147 --- /dev/null +++ b/files/cluster/manifests/console/auth-secret.yaml @@ -0,0 +1,113 @@ +# Mints the console's JWT signing key inside the cluster, once. +# +# This is what replaced the shipped Secret that carried +# `jwt-secret: default-jwt-secret-...`. That value was identical on every +# Bluefin Server ever installed, which made every console session forgeable by +# anyone who had read this repository. It does not come back: the key is +# generated here, on this machine, on first sync. +# +# Same idempotence contract as the PostgreSQL password: if the Secret exists the +# Job leaves it alone, so already-issued sessions survive a re-sync. +# +# GitHub OAuth credentials are deliberately absent. They are site-specific, and +# the console reads them with optional: true, so an operator who wants OAuth +# creates kubestellar-console-github-oauth themselves. +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubestellar-console-auth + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubestellar-console-auth + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubestellar-console-auth"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubestellar-console-auth + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubestellar-console-auth +subjects: + - kind: ServiceAccount + name: kubestellar-console-auth + namespace: kubestellar-console +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kubestellar-console-auth + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" + argocd.argoproj.io/sync-options: Replace=true +spec: + backoffLimit: 20 + template: + metadata: + labels: + app.kubernetes.io/name: kubestellar-console-auth + spec: + restartPolicy: OnFailure + serviceAccountName: kubestellar-console-auth + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: generate + image: docker.io/alpine/kubectl@sha256:8413f8890d19aa03f63851654f642957e65ba59654b0c9357ddc6ec0b05b63a6 + command: + - /bin/sh + - -c + - | + set -eu + if kubectl -n kubestellar-console get secret kubestellar-console-auth >/dev/null 2>&1; then + echo "kubestellar-console-auth already exists; existing sessions stay valid" + exit 0 + fi + jwt="$(tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 64)" + kubectl -n kubestellar-console create secret generic kubestellar-console-auth \ + --from-literal=jwt-secret="${jwt}" + echo "kubestellar-console-auth created" + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/console/console.yaml b/files/cluster/manifests/console/console.yaml new file mode 100644 index 00000000..38e5d95f --- /dev/null +++ b/files/cluster/manifests/console/console.yaml @@ -0,0 +1,125 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubestellar-console + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +rules: + - apiGroups: [""] + resources: ["nodes", "namespaces", "pods", "services", "events", "configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "statefulsets", "daemonsets", "replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes", "pods"] + verbs: ["get", "list"] + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubestellar-console +subjects: + - kind: ServiceAccount + name: kubestellar-console + namespace: kubestellar-console +--- +apiVersion: v1 +kind: Service +metadata: + name: kubestellar-console + namespace: kubestellar-console + labels: + app.kubernetes.io/name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: 8080 + name: http + selector: + app.kubernetes.io/name: kubestellar-console +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubestellar-console + namespace: kubestellar-console + labels: + app.kubernetes.io/name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: kubestellar-console + template: + metadata: + labels: + app.kubernetes.io/name: kubestellar-console + spec: + serviceAccountName: kubestellar-console + containers: + - name: console + image: ghcr.io/kubestellar/console@sha256:f27adaf8d097a5403f60041615525497e0970151fcd10b90b7b6e903b395be32 + env: + # GitHub OAuth is opt-in and site-specific. Nothing in this tree + # creates kubestellar-console-github-oauth; an operator who wants + # OAuth creates it, and the console starts fine without it. + - name: GITHUB_CLIENT_ID + valueFrom: + secretKeyRef: + name: kubestellar-console-github-oauth + key: client-id + optional: true + - name: GITHUB_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: kubestellar-console-github-oauth + key: client-secret + optional: true + # Generated in-cluster at sync wave 3. Mandatory: a console that + # signs sessions with a key it did not generate is a console whose + # sessions anyone can forge. + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: kubestellar-console-auth + key: jwt-secret + optional: false + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/console/kustomization.yaml b/files/cluster/manifests/console/kustomization.yaml new file mode 100644 index 00000000..302610a6 --- /dev/null +++ b/files/cluster/manifests/console/kustomization.yaml @@ -0,0 +1,12 @@ +# The KubeStellar console: the appliance's only dashboard. +# +# Argo CD Core ships no web UI and no argocd CLI, so this is the single +# authenticated web surface on the machine. DEV_MODE, ALLOW_DEV_MODE_IN_CLUSTER +# and DEV_USER_LOGIN are gone: they turned that surface into an unauthenticated +# one on every installed appliance. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - auth-secret.yaml + - console.yaml diff --git a/files/cluster/manifests/controlplanes/controlplanes.yaml b/files/cluster/manifests/controlplanes/controlplanes.yaml new file mode 100644 index 00000000..fd9cb3ec --- /dev/null +++ b/files/cluster/manifests/controlplanes/controlplanes.yaml @@ -0,0 +1,33 @@ +# KubeFlex control planes. +# +# `local-cluster` registers the appliance's own cluster with KubeFlex so the +# console has something real to show the moment it comes up, rather than an +# empty dashboard that looks broken. type: host means "this is the cluster +# KubeFlex is running on" — no new API server is created for it. +# +# `cp1` is the default tenant control plane: a shared-backend Kubernetes control +# plane whose data lands in the PostgreSQL instance from sync wave 1. It is the +# thing an operator gets to experiment with without touching the host cluster. +--- +apiVersion: tenancy.kflex.kubestellar.org/v1alpha1 +kind: ControlPlane +metadata: + name: local-cluster + labels: + app.kubernetes.io/part-of: bluefin-server + annotations: + argocd.argoproj.io/sync-wave: "6" +spec: + type: host +--- +apiVersion: tenancy.kflex.kubestellar.org/v1alpha1 +kind: ControlPlane +metadata: + name: cp1 + labels: + app.kubernetes.io/part-of: bluefin-server + annotations: + argocd.argoproj.io/sync-wave: "6" +spec: + type: k8s + backend: shared diff --git a/files/cluster/manifests/controlplanes/kustomization.yaml b/files/cluster/manifests/controlplanes/kustomization.yaml new file mode 100644 index 00000000..fa541c5b --- /dev/null +++ b/files/cluster/manifests/controlplanes/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - controlplanes.yaml diff --git a/files/k0s/kiosk/kiosk-gate.css b/files/cluster/manifests/kiosk/assets/kiosk-gate.css similarity index 100% rename from files/k0s/kiosk/kiosk-gate.css rename to files/cluster/manifests/kiosk/assets/kiosk-gate.css diff --git a/files/k0s/kiosk/kiosk-gate.js b/files/cluster/manifests/kiosk/assets/kiosk-gate.js similarity index 100% rename from files/k0s/kiosk/kiosk-gate.js rename to files/cluster/manifests/kiosk/assets/kiosk-gate.js diff --git a/files/k0s/kiosk/nginx.conf b/files/cluster/manifests/kiosk/assets/nginx.conf similarity index 59% rename from files/k0s/kiosk/nginx.conf rename to files/cluster/manifests/kiosk/assets/nginx.conf index 462628bd..9ce1f9b4 100644 --- a/files/k0s/kiosk/nginx.conf +++ b/files/cluster/manifests/kiosk/assets/nginx.conf @@ -1,3 +1,9 @@ +# Kiosk proxy for the KubeStellar console. +# +# Terminates TLS with a cert-manager-issued certificate and injects the kc-agent +# gate into the console's HTML. Everything here is served from a ConfigMap and a +# Secret; nothing is read from the host filesystem. + pid /tmp/nginx.pid; events { @@ -8,12 +14,22 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; + # The container root filesystem is read-only, so every scratch path nginx + # might touch has to land on the /tmp emptyDir. + client_body_temp_path /tmp/client_body; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + server { listen 8080 ssl; server_name _; - ssl_certificate /etc/kubestellar-kiosk/cert.pem; - ssl_certificate_key /etc/kubestellar-kiosk/key.pem; + # Issued by the in-cluster CA and rotated by cert-manager. No key material + # is baked into any image. + ssl_certificate /etc/kubestellar-kiosk/tls/tls.crt; + ssl_certificate_key /etc/kubestellar-kiosk/tls/tls.key; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers HIGH:!aNULL:!MD5; @@ -21,12 +37,12 @@ http { error_page 497 https://$http_host$request_uri; location = /kiosk-gate.js { - alias /etc/kubestellar-kiosk/kiosk-gate.js; + alias /etc/kubestellar-kiosk/assets/kiosk-gate.js; add_header Cache-Control "no-store"; } location = /kiosk-gate.css { - alias /etc/kubestellar-kiosk/kiosk-gate.css; + alias /etc/kubestellar-kiosk/assets/kiosk-gate.css; add_header Cache-Control "no-store"; } diff --git a/files/cluster/manifests/kiosk/certificate.yaml b/files/cluster/manifests/kiosk/certificate.yaml new file mode 100644 index 00000000..98b899df --- /dev/null +++ b/files/cluster/manifests/kiosk/certificate.yaml @@ -0,0 +1,35 @@ +# TLS for the kiosk proxy, issued by the in-cluster CA. +# +# This is what replaced the openssl invocation in the old sysext element, which +# baked a 10-year self-signed key into every published image — the same private +# key on every appliance. cert-manager mints this one on the machine and renews +# it automatically. +# +# Short-lived on purpose: 90 days, renewed at 30 days remaining. A kiosk +# certificate that outlives the appliance's install is a certificate nobody +# notices has leaked. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kubestellar-kiosk-tls + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + secretName: kubestellar-kiosk-tls + commonName: KubeStellar Console + duration: 2160h # 90 days + renewBefore: 720h # 30 days + privateKey: + algorithm: ECDSA + size: 256 + rotationPolicy: Always + dnsNames: + - localhost + - kubestellar-kiosk-proxy.kubestellar-console.svc.cluster.local + ipAddresses: + - 127.0.0.1 + issuerRef: + name: bluefin-ca + kind: ClusterIssuer + group: cert-manager.io diff --git a/files/cluster/manifests/kiosk/deployment.yaml b/files/cluster/manifests/kiosk/deployment.yaml new file mode 100644 index 00000000..a1bb7257 --- /dev/null +++ b/files/cluster/manifests/kiosk/deployment.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubestellar-kiosk-proxy + namespace: kubestellar-console + labels: + app.kubernetes.io/name: kubestellar-kiosk-proxy + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: kubestellar-kiosk-proxy + template: + metadata: + labels: + app.kubernetes.io/name: kubestellar-kiosk-proxy + spec: + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + seccompProfile: + type: RuntimeDefault + containers: + - name: kiosk-proxy + image: docker.io/library/nginx@sha256:5616878291a2eed594aee8db4dade5878cf7edcb475e59193904b198d9b830de + command: + - nginx + args: + - -g + - daemon off; + - -c + - /etc/kubestellar-kiosk/assets/nginx.conf + ports: + - containerPort: 8080 + # Published on every interface. The previous hostIP: 127.0.0.1 + # binding meant the console was unreachable from the LAN and from + # anything but the one QEMU hostfwd, which is not what a kiosk on + # a server appliance is for. The endpoint is HTTPS-only and the + # console behind it authenticates, so loopback was doing security + # theatre rather than security. + hostPort: 8080 + name: https + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: assets + mountPath: /etc/kubestellar-kiosk/assets + readOnly: true + - name: tls + mountPath: /etc/kubestellar-kiosk/tls + readOnly: true + - name: tmp + mountPath: /tmp + - name: cache + mountPath: /var/cache/nginx + volumes: + # nginx.conf, kiosk-gate.js and kiosk-gate.css. These arrive from the + # cluster, not from a host directory seeded by tmpfiles.d: a workload + # that reads its configuration off the node cannot be reconciled, and + # drifts the moment someone edits the file in place. + - name: assets + configMap: + name: kubestellar-kiosk + - name: tls + secret: + secretName: kubestellar-kiosk-tls + - name: tmp + emptyDir: {} + - name: cache + emptyDir: {} diff --git a/files/cluster/manifests/kiosk/kustomization.yaml b/files/cluster/manifests/kiosk/kustomization.yaml new file mode 100644 index 00000000..16a8d0c9 --- /dev/null +++ b/files/cluster/manifests/kiosk/kustomization.yaml @@ -0,0 +1,25 @@ +# The kiosk proxy: TLS termination and the kc-agent gate in front of the +# console. +# +# The three asset files are a ConfigMap rather than a host directory. Its +# generated name carries a content hash, so editing nginx.conf or the gate +# rolls the Deployment instead of leaving a stale config mounted until someone +# notices. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - certificate.yaml + - deployment.yaml + - service.yaml + +configMapGenerator: + - name: kubestellar-kiosk + namespace: kubestellar-console + options: + annotations: + argocd.argoproj.io/sync-wave: "5" + files: + - assets/nginx.conf + - assets/kiosk-gate.js + - assets/kiosk-gate.css diff --git a/files/cluster/manifests/kiosk/service.yaml b/files/cluster/manifests/kiosk/service.yaml new file mode 100644 index 00000000..595e4de6 --- /dev/null +++ b/files/cluster/manifests/kiosk/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubestellar-kiosk-proxy + namespace: kubestellar-console + labels: + app.kubernetes.io/name: kubestellar-kiosk-proxy + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: https + name: https + selector: + app.kubernetes.io/name: kubestellar-kiosk-proxy diff --git a/files/cluster/manifests/kubeflex/crds.yaml b/files/cluster/manifests/kubeflex/crds.yaml new file mode 100644 index 00000000..d3b62a78 --- /dev/null +++ b/files/cluster/manifests/kubeflex/crds.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: controlplanes.tenancy.kflex.kubestellar.org + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + group: tenancy.kflex.kubestellar.org + names: + kind: ControlPlane + listKind: ControlPlaneList + plural: controlplanes + singular: controlplane + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: Type + type: string + jsonPath: .spec.type + - name: Age + type: date + jsonPath: .metadata.creationTimestamp diff --git a/files/cluster/manifests/kubeflex/kustomization.yaml b/files/cluster/manifests/kubeflex/kustomization.yaml new file mode 100644 index 00000000..79386e18 --- /dev/null +++ b/files/cluster/manifests/kubeflex/kustomization.yaml @@ -0,0 +1,13 @@ +# KubeFlex: the ControlPlane CRD, the operator that reconciles it, and the +# in-cluster generator for the PostgreSQL superuser password the operator's +# database needs. +# +# Sync waves, not file ordering, decide what lands when: CRD and Secret at wave +# 0, PostgreSQL at wave 1, operator at wave 2. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - crds.yaml + - postgres-secret.yaml + - operator.yaml diff --git a/files/cluster/manifests/kubeflex/operator.yaml b/files/cluster/manifests/kubeflex/operator.yaml new file mode 100644 index 00000000..b6a2c0ac --- /dev/null +++ b/files/cluster/manifests/kubeflex/operator.yaml @@ -0,0 +1,93 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeflex-controller-manager + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "2" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflex-controller-manager + annotations: + argocd.argoproj.io/sync-wave: "2" +rules: + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes/status", "controlplanes/finalizers"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["namespaces", "services", "secrets", "configmaps", "serviceaccounts"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["apps"] + resources: ["deployments", "statefulsets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeflex-controller-manager + annotations: + argocd.argoproj.io/sync-wave: "2" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflex-controller-manager +subjects: + - kind: ServiceAccount + name: kubeflex-controller-manager + namespace: kubeflex-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubeflex-controller-manager + namespace: kubeflex-system + labels: + app.kubernetes.io/name: kubeflex + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubeflex + template: + metadata: + labels: + app.kubernetes.io/name: kubeflex + spec: + serviceAccountName: kubeflex-controller-manager + containers: + - name: manager + image: ghcr.io/kubestellar/kubeflex/manager@sha256:ca31299554fa4f4c2f917d4c141761978ede64048914895beaef80f82f6fd177 + command: + - /ko-app/manager + args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/kubeflex/postgres-secret.yaml b/files/cluster/manifests/kubeflex/postgres-secret.yaml new file mode 100644 index 00000000..0db7f71f --- /dev/null +++ b/files/cluster/manifests/kubeflex/postgres-secret.yaml @@ -0,0 +1,105 @@ +# Mints the PostgreSQL superuser password inside the cluster, once. +# +# The password is never committed to git and never appears in a pod spec: this +# Job generates it on first sync and stores it in the kubeflex-postgres Secret, +# which the StatefulSet reads through secretKeyRef (projectbluefin/server#98). +# It replaces the first-boot host script that used to write a Secret manifest +# into the node's manifest directory, where it was readable by anyone with a +# root shell and invisible to the cluster's own RBAC. +# +# Idempotent by construction: if the Secret already exists the Job exits without +# touching it, so the initialised database stays reachable across reboots and +# across re-syncs. Replace=true lets a future change to this Job land at all — +# Job specs are immutable, so an in-place update would otherwise fail the sync. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeflex-postgres-secret + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubeflex-postgres-secret + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubeflex-postgres"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubeflex-postgres-secret + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeflex-postgres-secret +subjects: + - kind: ServiceAccount + name: kubeflex-postgres-secret + namespace: kubeflex-system +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kubeflex-postgres-secret + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" + argocd.argoproj.io/sync-options: Replace=true +spec: + backoffLimit: 20 + template: + metadata: + labels: + app.kubernetes.io/name: kubeflex-postgres-secret + spec: + restartPolicy: OnFailure + serviceAccountName: kubeflex-postgres-secret + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: generate + image: docker.io/alpine/kubectl@sha256:8413f8890d19aa03f63851654f642957e65ba59654b0c9357ddc6ec0b05b63a6 + command: + - /bin/sh + - -c + - | + set -eu + if kubectl -n kubeflex-system get secret kubeflex-postgres >/dev/null 2>&1; then + echo "kubeflex-postgres already exists; leaving the running database's password alone" + exit 0 + fi + password="$(tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 32)" + kubectl -n kubeflex-system create secret generic kubeflex-postgres \ + --from-literal=password="${password}" + echo "kubeflex-postgres created" + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/kubelet-csr-approver/deployment.yaml b/files/cluster/manifests/kubelet-csr-approver/deployment.yaml new file mode 100644 index 00000000..3a39c9b3 --- /dev/null +++ b/files/cluster/manifests/kubelet-csr-approver/deployment.yaml @@ -0,0 +1,84 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubelet-csr-approver + namespace: kube-system + labels: + app.kubernetes.io/name: kubelet-csr-approver + annotations: + argocd.argoproj.io/sync-wave: "-5" +spec: + # Single-node appliance, so leader election buys nothing and a second replica + # would stay Pending. + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubelet-csr-approver + template: + metadata: + labels: + app.kubernetes.io/name: kubelet-csr-approver + spec: + serviceAccountName: kubelet-csr-approver + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: kubelet-csr-approver + image: ghcr.io/postfinance/kubelet-csr-approver@sha256:6469ef517d2b6d3cb108fce8a913483533a48b1cd78b5d4af307fa09e04a7818 + args: + - -metrics-bind-address + - ":8080" + - -health-probe-bind-address + - ":8081" + env: + # Strict by shape rather than by a hardcoded name: the appliance's + # node name is whatever the operator sets as the hostname, but it is + # always a single DNS label. Refusing dotted names refuses the + # actual attack, which is a compromised kubelet asking for a serving + # certificate for some external FQDN. + - name: PROVIDER_REGEX + value: ^[a-z0-9][a-z0-9-]{0,62}$ + # One DNS SAN, the node's own name. Upstream default, restated + # because a silent bump to it would widen what gets approved. + - name: ALLOWED_DNS_NAMES + value: "1" + # 30 days. Upstream's default is 367 days; bounding it hard means a + # serving certificate issued to a node that is later removed stops + # being useful in a month rather than a year. + - name: MAX_EXPIRATION_SEC + value: "2592000" + # The appliance has no authoritative DNS for its own node name, so + # the forward-lookup check would deny every CSR. The regex above and + # the kubelet's own identity are what gate approval here. + - name: BYPASS_DNS_RESOLUTION + value: "true" + # Refuse CSRs whose requester is not system:node:. + - name: IGNORE_NON_SYSTEM_NODE + value: "false" + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8081 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 128Mi + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/kubelet-csr-approver/kustomization.yaml b/files/cluster/manifests/kubelet-csr-approver/kustomization.yaml new file mode 100644 index 00000000..853006a0 --- /dev/null +++ b/files/cluster/manifests/kubelet-csr-approver/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - rbac.yaml + - deployment.yaml diff --git a/files/cluster/manifests/kubelet-csr-approver/rbac.yaml b/files/cluster/manifests/kubelet-csr-approver/rbac.yaml new file mode 100644 index 00000000..30c68796 --- /dev/null +++ b/files/cluster/manifests/kubelet-csr-approver/rbac.yaml @@ -0,0 +1,73 @@ +# kubelet serving-certificate approver. +# +# kubeadm.yaml sets serverTLSBootstrap: true, so kubelets request their serving +# certificates through the certificates API. kube-controller-manager +# auto-approves kubelet *client* CSRs but deliberately never auto-approves +# *serving* CSRs, because it cannot verify that the requested SANs belong to the +# node. Without this component those CSRs sit Pending forever, the kubelet falls +# back to a self-signed serving certificate, and `kubectl logs` and +# `kubectl exec` fail TLS verification. It is mandatory, not optional. +# +# RBAC below is upstream's, from +# charts/kubelet-csr-approver/templates/clusterrole.yaml at v1.2.15. The approve +# permission is scoped to the kubernetes.io/kubelet-serving signer by +# resourceNames, so this ServiceAccount cannot approve any other kind of CSR. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubelet-csr-approver + namespace: kube-system + labels: + app.kubernetes.io/name: kubelet-csr-approver + annotations: + argocd.argoproj.io/sync-wave: "-5" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubelet-csr-approver + labels: + app.kubernetes.io/name: kubelet-csr-approver + annotations: + argocd.argoproj.io/sync-wave: "-5" +rules: + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + verbs: + - update + - apiGroups: + - certificates.k8s.io + resourceNames: + - kubernetes.io/kubelet-serving + resources: + - signers + verbs: + - approve +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubelet-csr-approver + labels: + app.kubernetes.io/name: kubelet-csr-approver + annotations: + argocd.argoproj.io/sync-wave: "-5" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubelet-csr-approver +subjects: + - kind: ServiceAccount + name: kubelet-csr-approver + namespace: kube-system diff --git a/files/cluster/manifests/kubestellar/core.yaml b/files/cluster/manifests/kubestellar/core.yaml new file mode 100644 index 00000000..cfcf4e56 --- /dev/null +++ b/files/cluster/manifests/kubestellar/core.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kubestellar + annotations: + argocd.argoproj.io/sync-wave: "3" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubestellar-controller-manager + namespace: kubestellar + annotations: + argocd.argoproj.io/sync-wave: "3" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubestellar-controller-manager + annotations: + argocd.argoproj.io/sync-wave: "3" +rules: + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes/status"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["namespaces", "secrets", "configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubestellar-controller-manager + annotations: + argocd.argoproj.io/sync-wave: "3" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubestellar-controller-manager +subjects: + - kind: ServiceAccount + name: kubestellar-controller-manager + namespace: kubestellar +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubestellar-controller-manager + namespace: kubestellar + labels: + app.kubernetes.io/name: kubestellar + annotations: + argocd.argoproj.io/sync-wave: "3" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubestellar + template: + metadata: + labels: + app.kubernetes.io/name: kubestellar + spec: + serviceAccountName: kubestellar-controller-manager + containers: + - name: manager + image: ghcr.io/kubestellar/kubestellar/controller-manager@sha256:53db459bc17af77e644fe34f1469a425cd4c089939567029fe4e96ef5b80a7ab + command: + - /ko-app/controller-manager + args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/kubestellar/kustomization.yaml b/files/cluster/manifests/kubestellar/kustomization.yaml new file mode 100644 index 00000000..9691381b --- /dev/null +++ b/files/cluster/manifests/kubestellar/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - core.yaml diff --git a/files/cluster/manifests/kustomization.yaml b/files/cluster/manifests/kustomization.yaml new file mode 100644 index 00000000..7b4cf1aa --- /dev/null +++ b/files/cluster/manifests/kustomization.yaml @@ -0,0 +1,36 @@ +# The Argo-owned tree. +# +# Staged into the image at /usr/share/bluefin/cluster/, committed to +# /var/lib/bluefin/cluster.git by bluefin-cluster-repo.service on first boot, +# served by the git-daemon from seed phase 15, and reconciled by the single root +# Application from seed phase 20. +# +# This is one Application, not a fan-out: ordering comes from +# argocd.argoproj.io/sync-wave annotations on the resources themselves, which +# Argo honours across the whole tree. +# +# -10 cert-manager +# -5 CA bootstrap chain, kubelet-csr-approver +# 0 KubeFlex CRD, PostgreSQL password generator +# 1 PostgreSQL +# 2 KubeFlex operator +# 3 KubeStellar core, console JWT generator +# 4 KubeStellar console +# 5 kiosk proxy and its certificate +# 6 ControlPlane custom resources +# +# Directory names carry no ordering. The sync waves are the only ordering +# statement, so there is nothing to keep in agreement. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cert-manager + - pki + - kubelet-csr-approver + - kubeflex + - postgres + - kubestellar + - console + - kiosk + - controlplanes diff --git a/files/cluster/manifests/pki/issuers.yaml b/files/cluster/manifests/pki/issuers.yaml new file mode 100644 index 00000000..b4a72c2e --- /dev/null +++ b/files/cluster/manifests/pki/issuers.yaml @@ -0,0 +1,54 @@ +# The appliance's certificate authority. +# +# No certificate or private key ships in any Bluefin Server image. The CA is +# minted in-cluster on first sync and rotated by cert-manager thereafter. This +# permanently replaces the build-time self-signed key that used to be generated +# into the sysext at image build, which meant every installed appliance shared +# one private key. +# +# SelfSigned ClusterIssuer -> CA Certificate -> CA ClusterIssuer is the upstream +# bootstrap chain: the SelfSigned issuer exists only to sign the CA once. +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: bluefin-selfsigned + annotations: + argocd.argoproj.io/sync-wave: "-5" +spec: + selfSigned: {} +--- +# cert-manager reads ClusterIssuer CA secrets from its own namespace +# (--cluster-resource-namespace, which defaults to the release namespace), so +# this Certificate must land in cert-manager. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: bluefin-ca + namespace: cert-manager + annotations: + argocd.argoproj.io/sync-wave: "-5" +spec: + isCA: true + commonName: Bluefin Server CA + secretName: bluefin-ca + duration: 87600h # 10 years + renewBefore: 8760h # 1 year + privateKey: + algorithm: ECDSA + size: 256 + rotationPolicy: Always + issuerRef: + name: bluefin-selfsigned + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: bluefin-ca + annotations: + argocd.argoproj.io/sync-wave: "-4" +spec: + ca: + secretName: bluefin-ca diff --git a/files/cluster/manifests/pki/kustomization.yaml b/files/cluster/manifests/pki/kustomization.yaml new file mode 100644 index 00000000..25a48402 --- /dev/null +++ b/files/cluster/manifests/pki/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - issuers.yaml diff --git a/files/cluster/manifests/postgres/kustomization.yaml b/files/cluster/manifests/postgres/kustomization.yaml new file mode 100644 index 00000000..99bd600a --- /dev/null +++ b/files/cluster/manifests/postgres/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - postgres.yaml diff --git a/files/cluster/manifests/postgres/postgres.yaml b/files/cluster/manifests/postgres/postgres.yaml new file mode 100644 index 00000000..b76aa181 --- /dev/null +++ b/files/cluster/manifests/postgres/postgres.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-postgresql + namespace: kubeflex-system + labels: + app.kubernetes.io/name: postgresql + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + type: ClusterIP + ports: + - name: tcp-postgresql + port: 5432 + targetPort: postgresql + selector: + app.kubernetes.io/name: postgresql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres-postgresql + namespace: kubeflex-system + labels: + app.kubernetes.io/name: postgresql + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + serviceName: postgres-postgresql + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgresql + template: + metadata: + labels: + app.kubernetes.io/name: postgresql + spec: + securityContext: + fsGroup: 70 + containers: + - name: postgresql + image: docker.io/library/postgres@sha256:3c5c8892d184f738f4fe282d14ddaa613a38f00f4189d2d94725ebe6f2909ddb + env: + - name: POSTGRES_USER + value: "postgres" + # The superuser password lives in the kubeflex-postgres Secret, + # generated in-cluster by the Job in ../kubeflex/postgres-secret.yaml + # at sync wave 0, so it is never committed to git and never appears + # in this pod spec. See projectbluefin/server#98. + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: kubeflex-postgres + key: password + - name: POSTGRES_DB + value: "postgres" + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgresql + containerPort: 5432 + readinessProbe: + exec: + command: + - pg_isready + - -U + - postgres + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + securityContext: + runAsNonRoot: true + runAsUser: 70 + runAsGroup: 70 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + volumes: + # Deliberately ephemeral. The appliance ships no CSI driver and no + # default StorageClass, so a volumeClaimTemplate would leave this + # StatefulSet Pending forever and the root Application would never + # report Healthy. Postgres re-initialises from POSTGRES_PASSWORD on + # restart, and that password survives in the kubeflex-postgres Secret, + # so the database comes back consistent — just empty. Durable storage + # is gated on a storage provisioner landing in this tree. + - name: data + emptyDir: {} diff --git a/files/cluster/seed/00-cilium/cilium-values.yaml b/files/cluster/seed/00-cilium/cilium-values.yaml new file mode 100644 index 00000000..3826225e --- /dev/null +++ b/files/cluster/seed/00-cilium/cilium-values.yaml @@ -0,0 +1,42 @@ +# Helm values used to render cilium.yaml. Reproduce the rendered manifest with: +# +# curl -LO https://helm.cilium.io/cilium-1.20.2.tgz +# echo "b2afd87b7f75f875f92a14559f14f59b7babbb479d968e3fd625a20bf30ec20e cilium-1.20.2.tgz" | sha256sum -c +# helm template cilium ./cilium-1.20.2.tgz \ +# --namespace kube-system --include-crds --values cilium-values.yaml > cilium.yaml +# +# Helm is a build-time tool only. Nothing in this repository ships or runs Helm +# on the appliance; the seed is plain YAML applied with `kubectl apply -k`. + +# kube-proxy replacement. kubeadm runs with --skip-phases=addon/kube-proxy, so +# Cilium is the only thing implementing Service load balancing. This also turns +# on Cilium's hostPort support, which the kiosk proxy relies on. +kubeProxyReplacement: "true" + +# Every Cilium workload below is hostNetwork, and Bluefin Server is a +# single-node appliance whose kube-apiserver listens on the same host. Pointing +# at the loopback endpoint removes the chicken-and-egg of reaching the +# kubernetes Service before Service load balancing exists. +# +# The chart's `k8sServiceHost: auto` is not usable here: it resolves the +# cluster-info ConfigMap through Helm's `lookup`, which returns nothing during +# `helm template` and fails the render. +k8sServiceHost: "localhost" +k8sServicePort: "6443" + +# Single control-plane node; the chart default of 2 would leave one operator +# replica Pending forever. +operator: + replicas: 1 + +# Must agree with networking.podSubnet in /usr/share/bluefin/kubeadm.yaml. +ipam: + operator: + clusterPoolIPv4PodCIDRList: + - "10.244.0.0/16" + +# Hubble's TLS material is generated by Helm at render time, which would bake +# private keys into the shipped image. No certificate or key material ships in +# any Bluefin Server image, so Hubble stays off. +hubble: + enabled: false diff --git a/files/cluster/seed/00-cilium/cilium.yaml b/files/cluster/seed/00-cilium/cilium.yaml new file mode 100644 index 00000000..319d11a4 --- /dev/null +++ b/files/cluster/seed/00-cilium/cilium.yaml @@ -0,0 +1,1782 @@ +# Cilium 1.20.2, rendered from the upstream Helm chart at build time. +# +# chart: https://helm.cilium.io/cilium-1.20.2.tgz +# sha256: b2afd87b7f75f875f92a14559f14f59b7babbb479d968e3fd625a20bf30ec20e +# values: cilium-values.yaml (in this directory) +# +# Regenerate with the command in the cilium-values.yaml header. Do not hand-edit +# this file: customisation belongs in cilium-values.yaml or in kustomization.yaml +# so that the next chart bump is a re-render rather than a merge. +# +# Cilium ships in the seed rather than the Argo-owned tree because Argo's own +# pods need pod networking: a node with no CNI stays NotReady and carries a +# NoSchedule taint. +# +# All three images already carry sha256 digests; the chart sets useDigest: true. +--- +# Source: cilium/templates/cilium-secrets-namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 + annotations: +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-envoy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-envoy" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd", "kvstore" or + # "doublewrite-readkvstore" / "doublewrite-readcrd". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful + # for seamless migrations from the kvstore mode to the crd mode. Consult the + # documentation for more information on how to perform the migration. + identity-allocation-mode: crd + + identity-heartbeat-timeout: "30m0s" + identity-gc-interval: "15m0s" + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + debug-verbose: "" + metrics-sampling-interval: "5m" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "default" + policy-cidr-match-mode: "" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + enable-policy-secrets-sync: "true" + policy-secrets-only-from-secrets-namespace: "true" + policy-secrets-namespace: "cilium-secrets" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "false" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-policy-stats-map-max specifies the maximum number of entries in global + # policy stats map + bpf-policy-stats-map-max: "65536" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + bpf-lb-external-clusterip: "false" + bpf-lb-source-range-all-types: "false" +# When Gateway API is enabled this is forced to "true" so per-backend weights +# on TCPRoute/UDPRoute take effect (Maglev honors the LB algorithm annotation only +# when enabled). Paired with `bpf-lb-sock-hostns-only`, which is forced "true" +# for the same reason. + bpf-lb-algorithm-annotation: "false" + bpf-lb-mode-annotation: "false" + + bpf-distributed-lru: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: "default" + # Unique ID of the cluster. Must be unique across all connected clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + tunnel-source-port-range: "0-0" + service-no-backend-response: "reject" + policy-deny-response: "none" + + + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + enable-ipv4-masquerade: "true" + enable-ipv4-big-tcp: "false" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-tcx: "true" + datapath-mode: "veth" + enable-masquerade-to-route-source: "false" + + enable-xt-socket-fallback: "true" + install-no-conntrack-iptables-rules: "false" + iptables-random-fully: "false" + + auto-direct-node-routes: "false" + direct-routing-skip-unreachable: "false" + + + + enable-host-firewall: "false" + # List of devices used to attach bpf_host.o (implements BPF NodePort, + # host-firewall and BPF masquerading) + devices: "" + + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: "" + enable-no-service-endpoints-routable: "true" + bpf-lb-sock: "false" + nodeport-addresses: "" + enable-health-check-nodeport: "true" + enable-dynamic-source-lookup-nodeport: "false" + enable-health-check-loadbalancer-ip: "false" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + bpf-lb-acceleration: "disabled" + enable-service-topology: "false" + enable-l2-neigh-discovery: "false" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + enable-k8s-networkpolicy: "true" + enable-endpoint-lockdown-on-policy-overflow: "false" + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist + cni-exclusive: "true" + cni-log-file: "/var/run/cilium/cilium-cni.log" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + health-check-icmp-failure-threshold: "3" + enable-well-known-identities: "false" + enable-node-selector-labels: "false" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + + enable-hubble: "false" + ipam: "cluster-pool" + ipam-cilium-node-update-rate: "15s" + cluster-pool-ipv4-cidr: "10.244.0.0/16" + cluster-pool-ipv4-mask-size: "24" + + default-lb-service-ipam: "lbipam" + egress-gateway-reconciliation-trigger-interval: "1s" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + + packetization-layer-pmtud-mode: "blackhole" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + + identity-management-mode: "agent" + enable-sctp: "false" + remove-cilium-node-taints: "true" + set-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15s" + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: "true" + dnsproxy-socket-linger-timeout: "10" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-proxy-response-max-delay: "100ms" + tofqdns-preallocate-identities: "true" + agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" + + mesh-auth-enabled: "false" + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + mesh-auth-gc-interval: "5m0s" + + proxy-xff-num-trusted-hops-ingress: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-connect-timeout: "2" + proxy-initial-fetch-timeout: "30" + proxy-max-active-downstream-connections: "50000" + proxy-max-requests-per-connection: "0" + proxy-max-connection-duration-seconds: "0" + proxy-idle-timeout-seconds: "60" + proxy-max-concurrent-retries: "128" + proxy-use-original-source-address: "true" + proxy-cluster-max-connections: "1024" + proxy-cluster-max-pending-requests: "1024" + proxy-cluster-max-requests: "1024" + http-retry-count: "3" + http-stream-idle-timeout: "300" + envoy-node-locality-enabled: "false" + + external-envoy-proxy: "true" + envoy-base-id: "0" + envoy-access-log-enabled: "true" + envoy-access-log-buffer-size: "4096" + envoy-keep-cap-netbindservice: "false" + envoy-xds-mode: "ads" + max-connected-clusters: "255" + clustermesh-cache-ttl: "0s" + clustermesh-enable-endpoint-sync: "false" + clustermesh-enable-mcs-api: "false" + clustermesh-mcs-api-install-crds: "true" + clustermesh-default-global-namespace: "true" + policy-default-local-cluster: "true" + + nat-map-stats-entries: "32" + nat-map-stats-interval: "30s" + enable-lb-ipam: "true" + enable-non-default-deny-policies: "true" + enable-source-ip-verification: "true" + enable-dynamic-config: "true" + enable-drift-checker: "true" + + enable-datapath-plugins: "false" + datapath-plugins-state-dir: "/var/run/cilium/plugins" + +# Extra config allows adding arbitrary properties to the cilium config. +# By putting it at the end of the ConfigMap, it's also possible to override existing properties. +--- +# Source: cilium/templates/cilium-envoy/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-envoy-config + namespace: kube-system +data: + # Keep the key name as bootstrap-config.json to avoid breaking changes + bootstrap-config.json: | + {"admin":{"address":{"pipe":{"mode":432,"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"adsConfig":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"cdsConfig":{"ads":{},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"ads":{},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + - ciliumdatapathplugins + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cilium-config + verbs: + # allow patching of the configmap to set annotations + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + - secrets + - serviceaccounts + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + verbs: + - get + - list + - watch +- apiGroups: + # Watch CNP / CCNPs for validation and external groups. + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups'. + # This was removed in v1.20, but we must keep the permissions until v1.21 for + # upgrade phasing. + - create + - update + - deletecollection + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Set validity status on CNP / CCNP. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + - ciliumcidrgroups + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - cilium.io + resources: + - ciliumbgpclusterconfigs/status + - ciliumbgppeerconfigs/status + verbs: + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io + - ciliumgatewayclassconfigs.cilium.io + - ciliumdatapathplugins.cilium.io +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions/status + verbs: + - update + resourceNames: + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgppeerconfigs + - ciliumdatapathplugins + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + verbs: + - deletecollection +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-operator/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch +--- +# Source: cilium/templates/cilium-operator/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-ztunnel + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +# ZTunnel DaemonSet management permissions +# Note: These permissions must always be granted (not conditional on encryption.type) +# because the controller needs to clean up stale DaemonSets when ztunnel is disabled. +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-ztunnel + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-ztunnel +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-envoy/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-envoy + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 + io.cilium/app: proxy +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: 9964 +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent + helm.sh/chart: cilium-1.20.2 +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: cilium-agent + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 + spec: + securityContext: + appArmorProfile: + type: Unconfined + seccompProfile: + type: Unconfined + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 300 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + - name: "require-k8s-connectivity" + value: "false" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + set -o errexit + set -o pipefail + set -o nounset + + # When running in AWS ENI mode, it's likely that 'aws-node' has + # had a chance to install SNAT iptables rules. These can result + # in dropped traffic, so we should attempt to remove them. + # We do it using a 'postStart' hook since this may need to run + # for nodes which might have already been init'ed but may still + # have dangling rules. This is safe because there are no + # dependencies on anything that is part of the startup script + # itself, and can be safely run multiple times per node (e.g. in + # case of a restart). + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; + then + echo 'Deleting iptables rules created by the AWS CNI VPC plugin' + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + fi + echo 'Done!' + + preStop: + exec: + command: + - /cni-uninstall.sh + ports: + - name: health + containerPort: 9879 + hostPort: 9879 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + - SYSLOG + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: cilium-netns + mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: tmp + mountPath: /tmp + + initContainers: + - name: config + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + command: + - cilium-dbg + - build-config + - "--k8s-api-server-urls=" + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + volumeMounts: + - name: tmp + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + capabilities: + add: + - NET_ADMIN + drop: + - ALL + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + command: + - bash + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - bash + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin # .Values.cni.install + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To exec into pod network namespaces + - name: cilium-netns + hostPath: + path: /var/run/netns + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host or apply sysctlfix + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + # note: we configure the volume for the kvstoremesh-specific certificate + # regardless of whether KVStoreMesh is enabled or not, so that it can be + # automatically mounted in case KVStoreMesh gets subsequently enabled, + # without requiring an agent restart. + - secret: + name: clustermesh-apiserver-local-cert + optional: true + items: + - key: tls.key + path: local-etcd-client.key + - key: tls.crt + path: local-etcd-client.crt + - key: ca.crt + path: local-etcd-client-ca.crt + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory +--- +# Source: cilium/templates/cilium-envoy/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-envoy + namespace: kube-system + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy + helm.sh/chart: cilium-1.20.2 +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-envoy + image: "quay.io/cilium/cilium-envoy:v1.37.6-1789133542-cbec91f666af0bf742da986d43832932dbb26b82@sha256:af7382699576b9e65e9184efa52eeca0b58aea70ad6e511bf260c91d9f740463" + imagePullPolicy: IfNotPresent + command: + - /usr/bin/cilium-envoy-starter + args: + - '--' + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + - '--log-level info' + + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + timeoutSeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + initialDelaySeconds: 0 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + initialDelaySeconds: 0 + timeoutSeconds: 5 + + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + + ports: + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium-envoy" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium-envoy + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "/var/run/cilium/envoy/artifacts" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: "cilium-envoy-config" + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + helm.sh/chart: cilium-1.20.2 +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 1 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + helm.sh/chart: cilium-1.20.2 + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.20.2@sha256:64d8798350e8569b8e7622563fed6e44dce2625f311e4651b774816516c744fc" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + ports: + - name: health + containerPort: 9234 + hostPort: 9234 + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.cilium.io/agent-not-ready + operator: Exists + + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/files/cluster/seed/00-cilium/kustomization.yaml b/files/cluster/seed/00-cilium/kustomization.yaml new file mode 100644 index 00000000..3495062b --- /dev/null +++ b/files/cluster/seed/00-cilium/kustomization.yaml @@ -0,0 +1,12 @@ +# Seed phase 00 — CNI. +# +# Applied by bluefin-cluster-bootstrap.service before anything else: +# `kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/00-cilium`. +# +# The seed is append-only and is never pruned; Argo owns pruning for everything +# downstream. See the "Closed decision: pruning" section of the cutover design. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cilium.yaml diff --git a/files/cluster/seed/10-argocd-core/kustomization.yaml b/files/cluster/seed/10-argocd-core/kustomization.yaml new file mode 100644 index 00000000..07acb458 --- /dev/null +++ b/files/cluster/seed/10-argocd-core/kustomization.yaml @@ -0,0 +1,33 @@ +# Seed phase 10 — Argo CD Core. +# +# core-install.yaml is the upstream manifest, fetched and sha256-pinned by +# elements/bluefin-server/os-cluster-manifests.bst: +# +# https://raw.githubusercontent.com/argoproj/argo-cd/v3.5.3/manifests/core-install.yaml +# sha256: 1a87025d8eb2eae621653fd312fb9ca51df1b4b3b6992a030e3a9ef38e45c448 +# +# It is deliberately not vendored into this repository. A hand-maintained subset +# drifts from upstream and silently drops CRDs and RBAC; pinning the real file +# keeps the next Argo bump a two-line change. +# +# core-install.yaml carries no namespace on any object and ships no Namespace, +# so both are supplied here. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: argocd + +resources: + - namespace.yaml + - core-install.yaml + +patches: + # One root Application needs no template generator. The CRD stays in place; + # only the controller Deployment goes. + - path: remove-applicationset-controller.yaml + +images: + - name: quay.io/argoproj/argocd + digest: sha256:dd3f47d5a5e4da563a7a398506e892481b358a7cec50abdf320c71aa55904bfa + - name: public.ecr.aws/docker/library/redis + digest: sha256:08ad0b1d280850169a790dba1393ff7a90aef951fc19632cf4d3ce4f78e679ba diff --git a/files/cluster/seed/10-argocd-core/namespace.yaml b/files/cluster/seed/10-argocd-core/namespace.yaml new file mode 100644 index 00000000..15de758f --- /dev/null +++ b/files/cluster/seed/10-argocd-core/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: argocd + labels: + app.kubernetes.io/part-of: argocd diff --git a/files/cluster/seed/10-argocd-core/remove-applicationset-controller.yaml b/files/cluster/seed/10-argocd-core/remove-applicationset-controller.yaml new file mode 100644 index 00000000..3ff140b4 --- /dev/null +++ b/files/cluster/seed/10-argocd-core/remove-applicationset-controller.yaml @@ -0,0 +1,18 @@ +# Delete the ApplicationSet controller from upstream core-install.yaml. +# +# ApplicationSet generates Applications from templates. This appliance declares +# exactly one root Application by hand, so the controller has nothing to do and +# is a third idle workload plus a cluster-wide write surface. +# +# applicationsets.argoproj.io stays installed: an orphaned CRD costs nothing and +# removing it from the upstream manifest would fight every Argo bump. +# +# No namespace is named here on purpose: kustomize matches patches against the +# resource as written, and upstream core-install.yaml carries no namespace on +# any object. The `namespace: argocd` transformer in kustomization.yaml runs +# afterwards. +$patch: delete +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-applicationset-controller diff --git a/files/cluster/seed/15-gitd/deployment.yaml b/files/cluster/seed/15-gitd/deployment.yaml new file mode 100644 index 00000000..f643162f --- /dev/null +++ b/files/cluster/seed/15-gitd/deployment.yaml @@ -0,0 +1,116 @@ +# Serves /var/lib/bluefin/cluster.git over the cluster network. +# +# Argo's Application.source speaks git, Helm or OCI. There is no filesystem +# source, so Argo cannot read /usr/share/bluefin/cluster/ directly. This +# Deployment is the bridge: bluefin-cluster-repo.service initialises the bare +# repository from the image's manifest tree on first boot, and git-daemon +# publishes it to argocd-repo-server. +# +# It lives in the seed and cannot be an Argo manifest: the root Application +# needs the repository served before Argo can read anything. Same class of +# circular dependency as Cilium. +# +# The image is the Argo CD image that 10-argocd-core already pulls onto this +# node. It carries git (Debian's git package owns /usr/lib/git-core/git-daemon), +# so the bridge adds no second image to pull — which matters for a node that is +# expected to come up without a registry reachable. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bluefin-git-daemon + namespace: bluefin-system + labels: + app.kubernetes.io/name: bluefin-git-daemon + app.kubernetes.io/part-of: bluefin-server +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: bluefin-git-daemon + template: + metadata: + labels: + app.kubernetes.io/name: bluefin-git-daemon + app.kubernetes.io/part-of: bluefin-server + spec: + # The bare repository is a host path, so the pod is pinned to the node + # that owns it. Single-node appliance; this is a statement of fact, not a + # constraint being imposed. + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: git-daemon + image: quay.io/argoproj/argocd@sha256:dd3f47d5a5e4da563a7a398506e892481b358a7cec50abdf320c71aa55904bfa + command: + - git + - daemon + - --verbose + - --informative-errors + - --reuseaddr + - --export-all + - --listen=0.0.0.0 + - --port=9418 + - --base-path=/srv/git + - /srv/git + env: + # The bare repo is root-owned on the host and git-daemon runs as + # uid 999, which trips git's dubious-ownership guard. The mount is + # read-only, so there is nothing for an owner mismatch to protect. + - name: GIT_CONFIG_COUNT + value: "1" + - name: GIT_CONFIG_KEY_0 + value: safe.directory + - name: GIT_CONFIG_VALUE_0 + value: "*" + - name: HOME + value: /tmp + ports: + - name: git + containerPort: 9418 + readinessProbe: + tcpSocket: + port: git + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + tcpSocket: + port: git + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: cluster-repo + mountPath: /srv/git/cluster.git + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: cluster-repo + hostPath: + path: /var/lib/bluefin/cluster.git + type: Directory + - name: tmp + emptyDir: {} diff --git a/files/cluster/seed/15-gitd/kustomization.yaml b/files/cluster/seed/15-gitd/kustomization.yaml new file mode 100644 index 00000000..fba9a756 --- /dev/null +++ b/files/cluster/seed/15-gitd/kustomization.yaml @@ -0,0 +1,12 @@ +# Seed phase 15 — manifest source bridge. +# +# Applied after Argo CD Core so the git URL exists before the root Application +# in phase 20 is created. See deployment.yaml for why this cannot be an +# Argo-managed manifest. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - deployment.yaml + - service.yaml diff --git a/files/cluster/seed/15-gitd/namespace.yaml b/files/cluster/seed/15-gitd/namespace.yaml new file mode 100644 index 00000000..5353822a --- /dev/null +++ b/files/cluster/seed/15-gitd/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: bluefin-system + labels: + app.kubernetes.io/part-of: bluefin-server diff --git a/files/cluster/seed/15-gitd/service.yaml b/files/cluster/seed/15-gitd/service.yaml new file mode 100644 index 00000000..ad8b3420 --- /dev/null +++ b/files/cluster/seed/15-gitd/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: bluefin-git-daemon + namespace: bluefin-system + labels: + app.kubernetes.io/name: bluefin-git-daemon + app.kubernetes.io/part-of: bluefin-server +spec: + type: ClusterIP + ports: + - name: git + port: 9418 + targetPort: git + selector: + app.kubernetes.io/name: bluefin-git-daemon diff --git a/files/cluster/seed/20-root-app/application.yaml b/files/cluster/seed/20-root-app/application.yaml new file mode 100644 index 00000000..3ed5a039 --- /dev/null +++ b/files/cluster/seed/20-root-app/application.yaml @@ -0,0 +1,45 @@ +# The one Application. Everything downstream of here is Argo's. +# +# repoURL points at the in-cluster git-daemon from seed phase 15, which serves +# the bare repository that bluefin-cluster-repo.service initialises from +# /usr/share/bluefin/cluster/. No external git host is contacted, so the +# manifest tree stays available with the network unplugged. +# +# `.` is a kustomization that pulls in every component directory in order, so +# the whole appliance is one sync unit with sync waves for ordering rather than +# a fan-out of Applications that would each need their own health gate. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: bluefin-cluster + namespace: argocd + labels: + app.kubernetes.io/part-of: bluefin-server +spec: + project: bluefin + source: + repoURL: git://bluefin-git-daemon.bluefin-system.svc.cluster.local/cluster.git + targetRevision: main + path: . + destination: + server: https://kubernetes.default.svc + syncPolicy: + automated: + # Upstream Kubernetes ships no addon manager, so nothing else removes a + # resource that leaves this tree. Argo is the only pruner in the system: + # the seed itself is append-only and never pruned. + prune: true + # ...and the only thing correcting drift. + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + # cert-manager's CRDs exceed the annotation size limit that + # client-side apply's last-applied-configuration would need. + - Replace=false + retry: + limit: 10 + backoff: + duration: 10s + factor: 2 + maxDuration: 5m diff --git a/files/cluster/seed/20-root-app/appproject.yaml b/files/cluster/seed/20-root-app/appproject.yaml new file mode 100644 index 00000000..e01717bf --- /dev/null +++ b/files/cluster/seed/20-root-app/appproject.yaml @@ -0,0 +1,42 @@ +# Scopes what the root Application is allowed to touch. +# +# The tree it syncs installs cluster-scoped CRDs and webhooks and spans several +# namespaces, so the destination and cluster-resource allowlists are wide. They +# are still narrower than "*": a project that cannot create a new AppProject or +# a new Application cannot be used to escalate out of itself. +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: bluefin + namespace: argocd + labels: + app.kubernetes.io/part-of: bluefin-server +spec: + description: Bluefin Server appliance workload + sourceRepos: + - git://bluefin-git-daemon.bluefin-system.svc.cluster.local/cluster.git + destinations: + - server: https://kubernetes.default.svc + namespace: "*" + clusterResourceWhitelist: + - group: "" + kind: Namespace + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: cert-manager.io + kind: ClusterIssuer + - group: tenancy.kflex.kubestellar.org + kind: ControlPlane + namespaceResourceBlacklist: + - group: argoproj.io + kind: AppProject + - group: argoproj.io + kind: Application diff --git a/files/cluster/seed/20-root-app/kustomization.yaml b/files/cluster/seed/20-root-app/kustomization.yaml new file mode 100644 index 00000000..e35449e7 --- /dev/null +++ b/files/cluster/seed/20-root-app/kustomization.yaml @@ -0,0 +1,12 @@ +# Seed phase 20 — hand-off to Argo. +# +# Last seed phase. After this the seed is done: everything else in the +# appliance is reconciled by Argo CD Core from the git bridge. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: argocd + +resources: + - appproject.yaml + - application.yaml diff --git a/files/installer/repart.d/30-var.conf b/files/installer/repart.d/30-var.conf index 7f2ec345..2fbb3a27 100644 --- a/files/installer/repart.d/30-var.conf +++ b/files/installer/repart.d/30-var.conf @@ -4,10 +4,14 @@ # creating /var entirely. # GrowFileSystem=yes: XFS grows to fill all remaining disk space after # the ESP and root-a partitions are placed. +# CopyFiles seeds the Kubernetes sysext straight into systemd-sysext's own +# scan directory, so the first boot merges it without a network fetch. +# systemd-sysupdate later replaces this file through the kubernetes.raw +# CurrentSymlink in files/os/sysupdate.kubernetes.d/70-kubernetes.transfer. [Partition] Type=var Label=var Format=xfs GrowFileSystem=yes -CopyFiles=/k0s.raw:/lib/k0s/k0s.raw +CopyFiles=/kubernetes.raw:/lib/extensions/kubernetes.raw SizeMinBytes=4G diff --git a/files/k0s/k0s.yaml b/files/k0s/k0s.yaml deleted file mode 100644 index 74597e05..00000000 --- a/files/k0s/k0s.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: k0s.k0sproject.io/v1beta1 -kind: ClusterConfig -metadata: - name: k0s - namespace: kube-system -spec: - network: - provider: kuberouter - kuberouter: - autoMTU: true - hairpin: Enabled - metricsPort: 8088 diff --git a/files/k0s/kubeflex/generate-postgres-secret.sh b/files/k0s/kubeflex/generate-postgres-secret.sh deleted file mode 100755 index 7e7ea1b8..00000000 --- a/files/k0s/kubeflex/generate-postgres-secret.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# First-boot generator for the KubeStellar postgres superuser password. -# -# The postgres StatefulSet (20-postgres.yaml) reads its superuser password from -# the kubeflex-postgres Secret via secretKeyRef. This script creates that Secret -# once, with a random password, so no usable credential is committed to git or -# exposed in the pod spec (see projectbluefin/server#98). -# -# Idempotent: the password is generated only if the Secret already exists, so -# the initialized database stays accessible across re-boots. Run once per boot -# by k0s-first-boot.service, before k0s applies the manifests (its 15- name -# sorts before 20-postgres.yaml, so the Secret exists first). -set -euo pipefail - -manifest_dir="${KUBEFLEX_MANIFEST_DIR:-/var/lib/k0s/manifests/kubestellar}" -secret_file="$manifest_dir/15-kubeflex-postgres-secret.yaml" - -if [ -e "$secret_file" ]; then - exit 0 -fi - -mkdir -p "$manifest_dir" -password="$(head -c 32 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | cut -c1-30)" - -cat > "$secret_file" < k0s system extension raw image not found in /var/lib/extensions/" - echo "==> Running systemd-sysupdate to fetch and install the k0s sysext OTA..." - # Ignore errors if systemd-sysupdate can't connect (e.g. offline dev test) - systemd-sysupdate update || true + + # 0. Reject unsupported roles before touching anything on the host. + if [ "{{ROLE}}" != "controller" ] && [ "{{ROLE}}" != "server" ]; then + echo "ERROR: Invalid role '{{ROLE}}'. Supported: controller" >&2 + exit 1 fi - # 2. Setup k0s configuration directory - install -d -m 0755 /etc/k0s + # 1. Fetch the Kubernetes systemd-sysext if it is not installed yet. + # 70-kubernetes.transfer lands it in /var/lib/extensions/ behind the + # stable kubernetes.raw symlink. + if [ ! -e /var/lib/extensions/kubernetes.raw ]; then + echo "==> Kubernetes system extension not found in /var/lib/extensions/" + echo "==> Running systemd-sysupdate to fetch and install the Kubernetes sysext OTA..." + # Ignore errors if systemd-sysupdate can't connect (e.g. offline dev test) + systemd-sysupdate --component=kubernetes update || true + fi - # 3. Trigger systemd-sysext to merge extensions into /usr immediately - echo "==> Merging k0s system extension..." + # 2. Merge the extension into /usr now rather than waiting for a reboot. + echo "==> Merging the Kubernetes system extension..." systemctl enable --now systemd-sysext.service || true systemd-sysext merge || true - # 4. Trigger systemd-tmpfiles to seed declarative manifests - systemd-tmpfiles --create /usr/lib/tmpfiles.d/k0s-manifests.conf || true + # 3. A merge makes unit files appear under /usr; systemd does not notice + # them on its own. + systemctl daemon-reload - # 5. Enable and start the selected systemd service - if [ "{{ROLE}}" = "controller" ] || [ "{{ROLE}}" = "server" ]; then - echo "==> Enabling and starting k0s controller service..." - systemctl enable --now k0scontroller.service - else - echo "ERROR: Invalid role '{{ROLE}}'. Supported: controller" >&2 - exit 1 - fi + # 4. Bring the cluster up. bluefin-cluster-bootstrap.service carries + # Requires=/After= on kubeadm-init.service and bluefin-cluster-repo.service, + # so starting it runs kubeadm init and then the seed phases in order. + echo "==> Starting cluster bootstrap..." + systemctl enable --now bluefin-cluster-bootstrap.service - echo "==> k0s Kubernetes has been successfully configured and started!" + echo "==> Kubernetes has been successfully configured and started!" + echo "==> Inspect with: kubectl --kubeconfig /etc/kubernetes/admin.conf get nodes" diff --git a/files/os/libexec/bluefin-cluster-repo b/files/os/libexec/bluefin-cluster-repo new file mode 100755 index 00000000..2a134d47 --- /dev/null +++ b/files/os/libexec/bluefin-cluster-repo @@ -0,0 +1,75 @@ +#!/bin/bash +# Publish /usr/share/bluefin/cluster as a bare git repository that the +# in-cluster git-daemon serves to Argo CD. +# +# Argo CD's Application.source accepts git, Helm or OCI and has no filesystem +# type, so the image-delivered manifest tree cannot be read from /usr directly. +# This script bridges the two: it mirrors the read-only tree into a bare +# repository on /var, which the git-daemon Deployment exports over the cluster +# network. No external git host is ever contacted. +# +# It runs on every boot and is idempotent. Re-running after a system update +# publishes the manifests that update shipped; re-running with no change +# publishes nothing and creates no empty commit. +set -euo pipefail + +# git-daemon serves the repository from a non-root uid through a read-only +# hostPath mount, so every directory and object it walks must stay readable. +umask 022 + +SEED=/usr/share/bluefin/cluster +REPO=/var/lib/bluefin/cluster.git +BRANCH=main + +if [ ! -d "${SEED}" ]; then + echo "ERROR: cluster manifest tree is missing at ${SEED}" >&2 + exit 1 +fi + +# Keep the result independent of any stray host or user git configuration. +export GIT_CONFIG_SYSTEM=/dev/null +export GIT_CONFIG_GLOBAL=/dev/null +export GIT_AUTHOR_NAME="Bluefin Server" +export GIT_AUTHOR_EMAIL="root@localhost" +export GIT_COMMITTER_NAME="${GIT_AUTHOR_NAME}" +export GIT_COMMITTER_EMAIL="${GIT_AUTHOR_EMAIL}" + +mkdir -p "${REPO}" +if [ ! -e "${REPO}/HEAD" ]; then + git init --bare --initial-branch="${BRANCH}" "${REPO}" +fi + +# git-daemon refuses to serve a repository that is not explicitly exported. +touch "${REPO}/git-daemon-export-ok" + +# Build the tree through a scratch index so a failure leaves no partial state +# in the repository the daemon is serving. core.bare is overridden per +# invocation because `git add` refuses to run against a bare repository. +INDEX="${REPO}/bluefin-seed.index" +rm -f "${INDEX}" +GIT_INDEX_FILE="${INDEX}" git --git-dir="${REPO}" -c core.bare=false \ + --work-tree="${SEED}" add --all +TREE="$(GIT_INDEX_FILE="${INDEX}" git --git-dir="${REPO}" write-tree)" +rm -f "${INDEX}" + +PUBLISHED="" +if git --git-dir="${REPO}" rev-parse --verify --quiet "refs/heads/${BRANCH}^{tree}" >/dev/null; then + PUBLISHED="$(git --git-dir="${REPO}" rev-parse --verify "refs/heads/${BRANCH}^{tree}")" +fi + +if [ "${TREE}" = "${PUBLISHED}" ]; then + echo "cluster repository already matches ${SEED}" +else + if [ -n "${PUBLISHED}" ]; then + COMMIT="$(git --git-dir="${REPO}" commit-tree "${TREE}" \ + -p "refs/heads/${BRANCH}" -m "Bluefin cluster manifests")" + else + COMMIT="$(git --git-dir="${REPO}" commit-tree "${TREE}" \ + -m "Bluefin cluster manifests")" + fi + git --git-dir="${REPO}" update-ref "refs/heads/${BRANCH}" "${COMMIT}" + echo "published ${SEED} as ${BRANCH} ${COMMIT}" +fi + +git --git-dir="${REPO}" symbolic-ref HEAD "refs/heads/${BRANCH}" +git --git-dir="${REPO}" update-server-info diff --git a/files/os/systemd/system-preset/zz-enable-cluster-bootstrap.preset b/files/os/systemd/system-preset/zz-enable-cluster-bootstrap.preset new file mode 100644 index 00000000..dc9246b0 --- /dev/null +++ b/files/os/systemd/system-preset/zz-enable-cluster-bootstrap.preset @@ -0,0 +1,4 @@ +enable kubelet.service +enable kubeadm-init.service +enable bluefin-cluster-repo.service +enable bluefin-cluster-bootstrap.service diff --git a/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset b/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset deleted file mode 100644 index 60ae94e5..00000000 --- a/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset +++ /dev/null @@ -1 +0,0 @@ -enable k0s-first-boot.service diff --git a/files/os/systemd/system/bluefin-cluster-bootstrap.service b/files/os/systemd/system/bluefin-cluster-bootstrap.service new file mode 100644 index 00000000..d4cb71c8 --- /dev/null +++ b/files/os/systemd/system/bluefin-cluster-bootstrap.service @@ -0,0 +1,40 @@ +[Unit] +Description=Seed the Bluefin cluster: CNI, Argo CD Core, manifest bridge, root Application +Documentation=file:///usr/share/bluefin/seed +# Deliberately no ConditionPathExists. A failed condition marks a unit +# *skipped* rather than failed, which would silently no-op the seed on first +# boot and leave a node with no CNI and no reconciler. Every phase below is a +# server-side apply and is idempotent on its own, so re-running is free. +Requires=kubeadm-init.service +After=kubeadm-init.service +Requires=bluefin-cluster-repo.service +After=bluefin-cluster-repo.service +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +Restart=on-failure +RestartSec=30s +TimeoutStartSec=1800 +Environment=KUBECONFIG=/etc/kubernetes/admin.conf +# The seed is append-only and is never pruned: kubectl's allowlist --prune is +# alpha and incompatible with --server-side, and --applyset cannot span the +# namespaces this seed touches. Argo CD owns pruning for everything downstream. +# Phase 1: CNI. Nothing else can schedule until the node leaves NotReady. +ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/00-cilium +ExecStart=/usr/bin/kubectl wait --for=condition=Ready node --all --timeout=300s +# Phase 2: the reconciler itself. The application controller is a StatefulSet, +# so `wait deployment --all` would not cover it. +ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/10-argocd-core +ExecStart=/usr/bin/kubectl wait --for=condition=Available deployment --all --namespace argocd --timeout=600s +ExecStart=/usr/bin/kubectl rollout status statefulset/argocd-application-controller --namespace argocd --timeout=600s +# Phase 3: the git bridge that lets Argo read /var/lib/bluefin/cluster.git. +ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/15-gitd +ExecStart=/usr/bin/kubectl wait --for=condition=Available deployment/bluefin-git-daemon --namespace bluefin-system --timeout=600s +# Phase 4: hand the cluster over. Argo owns everything from here. +ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/20-root-app + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/bluefin-cluster-repo.service b/files/os/systemd/system/bluefin-cluster-repo.service new file mode 100644 index 00000000..fa9c0574 --- /dev/null +++ b/files/os/systemd/system/bluefin-cluster-repo.service @@ -0,0 +1,16 @@ +[Unit] +Description=Publish the Bluefin cluster manifests as a local git repository +Documentation=file:///usr/share/bluefin/cluster +# Argo CD has no filesystem source type, so the image-delivered manifest tree +# has to be reachable over git before the root Application can sync. +RequiresMountsFor=/var/lib/bluefin +Before=bluefin-cluster-bootstrap.service + +[Service] +Type=oneshot +RemainAfterExit=yes +StateDirectory=bluefin +ExecStart=/usr/libexec/bluefin-cluster-repo + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/k0s-first-boot-fetch.service b/files/os/systemd/system/k0s-first-boot-fetch.service deleted file mode 100644 index a8cee552..00000000 --- a/files/os/systemd/system/k0s-first-boot-fetch.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Fetch the k0s sysext when installation did not seed it -ConditionPathExists=!/var/lib/k0s/k0s.raw -Wants=network-online.target -After=network-online.target -Before=k0s-first-boot.service - -[Service] -Type=oneshot -Restart=on-failure -RestartSec=30s -ExecStart=/usr/bin/systemd-sysupdate --component=k0s update diff --git a/files/os/systemd/system/k0s-first-boot.service b/files/os/systemd/system/k0s-first-boot.service deleted file mode 100644 index 6e61f470..00000000 --- a/files/os/systemd/system/k0s-first-boot.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=Activate k0s sysext on boot -Wants=network-online.target -Wants=k0s-first-boot-fetch.service -After=network-online.target -After=k0s-first-boot-fetch.service -RequiresMountsFor=/var/lib/k0s - -[Service] -Type=oneshot -StateDirectory=k0s -Restart=on-failure -RestartSec=30s -ExecStartPre=/usr/bin/test -e /var/lib/k0s/k0s.raw -ExecStart=/usr/bin/install -D -m 0644 /var/lib/k0s/k0s.raw /run/extensions/k0s.raw -ExecStart=/usr/bin/systemd-sysext merge -ExecStart=/usr/bin/systemd-tmpfiles --create /usr/lib/tmpfiles.d/k0s-manifests.conf -ExecStart=/usr/share/k0s/kubeflex/generate-postgres-secret.sh -ExecStart=/usr/bin/systemctl daemon-reload -ExecStart=/usr/bin/systemctl enable --now k0scontroller.service -[Install] -WantedBy=multi-user.target diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service new file mode 100644 index 00000000..5c28f1f6 --- /dev/null +++ b/files/os/systemd/system/kubeadm-init.service @@ -0,0 +1,30 @@ +[Unit] +Description=Initialise the Kubernetes control plane with kubeadm +Documentation=https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/ +# The idempotency guard belongs here and nowhere else. A unit whose condition +# fails is marked *skipped*, and systemd treats a skipped unit as satisfied for +# everything ordered after it, so bluefin-cluster-bootstrap.service still runs. +ConditionPathExists=!/etc/kubernetes/admin.conf +# /usr/bin/kubeadm and containerd.service both arrive with the sysext merge. +Requires=systemd-sysext.service +After=systemd-sysext.service +Requires=containerd.service +After=containerd.service +Wants=network-online.target +After=network-online.target +RequiresMountsFor=/var/lib/kubelet + +[Service] +Type=oneshot +# RemainAfterExit is what makes the ordering edge useful: the unit stays active +# after kubeadm exits, so dependents see a control plane that is still up. +RemainAfterExit=yes +Restart=on-failure +RestartSec=30s +TimeoutStartSec=900 +# Cilium provides the kube-proxy replacement; installing kube-proxy as well +# would leave two datapaths fighting over the same Services. +ExecStart=/usr/bin/kubeadm init --config /usr/share/bluefin/kubeadm.yaml --skip-phases=addon/kube-proxy + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/kubelet.service b/files/os/systemd/system/kubelet.service new file mode 100644 index 00000000..c9f7cfe9 --- /dev/null +++ b/files/os/systemd/system/kubelet.service @@ -0,0 +1,28 @@ +[Unit] +Description=kubelet: the Kubernetes node agent +Documentation=https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ +# /usr/bin/kubelet ships in the Kubernetes sysext, not in the base DDI, so the +# merge must complete before this unit can start. +Requires=systemd-sysext.service +After=systemd-sysext.service +Wants=network-online.target +After=network-online.target +After=containerd.service +# kubeadm writes the kubelet configuration, so the kubelet restarts until +# `kubeadm init` has run. Never give up on it. +StartLimitIntervalSec=0 + +[Service] +# The Kubernetes sysext ships the upstream CNI plugins at /usr/local/bin/cni, +# matching the flatcar/sysext-bakery layout. containerd and Cilium both look in +# /opt/cni/bin. Without this bridge the `loopback` plugin is missing, every pod +# sandbox fails to get a network, and the node never leaves NotReady — silently. +# -n preserves the cilium binary that the agent installs into the same directory. +ExecStartPre=/usr/bin/install -d -m 0755 /opt/cni/bin +ExecStartPre=/bin/sh -c 'cp -an /usr/local/bin/cni/. /opt/cni/bin/' +ExecStart=/usr/bin/kubelet +Restart=always +RestartSec=10s + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/kubelet.service.d/10-kubeadm.conf b/files/os/systemd/system/kubelet.service.d/10-kubeadm.conf new file mode 100644 index 00000000..56d23d0d --- /dev/null +++ b/files/os/systemd/system/kubelet.service.d/10-kubeadm.conf @@ -0,0 +1,10 @@ +# Upstream kubeadm kubelet drop-in. kubeadm rewrites +# /var/lib/kubelet/kubeadm-flags.env on every `kubeadm init`, so node-specific +# flags are derived rather than baked into the image. +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS diff --git a/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf b/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf new file mode 100644 index 00000000..3e12ecb3 --- /dev/null +++ b/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf @@ -0,0 +1,6 @@ +# A sysext merge makes unit files appear under /usr; it does not by itself make +# systemd notice them. Without this reload, ordering edges on containerd.service +# and kubelet.service resolve against units systemd has never read, and the +# failure mode is silent: image merged, nothing running. +[Service] +ExecStartPost=/usr/bin/systemctl daemon-reload diff --git a/files/os/sysupdate.k0s.d/70-k0s.transfer b/files/os/sysupdate.k0s.d/70-k0s.transfer deleted file mode 100644 index fa3cab9a..00000000 --- a/files/os/sysupdate.k0s.d/70-k0s.transfer +++ /dev/null @@ -1,13 +0,0 @@ -[Transfer] - -[Source] -Type=url-file -Path=https://github.com/projectbluefin/server/releases/latest/download/ -MatchPattern=k0s-@v.raw.zst - -[Target] -Type=regular-file -Path=/var/lib/k0s -MatchPattern=k0s-@v.raw -CurrentSymlink=k0s.raw -Mode=0644 diff --git a/files/os/sysupdate.kubernetes.d/70-kubernetes.transfer b/files/os/sysupdate.kubernetes.d/70-kubernetes.transfer new file mode 100644 index 00000000..fff4150c --- /dev/null +++ b/files/os/sysupdate.kubernetes.d/70-kubernetes.transfer @@ -0,0 +1,24 @@ +# systemd-sysupdate transfer for the Kubernetes sysext. +# +# The image lands directly in /var/lib/extensions, systemd-sysext's own scan +# directory, so systemd-sysext.service merges it at boot with no copy step. +# +# Upgrade track: Kubernetes does not support unattended cross-minor in-place +# upgrades, so this transfer must never carry the node across a minor. The +# asset name is fixed by .github/scripts/check-kubernetes-version.py and +# systemd-sysupdate has no version-range filter, so the minor track is held by +# what the release feed publishes: bumping include/kubernetes.yml across a +# minor is a deliberate release action, not something this file can drift into. +[Transfer] + +[Source] +Type=url-file +Path=https://github.com/projectbluefin/server/releases/latest/download/ +MatchPattern=kubernetes-@v.raw.zst + +[Target] +Type=regular-file +Path=/var/lib/extensions +MatchPattern=kubernetes-@v.raw +CurrentSymlink=kubernetes.raw +Mode=0644 diff --git a/include/aliases.yml b/include/aliases.yml index 6ff26ed3..aef91c40 100644 --- a/include/aliases.yml +++ b/include/aliases.yml @@ -3,7 +3,9 @@ aliases: pypi: https://files.pythonhosted.org/packages/ github: https://github.com/ + githubraw: https://raw.githubusercontent.com/ gitlab: https://gitlab.com/ gnome: https://gitlab.gnome.org/GNOME/ crates: https://static.crates.io/ + k8s: https://dl.k8s.io/ flatcar: https://flatcar.cdn.cncf.io/ diff --git a/include/arch.yml b/include/arch.yml index a8a79d60..871dfaec 100644 --- a/include/arch.yml +++ b/include/arch.yml @@ -13,7 +13,7 @@ # architectures mapped below, so an unmapped architecture cannot be selected. # # Included by: -# elements/oci/k0s-sysext.bst -> %{systemd-arch} +# elements/oci/kubernetes-sysext.bst -> %{systemd-arch} # elements/bluefin-server/os-release-flatcar.bst -> %{flatcar-board} variables: (?): diff --git a/include/k0s.yml b/include/k0s.yml deleted file mode 100644 index 5dd0fd28..00000000 --- a/include/k0s.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Single source of truth for the k0s version axis. -# -# k0s is an independently-pinned third-party payload, not an OS asset. Its -# version therefore does NOT live on the OS `release-version` axis declared in -# project.conf — that axis names the FSDK point release and is bumped by -# unrelated OS work. -# -# Only the two atoms below are edited when k0s is bumped. Every other spelling -# of the k0s version in this repository is derived from them: -# -# %{k0s-k8s-version} upstream Kubernetes version 1.36.4 -# %{k0s-patch} upstream k0s patch suffix number 0 -# %{k0s-upstream-tag} GitHub release tag, URL-escaped v1.36.4%2Bk0s.0 -# %{k0s-version} asset / extension-release version 1.36.4-k0s.0 -variables: - k0s-k8s-version: "1.36.4" - k0s-patch: "0" - - k0s-upstream-tag: "v%{k0s-k8s-version}%2Bk0s.%{k0s-patch}" - k0s-version: "%{k0s-k8s-version}-k0s.%{k0s-patch}" diff --git a/include/kubernetes.yml b/include/kubernetes.yml new file mode 100644 index 00000000..11b8be89 --- /dev/null +++ b/include/kubernetes.yml @@ -0,0 +1,29 @@ +# Single source of truth for the Kubernetes version axis. +# +# Kubernetes is an independently-pinned third-party payload, not an OS asset. +# Its version therefore does NOT live on the OS `release-version` axis declared +# in project.conf — that axis names the FSDK point release and is bumped by +# unrelated OS work. +# +# Only the two atoms below are edited when Kubernetes is bumped. Every other +# spelling of the Kubernetes version in this repository is derived from them: +# +# %{k8s-version} upstream Kubernetes version 1.36.4 +# %{cni-version} containernetworking/plugins tag 1.9.1 +# %{k8s-upstream-tag} dl.k8s.io release directory v1.36.4 +# %{cni-upstream-tag} CNI plugins GitHub release tag v1.9.1 +# +# The sysext asset is named `kubernetes-%{k8s-version}.raw`, which is the +# version oracle systemd-sysupdate parses back out through the `@v` wildcard +# in files/os/sysupdate.kubernetes.d/70-kubernetes.transfer. +# +# Included by: +# elements/kubernetes/kubernetes-bin.bst -> %{k8s-upstream-tag} +# elements/kubernetes/cni-plugins.bst -> %{cni-upstream-tag} +# elements/oci/kubernetes-sysext.bst -> %{k8s-version}, %{cni-version} +variables: + k8s-version: "1.36.4" + cni-version: "1.9.1" + + k8s-upstream-tag: "v%{k8s-version}" + cni-upstream-tag: "v%{cni-version}" diff --git a/scripts/lima-e2e-kubestellar-test.sh b/scripts/lima-e2e-kubestellar-test.sh index 79ecff63..cc604288 100755 --- a/scripts/lima-e2e-kubestellar-test.sh +++ b/scripts/lima-e2e-kubestellar-test.sh @@ -41,9 +41,9 @@ limactl start --tty=false "${INSTANCE_NAME}" & LIMA_START_PID=$! echo "==> Starting local kc-agent client connection..." -./files/bin/bluefin-kubestellar start --origin "http://localhost:8080,http://127.0.0.1:8080" +./files/bin/bluefin-kubestellar start --origin "https://localhost:8080,https://127.0.0.1:8080" echo "==> Running automated browser verification test..." -python3 tests/e2e/test_kubestellar_browser_login.py --console-url "http://127.0.0.1:8080" --agent-url "http://127.0.0.1:8585" --timeout 15 +python3 tests/e2e/test_kubestellar_browser_login.py --console-url "https://127.0.0.1:8080" --agent-url "http://127.0.0.1:8585" --timeout 15 echo "==> Lima VM end-to-end verification passed!" diff --git a/tests/e2e/test_kubestellar_browser_login.py b/tests/e2e/test_kubestellar_browser_login.py index bf975b32..e7fbee20 100755 --- a/tests/e2e/test_kubestellar_browser_login.py +++ b/tests/e2e/test_kubestellar_browser_login.py @@ -2,7 +2,7 @@ """Automated end-to-end browser test for KubeStellar Console. Tests: -1. Console availability at http://127.0.0.1:8080/ +1. Console availability at https://127.0.0.1:8080/ 2. Automated login flow (Cluster Access / GitHub OAuth / Dev Mode / Token Login) 3. Dashboard navigation and DOM rendering 4. Kiosk gate overlay and kc-agent interaction on port 8585 @@ -27,8 +27,8 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="KubeStellar Console automated browser verification") parser.add_argument( "--console-url", - default="http://127.0.0.1:8080", - help="Base URL of KubeStellar console (default: http://127.0.0.1:8080)", + default="https://127.0.0.1:8080", + help="Base URL of KubeStellar console (default: https://127.0.0.1:8080)", ) parser.add_argument( "--agent-url", diff --git a/tests/unit/os-justfile_test.bats b/tests/unit/os-justfile_test.bats old mode 100644 new mode 100755 index a40d906b..767451d1 --- a/tests/unit/os-justfile_test.bats +++ b/tests/unit/os-justfile_test.bats @@ -2,11 +2,11 @@ # # Unit tests for the `k8s` recipe in files/os/justfile. # -# The recipe interacts with /etc/k0s and /var/lib/extensions, so every +# The recipe interacts with /var/lib/extensions and enables host units, so every # invocation runs inside an unprivileged user + mount namespace with tmpfs # masks over /etc and /var/lib. Nothing on the host is touched. systemctl, -# systemd-sysext, systemd-sysupdate, and systemd-tmpfiles are replaced by -# logging stubs on PATH, so no unit is ever enabled and no OTA is ever fetched. +# systemd-sysext and systemd-sysupdate are replaced by logging stubs on PATH, +# so no unit is ever enabled and no OTA is ever fetched. # # Results that must outlive the namespace (the call log) are copied back # into BATS_TEST_TMPDIR, which is outside both tmpfs masks. @@ -36,11 +36,10 @@ setup() { JUSTFILE_PARSES=0 fi - # Defaults: every helper succeeds, and the k0s sysext is not yet present. + # Defaults: every helper succeeds, and the sysext is not yet installed. make_stub systemctl 0 make_stub systemd-sysext 0 make_stub systemd-sysupdate 0 - make_stub systemd-tmpfiles 0 SYSEXT_PRESENT=0 } @@ -69,7 +68,7 @@ set -u mount -t tmpfs tmpfs /etc mount -t tmpfs tmpfs /var/lib mkdir -p /var/lib/extensions -[ "$SYSEXT_PRESENT" = "1" ] && : > /var/lib/extensions/k0s.raw +[ "$SYSEXT_PRESENT" = "1" ] && : > /var/lib/extensions/kubernetes.raw just --justfile "$JUSTFILE" --working-directory "$WORKDIR" k8s "$@" rc=$? @@ -83,11 +82,6 @@ calls() { # --- the recipe file itself -------------------------------------------------- -# This is the parse gate. It fails on the current tree: the `cat <> extension-release.k0s -""" - -GOOD_EXTENSION_RELEASE = "NAME=k0s\nID=_any\n" - -GOOD_TRANSFER = """[Source] -Type=url-file -MatchPattern=k0s-@v.raw.zst - -[Target] -MatchPattern=k0s-@v.raw -""" - -EXPECTED_VERSION = "1.36.4-k0s.0" - - -def _load_module(): - spec = importlib.util.spec_from_file_location("check_k0s_version", SCRIPT_PATH) - module = importlib.util.module_from_spec(spec) - sys.modules["check_k0s_version"] = module - spec.loader.exec_module(module) - return module - - -@pytest.fixture -def checker(tmp_path): - """Fresh module instance with its path constants rooted at tmp_path. - - ``ROOT`` and the five consumer paths are module-level globals derived from - the script's own location, so each test gets a re-imported copy repointed at - an isolated tree pre-populated with a passing configuration. - """ - module = _load_module() - module.ROOT = tmp_path - module.K0S_INCLUDE = tmp_path / "include" / "k0s.yml" - module.K0S_BIN = tmp_path / "elements" / "k0s" / "k0s-bin.bst" - module.K0S_SYSEXT = tmp_path / "elements" / "oci" / "k0s-sysext.bst" - module.EXTENSION_RELEASE = ( - tmp_path / "files" / "k0s" / "sysext" / "extension-release.k0s" - ) - module.TRANSFER = tmp_path / "files" / "os" / "sysupdate.d" / "70-k0s.transfer" - for path, text in ( - (module.K0S_INCLUDE, GOOD_INCLUDE), - (module.K0S_BIN, GOOD_BIN), - (module.K0S_SYSEXT, GOOD_SYSEXT), - (module.EXTENSION_RELEASE, GOOD_EXTENSION_RELEASE), - (module.TRANSFER, GOOD_TRANSFER), - ): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text, encoding="utf-8") - yield module - sys.modules.pop("check_k0s_version", None) - - -def _run(checker): - with pytest.raises(SystemExit) as excinfo: - checker.main() - return str(excinfo.value) - - -# --- read() --------------------------------------------------------------- - - -def test_read_returns_file_contents(checker, tmp_path): - target = tmp_path / "include" / "k0s.yml" - assert checker.read(target) == GOOD_INCLUDE - - -def test_read_exits_on_missing_file(checker, tmp_path): - missing = tmp_path / "include" / "gone.yml" - with pytest.raises(SystemExit) as excinfo: - checker.read(missing) - assert "expected file not found" in str(excinfo.value) - assert "include/gone.yml" in str(excinfo.value) - - -# --- scalar() ------------------------------------------------------------- - - -def test_scalar_reads_double_quoted_value(checker): - assert checker.scalar(GOOD_INCLUDE, "k0s-patch", "where") == "0" - - -def test_scalar_reads_single_quoted_value(checker): - text = "variables:\n k0s-patch: '3'\n" - assert checker.scalar(text, "k0s-patch", "where") == "3" - - -def test_scalar_ignores_unquoted_value(checker): - with pytest.raises(SystemExit) as excinfo: - checker.scalar("variables:\n k0s-patch: 0\n", "k0s-patch", "include/k0s.yml") - assert "does not declare a 'k0s-patch:' variable" in str(excinfo.value) - - -def test_scalar_exits_when_name_absent(checker): - with pytest.raises(SystemExit) as excinfo: - checker.scalar(GOOD_INCLUDE, "k0s-nope", "include/k0s.yml") - assert "include/k0s.yml does not declare a 'k0s-nope:' variable" in str( - excinfo.value - ) - - -# --- expand() ------------------------------------------------------------- - - -def test_expand_returns_literal_unchanged(checker): - assert checker.expand("1.36.4", {}, "where") == "1.36.4" - - -def test_expand_substitutes_nested_references(checker): - variables = {"a": "%{b}", "b": "leaf"} - assert checker.expand("%{a}", variables, "where") == "leaf" - - -def test_expand_exits_on_undefined_reference(checker): - with pytest.raises(SystemExit) as excinfo: - checker.expand("%{missing}", {}, "include/k0s.yml x") - assert "references undefined variable '%{missing}'" in str(excinfo.value) - - -def test_expand_exits_on_circular_reference(checker): - with pytest.raises(SystemExit) as excinfo: - checker.expand("%{a}", {"a": "%{b}", "b": "%{a}"}, "include/k0s.yml x") - assert "circular variable reference" in str(excinfo.value) - - -# --- fail() --------------------------------------------------------------- - - -def test_fail_reports_problem_and_fix(checker): - with pytest.raises(SystemExit) as excinfo: - checker.fail("something drifted", "do the thing") - message = str(excinfo.value) - assert "ERROR: something drifted" in message - assert "Fix: do the thing" in message - - -# --- module-level regexes ------------------------------------------------- - - -@pytest.mark.parametrize( - "candidate", - ["1.36.4-k0s.0", "v1.36.4", "1.36.4+k0s.0", "release~1^2"], -) -def test_safe_version_re_accepts_filename_safe_versions(checker, candidate): - assert checker.SAFE_VERSION_RE.match(candidate) - - -@pytest.mark.parametrize("candidate", ["1.36.4 k0s", "1.36.4/0", "", "a:b"]) -def test_safe_version_re_rejects_unsafe_versions(checker, candidate): - assert not checker.SAFE_VERSION_RE.match(candidate) - - -@pytest.mark.parametrize( - "candidate", - ["v1.36.4+k0s.0", "1.36.4%2Bk0s.0", "1.36.4-k0s.12"], -) -def test_literal_version_re_matches_every_spelling(checker, candidate): - assert checker.LITERAL_VERSION_RE.search(candidate) - - -def test_literal_version_re_ignores_templated_reference(checker): - assert not checker.LITERAL_VERSION_RE.search("%{k0s-upstream-tag}") - - -# --- main(): the passing path -------------------------------------------- - - -def test_main_accepts_a_fully_derived_configuration(checker, capsys): - checker.main() - out = capsys.readouterr().out - assert "OK: k0s v1.36.4%2Bk0s.0 pinned in include/k0s.yml" in out - assert f"asset k0s-{EXPECTED_VERSION}.raw.zst" in out - assert f"VERSION_ID={EXPECTED_VERSION}" in out - - -def test_main_tolerates_literal_version_inside_comments(checker, capsys): - checker.K0S_BIN.write_text( - "# pinned from the v1.36.4+k0s.0 release notes\n" + GOOD_BIN, - encoding="utf-8", - ) - checker.main() - assert "OK: k0s" in capsys.readouterr().out - - -# --- main(): version safety ---------------------------------------------- - - -def test_main_rejects_version_that_is_not_filename_safe(checker): - checker.K0S_INCLUDE.write_text( - GOOD_INCLUDE.replace('k0s-patch: "0"', 'k0s-patch: "0 rc1"'), - encoding="utf-8", - ) - assert "is not filename-safe" in _run(checker) - - -# --- main(): upstream download URL --------------------------------------- - - -def test_main_rejects_missing_source_url(checker): - checker.K0S_BIN.write_text("kind: manual\nsources: []\n", encoding="utf-8") - assert "declares no source 'url:'" in _run(checker) - - -def test_main_rejects_hardcoded_upstream_tag_in_url(checker): - checker.K0S_BIN.write_text( - "sources:\n" - " - kind: remote\n" - " url: github:k0sproject/k0s/releases/download/v1.36.4%2Bk0s.0/k0s-amd64\n", - encoding="utf-8", - ) - assert "hardcodes the upstream k0s release tag" in _run(checker) - - -# --- main(): sysext asset filename --------------------------------------- - - -def test_main_rejects_missing_fname(checker): - checker.K0S_SYSEXT.write_text( - 'echo "VERSION_ID=%{k0s-version}"\n', encoding="utf-8" - ) - assert 'declares no FNAME="..." asset name' in _run(checker) - - -def test_main_rejects_fname_on_the_os_release_axis(checker): - checker.K0S_SYSEXT.write_text( - GOOD_SYSEXT.replace("%{k0s-version}.raw", "%{release-version}.raw", 1), - encoding="utf-8", - ) - assert "names the k0s sysext on the OS release" in _run(checker) - - -def test_main_rejects_fname_not_derived_from_k0s_version(checker): - checker.K0S_SYSEXT.write_text( - GOOD_SYSEXT.replace('FNAME="k0s-%{k0s-version}.raw"', 'FNAME="k0s.raw"'), - encoding="utf-8", - ) - assert "does not derive the sysext asset name" in _run(checker) - - -def test_main_rejects_missing_generated_version_id(checker): - checker.K0S_SYSEXT.write_text( - 'FNAME="k0s-%{k0s-version}.raw"\n', encoding="utf-8" - ) - assert "does not generate VERSION_ID= from" in _run(checker) - - -# --- main(): extension-release.k0s --------------------------------------- - - -def test_main_rejects_hardcoded_version_id_in_extension_release(checker): - checker.EXTENSION_RELEASE.write_text( - "NAME=k0s\nID=_any\nVERSION_ID=1.36.4-k0s.0\n", encoding="utf-8" - ) - message = _run(checker) - assert "hardcodes a version" in message - assert "VERSION_ID=1.36.4-k0s.0" in message - - -def test_main_rejects_indented_hardcoded_version_id(checker): - checker.EXTENSION_RELEASE.write_text( - "NAME=k0s\n VERSION_ID=9.9.9\n", encoding="utf-8" - ) - assert "hardcodes a version" in _run(checker) - - -# --- main(): literal version scan ---------------------------------------- - - -@pytest.mark.parametrize( - "attr", - ["K0S_BIN", "K0S_SYSEXT", "EXTENSION_RELEASE"], -) -def test_main_rejects_literal_version_in_any_consumer(checker, attr): - path = getattr(checker, attr) - path.write_text( - path.read_text(encoding="utf-8") + "\nnote: shipped v1.36.4+k0s.0\n", - encoding="utf-8", - ) - message = _run(checker) - assert "restates a literal k0s version" in message - assert "Literal: v1.36.4+k0s.0" in message - - -# --- main(): sysupdate MatchPattern -------------------------------------- - - -def test_main_rejects_missing_match_pattern(checker): - checker.TRANSFER.write_text("[Source]\nType=url-file\n", encoding="utf-8") - assert "declares no [Source]" in _run(checker) - - -def test_main_rejects_match_pattern_without_zst_suffix(checker): - checker.TRANSFER.write_text( - "[Source]\nMatchPattern=k0s-@v.raw\n", encoding="utf-8" - ) - assert "declares no [Source]" in _run(checker) - - -def test_main_rejects_prefix_drift_between_fname_and_match_pattern(checker): - checker.TRANSFER.write_text( - "[Source]\nMatchPattern=kzeros-@v.raw.zst\n", encoding="utf-8" - ) - message = _run(checker) - assert "is not the pinned k0s version" in message - assert "'@v' captures : None" in message - - -def test_main_rejects_suffix_drift_between_fname_and_match_pattern(checker): - checker.K0S_SYSEXT.write_text( - GOOD_SYSEXT.replace( - 'FNAME="k0s-%{k0s-version}.raw"', 'FNAME="k0s-%{k0s-version}.erofs"' - ), - encoding="utf-8", - ) - assert "is not the pinned k0s version" in _run(checker) - - -def test_main_rejects_match_pattern_that_captures_extra_text(checker): - checker.TRANSFER.write_text( - "[Source]\nMatchPattern=k@v.raw.zst\n", encoding="utf-8" - ) - message = _run(checker) - assert "is not the pinned k0s version" in message - assert f"'@v' captures : '0s-{EXPECTED_VERSION}'" in message diff --git a/tests/unit/test_check_kubernetes_version.py b/tests/unit/test_check_kubernetes_version.py new file mode 100644 index 00000000..0843d8de --- /dev/null +++ b/tests/unit/test_check_kubernetes_version.py @@ -0,0 +1,481 @@ +"""Unit coverage for .github/scripts/check-kubernetes-version.py. + +The script is the only gate holding the Kubernetes version axis together. It is +run by ``just validate`` and by the ``check-kubernetes-version`` pre-commit +hook, so every fail-closed branch needs direct coverage: without it a branch +can regress to fail-open and nothing notices until the sysext silently stops +updating on deployed hosts. + +Covered here: ``read()``, ``scalar()``, ``expand()``, ``fail()``, +``source_urls()`` and each rejection path of ``main()`` — non-filename-safe +version, missing/hardcoded node-binary URLs, missing/hardcoded CNI URL, +missing/OS-axis/underived ``FNAME``, missing generated ``VERSION_ID``, missing +``/usr/local/share`` version stamps, a hardcoded ``VERSION_ID=`` in +extension-release.kubernetes, literal versions restated in any consumer, a +missing sysupdate ``MatchPattern``, and an asset name whose ``@v`` capture +disagrees with ``%{k8s-version}``. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_PATH = REPO_ROOT / ".github" / "scripts" / "check-kubernetes-version.py" + +GOOD_INCLUDE = """variables: + k8s-version: "1.36.4" + cni-version: "1.9.1" + + k8s-upstream-tag: "v%{k8s-version}" + cni-upstream-tag: "v%{cni-version}" +""" + +GOOD_BIN = """kind: manual +sources: + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubectl + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubeadm + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubelet +""" + +GOOD_CNI = """kind: manual +sources: + - kind: tar + url: github:containernetworking/plugins/releases/download/%{cni-upstream-tag}/cni-plugins-linux-amd64-%{cni-upstream-tag}.tgz +""" + +GOOD_SYSEXT = """kind: manual +config: + install-commands: + - | + FNAME="kubernetes-%{k8s-version}.raw" + echo "%{k8s-version}" > sysext/usr/local/share/kubernetes-version + echo "%{cni-version}" > sysext/usr/local/share/kubernetes-cni-version + echo "VERSION_ID=%{k8s-version}" >> extension-release.kubernetes +""" + +GOOD_EXTENSION_RELEASE = "NAME=kubernetes\nID=_any\n" + +GOOD_TRANSFER = """[Source] +Type=url-file +MatchPattern=kubernetes-@v.raw.zst + +[Target] +MatchPattern=kubernetes-@v.raw +""" + +EXPECTED_VERSION = "1.36.4" + + +def _load_module(): + spec = importlib.util.spec_from_file_location( + "check_kubernetes_version", SCRIPT_PATH + ) + module = importlib.util.module_from_spec(spec) + sys.modules["check_kubernetes_version"] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def checker(tmp_path): + """Fresh module instance with its path constants rooted at tmp_path. + + ``ROOT`` and the six consumer paths are module-level globals derived from + the script's own location, so each test gets a re-imported copy repointed at + an isolated tree pre-populated with a passing configuration. + """ + module = _load_module() + module.ROOT = tmp_path + module.K8S_INCLUDE = tmp_path / "include" / "kubernetes.yml" + module.K8S_BIN = tmp_path / "elements" / "kubernetes" / "kubernetes-bin.bst" + module.CNI_BIN = tmp_path / "elements" / "kubernetes" / "cni-plugins.bst" + module.K8S_SYSEXT = tmp_path / "elements" / "oci" / "kubernetes-sysext.bst" + module.EXTENSION_RELEASE = ( + tmp_path / "files" / "kubernetes" / "sysext" / "extension-release.kubernetes" + ) + module.TRANSFER = ( + tmp_path / "files" / "os" / "sysupdate.kubernetes.d" / "70-kubernetes.transfer" + ) + for path, text in ( + (module.K8S_INCLUDE, GOOD_INCLUDE), + (module.K8S_BIN, GOOD_BIN), + (module.CNI_BIN, GOOD_CNI), + (module.K8S_SYSEXT, GOOD_SYSEXT), + (module.EXTENSION_RELEASE, GOOD_EXTENSION_RELEASE), + (module.TRANSFER, GOOD_TRANSFER), + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + yield module + sys.modules.pop("check_kubernetes_version", None) + + +def _run(checker): + with pytest.raises(SystemExit) as excinfo: + checker.main() + return str(excinfo.value) + + +# --- read() --------------------------------------------------------------- + + +def test_read_returns_file_contents(checker, tmp_path): + target = tmp_path / "include" / "kubernetes.yml" + assert checker.read(target) == GOOD_INCLUDE + + +def test_read_exits_on_missing_file(checker, tmp_path): + missing = tmp_path / "include" / "gone.yml" + with pytest.raises(SystemExit) as excinfo: + checker.read(missing) + assert "expected file not found" in str(excinfo.value) + assert "include/gone.yml" in str(excinfo.value) + + +# --- scalar() ------------------------------------------------------------- + + +def test_scalar_reads_double_quoted_value(checker): + assert checker.scalar(GOOD_INCLUDE, "cni-version", "where") == "1.9.1" + + +def test_scalar_reads_single_quoted_value(checker): + text = "variables:\n cni-version: '1.9.2'\n" + assert checker.scalar(text, "cni-version", "where") == "1.9.2" + + +def test_scalar_ignores_unquoted_value(checker): + with pytest.raises(SystemExit) as excinfo: + checker.scalar( + "variables:\n cni-version: 1.9.1\n", + "cni-version", + "include/kubernetes.yml", + ) + assert "does not declare a 'cni-version:' variable" in str(excinfo.value) + + +def test_scalar_exits_when_name_absent(checker): + with pytest.raises(SystemExit) as excinfo: + checker.scalar(GOOD_INCLUDE, "k8s-nope", "include/kubernetes.yml") + assert "include/kubernetes.yml does not declare a 'k8s-nope:' variable" in str( + excinfo.value + ) + + +# --- expand() ------------------------------------------------------------- + + +def test_expand_returns_literal_unchanged(checker): + assert checker.expand("1.36.4", {}, "where") == "1.36.4" + + +def test_expand_substitutes_nested_references(checker): + variables = {"a": "%{b}", "b": "leaf"} + assert checker.expand("%{a}", variables, "where") == "leaf" + + +def test_expand_exits_on_undefined_reference(checker): + with pytest.raises(SystemExit) as excinfo: + checker.expand("%{missing}", {}, "include/kubernetes.yml x") + assert "references undefined variable '%{missing}'" in str(excinfo.value) + + +def test_expand_exits_on_circular_reference(checker): + with pytest.raises(SystemExit) as excinfo: + checker.expand("%{a}", {"a": "%{b}", "b": "%{a}"}, "include/kubernetes.yml x") + assert "circular variable reference" in str(excinfo.value) + + +# --- fail() --------------------------------------------------------------- + + +def test_fail_reports_problem_and_fix(checker): + with pytest.raises(SystemExit) as excinfo: + checker.fail("something drifted", "do the thing") + message = str(excinfo.value) + assert "ERROR: something drifted" in message + assert "Fix: do the thing" in message + + +# --- source_urls() -------------------------------------------------------- + + +def test_source_urls_collects_every_declared_url(checker): + assert checker.source_urls(GOOD_BIN) == [ + "k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubectl", + "k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubeadm", + "k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubelet", + ] + + +def test_source_urls_returns_empty_without_sources(checker): + assert checker.source_urls("kind: manual\nsources: []\n") == [] + + +# --- module-level regexes ------------------------------------------------- + + +@pytest.mark.parametrize( + "candidate", + ["1.36.4", "v1.36.4", "1.9.1", "release~1^2"], +) +def test_safe_version_re_accepts_filename_safe_versions(checker, candidate): + assert checker.SAFE_VERSION_RE.match(candidate) + + +@pytest.mark.parametrize("candidate", ["1.36.4 rc1", "1.36.4/0", "", "a:b"]) +def test_safe_version_re_rejects_unsafe_versions(checker, candidate): + assert not checker.SAFE_VERSION_RE.match(candidate) + + +@pytest.mark.parametrize( + "candidate", + [ + "url: k8s:v1.36.4/bin/linux/amd64/kubeadm", + "cni-plugins-linux-amd64-v1.9.1.tgz", + "FNAME=kubernetes-1.36.4.raw", + ], +) +def test_literal_version_re_matches_every_spelling(checker, candidate): + assert checker.LITERAL_VERSION_RE.search(candidate) + + +@pytest.mark.parametrize( + "candidate", + [ + "%{k8s-upstream-tag}", + 'FNAME="kubernetes-%{k8s-version}.raw"', + "ref: b98f74a0f8522f0a83867178729c1aa70f2158f90c45a2ca8fa791db1c76b303", + "install -D -m 0755 cni-src/host-local", + ], +) +def test_literal_version_re_ignores_derived_and_incidental_text(checker, candidate): + assert not checker.LITERAL_VERSION_RE.search(candidate) + + +# --- main(): the passing path -------------------------------------------- + + +def test_main_accepts_a_fully_derived_configuration(checker, capsys): + checker.main() + out = capsys.readouterr().out + assert "OK: Kubernetes v1.36.4 and CNI plugins v1.9.1" in out + assert f"asset kubernetes-{EXPECTED_VERSION}.raw.zst" in out + assert f"VERSION_ID={EXPECTED_VERSION}" in out + + +def test_main_tolerates_literal_version_inside_comments(checker, capsys): + checker.K8S_BIN.write_text( + "# pinned from the v1.36.4 release notes\n" + GOOD_BIN, + encoding="utf-8", + ) + checker.main() + assert "OK: Kubernetes" in capsys.readouterr().out + + +# --- main(): version safety ---------------------------------------------- + + +def test_main_rejects_version_that_is_not_filename_safe(checker): + checker.K8S_INCLUDE.write_text( + GOOD_INCLUDE.replace('k8s-version: "1.36.4"', 'k8s-version: "1.36.4 rc1"'), + encoding="utf-8", + ) + assert "is not filename-safe" in _run(checker) + + +# --- main(): node binary download URLs ----------------------------------- + + +@pytest.mark.parametrize("binary", ["kubectl", "kubeadm", "kubelet"]) +def test_main_rejects_a_missing_node_binary_url(checker, binary): + checker.K8S_BIN.write_text( + "\n".join( + line for line in GOOD_BIN.splitlines() if not line.endswith(f"/{binary}") + ) + + "\n", + encoding="utf-8", + ) + message = _run(checker) + assert "declares no source 'url:'" in message + assert f"'/{binary}'" in message + + +def test_main_rejects_hardcoded_upstream_tag_in_node_binary_url(checker): + checker.K8S_BIN.write_text( + GOOD_BIN.replace("%{k8s-upstream-tag}", "v1.36.4", 1), encoding="utf-8" + ) + assert "hardcodes the upstream\n Kubernetes release tag" in _run(checker) + + +# --- main(): CNI plugins download URL ------------------------------------ + + +def test_main_rejects_missing_cni_url(checker): + checker.CNI_BIN.write_text("kind: manual\nsources: []\n", encoding="utf-8") + assert "cni-plugins.bst declares no source 'url:'" in _run(checker) + + +def test_main_rejects_hardcoded_cni_tag(checker): + checker.CNI_BIN.write_text( + GOOD_CNI.replace("%{cni-upstream-tag}", "v1.9.1"), encoding="utf-8" + ) + assert "hardcodes the upstream CNI" in _run(checker) + + +# --- main(): sysext asset filename --------------------------------------- + + +def test_main_rejects_missing_fname(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace('FNAME="kubernetes-%{k8s-version}.raw"', ""), + encoding="utf-8", + ) + assert 'declares no FNAME="..." asset name' in _run(checker) + + +def test_main_rejects_fname_on_the_os_release_axis(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace("%{k8s-version}.raw", "%{release-version}.raw", 1), + encoding="utf-8", + ) + assert "names the Kubernetes sysext on the\n OS release axis" in _run(checker) + + +def test_main_rejects_fname_not_derived_from_k8s_version(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace( + 'FNAME="kubernetes-%{k8s-version}.raw"', 'FNAME="kubernetes.raw"' + ), + encoding="utf-8", + ) + assert "does not derive the sysext asset\n name" in _run(checker) + + +def test_main_rejects_missing_generated_version_id(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace( + 'echo "VERSION_ID=%{k8s-version}" >> extension-release.kubernetes', "" + ), + encoding="utf-8", + ) + assert "does not generate VERSION_ID= from" in _run(checker) + + +# --- main(): /usr/local/share version stamps ----------------------------- + + +@pytest.mark.parametrize( + "stamp", ["kubernetes-version", "kubernetes-cni-version"] +) +def test_main_rejects_a_missing_version_stamp(checker, stamp): + checker.K8S_SYSEXT.write_text( + "\n".join( + line + for line in GOOD_SYSEXT.splitlines() + if not line.strip().endswith(f"sysext/usr/local/share/{stamp}") + ) + + "\n", + encoding="utf-8", + ) + message = _run(checker) + assert "does not generate" in message + assert f"/usr/local/share/{stamp}" in message + + +def test_main_rejects_cni_stamp_written_from_the_kubernetes_atom(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace( + 'echo "%{cni-version}" > sysext/usr/local/share/kubernetes-cni-version', + 'echo "%{k8s-version}" > sysext/usr/local/share/kubernetes-cni-version', + ), + encoding="utf-8", + ) + assert "/usr/local/share/kubernetes-cni-version" in _run(checker) + + +# --- main(): extension-release.kubernetes -------------------------------- + + +def test_main_rejects_hardcoded_version_id_in_extension_release(checker): + checker.EXTENSION_RELEASE.write_text( + "NAME=kubernetes\nID=_any\nVERSION_ID=1.36.4\n", encoding="utf-8" + ) + message = _run(checker) + assert "hardcodes a\n version" in message + assert "VERSION_ID=1.36.4" in message + + +def test_main_rejects_indented_hardcoded_version_id(checker): + checker.EXTENSION_RELEASE.write_text( + "NAME=kubernetes\n VERSION_ID=9.9.9\n", encoding="utf-8" + ) + assert "hardcodes a\n version" in _run(checker) + + +# --- main(): literal version scan ---------------------------------------- + + +@pytest.mark.parametrize( + "attr", + ["K8S_BIN", "CNI_BIN", "K8S_SYSEXT", "EXTENSION_RELEASE"], +) +def test_main_rejects_literal_version_in_any_consumer(checker, attr): + path = getattr(checker, attr) + path.write_text( + path.read_text(encoding="utf-8") + "\nnote: shipped v1.36.4\n", + encoding="utf-8", + ) + message = _run(checker) + assert "restates a literal Kubernetes or CNI" in message + assert "Literal: v1.36.4" in message + + +# --- main(): sysupdate MatchPattern -------------------------------------- + + +def test_main_rejects_missing_match_pattern(checker): + checker.TRANSFER.write_text("[Source]\nType=url-file\n", encoding="utf-8") + assert "declares no\n [Source]" in _run(checker) + + +def test_main_rejects_match_pattern_without_zst_suffix(checker): + checker.TRANSFER.write_text( + "[Source]\nMatchPattern=kubernetes-@v.raw\n", encoding="utf-8" + ) + assert "declares no\n [Source]" in _run(checker) + + +def test_main_rejects_prefix_drift_between_fname_and_match_pattern(checker): + checker.TRANSFER.write_text( + "[Source]\nMatchPattern=k8s-@v.raw.zst\n", encoding="utf-8" + ) + message = _run(checker) + assert "is not the pinned Kubernetes version" in message + assert "'@v' captures : None" in message + + +def test_main_rejects_suffix_drift_between_fname_and_match_pattern(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace( + 'FNAME="kubernetes-%{k8s-version}.raw"', + 'FNAME="kubernetes-%{k8s-version}.erofs"', + ), + encoding="utf-8", + ) + assert "is not the pinned Kubernetes version" in _run(checker) + + +def test_main_rejects_match_pattern_that_captures_extra_text(checker): + checker.TRANSFER.write_text( + "[Source]\nMatchPattern=kuber@v.raw.zst\n", encoding="utf-8" + ) + message = _run(checker) + assert "is not the pinned Kubernetes version" in message + assert f"'@v' captures : 'netes-{EXPECTED_VERSION}'" in message diff --git a/tests/unit/test_cluster_bootstrap.py b/tests/unit/test_cluster_bootstrap.py new file mode 100644 index 00000000..202b0c6b --- /dev/null +++ b/tests/unit/test_cluster_bootstrap.py @@ -0,0 +1,363 @@ +"""Contracts for Kubernetes control-plane bring-up and cluster seeding. + +``systemd-analyze verify`` is not available in this repo's test environment, so +the unit invariants that a verifier would catch are asserted here instead: +ordering edges, the placement of the idempotency guard, and the fact that every +unit the preset enables can actually be enabled. + +The guard placement is the subtle one. systemd marks a unit whose ``Condition*`` +fails as *skipped*, not failed, and treats it as satisfied for everything +ordered after it. That is exactly what ``kubeadm-init.service`` wants on a +second boot, and exactly what ``bluefin-cluster-bootstrap.service`` must never +have: a condition there would silently no-op the seed. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +SYSTEM = ROOT / "files" / "os" / "systemd" / "system" +PRESET_DIR = ROOT / "files" / "os" / "systemd" / "system-preset" + +KUBEADM_INIT = SYSTEM / "kubeadm-init.service" +CLUSTER_REPO = SYSTEM / "bluefin-cluster-repo.service" +CLUSTER_BOOTSTRAP = SYSTEM / "bluefin-cluster-bootstrap.service" +KUBELET = SYSTEM / "kubelet.service" +KUBELET_DROPIN = SYSTEM / "kubelet.service.d" / "10-kubeadm.conf" +SYSEXT_DROPIN = ( + ROOT + / "files" + / "os" + / "systemd" + / "systemd-sysext.service.d" + / "10-daemon-reload.conf" +) +PRESET = PRESET_DIR / "zz-enable-cluster-bootstrap.preset" +REPO_HELPER = ROOT / "files" / "os" / "libexec" / "bluefin-cluster-repo" +KUBEADM_CONFIG = ROOT / "files" / "kubernetes" / "kubeadm.yaml" + +NETWORK = SYSTEM.parent / "network" / "20-wired.network" +NETWORK_PRESET = PRESET_DIR / "zz-enable-networkd.preset" + +ELEMENT = ROOT / "elements" / "bluefin-server" / "os-cluster-bootstrap.bst" +NETWORK_ELEMENT = ROOT / "elements" / "bluefin-server" / "os-networkd.bst" +SYSUPDATE_ELEMENT = ( + ROOT / "elements" / "bluefin-server" / "os-kubernetes-sysupdate.bst" +) +STACK = ROOT / "elements" / "bluefin-server" / "os-stack.bst" + +SEED_ROOT = "/usr/share/bluefin/seed" +SEED_PHASES = ("00-cilium", "10-argocd-core", "15-gitd", "20-root-app") + +UNITS = (KUBEADM_INIT, CLUSTER_REPO, CLUSTER_BOOTSTRAP, KUBELET) + + +def unit(path: Path) -> dict[str, dict[str, list[str]]]: + """Parse a unit file into ``{section: {key: [values...]}}``. + + ``configparser`` collapses repeated keys, and repeated keys are the whole + point of a unit file: ``ExecStart=`` order *is* the seed order. + """ + sections: dict[str, dict[str, list[str]]] = {} + current: dict[str, list[str]] | None = None + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith(("#", ";")): + continue + if line.startswith("[") and line.endswith("]"): + current = sections.setdefault(line[1:-1], {}) + continue + assert current is not None, f"{path.name}: {line!r} precedes any section" + key, _, value = line.partition("=") + current.setdefault(key.strip(), []).append(value.strip()) + return sections + + +def exec_starts(path: Path) -> list[str]: + return unit(path)["Service"].get("ExecStart", []) + + +@pytest.mark.parametrize("path", UNITS, ids=lambda p: p.name) +def test_every_unit_parses_and_can_be_enabled(path: Path) -> None: + sections = unit(path) + assert "Unit" in sections and "Service" in sections + assert sections["Install"]["WantedBy"] == ["multi-user.target"], ( + f"{path.name} has no [Install] WantedBy, so the preset cannot enable it" + ) + + +def test_preset_enables_exactly_the_units_that_are_shipped() -> None: + enabled = [ + line.removeprefix("enable ") + for line in PRESET.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + assert enabled == [ + "kubelet.service", + "kubeadm-init.service", + "bluefin-cluster-repo.service", + "bluefin-cluster-bootstrap.service", + ] + for name in enabled: + assert (SYSTEM / name).is_file(), f"preset enables missing unit {name}" + + +def test_the_idempotency_guard_lives_on_kubeadm_init_alone() -> None: + """A skipped unit satisfies its dependents; a skipped seed is a dead node.""" + init = unit(KUBEADM_INIT) + assert init["Unit"]["ConditionPathExists"] == ["!/etc/kubernetes/admin.conf"] + + for path in (CLUSTER_BOOTSTRAP,): + guards = [ + key + for key in unit(path)["Unit"] + if key.startswith(("Condition", "Assert")) + ] + assert guards == [], ( + f"{path.name} carries {guards}; a failed condition marks the unit " + "skipped rather than failed, silently no-opping the first boot seed" + ) + + +def test_kubeadm_init_waits_for_the_sysext_merge_and_the_runtime() -> None: + init = unit(KUBEADM_INIT)["Unit"] + requires = set(init.get("Requires", [])) + after = set(init.get("After", [])) + + # kubeadm, kubelet and containerd.service all arrive with the merge. + assert "systemd-sysext.service" in requires + assert "systemd-sysext.service" in after + assert "containerd.service" in requires + assert "containerd.service" in after + assert "network-online.target" in after + + +def test_kubeadm_init_stays_active_so_dependents_can_order_on_it() -> None: + service = unit(KUBEADM_INIT)["Service"] + assert service["Type"] == ["oneshot"] + assert service["RemainAfterExit"] == ["yes"], ( + "without RemainAfterExit the unit deactivates immediately and ordering " + "on it tells a dependent nothing about the control plane" + ) + + +def test_kubeadm_init_reads_the_staged_config_and_skips_kube_proxy() -> None: + commands = exec_starts(KUBEADM_INIT) + assert commands == [ + "/usr/bin/kubeadm init --config /usr/share/bluefin/kubeadm.yaml " + "--skip-phases=addon/kube-proxy" + ], "Cilium supplies the kube-proxy replacement; installing both fights" + + +def test_sysext_merge_reloads_systemd_so_merged_units_become_visible() -> None: + dropin = unit(SYSEXT_DROPIN)["Service"] + assert dropin["ExecStartPost"] == ["/usr/bin/systemctl daemon-reload"] + + +def test_kubelet_is_configured_by_kubeadm_rather_than_by_the_image() -> None: + service = unit(KUBELET)["Service"] + assert service["Restart"] == ["always"] + assert unit(KUBELET)["Unit"]["StartLimitIntervalSec"] == ["0"], ( + "the kubelet restarts until kubeadm has written its config; a start " + "limit would leave the node permanently down" + ) + + dropin = unit(KUBELET_DROPIN)["Service"] + assert dropin["ExecStart"][0] == "", ( + "the drop-in must clear the inherited ExecStart before adding its own" + ) + assert "$KUBELET_KUBEADM_ARGS" in dropin["ExecStart"][1] + assert "-/var/lib/kubelet/kubeadm-flags.env" in dropin["EnvironmentFile"] + + +def test_cluster_repo_is_ordered_before_the_seed_and_is_re_runnable() -> None: + repo = unit(CLUSTER_REPO) + assert repo["Unit"]["Before"] == ["bluefin-cluster-bootstrap.service"] + assert repo["Service"]["Type"] == ["oneshot"] + assert repo["Service"]["StateDirectory"] == ["bluefin"] + assert exec_starts(CLUSTER_REPO) == ["/usr/libexec/bluefin-cluster-repo"] + + +def test_cluster_repo_helper_publishes_a_servable_bare_repository() -> None: + helper = REPO_HELPER.read_text(encoding="utf-8") + assert REPO_HELPER.stat().st_mode & 0o111, "the helper must be executable" + assert "REPO=/var/lib/bluefin/cluster.git" in helper + assert "SEED=/usr/share/bluefin/cluster" in helper + assert "BRANCH=main" in helper + assert "git init --bare" in helper + assert 'touch "${REPO}/git-daemon-export-ok"' in helper, ( + "git-daemon refuses to serve a repository that is not exported" + ) + assert "umask 022" in helper, ( + "git-daemon reads the hostPath mount as a non-root uid" + ) + + +def test_seed_runs_every_phase_in_order_under_the_admin_kubeconfig() -> None: + bootstrap = unit(CLUSTER_BOOTSTRAP) + assert bootstrap["Service"]["Environment"] == [ + "KUBECONFIG=/etc/kubernetes/admin.conf" + ] + applied = [ + command.rsplit(" ", 1)[-1] + for command in exec_starts(CLUSTER_BOOTSTRAP) + if " apply " in command + ] + assert applied == [f"{SEED_ROOT}/{phase}" for phase in SEED_PHASES] + + +def test_seed_applies_server_side_and_never_prunes() -> None: + for command in exec_starts(CLUSTER_BOOTSTRAP): + assert "prune" not in command, ( + f"{command}: kubectl rejects --prune alongside --server-side, and " + "the seed is append-only by design; Argo CD owns pruning downstream" + ) + if " apply " not in command: + continue + assert "--server-side --force-conflicts" in command, command + assert "-k " in command, f"{command} must apply a kustomization" + + +def test_seed_waits_for_the_node_to_leave_notready_before_argo() -> None: + commands = exec_starts(CLUSTER_BOOTSTRAP) + cilium = next(i for i, c in enumerate(commands) if c.endswith("/00-cilium")) + argocd = next( + i for i, c in enumerate(commands) if c.endswith("/10-argocd-core") + ) + waits = [ + c for c in commands[cilium + 1 : argocd] if "wait" in c and " node " in c + ] + assert waits and "--for=condition=Ready" in waits[0], ( + "without CNI the node stays NotReady and carries a NoSchedule taint, " + "so every Argo pod would be Pending forever" + ) + + +def test_seed_waits_for_the_git_bridge_before_handing_over_to_argo() -> None: + commands = exec_starts(CLUSTER_BOOTSTRAP) + gitd = next(i for i, c in enumerate(commands) if c.endswith("/15-gitd")) + root_app = next( + i for i, c in enumerate(commands) if c.endswith("/20-root-app") + ) + waits = commands[gitd + 1 : root_app] + assert any( + "deployment/bluefin-git-daemon" in c and "--namespace bluefin-system" in c + for c in waits + ), ( + "the root Application cannot sync until the daemon serving " + "/var/lib/bluefin/cluster.git is up" + ) + + +def test_seed_requires_both_prerequisite_units() -> None: + bootstrap = unit(CLUSTER_BOOTSTRAP)["Unit"] + for name in ("kubeadm-init.service", "bluefin-cluster-repo.service"): + assert name in bootstrap["Requires"] + assert name in bootstrap["After"] + + +def test_kubeadm_config_bootstraps_every_certificate_it_uses() -> None: + documents = [ + document + for document in yaml.safe_load_all( + KUBEADM_CONFIG.read_text(encoding="utf-8") + ) + if document + ] + kinds = {document["kind"]: document for document in documents} + assert set(kinds) == { + "InitConfiguration", + "ClusterConfiguration", + "KubeletConfiguration", + } + + kubelet = kinds["KubeletConfiguration"] + assert kubelet["serverTLSBootstrap"] is True, ( + "kubelet would otherwise generate a self-signed serving certificate" + ) + assert kubelet["rotateCertificates"] is True + assert kubelet["cgroupDriver"] == "systemd" + + +def test_kubeadm_config_does_not_restate_the_kubernetes_version() -> None: + cluster = next( + document + for document in yaml.safe_load_all( + KUBEADM_CONFIG.read_text(encoding="utf-8") + ) + if document and document["kind"] == "ClusterConfiguration" + ) + assert "kubernetesVersion" not in cluster, ( + "kubeadm derives the version from its own binary; restating it here " + "forks the single source of truth in include/kubernetes.yml" + ) + + +def test_kubeadm_config_schedules_workload_on_the_only_node() -> None: + init = next( + document + for document in yaml.safe_load_all( + KUBEADM_CONFIG.read_text(encoding="utf-8") + ) + if document and document["kind"] == "InitConfiguration" + ) + assert init["nodeRegistration"]["taints"] == [], ( + "the default control-plane NoSchedule taint leaves every workload " + "Pending on a single-node appliance" + ) + assert ( + init["nodeRegistration"]["criSocket"] + == "unix:///run/containerd/containerd.sock" + ) + + +def test_cluster_bootstrap_element_stages_units_config_and_helper() -> None: + element = ELEMENT.read_text(encoding="utf-8") + assert "path: files/os/systemd/system" in element + assert "path: files/os/systemd/systemd-sysext.service.d" in element + assert "path: files/kubernetes/kubeadm.yaml" in element + assert "path: files/os/libexec" in element + assert '"${OUT}/usr/share/bluefin/kubeadm.yaml"' in element + assert '"${OUT}/usr/libexec/bluefin-cluster-repo"' in element + assert "bluefin-server/os-cluster-bootstrap.bst" in STACK.read_text( + encoding="utf-8" + ) + + +def test_stack_pulls_in_git_for_the_manifest_bridge() -> None: + stack = STACK.read_text(encoding="utf-8") + assert "freedesktop-sdk.bst:components/git.bst" in stack, ( + "bluefin-cluster-repo.service shells out to git; without it the root " + "Application has nothing to sync from" + ) + assert "bluefin-server/os-cluster-manifests.bst" in stack + + +def test_kubernetes_sysupdate_transfer_is_packaged_as_a_component() -> None: + element = SYSUPDATE_ELEMENT.read_text(encoding="utf-8") + assert "path: files/os/sysupdate.kubernetes.d" in element + assert "target: /usr/lib/sysupdate.kubernetes.d" in element + assert "bluefin-server/os-kubernetes-sysupdate.bst" in STACK.read_text( + encoding="utf-8" + ) + + +def test_installed_os_configures_wired_dhcp_with_networkd() -> None: + assert NETWORK.read_text(encoding="utf-8") == ( + "[Match]\n" + "Name=e*\n" + "\n" + "[Network]\n" + "DHCP=ipv4\n" + ) + assert NETWORK_PRESET.read_text(encoding="utf-8") == ( + "enable systemd-networkd.service\n" + ) + network_element = NETWORK_ELEMENT.read_text(encoding="utf-8") + assert "path: files/os/systemd/network" in network_element + assert "target: /usr/lib/systemd/network" in network_element + assert "bluefin-server/os-networkd.bst" in STACK.read_text(encoding="utf-8") diff --git a/tests/unit/test_cluster_manifests.py b/tests/unit/test_cluster_manifests.py new file mode 100644 index 00000000..2eafb3c5 --- /dev/null +++ b/tests/unit/test_cluster_manifests.py @@ -0,0 +1,400 @@ +"""Contracts for the cluster bootstrap seed and the Argo CD-owned manifest tree. + +The appliance has no addon manager. Ordering, drift correction and pruning all +come from four seed phases plus one root Argo Application, so the properties +worth defending here are: every overlay still builds, every image is immutable, +and nothing in the tree is a credential someone could load. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +CLUSTER = ROOT / "files" / "cluster" +SEED = CLUSTER / "seed" +MANIFESTS = CLUSTER / "manifests" +ELEMENT = ROOT / "elements" / "bluefin-server" / "os-cluster-manifests.bst" + +SEED_PHASES = ("00-cilium", "10-argocd-core", "15-gitd", "20-root-app") + +# Upstream manifests are fetched and sha256-pinned by the BuildStream element +# rather than vendored, so they are absent from a source checkout. Each entry +# maps the missing file to a stand-in carrying the resources the overlay +# actually acts on — the ApplicationSet controller it deletes, and every image +# name its `images:` block claims to pin. Re-derive these when the pinned +# upstream version moves: a stand-in that no longer matches upstream is how a +# silently-unpinned image would slip through. +STAND_INS: dict[str, str] = { + "seed/10-argocd-core/core-install.yaml": """ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: applicationsets.argoproj.io +spec: + group: argoproj.io + names: + kind: ApplicationSet + plural: applicationsets + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-applicationset-controller +spec: + selector: + matchLabels: + app.kubernetes.io/name: argocd-applicationset-controller + template: + metadata: + labels: + app.kubernetes.io/name: argocd-applicationset-controller + spec: + containers: + - name: argocd-applicationset-controller + image: quay.io/argoproj/argocd:v3.5.3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-repo-server +spec: + selector: + matchLabels: + app.kubernetes.io/name: argocd-repo-server + template: + metadata: + labels: + app.kubernetes.io/name: argocd-repo-server + spec: + containers: + - name: argocd-repo-server + image: quay.io/argoproj/argocd:v3.5.3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-redis +spec: + selector: + matchLabels: + app.kubernetes.io/name: argocd-redis + template: + metadata: + labels: + app.kubernetes.io/name: argocd-redis + spec: + containers: + - name: redis + image: public.ecr.aws/docker/library/redis:8.2.3-alpine +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: argocd-application-controller +spec: + serviceName: argocd-application-controller + selector: + matchLabels: + app.kubernetes.io/name: argocd-application-controller + template: + metadata: + labels: + app.kubernetes.io/name: argocd-application-controller + spec: + containers: + - name: argocd-application-controller + image: quay.io/argoproj/argocd:v3.5.3 +""", + "manifests/cert-manager/cert-manager.yaml": """ +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager + namespace: cert-manager +spec: + selector: + matchLabels: + app.kubernetes.io/name: cert-manager + template: + metadata: + labels: + app.kubernetes.io/name: cert-manager + spec: + containers: + - name: cert-manager-controller + image: "quay.io/jetstack/cert-manager-controller:v1.21.2" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-webhook + namespace: cert-manager +spec: + selector: + matchLabels: + app.kubernetes.io/name: webhook + template: + metadata: + labels: + app.kubernetes.io/name: webhook + spec: + containers: + - name: cert-manager-webhook + image: "quay.io/jetstack/cert-manager-webhook:v1.21.2" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-cainjector + namespace: cert-manager +spec: + selector: + matchLabels: + app.kubernetes.io/name: cainjector + template: + metadata: + labels: + app.kubernetes.io/name: cainjector + spec: + containers: + - name: cert-manager-cainjector + image: "quay.io/jetstack/cert-manager-cainjector:v1.21.2" +""", +} + +KUBECTL = shutil.which("kubectl") +requires_kubectl = pytest.mark.skipif( + KUBECTL is None, reason="kubectl is needed to build the kustomize overlays" +) + + +def _overlays() -> list[Path]: + """Every directory in files/cluster/ carrying a kustomization.yaml.""" + return sorted(p.parent for p in CLUSTER.rglob("kustomization.yaml")) + + +@pytest.fixture(scope="module") +def tree(tmp_path_factory: pytest.TempPathFactory) -> Path: + """files/cluster/ with the fetch-pinned upstream manifests stood in for.""" + staged = tmp_path_factory.mktemp("cluster") + shutil.copytree(CLUSTER, staged, dirs_exist_ok=True) + for relative, content in STAND_INS.items(): + target = staged / relative + assert not target.exists(), ( + f"{relative} is vendored into the repository; it is meant to be " + "fetched and sha256-pinned by os-cluster-manifests.bst" + ) + target.write_text(content, encoding="utf-8") + return staged + + +def _build(directory: Path) -> list[dict]: + result = subprocess.run( + [KUBECTL, "kustomize", str(directory)], + capture_output=True, + text=True, + check=False, + # kustomize reaches the network for remote bases and helmCharts. The + # tree is meant to be buildable on a machine with no network at all. + env={"PATH": "/usr/bin:/bin", "HOME": str(directory), "HTTP_PROXY": "http://127.0.0.1:1", "HTTPS_PROXY": "http://127.0.0.1:1"}, + ) + assert result.returncode == 0, f"kustomize build failed for {directory}:\n{result.stderr}" + return [doc for doc in yaml.safe_load_all(result.stdout) if doc] + + +def _images(docs: list[dict]) -> list[str]: + found: list[str] = [] + + def walk(node: object) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if key == "image" and isinstance(value, str): + found.append(value) + else: + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(docs) + return found + + +def test_seed_has_exactly_the_four_documented_phases() -> None: + # bluefin-cluster-bootstrap.service applies these by name, in this order. + # A fifth directory, or a renamed one, silently changes what first boot does. + assert sorted(p.name for p in SEED.iterdir() if p.is_dir()) == list(SEED_PHASES) + for phase in SEED_PHASES: + assert (SEED / phase / "kustomization.yaml").is_file(), ( + f"{phase} is applied with `kubectl apply -k` and needs a kustomization.yaml" + ) + + +@requires_kubectl +@pytest.mark.parametrize("overlay", _overlays(), ids=lambda p: str(p.relative_to(CLUSTER))) +def test_every_overlay_builds_offline(overlay: Path, tree: Path) -> None: + _build(tree / overlay.relative_to(CLUSTER)) + + +@requires_kubectl +@pytest.mark.parametrize("overlay", [*SEED_PHASES, "."], ids=lambda name: f"seed-{name}") +def test_every_seed_image_is_digest_pinned(overlay: str, tree: Path) -> None: + directory = tree / "manifests" if overlay == "." else tree / "seed" / overlay + for image in _images(_build(directory)): + assert "@sha256:" in image, f"{image} is a mutable reference in {directory.name}" + + +@requires_kubectl +def test_applicationset_controller_is_patched_out_but_its_crd_stays(tree: Path) -> None: + docs = _build(tree / "seed" / "10-argocd-core") + workloads = {(d["kind"], d["metadata"]["name"]) for d in docs} + assert ("Deployment", "argocd-applicationset-controller") not in workloads + assert ("CustomResourceDefinition", "applicationsets.argoproj.io") in workloads + # Deleting the wrong thing is the failure mode that matters: Core's other + # three workloads have to survive the patch. + assert ("Deployment", "argocd-repo-server") in workloads + assert ("Deployment", "argocd-redis") in workloads + assert ("StatefulSet", "argocd-application-controller") in workloads + + +@requires_kubectl +def test_root_application_owns_pruning_and_drift(tree: Path) -> None: + docs = _build(tree / "seed" / "20-root-app") + app = next(d for d in docs if d["kind"] == "Application") + project = next(d for d in docs if d["kind"] == "AppProject") + + automated = app["spec"]["syncPolicy"]["automated"] + assert automated["prune"] is True + assert automated["selfHeal"] is True + + # The manifest source must be the in-cluster bridge. An external repoURL + # would make first boot depend on a git host being reachable. + repo = app["spec"]["source"]["repoURL"] + assert repo.startswith("git://bluefin-git-daemon.bluefin-system.svc.cluster.local/") + assert repo in project["spec"]["sourceRepos"] + assert app["spec"]["project"] == project["metadata"]["name"] + + # A project that can mint Applications or AppProjects can escape its own + # scope, so those two kinds stay denied. + denied = {(e["group"], e["kind"]) for e in project["spec"]["namespaceResourceBlacklist"]} + assert ("argoproj.io", "Application") in denied + assert ("argoproj.io", "AppProject") in denied + + +def test_git_daemon_serves_the_bare_repo_read_only() -> None: + docs = list(yaml.safe_load_all((SEED / "15-gitd" / "deployment.yaml").read_text())) + pod = next(d for d in docs if d["kind"] == "Deployment")["spec"]["template"]["spec"] + volume = next(v for v in pod["volumes"] if v["name"] == "cluster-repo") + mount = next( + m for m in pod["containers"][0]["volumeMounts"] if m["name"] == "cluster-repo" + ) + + assert volume["hostPath"]["path"] == "/var/lib/bluefin/cluster.git" + assert mount["readOnly"] is True, "Argo must never be able to write to its own source" + assert pod["securityContext"]["runAsNonRoot"] is True + + +def test_no_loadable_secret_material_in_the_tree() -> None: + # No certificate or private key ships in any image, and no Secret in the + # tree carries a value. Credentials are generated in-cluster on first sync. + pem_markers = ("BEGIN PRIVATE KEY", "BEGIN RSA PRIVATE KEY", "BEGIN EC PRIVATE KEY", "BEGIN CERTIFICATE") + offenders: list[str] = [] + + for path in sorted(CLUSTER.rglob("*")): + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8") + relative = path.relative_to(ROOT) + for marker in pem_markers: + if marker in text: + offenders.append(f"{relative}: contains {marker}") + if path.suffix not in (".yaml", ".yml"): + continue + for doc in yaml.safe_load_all(text): + if not isinstance(doc, dict) or doc.get("kind") != "Secret": + continue + name = doc.get("metadata", {}).get("name") + if doc.get("data") or doc.get("stringData"): + offenders.append(f"{relative}: Secret/{name} carries a value") + + assert offenders == [] + + +def test_console_has_no_developer_bypass() -> None: + # DEV_MODE made the appliance's only dashboard unauthenticated on every + # installed machine, and the shipped jwt-secret made its sessions forgeable + # by anyone who read this repository. + console = (MANIFESTS / "console" / "console.yaml").read_text(encoding="utf-8") + docs = list(yaml.safe_load_all(console)) + deployment = next(d for d in docs if d["kind"] == "Deployment") + env = {e["name"]: e for e in deployment["spec"]["template"]["spec"]["containers"][0]["env"]} + + assert "DEV_MODE" not in env + assert "ALLOW_DEV_MODE_IN_CLUSTER" not in env + assert "DEV_USER_LOGIN" not in env + + jwt = env["JWT_SECRET"]["valueFrom"]["secretKeyRef"] + assert "value" not in env["JWT_SECRET"], "the signing key must never be inline" + assert jwt["optional"] is False, "a console that starts without a signing key is a console with no sessions" + assert jwt["name"] == "kubestellar-console-auth" + + +def test_generated_credentials_are_never_overwritten() -> None: + # Re-running either generator must not rotate a live credential: the + # PostgreSQL password would lock the initialised database out, and the JWT + # key would invalidate every open console session on every re-sync. + for manifest, secret in ( + (MANIFESTS / "kubeflex" / "postgres-secret.yaml", "kubeflex-postgres"), + (MANIFESTS / "console" / "auth-secret.yaml", "kubestellar-console-auth"), + ): + job = next( + d for d in yaml.safe_load_all(manifest.read_text()) if d["kind"] == "Job" + ) + script = job["spec"]["template"]["spec"]["containers"][0]["command"][-1] + assert f"get secret {secret}" in script + assert "exit 0" in script + assert "/dev/urandom" in script + + +def test_element_pins_every_manifest_the_overlays_expect() -> None: + # The overlays name these files in their `resources:`; the element is what + # puts them there. A rename on either side is a build that succeeds and an + # appliance that never syncs. + element = yaml.safe_load(ELEMENT.read_text(encoding="utf-8")) + refs = {s["filename"]: s["ref"] for s in element["sources"] if s["kind"] == "remote"} + + assert refs == { + "core-install.yaml": "1a87025d8eb2eae621653fd312fb9ca51df1b4b3b6992a030e3a9ef38e45c448", + "cert-manager.yaml": "e03b668ec8675214af6b0a671699d088f2601fa3878e0dbe1b41d3feafd1879f", + } + + install = "\n".join(element["config"]["install-commands"]) + for relative in STAND_INS: + destination = relative.replace("seed/", "${SEED}/").replace( + "manifests/", "${CLUSTER}/" + ) + assert destination in install, f"element never stages {relative}" + + # core-install.yaml, not install.yaml: Core drops argocd-server and with it + # a second authenticated web surface on the appliance. + urls = [s["url"] for s in element["sources"] if s["kind"] == "remote"] + assert any(u.endswith("manifests/core-install.yaml") for u in urls) diff --git a/tests/unit/test_console_issue.py b/tests/unit/test_console_issue.py index af83ead1..70513b4a 100644 --- a/tests/unit/test_console_issue.py +++ b/tests/unit/test_console_issue.py @@ -15,7 +15,7 @@ def test_issue_file_exists_and_contains_kubestellar_url() -> None: assert ISSUE_FILE.is_file(), f"{ISSUE_FILE} must exist" content = ISSUE_FILE.read_text(encoding="utf-8") assert "Bluefin Server" in content - assert "KubeStellar Console: http://\\4:8080/" in content + assert "KubeStellar Console: https://\\4:8080/" in content def test_os_issue_element_target_usr_lib_issue_d() -> None: diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index c8fe001d..e9b0a4a1 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -62,25 +62,27 @@ def test_installer_wrapper_reads_kernel_command_line_without_cat() -> None: assert 'CMDLINE="$(cat /proc/cmdline' not in installer_element -def test_installer_stages_uncompressed_k0s_before_packing_cpio() -> None: +def test_installer_stages_uncompressed_sysext_before_packing_cpio() -> None: + """The sysext has to be in /layer before the initrd is packed, or the + offline installer ships no Kubernetes at all.""" installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") data = yaml.safe_load(installer_element) - k0s_dependency = next( + sysext_dependency = next( ( dependency for dependency in data["build-depends"] if isinstance(dependency, dict) - and dependency.get("filename") == "oci/k0s-sysext.bst" + and dependency.get("filename") == "oci/kubernetes-sysext.bst" ), None, ) - assert k0s_dependency == { - "filename": "oci/k0s-sysext.bst", - "config": {"location": "/k0s"}, + assert sysext_dependency == { + "filename": "oci/kubernetes-sysext.bst", + "config": {"location": "/kubernetes"}, } - seed_command = "cp /k0s/k0s-*.raw /layer/k0s.raw" + seed_command = "cp /kubernetes/kubernetes-*.raw /layer/kubernetes.raw" cpio_command = "| cpio --null --create --format=newc" assert seed_command in installer_element assert installer_element.index(seed_command) < installer_element.index( diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py deleted file mode 100644 index a185a611..00000000 --- a/tests/unit/test_k0s_first_boot.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Contracts for retry-safe k0s sysext first-boot activation.""" - -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -SERVICE = ( - ROOT / "files" / "os" / "systemd" / "system" / "k0s-first-boot.service" -) -FETCH_SERVICE = ( - ROOT - / "files" - / "os" - / "systemd" - / "system" - / "k0s-first-boot-fetch.service" -) -PRESET = ( - ROOT - / "files" - / "os" - / "systemd" - / "system-preset" - / "zz-enable-k0s-first-boot.preset" -) -NETWORK = ROOT / "files" / "os" / "systemd" / "network" / "20-wired.network" -NETWORK_PRESET = ( - ROOT - / "files" - / "os" - / "systemd" - / "system-preset" - / "zz-enable-networkd.preset" -) -ELEMENT = ROOT / "elements" / "bluefin-server" / "os-k0s-first-boot.bst" -NETWORK_ELEMENT = ROOT / "elements" / "bluefin-server" / "os-networkd.bst" -K0S_UPDATE_ELEMENT = ( - ROOT / "elements" / "bluefin-server" / "os-k0s-sysupdate.bst" -) -STACK = ROOT / "elements" / "bluefin-server" / "os-stack.bst" - - -def test_seeded_install_skips_the_network_fetch() -> None: - assert FETCH_SERVICE.is_file(), "the conditional k0s fetch unit is missing" - fetch_service = FETCH_SERVICE.read_text(encoding="utf-8") - first_boot = SERVICE.read_text(encoding="utf-8") - - assert "ConditionPathExists=!/var/lib/k0s/k0s.raw" in fetch_service - assert ( - "ExecStart=/usr/bin/systemd-sysupdate --component=k0s update" - in fetch_service - ) - assert "systemd-sysupdate" not in first_boot - - -def test_missing_seed_fetches_before_activation_without_blocking_activation_retry() -> None: - assert FETCH_SERVICE.is_file(), "the conditional k0s fetch unit is missing" - fetch_service = FETCH_SERVICE.read_text(encoding="utf-8") - first_boot = SERVICE.read_text(encoding="utf-8") - - assert "Type=oneshot" in fetch_service - assert "Restart=on-failure" in fetch_service - assert "ConditionPathExists=!/var/lib/k0s/k0s.raw" in fetch_service - assert "Before=k0s-first-boot.service" in fetch_service - assert "Wants=k0s-first-boot-fetch.service" in first_boot - assert "Requires=k0s-first-boot-fetch.service" not in first_boot - assert "After=k0s-first-boot-fetch.service" in first_boot - assert ( - "ExecStartPre=/usr/bin/test -e /var/lib/k0s/k0s.raw" - in first_boot - ) - assert ( - "ExecStart=/usr/bin/install -D -m 0644 " - "/var/lib/k0s/k0s.raw /run/extensions/k0s.raw" - ) in first_boot - assert "ExecStart=/usr/bin/systemd-sysext merge" in first_boot - - -def test_k0s_first_boot_retries_until_controller_starts() -> None: - service = SERVICE.read_text(encoding="utf-8") - - assert "ConditionPathExists=" not in service - assert "Wants=network-online.target" in service - assert "After=network-online.target" in service - assert "Before=multi-user.target" not in service - assert "Type=oneshot" in service - assert "StateDirectory=k0s" in service - assert "Restart=on-failure" in service - assert ( - "ExecStart=/usr/bin/systemd-sysupdate --component=k0s update" - not in service - ) - assert "ExecStart=/usr/bin/systemd-sysupdate update" not in service - assert "ExecStart=/usr/bin/systemctl enable --now systemd-sysext.service" not in service - assert "ExecStart=/usr/bin/systemd-sysext merge" in service - assert ( - "ExecStart=/usr/bin/systemd-tmpfiles --create " - "/usr/lib/tmpfiles.d/k0s-manifests.conf" - ) in service - assert "ExecStart=/usr/bin/systemctl daemon-reload" in service - assert ( - "ExecStart=/usr/bin/systemctl enable --now k0scontroller.service" - in service - ) - assert ( - "ExecStartPost=/usr/bin/touch /var/lib/k0s/.first-boot-complete" - not in service - ) - assert "ConditionFirstBoot" not in service - - -def test_k0s_first_boot_is_packaged_and_enabled() -> None: - assert ( - PRESET.read_text(encoding="utf-8") - == "enable k0s-first-boot.service\n" - ) - assert "path: files/os/systemd/system" in ELEMENT.read_text(encoding="utf-8") - assert "target: /usr/lib/systemd/system" in ELEMENT.read_text(encoding="utf-8") - assert ( - "bluefin-server/os-k0s-first-boot.bst" - in STACK.read_text(encoding="utf-8") - ) - - -def test_installed_os_configures_wired_dhcp_with_networkd() -> None: - assert NETWORK.read_text(encoding="utf-8") == ( - "[Match]\n" - "Name=e*\n" - "\n" - "[Network]\n" - "DHCP=ipv4\n" - ) - assert NETWORK_PRESET.read_text(encoding="utf-8") == ( - "enable systemd-networkd.service\n" - ) - network_element = NETWORK_ELEMENT.read_text(encoding="utf-8") - assert "path: files/os/systemd/network" in network_element - assert "target: /usr/lib/systemd/network" in network_element - assert "bluefin-server/os-networkd.bst" in STACK.read_text(encoding="utf-8") - - -def test_k0s_sysupdate_transfer_is_packaged_as_a_component() -> None: - element = K0S_UPDATE_ELEMENT.read_text(encoding="utf-8") - assert "path: files/os/sysupdate.k0s.d" in element - assert "target: /usr/lib/sysupdate.k0s.d" in element - assert ( - "bluefin-server/os-k0s-sysupdate.bst" - in STACK.read_text(encoding="utf-8") - ) diff --git a/tests/unit/test_k0s_manifests.py b/tests/unit/test_k0s_manifests.py deleted file mode 100644 index 8e37bc8c..00000000 --- a/tests/unit/test_k0s_manifests.py +++ /dev/null @@ -1,104 +0,0 @@ -from pathlib import Path - -import os -import subprocess - -import yaml - -ROOT = Path(__file__).resolve().parents[2] - - -def test_k0s_service_unit(): - unit = ROOT / "files" / "k0s" / "sysext" / "k0scontroller.service" - assert unit.is_file(), "k0scontroller.service missing" - text = unit.read_text() - assert "--disable-components=helm,autopilot" in text - assert "--enable-worker" in text - assert "--single" in text - - -def test_k0s_manifests_conf(): - conf = ROOT / "files" / "k0s" / "sysext" / "k0s-manifests.conf" - assert conf.is_file(), "k0s-manifests.conf missing" - text = conf.read_text() - assert "d /var/lib/k0s/manifests 0755 root root - -" in text - assert "C+ /var/lib/k0s/manifests/argocd - - - - /usr/share/k0s/manifests/argocd" in text - assert "C+ /var/lib/k0s/manifests/kubestellar - - - - /usr/share/k0s/manifests/kubestellar" in text - assert "d /etc/k0s 0755 root root - -" in text - assert "C+ /etc/k0s/k0s.yaml - - - - /usr/share/k0s/k0s.yaml" in text - - -def test_k0s_manifest_files(): - argo_yaml = ROOT / "files" / "k0s" / "manifests" / "argocd" / "install.yaml" - assert argo_yaml.is_file(), "argocd install.yaml missing" - assert "namespace: argocd" in argo_yaml.read_text() - - ks_dir = ROOT / "files" / "k0s" / "manifests" / "kubestellar" - assert (ks_dir / "00-kubeflex-crds.yaml").is_file() - assert (ks_dir / "10-kubeflex-operator.yaml").is_file() - assert (ks_dir / "20-postgres.yaml").is_file() - assert (ks_dir / "30-kubestellar-core.yaml").is_file() - assert (ks_dir / "40-kubestellar-console.yaml").is_file() - assert (ks_dir / "41-kubestellar-kiosk-proxy.yaml").is_file() - k0s_yaml = ROOT / "files" / "k0s" / "k0s.yaml" - assert k0s_yaml.is_file(), "files/k0s/k0s.yaml missing" - data = yaml.safe_load(k0s_yaml.read_text()) - assert data["spec"]["network"]["kuberouter"]["metricsPort"] == 8088 - - -def test_flatcar_zfs_includes_sed_build_dep(): - zfs_bst = ROOT / "elements" / "flatcar" / "flatcar-zfs.bst" - data = yaml.safe_load(zfs_bst.read_text(encoding="utf-8")) - assert "freedesktop-sdk.bst:components/sed.bst" in data.get("build-depends", []) - - -def test_postgres_password_not_hardcoded(): - # #98: the postgres superuser password must not be committed to git in - # plaintext, and must not be the publicly documented kubeflex default. - manifest = ROOT / "files" / "k0s" / "manifests" / "kubestellar" / "20-postgres.yaml" - text = manifest.read_text() - assert "kubeflex" not in text.lower().replace("kubeflex-system", "").replace("kubeflex-postgres", "") - assert 'value: "kubeflex"' not in text - assert "POSTGRES_PASSWORD" in text - - -def test_postgres_password_from_secret(): - # The StatefulSet reads the password from a Secret, not a plaintext env. - docs = list(yaml.safe_load_all((ROOT / "files" / "k0s" / "manifests" / "kubestellar" / "20-postgres.yaml").read_text())) - statefulset = next(d for d in docs if d and d.get("kind") == "StatefulSet") - container = statefulset["spec"]["template"]["spec"]["containers"][0] - env = {e["name"]: e for e in container["env"]} - assert "value" not in env["POSTGRES_PASSWORD"] - ref = env["POSTGRES_PASSWORD"]["valueFrom"]["secretKeyRef"] - assert ref == {"name": "kubeflex-postgres", "key": "password"} - - -def test_k0s_first_boot_generates_postgres_secret_before_k0s(): - # The Secret must be staged before k0s applies the manifests, and the - # generated 15- file must sort before 20-postgres.yaml. - unit = ROOT / "files" / "os" / "systemd" / "system" / "k0s-first-boot.service" - text = unit.read_text() - lines = [l for l in text.splitlines() if l.startswith("ExecStart")] - gen = next((i for i, l in enumerate(lines) if "generate-postgres-secret.sh" in l), None) - k0s = next((i for i, l in enumerate(lines) if "k0scontroller.service" in l), None) - assert gen is not None, "first-boot service never runs the postgres secret generator" - assert k0s is not None, "first-boot service never starts k0scontroller" - assert gen < k0s, "postgres secret generator must run before k0s applies manifests" - - -def test_generate_postgres_secret_is_idempotent(tmp_path): - # Running the generator twice must not change an already-created password, - # so the initialized database stays accessible across re-boots. - script = ROOT / "files" / "k0s" / "kubeflex" / "generate-postgres-secret.sh" - env = dict(os.environ, KUBEFLEX_MANIFEST_DIR=str(tmp_path)) - run = lambda: subprocess.run(["/bin/bash", str(script)], env=env, check=True, capture_output=True, text=True) - run() - secret_file = tmp_path / "15-kubeflex-postgres-secret.yaml" - assert secret_file.is_file() - secret = yaml.safe_load(secret_file.read_text()) - assert secret["kind"] == "Secret" - assert secret["metadata"]["name"] == "kubeflex-postgres" - assert secret["stringData"]["password"] - first = secret_file.read_text() - run() - assert secret_file.read_text() == first, "password changed on re-run; DB would lose access" diff --git a/tests/unit/test_k0s_version.py b/tests/unit/test_k0s_version.py deleted file mode 100644 index ddf83633..00000000 --- a/tests/unit/test_k0s_version.py +++ /dev/null @@ -1,52 +0,0 @@ -import importlib.util -import re -from pathlib import Path -import yaml - -ROOT = Path(__file__).resolve().parents[2] - - -def test_k0s_version_ssot(): - k0s_yml = ROOT / "include" / "k0s.yml" - assert k0s_yml.is_file(), "include/k0s.yml missing" - data = yaml.safe_load(k0s_yml.read_text()) - vars_ = data.get("variables", {}) - assert vars_.get("k0s-k8s-version") == "1.36.4" - assert vars_.get("k0s-patch") == "0" - assert vars_.get("k0s-version") == "%{k0s-k8s-version}-k0s.%{k0s-patch}" - - -def test_k0s_bin_element(): - bin_bst = ROOT / "elements" / "k0s" / "k0s-bin.bst" - assert bin_bst.is_file(), "elements/k0s/k0s-bin.bst missing" - content = bin_bst.read_text() - assert "base/base-stack.bst" in content - assert "github:k0sproject/k0s/releases/download/" in content - assert "ca1e9e68107335846e8296777fce2ccd654284e6265b4b5d32c34ead872af98f" in content - - -def test_k0s_version_checker_reads_the_component_transfer(): - checker = ROOT / ".github" / "scripts" / "check-k0s-version.py" - spec = importlib.util.spec_from_file_location("check_k0s_version", checker) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - assert module.TRANSFER == ( - ROOT / "files" / "os" / "sysupdate.k0s.d" / "70-k0s.transfer" - ) - - -def test_k0s_version_pre_commit_hook_selects_the_relocated_transfer(): - config = yaml.safe_load((ROOT / ".pre-commit-config.yaml").read_text()) - hook = next( - hook - for repo in config["repos"] - if repo.get("repo") == "local" - for hook in repo.get("hooks", []) - if hook.get("id") == "check-k0s-version" - ) - selector = re.compile(hook["files"]) - assert ( - selector.fullmatch("files/os/sysupdate.k0s.d/70-k0s.transfer") is not None - ) - assert selector.fullmatch("files/os/sysupdate.d/70-k0s.transfer") is None diff --git a/tests/unit/test_kubernetes_version.py b/tests/unit/test_kubernetes_version.py new file mode 100644 index 00000000..3939075b --- /dev/null +++ b/tests/unit/test_kubernetes_version.py @@ -0,0 +1,146 @@ +"""Version-axis invariants for the Kubernetes sysext. + +These assert the properties that hold the axis together — that every consumer +derives its version from ``include/kubernetes.yml``, that the derived spellings +agree with each other, and that the pre-commit gate actually fires on every +file the checker reads — rather than restating the current contents of any one +file. +""" + +import importlib.util +import re +import subprocess +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / ".github" / "scripts" / "check-kubernetes-version.py" + +K8S_VERSION = "1.36.4" +CNI_VERSION = "1.9.1" + + +def _checker(): + spec = importlib.util.spec_from_file_location("check_kubernetes_version", CHECKER) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _atoms(): + data = yaml.safe_load((ROOT / "include" / "kubernetes.yml").read_text()) + return data["variables"] + + +def test_include_declares_the_two_atoms_the_axis_is_built_from(): + variables = _atoms() + assert variables["k8s-version"] == K8S_VERSION + assert variables["cni-version"] == CNI_VERSION + + +def test_derived_tags_expand_to_the_pinned_upstream_identifiers(): + checker = _checker() + variables = _atoms() + atoms = {k: variables[k] for k in ("k8s-version", "cni-version")} + assert ( + checker.expand(variables["k8s-upstream-tag"], atoms, "test") + == f"v{K8S_VERSION}" + ) + assert ( + checker.expand(variables["cni-upstream-tag"], atoms, "test") + == f"v{CNI_VERSION}" + ) + + +def test_version_axis_is_closed_over_its_own_atoms(): + """The FSDK point release must not be able to drag Kubernetes with it. + + Every derived spelling has to resolve using only k8s-version and + cni-version. If one reached for %{release-version} — or any other + project.conf variable — bumping the OS would move the Kubernetes sysext. + """ + checker = _checker() + variables = _atoms() + atoms = {k: variables[k] for k in ("k8s-version", "cni-version")} + for name, value in variables.items(): + if name in atoms: + continue + checker.expand(value, atoms, f"include/kubernetes.yml {name}") + + +def test_every_node_binary_is_fetched_from_the_same_derived_release(): + """kubeadm, kubelet and kubectl skewing apart is a supported-skew violation.""" + checker = _checker() + urls = checker.source_urls(checker.read(checker.K8S_BIN)) + fetched = {url.rsplit("/", 1)[1] for url in urls} + assert fetched == set(checker.NODE_BINARIES) + assert all("%{k8s-upstream-tag}" in url for url in urls) + + +def test_no_consumer_restates_a_literal_version(): + checker = _checker() + for path in ( + checker.K8S_BIN, + checker.CNI_BIN, + checker.K8S_SYSEXT, + checker.EXTENSION_RELEASE, + ): + for line in checker.read(path).splitlines(): + if line.lstrip().startswith("#"): + continue + assert not checker.LITERAL_VERSION_RE.search(line), ( + f"{path.relative_to(ROOT)}: {line.strip()}" + ) + + +def test_extension_release_defers_version_id_to_build_time(): + """A literal VERSION_ID would silently win over the generated one.""" + checker = _checker() + text = checker.read(checker.EXTENSION_RELEASE) + assert "NAME=kubernetes" in text + assert not any( + line.strip().startswith("VERSION_ID=") for line in text.splitlines() + ) + + +def test_checker_reads_the_component_scoped_transfer(): + checker = _checker() + assert checker.TRANSFER == ( + ROOT / "files" / "os" / "sysupdate.kubernetes.d" / "70-kubernetes.transfer" + ) + + +def test_pre_commit_hook_fires_on_every_file_the_checker_reads(): + checker = _checker() + config = yaml.safe_load((ROOT / ".pre-commit-config.yaml").read_text()) + hook = next( + hook + for repo in config["repos"] + if repo.get("repo") == "local" + for hook in repo.get("hooks", []) + if hook.get("id") == "check-kubernetes-version" + ) + selector = re.compile(hook["files"]) + consumers = [ + checker.K8S_INCLUDE, + checker.K8S_BIN, + checker.CNI_BIN, + checker.K8S_SYSEXT, + checker.EXTENSION_RELEASE, + checker.TRANSFER, + Path(CHECKER), + ] + for path in consumers: + relative = path.relative_to(ROOT).as_posix() + assert selector.fullmatch(relative) is not None, relative + assert selector.fullmatch("files/os/sysupdate.d/70-kubernetes.transfer") is None + + +def test_repository_satisfies_the_version_invariant(): + result = subprocess.run( + [sys.executable, str(CHECKER)], capture_output=True, text=True + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/unit/test_kubestellar_kiosk.py b/tests/unit/test_kubestellar_kiosk.py index b9081fc5..3a1ff2ec 100644 --- a/tests/unit/test_kubestellar_kiosk.py +++ b/tests/unit/test_kubestellar_kiosk.py @@ -1,117 +1,142 @@ -"""Contracts for the KubeStellar kiosk proxy and authenticated agent gate.""" +"""Contracts for the KubeStellar kiosk proxy and its authenticated agent gate. +The kiosk is the appliance's only dashboard and its only public port. What is +defended here is that it is reachable, that it terminates TLS with material it +generated itself, that the gate it injects actually reaches the browser, and +that it reads nothing off the node. +""" + +from __future__ import annotations + +import re from pathlib import Path +import yaml + ROOT = Path(__file__).resolve().parents[2] -KIOSK = ROOT / "files" / "k0s" / "kiosk" -KIOSK_CONF = KIOSK / "nginx.conf" -KIOSK_JS = KIOSK / "kiosk-gate.js" -KIOSK_CSS = KIOSK / "kiosk-gate.css" -TMPFILES = ROOT / "files" / "k0s" / "sysext" / "k0s-manifests.conf" -SYSEXT = ROOT / "elements" / "oci" / "k0s-sysext.bst" -CONSOLE_MANIFEST = ( - ROOT - / "files" - / "k0s" - / "manifests" - / "kubestellar" - / "40-kubestellar-console.yaml" -) -PROXY_MANIFEST = ( - ROOT - / "files" - / "k0s" - / "manifests" - / "kubestellar" - / "41-kubestellar-kiosk-proxy.yaml" -) - - -def test_kiosk_assets_are_packaged_and_seeded() -> None: - sysext = SYSEXT.read_text(encoding="utf-8") - tmpfiles = TMPFILES.read_text(encoding="utf-8") - - assert KIOSK_CONF.is_file() - assert KIOSK_JS.is_file() - assert KIOSK_CSS.is_file() - assert "freedesktop-sdk.bst:components/openssl.bst" in sysext - assert "keyout sysext/usr/share/k0s/kiosk/key.pem" in sysext - assert "path: files/k0s/kiosk" in sysext - assert "directory: kiosk-src" in sysext - assert "cp -a kiosk-src/. sysext/usr/share/k0s/kiosk/" in sysext - assert "C+ /var/lib/k0s/kiosk - - - - /usr/share/k0s/kiosk" in tmpfiles - - -def test_proxy_injects_only_csp_safe_same_origin_assets() -> None: - nginx = KIOSK_CONF.read_text(encoding="utf-8") - - assert "proxy_pass $upstream_endpoint;" in nginx - assert ( - "set $upstream_endpoint " - "http://kubestellar-console.kubestellar-console.svc.cluster.local:8080;" - ) in nginx - assert "resolver 10.96.0.10" in nginx - assert "proxy_redirect" in nginx - assert 'proxy_set_header Accept-Encoding "";' in nginx - assert "sub_filter_types" not in nginx - assert ( - 'sub_filter \'\' ' - '\'\';' - ) in nginx +KIOSK = ROOT / "files" / "cluster" / "manifests" / "kiosk" +ASSETS = KIOSK / "assets" +NGINX_CONF = ASSETS / "nginx.conf" +GATE_JS = ASSETS / "kiosk-gate.js" +GATE_CSS = ASSETS / "kiosk-gate.css" + + +def _doc(path: Path, kind: str) -> dict: + return next(d for d in yaml.safe_load_all(path.read_text(encoding="utf-8")) if d["kind"] == kind) + + +def test_assets_are_delivered_from_the_cluster_not_the_node() -> None: + # These three files used to arrive through a hostPath mount seeded by + # tmpfiles.d, which meant the workload's configuration lived outside the + # cluster and could not be reconciled or rolled. + kustomization = yaml.safe_load((KIOSK / "kustomization.yaml").read_text(encoding="utf-8")) + generator = kustomization["configMapGenerator"][0] + assert sorted(generator["files"]) == [ + "assets/kiosk-gate.css", + "assets/kiosk-gate.js", + "assets/nginx.conf", + ] + + pod = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"] + assert all("hostPath" not in volume for volume in pod["volumes"]) + assets = next(v for v in pod["volumes"] if v["name"] == "assets") + assert assets["configMap"]["name"] == generator["name"] + + +def test_nginx_writes_only_to_mounted_volumes() -> None: + # The container root filesystem is read-only. nginx opens its pid file and + # its proxy scratch paths at startup, so any path that is not on a writable + # mount is a pod that crash-loops before it ever serves a request. + container = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"]["containers"][0] + assert container["securityContext"]["readOnlyRootFilesystem"] is True + + writable = { + mount["mountPath"] + for mount in container["volumeMounts"] + if not mount.get("readOnly", False) + } + assert writable, "a read-only root filesystem needs somewhere for nginx to write" + + conf = NGINX_CONF.read_text(encoding="utf-8") + paths = re.findall(r"^\s*(?:pid|\w+_temp_path)\s+(\S+?);", conf, flags=re.MULTILINE) + assert paths, "nginx.conf declares no pid or temp paths to check" + for path in paths: + assert any(path.startswith(f"{root}/") for root in writable), ( + f"{path} is not on a writable mount ({sorted(writable)})" + ) + + +def test_proxy_injects_the_gate_same_origin_into_live_html() -> None: + conf = NGINX_CONF.read_text(encoding="utf-8") + + # The gate is injected by rewriting the console's HTML. nginx cannot rewrite + # a compressed response body, so upstream compression has to be refused or + # sub_filter silently does nothing and the gate never appears. + assert 'proxy_set_header Accept-Encoding "";' in conf + assert '' in conf + assert '' in conf + + # Both assets are served by this proxy on its own origin, so no CSP the + # console sets can block them and no third-party host is contacted. + for asset in ("kiosk-gate.js", "kiosk-gate.css"): + assert f"location = /{asset} {{" in conf + assert f"alias /etc/kubestellar-kiosk/assets/{asset};" in conf + + # CoreDNS on the kubeadm service CIDR; resolved at request time so the + # console Service moving does not require restarting the proxy. + assert "resolver 10.96.0.10" in conf assert ( - 'sub_filter \'\' ' - '\'\';' - ) in nginx - assert "Content-Security-Policy" not in nginx + "http://kubestellar-console.kubestellar-console.svc.cluster.local:8080" in conf + ) -def test_gate_waits_for_session_and_blocks_until_agent_health() -> None: - script = KIOSK_JS.read_text(encoding="utf-8") - css = KIOSK_CSS.read_text(encoding="utf-8") +def test_gate_blocks_the_console_until_the_agent_is_reachable() -> None: + script = GATE_JS.read_text(encoding="utf-8") + css = GATE_CSS.read_text(encoding="utf-8") assert "kc-has-session" in script assert "http://127.0.0.1:8585/health" in script - assert "brew tap kubestellar/tap && brew install kc-agent" in script - assert ( - 'KAGENTI_CONTROLLER_URL="none" kc-agent -allowed-origins' - " ${window.location.origin}" - ) in script assert "aria-modal" in script + # The overlay has to actually trap interaction, or it is decoration. assert "addEventListener('keydown'" in script assert "pointer-events: auto" in css assert "z-index: 2147483647" in css -def test_console_provides_local_and_oauth_login_options() -> None: - console = CONSOLE_MANIFEST.read_text(encoding="utf-8") - - assert "kind: ServiceAccount" in console - assert "kind: ClusterRole" in console - assert "kind: ClusterRoleBinding" in console - assert "serviceAccountName: kubestellar-console" in console - assert "resources: [\"nodes\", \"namespaces\", \"pods\", \"services\", \"events\", \"configmaps\"]" in console - assert "name: DEV_MODE" in console - assert 'value: "true"' in console - assert "name: ALLOW_DEV_MODE_IN_CLUSTER" in console - assert "hostPort:" not in console - assert "name: GITHUB_CLIENT_ID" in console - assert "name: GITHUB_CLIENT_SECRET" in console - assert "name: kubestellar-console-github-oauth" in console - assert "key: client-id" in console - assert "key: client-secret" in console - assert "optional: true" in console - - -def test_proxy_is_the_only_public_console_endpoint() -> None: - proxy = PROXY_MANIFEST.read_text(encoding="utf-8") - - assert "name: kubestellar-kiosk-proxy" in proxy - assert "hostPort: 8080" in proxy - assert "hostIP: 127.0.0.1" in proxy - assert "mountPath: /etc/kubestellar-kiosk" in proxy - assert "hostPath:\n path: /var/lib/k0s/kiosk" in proxy - assert "readOnly: true" in proxy - assert ( - "nginx@sha256:62223d644fa234c3a1cc785ee14242ec47a77364226f1c811d2f669f96dc2ac8" - in proxy - ) +def test_tls_is_issued_in_cluster_and_short_lived() -> None: + # Replaces the 10-year self-signed key that was generated at image build + # time and was therefore identical on every installed appliance. + certificate = _doc(KIOSK / "certificate.yaml", "Certificate")["spec"] + assert certificate["issuerRef"]["kind"] == "ClusterIssuer" + assert certificate["issuerRef"]["name"] == "bluefin-ca" + assert certificate["privateKey"]["rotationPolicy"] == "Always" + + hours = int(certificate["duration"].removesuffix("h")) + assert hours <= 2400, "the kiosk certificate must stay under 100 days" + assert int(certificate["renewBefore"].removesuffix("h")) < hours + + container = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"]["containers"][0] + tls = next(m for m in container["volumeMounts"] if m["name"] == "tls") + conf = NGINX_CONF.read_text(encoding="utf-8") + assert f"ssl_certificate {tls['mountPath']}/tls.crt;" in conf + assert f"ssl_certificate_key {tls['mountPath']}/tls.key;" in conf + assert "listen 8080 ssl;" in conf + # Anyone arriving over plain HTTP on the TLS port gets redirected rather + # than an nginx error page. + assert "error_page 497 https://$http_host$request_uri;" in conf + + +def test_proxy_is_published_on_every_interface() -> None: + # Binding hostIP 127.0.0.1 made the console unreachable from the LAN and + # from anything but a single QEMU hostfwd. The endpoint is HTTPS-only and + # the console behind it authenticates. + container = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"]["containers"][0] + port = next(p for p in container["ports"] if p.get("hostPort")) + assert port["hostPort"] == 8080 + assert port["containerPort"] == 8080 + assert "hostIP" not in port + + +def test_proxy_image_is_immutable() -> None: + container = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"]["containers"][0] + assert "@sha256:" in container["image"] diff --git a/tests/unit/test_repart_layout.py b/tests/unit/test_repart_layout.py index e21e5b46..62239973 100644 --- a/tests/unit/test_repart_layout.py +++ b/tests/unit/test_repart_layout.py @@ -218,9 +218,16 @@ def test_var_is_a_growing_xfs_tail(): ) -def test_var_seeds_the_offline_k0s_sysext(): +def test_var_seeds_the_sysext_into_the_systemd_sysext_scan_directory(): + """An offline install must boot into a working cluster with no fetch. + + The destination is relative to the /var partition root, so + ``/lib/extensions/kubernetes.raw`` is ``/var/lib/extensions/kubernetes.raw`` + on the running system — one of systemd-sysext's own search paths, which is + why no unit has to copy the image anywhere before the merge. + """ var = next(s for s in partitions().values() if s["Type"] == "var") - assert var["CopyFiles"] == "/k0s.raw:/lib/k0s/k0s.raw" + assert var["CopyFiles"] == "/kubernetes.raw:/lib/extensions/kubernetes.raw" def test_root_partition_label_is_matched_by_the_sysupdate_root_transfer(): diff --git a/tests/unit/test_sysext_export_contract.py b/tests/unit/test_sysext_export_contract.py deleted file mode 100644 index 30f2fd07..00000000 --- a/tests/unit/test_sysext_export_contract.py +++ /dev/null @@ -1,13 +0,0 @@ -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[2] -JUSTFILE = ROOT / "Justfile" - - -def test_export_sysext_does_not_emit_legacy_k3s_alias() -> None: - justfile = JUSTFILE.read_text(encoding="utf-8") - start = justfile.index("export-sysext:") - end = justfile.index("\n# Write the raw GPT installer image", start) - - assert "k3s-" not in justfile[start:end] diff --git a/tests/unit/test_sysupdate_transfers.py b/tests/unit/test_sysupdate_transfers.py index 5cf22c61..493828d9 100644 --- a/tests/unit/test_sysupdate_transfers.py +++ b/tests/unit/test_sysupdate_transfers.py @@ -2,9 +2,9 @@ ``tests/unit/test_repart_layout.py`` already pins ``50-root.transfer`` against the installer repart config. The other two OTA transfer definitions — -``60-uki.transfer`` (the boot UKI) and ``70-k0s.transfer`` (the k0s sysext) — -have no coverage at all, and neither does the source-side artifact naming that -systemd-sysupdate matches against. +``60-uki.transfer`` (the boot UKI) and ``70-kubernetes.transfer`` (the +Kubernetes sysext) — have no coverage at all, and neither does the source-side +artifact naming that systemd-sysupdate matches against. These tests assert the drift-prone contracts: @@ -12,9 +12,10 @@ * every source pulls from this repo's own release feed over https * every source ``MatchPattern`` corresponds to an artifact name that some element under ``elements/`` actually emits (``bluefin-server-ddi-.raw.zst``, - ``bluefin-server-.efi``, ``k0s-.raw.zst``) -* the k0s sysext transfer lands in ``/var/lib/extensions`` under a name that - ``files/k0s/sysext/extension-release.k0s`` (``NAME=k0s``) can merge + ``bluefin-server-.efi``, ``kubernetes-.raw.zst``) +* the Kubernetes sysext transfer lands in ``/var/lib/extensions`` under a name + that ``files/kubernetes/sysext/extension-release.kubernetes`` + (``NAME=kubernetes``) can merge """ import configparser @@ -25,17 +26,22 @@ REPO_ROOT = Path(__file__).resolve().parents[2] SYSUPDATE_DIR = REPO_ROOT / "files" / "os" / "sysupdate.d" -K0S_SYSUPDATE_DIR = REPO_ROOT / "files" / "os" / "sysupdate.k0s.d" -K0S_TRANSFER = K0S_SYSUPDATE_DIR / "70-k0s.transfer" +KUBERNETES_SYSUPDATE_DIR = REPO_ROOT / "files" / "os" / "sysupdate.kubernetes.d" +KUBERNETES_TRANSFER = KUBERNETES_SYSUPDATE_DIR / "70-kubernetes.transfer" ELEMENTS_DIR = REPO_ROOT / "elements" -EXTENSION_RELEASE = REPO_ROOT / "files" / "k0s" / "sysext" / "extension-release.k0s" +EXTENSION_RELEASE = ( + REPO_ROOT / "files" / "kubernetes" / "sysext" / "extension-release.kubernetes" +) RELEASE_FEED = "https://github.com/projectbluefin/server/releases/latest/download/" def transfer_paths() -> list[Path]: return sorted( - (*SYSUPDATE_DIR.glob("*.transfer"), *K0S_SYSUPDATE_DIR.glob("*.transfer")) + ( + *SYSUPDATE_DIR.glob("*.transfer"), + *KUBERNETES_SYSUPDATE_DIR.glob("*.transfer"), + ) ) @@ -63,9 +69,11 @@ def split_match_pattern(pattern: str) -> tuple[str, str]: def test_sysupdate_directories_are_populated(): generic = sorted(p.name for p in SYSUPDATE_DIR.glob("*.transfer")) - k0s = sorted(p.name for p in K0S_SYSUPDATE_DIR.glob("*.transfer")) + kubernetes = sorted( + p.name for p in KUBERNETES_SYSUPDATE_DIR.glob("*.transfer") + ) assert generic == ["50-root.transfer", "60-uki.transfer"] - assert k0s == ["70-k0s.transfer"] + assert kubernetes == ["70-kubernetes.transfer"] @pytest.mark.parametrize("path", transfer_paths(), ids=lambda p: p.name) @@ -126,7 +134,7 @@ def test_every_source_artifact_is_produced_by_an_element(path: Path): load_transfer(path)["Source"]["MatchPattern"] ) # Elements name artifacts with a BuildStream variable in the version slot, - # e.g. FNAME="k0s-%{k0s-version}.raw" plus a .zst compression step. + # e.g. FNAME="kubernetes-%{k8s-version}.raw" plus a .zst compression step. produced = re.compile( re.escape(prefix) + r"%\{[a-z0-9-]+\}" + re.escape(suffix.removesuffix(".zst")) ) @@ -158,27 +166,33 @@ def test_uki_source_and_target_names_agree(): ) -def test_k0s_sysext_transfer_lands_in_the_system_extension_directory(): - target = load_transfer(K0S_TRANSFER)["Target"] +def test_kubernetes_sysext_transfer_lands_where_systemd_sysext_scans() -> None: + """The image must be merged by systemd-sysext.service, not by a copy step. + + ``/var/lib/extensions`` is one of systemd-sysext's own search paths, so an + image landing there is merged at boot with no unit shuffling it into + ``/run/extensions`` first. + """ + target = load_transfer(KUBERNETES_TRANSFER)["Target"] assert target.get("Type") == "regular-file", ( - "the k0s sysext is delivered as a decompressed regular file" + "the Kubernetes sysext is delivered as a decompressed regular file" ) - assert target.get("Path") == "/var/lib/k0s", ( - f"k0s sysext target path is {target.get('Path')!r}; the persistent " - "staging path must remain outside systemd-sysext's early scan" + assert target.get("Path") == "/var/lib/extensions", ( + f"sysext target path is {target.get('Path')!r}; anywhere else and " + "systemd-sysext.service never sees the image" ) assert target.get("Mode") == "0644", ( - f"k0s sysext mode is {target.get('Mode')!r}; the image must be readable " + f"sysext mode is {target.get('Mode')!r}; the image must be readable " "by systemd-sysext at merge time" ) -def test_k0s_sysext_transfer_maintains_a_stable_current_symlink(): - target = load_transfer(K0S_TRANSFER)["Target"] +def test_kubernetes_sysext_transfer_maintains_a_stable_current_symlink(): + target = load_transfer(KUBERNETES_TRANSFER)["Target"] symlink = target.get("CurrentSymlink") - assert symlink == "k0s.raw", ( - f"CurrentSymlink is {symlink!r}; the boot activation unit requires a " - "stable filename, so a version bump otherwise stops merging k0s" + assert symlink == "kubernetes.raw", ( + f"CurrentSymlink is {symlink!r}; the merge name must be stable, so a " + "version bump otherwise stops merging the sysext" ) prefix, suffix = split_match_pattern(target["MatchPattern"]) assert symlink == f"{prefix.rstrip('-')}{suffix}", ( @@ -187,23 +201,28 @@ def test_k0s_sysext_transfer_maintains_a_stable_current_symlink(): ) -def test_k0s_sysext_transfer_decompresses_the_release_asset(): - parser = load_transfer(K0S_TRANSFER) +def test_kubernetes_sysext_transfer_decompresses_the_release_asset(): + parser = load_transfer(KUBERNETES_TRANSFER) source = parser["Source"]["MatchPattern"] target = parser["Target"]["MatchPattern"] - assert source.endswith(".raw.zst"), f"k0s release asset {source!r} is not zstd" + assert source == "kubernetes-@v.raw.zst", ( + f"release asset pattern is {source!r}; " + ".github/scripts/check-kubernetes-version.py cross-checks this exact " + "spelling against the name elements/oci/kubernetes-sysext.bst emits" + ) assert target == source.removesuffix(".zst"), ( - f"k0s sysext installs as {target!r} but downloads {source!r}; a still " + f"sysext installs as {target!r} but downloads {source!r}; a still " "compressed image cannot be mounted by systemd-sysext" ) -def test_k0s_sysext_image_name_matches_its_extension_release_name(): +def test_kubernetes_sysext_image_name_matches_its_extension_release_name(): """systemd-sysext requires ``extension-release.`` to agree. - The image installed by ``70-k0s.transfer`` is merged through the stable - ``k0s.raw`` symlink, so ``NAME=`` in ``files/k0s/sysext/extension-release.k0s`` - must be ``k0s`` or the merge is rejected at boot. + The image installed by ``70-kubernetes.transfer`` is merged through the + stable ``kubernetes.raw`` symlink, so ``NAME=`` in + ``files/kubernetes/sysext/extension-release.kubernetes`` must be + ``kubernetes`` or the merge is rejected at boot. """ assert EXTENSION_RELEASE.is_file(), f"{EXTENSION_RELEASE} missing" fields = dict( @@ -211,9 +230,7 @@ def test_k0s_sysext_image_name_matches_its_extension_release_name(): for line in EXTENSION_RELEASE.read_text().splitlines() if "=" in line and not line.startswith("#") ) - symlink = load_transfer(K0S_TRANSFER)["Target"][ - "CurrentSymlink" - ] + symlink = load_transfer(KUBERNETES_TRANSFER)["Target"]["CurrentSymlink"] image_name = symlink.removesuffix(".raw") assert EXTENSION_RELEASE.name == f"extension-release.{image_name}", ( f"{EXTENSION_RELEASE.name} does not match the installed image name " @@ -239,11 +256,3 @@ def test_every_source_artifact_is_staged_in_release_workflow(path: Path): assert re.search(rf"cp\s+.*{re.escape(prefix)}\*.*dist/release/", build_yml), ( f"{path.name} source asset prefix {prefix!r} is not staged to dist/release/ in .github/workflows/build.yml" ) - - -def test_k0s_release_staging_does_not_use_legacy_k3s_name() -> None: - build_yml = (REPO_ROOT / ".github" / "workflows" / "build.yml").read_text() - assert re.search(r"cp\s+.*k0s-\*\.raw\.zst.*dist/release/", build_yml) - assert not re.search( - r"cp\s+.*k3s-\*\.raw\.zst.*dist/release/", build_yml - ) diff --git a/tests/unit/test_unit_dependency_provenance.py b/tests/unit/test_unit_dependency_provenance.py new file mode 100644 index 00000000..7e772ccb --- /dev/null +++ b/tests/unit/test_unit_dependency_provenance.py @@ -0,0 +1,126 @@ +"""Every hard dependency of a shipped unit must have a named provider. + +`just validate` resolves the BuildStream graph, but a systemd `Requires=` is not +a bst node — the graph cannot see it. A unit that requires a service nothing +ships still builds, still validates, and then fails at boot with a job that +cannot be resolved. That is a silent, boot-only failure mode. + +This module closes it. Every unit named by `Requires=` or `BindsTo=` in a unit +this repository stages must be either: + + * staged by this repository (a peer file in files/os/systemd/system), or + * listed in EXTERNAL_PROVIDERS with the component that supplies it. + +Adding a hard dependency on something nothing provides therefore fails here +rather than on a running machine. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +UNIT_DIR = ROOT / "files" / "os" / "systemd" / "system" + +# Units this repository does not stage, mapped to what does supply them. +# +# containerd.service is the load-bearing entry. Flatcar ships containerd as a +# sysext blob inside its /usr tree (usr/share/flatcar/sysext/containerd-flatcar.raw, +# 26 MiB, verified against the 4593.2.5 image contents listing). That tree only +# reaches this image through elements/flatcar/flatcar-usr.bst, which is NOT on +# main yet — it arrives with PR #140, and os-stack.bst consumes it in PR #132. +# +# Until both land, kubeadm-init.service has a hard requirement on a unit no +# element supplies, so the control plane does not come up on a real boot. The +# cutover is structurally complete but inert until then. This entry records that +# deliberately; delete the note, not the entry, once #140 and #132 have landed. +EXTERNAL_PROVIDERS = { + "containerd.service": ( + "Flatcar /usr via elements/flatcar/flatcar-usr.bst — PENDING PR #140/#132" + ), + "systemd-sysext.service": "systemd, present in the base image", + "network-online.target": "systemd", + "systemd-networkd.service": "systemd", + "var.mount": "generated from the installer's /var partition", +} + +HARD_DEPENDENCY_KEYS = ("Requires=", "BindsTo=") + + +def staged_units() -> set[str]: + return {path.name for path in UNIT_DIR.iterdir() if path.is_file()} + + +def hard_dependencies() -> list[tuple[str, str]]: + """Return (unit_file, required_unit) for every hard dependency declared.""" + found: list[tuple[str, str]] = [] + for unit in sorted(UNIT_DIR.glob("*.service")): + for raw in unit.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + for key in HARD_DEPENDENCY_KEYS: + if line.startswith(key): + # systemd accepts a space-separated list and repeated keys. + for name in line[len(key):].split(): + found.append((unit.name, name)) + return found + + +def test_units_declare_at_least_one_hard_dependency() -> None: + """Guard the parser itself: a silent zero-match would make this suite vacuous.""" + assert hard_dependencies(), "parsed no Requires=/BindsTo= at all; parser is broken" + + +@pytest.mark.parametrize( + ("unit", "required"), + hard_dependencies(), + ids=lambda value: value.replace(".service", ""), +) +def test_hard_dependency_has_a_provider(unit: str, required: str) -> None: + local = staged_units() + assert required in local or required in EXTERNAL_PROVIDERS, ( + f"{unit} declares a hard dependency on {required!r}, which this repository " + f"does not stage and EXTERNAL_PROVIDERS does not account for. Either ship " + f"the unit, or record who supplies it — a Requires= on a unit nothing " + f"provides fails only at boot, where nothing else catches it." + ) + + +def test_containerd_provider_is_recorded_as_pending() -> None: + """The cutover is inert until Flatcar's /usr lands; keep that visible. + + kubeadm-init.service requires containerd.service. Nothing on main supplies + it. This asserts the gap stays documented rather than quietly forgotten. + """ + assert "containerd.service" in EXTERNAL_PROVIDERS + assert "flatcar-usr.bst" in EXTERNAL_PROVIDERS["containerd.service"], ( + "containerd.service must name flatcar-usr.bst as its provider" + ) + + +def test_kubelet_bridges_the_cni_plugin_directory() -> None: + """Plugins ship at the bakery path; containerd and Cilium read /opt/cni/bin. + + Without the bridge the loopback plugin is absent, every pod sandbox fails to + get a network, and the node never leaves NotReady with no obvious cause. + """ + kubelet = (UNIT_DIR / "kubelet.service").read_text(encoding="utf-8") + + assert "/opt/cni/bin" in kubelet, "kubelet does not populate /opt/cni/bin" + assert "/usr/local/bin/cni" in kubelet, ( + "kubelet does not read the sysext's CNI plugin directory" + ) + + bridge = next( + line for line in kubelet.splitlines() if "cp -a" in line and "cni" in line + ) + assert "-an" in bridge or "--no-clobber" in bridge, ( + "the copy must not clobber: the cilium agent installs its own binary " + "into /opt/cni/bin and overwriting it breaks the CNI chain" + ) + + body = kubelet.split("[Service]", 1)[1] + assert body.index("/opt/cni/bin") < body.index("ExecStart=/usr/bin/kubelet"), ( + "the bridge must run before kubelet starts" + ) diff --git a/workflows/server-release-watch-and-fix.md b/workflows/server-release-watch-and-fix.md index fa352027..ffa05089 100644 --- a/workflows/server-release-watch-and-fix.md +++ b/workflows/server-release-watch-and-fix.md @@ -28,11 +28,11 @@ Specifies the automated loop that monitors GitHub Actions release runs on `proje #### Stage B: Release Staging & Contract Audit While the remote build is compiling in CI, audit the local tree against the release contract: -1. **Element Graph**: `just validate` (checks release version and k0s version alignment). +1. **Element Graph**: `just validate` (checks release version and Kubernetes version alignment). 2. **Unit Suite**: `pytest tests/unit` and `bats tests/unit`. 3. **Docs & Skills**: `python3 .github/scripts/docs-checks.py`. 4. **Sysupdate Asset Parity**: Verify every `.transfer` target pattern in - `files/os/sysupdate.d/*.transfer` and `files/os/sysupdate.k0s.d/*.transfer` + `files/os/sysupdate.d/*.transfer` and `files/os/sysupdate.kubernetes.d/*.transfer` is emitted by an element under `elements/` AND staged into `dist/release/` in `.github/workflows/build.yml`. @@ -66,7 +66,7 @@ Once the GitHub Actions `release` job finishes: - `bluefin-server-installer-*.raw.zst` - `bluefin-server-*.efi` - `bluefin-server-ddi-*.raw.zst` - - `k0s-*.raw.zst` + - `kubernetes-*.raw.zst` - `SHA256SUMS` - `SHA256SUMS.gpg` 3. Verify GPG signature against `SHA256SUMS`. From 4def5f1ac9a649a79ece43e603bd3c6f7e3870ac Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 06:30:50 -0400 Subject: [PATCH 19/46] fix(elements): disable the stripper on the staging-only cluster elements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `os-cluster-bootstrap.bst` and `os-cluster-manifests.bst` are kind: manual with no build-commands — they only copy units, YAML and a shell helper into %{install-root}. FSDK's default strip-commands still ran, shelling out to freedesktop-sdk-stripper, which base/base-stack.bst does not put in the sandbox: sh: line 11: freedesktop-sdk-stripper: command not found Command '...' failed with exitcode 127 FAILURE bluefin-server/os-cluster-manifests.bst: Running commands Both elements now set `variables: strip-binaries: ""`, matching os-countme.bst, os-image-info.bst and os-release-flatcar.bst. `just validate` could not catch this: it runs `bst show --deps all`, which resolves the graph without executing a build command, so the failure only appeared eight minutes into `Build DDI artifacts`. Adds tests/unit/test_manual_element_strip_contract.py, which asserts the invariant directly — every kind: manual element without config.build-commands disables the stripper, and every one with build-commands keeps it. Mutation-tested: removing either override fails the gate. Verified: `just bst build bluefin-server/os-cluster-manifests.bst bluefin-server/os-cluster-bootstrap.bst` succeeds, having failed at exit 127 before this change. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../bluefin-server/os-cluster-bootstrap.bst | 8 ++ .../bluefin-server/os-cluster-manifests.bst | 7 ++ .../test_manual_element_strip_contract.py | 103 ++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 tests/unit/test_manual_element_strip_contract.py diff --git a/elements/bluefin-server/os-cluster-bootstrap.bst b/elements/bluefin-server/os-cluster-bootstrap.bst index 09ea60d9..519123f3 100644 --- a/elements/bluefin-server/os-cluster-bootstrap.bst +++ b/elements/bluefin-server/os-cluster-bootstrap.bst @@ -28,6 +28,14 @@ sources: path: files/os/libexec directory: libexec-src +# This element stages systemd units, a YAML config and a shell helper. FSDK's +# default strip-commands shell out to freedesktop-sdk-stripper, which +# base/base-stack.bst does not provide, so the default turns a no-op into +# `command not found` (exit 127). Same contract as os-countme.bst, +# os-image-info.bst and os-release-flatcar.bst. +variables: + strip-binaries: "" + config: install-commands: - | diff --git a/elements/bluefin-server/os-cluster-manifests.bst b/elements/bluefin-server/os-cluster-manifests.bst index f1c8abc1..f0200f8a 100644 --- a/elements/bluefin-server/os-cluster-manifests.bst +++ b/elements/bluefin-server/os-cluster-manifests.bst @@ -44,6 +44,13 @@ sources: directory: certmanager-src ref: e03b668ec8675214af6b0a671699d088f2601fa3878e0dbe1b41d3feafd1879f +# This element stages YAML only. FSDK's default strip-commands shell out to +# freedesktop-sdk-stripper, which base/base-stack.bst does not provide, so the +# default turns a no-op into `command not found` (exit 127). Same contract as +# os-countme.bst, os-image-info.bst and os-release-flatcar.bst. +variables: + strip-binaries: "" + config: install-commands: - | diff --git a/tests/unit/test_manual_element_strip_contract.py b/tests/unit/test_manual_element_strip_contract.py new file mode 100644 index 00000000..574312a7 --- /dev/null +++ b/tests/unit/test_manual_element_strip_contract.py @@ -0,0 +1,103 @@ +"""A staging-only `kind: manual` element must disable the binary stripper. + +FSDK's `manual` element runs default `strip-commands` after install. Those +commands shell out to `freedesktop-sdk-stripper`, a tool that arrives in the +sandbox with the FSDK build-system stacks — not with `base/base-stack.bst`. + +An element that only copies files into `%{install-root}` therefore has nothing +to strip *and* no stripper to run it, so the default turns a no-op into +`sh: freedesktop-sdk-stripper: command not found` and the build dies with +exit 127. The repository's answer is `variables: strip-binaries: ""`. + +`just validate` cannot catch this. It runs `bst show --deps all`, which +resolves the graph without executing a single build command, so a missing +`strip-binaries` override shows up only after a full `bst build` — eight +minutes into CI, and only for whoever pushes next. + +That is not hypothetical: `os-cluster-bootstrap.bst` and +`os-cluster-manifests.bst` both shipped without the override and both failed +at exit 127 in the `Build DDI artifacts` run for the Kubernetes cutover, after +`just validate` had passed clean. + +The discriminator is `config.build-commands`. An element that declares build +commands ran a compiler, so it has real ELF output and wants the stripper — +`uutils-coreutils.bst` is the one such element here, and it correctly leaves +the default alone. An element with no build commands only stages, and must +opt out. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ELEMENTS_DIR = ROOT / "elements" + + +def _manual_elements(): + for path in sorted(ELEMENTS_DIR.glob("**/*.bst")): + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if data.get("kind") == "manual": + yield path, data + + +STAGING_ONLY = [ + (path, data) + for path, data in _manual_elements() + if not (data.get("config") or {}).get("build-commands") +] + +COMPILING = [ + (path, data) + for path, data in _manual_elements() + if (data.get("config") or {}).get("build-commands") +] + + +@pytest.mark.parametrize( + "path,data", STAGING_ONLY, ids=[p.relative_to(ROOT).as_posix() for p, _ in STAGING_ONLY] +) +def test_staging_only_manual_elements_disable_the_stripper(path: Path, data: dict) -> None: + strip = (data.get("variables") or {}).get("strip-binaries", None) + + assert strip == "", ( + f"{path.relative_to(ROOT)} is a kind: manual element with no " + f"config.build-commands, so it only stages files and no stripper is in " + f"its sandbox. It must set:\n\n" + f" variables:\n" + f" strip-binaries: \"\"\n\n" + f"Without it the build fails with " + f"'freedesktop-sdk-stripper: command not found' (exit 127), which " + f"`just validate` cannot detect because it never runs a build command. " + f"Got strip-binaries={strip!r}." + ) + + +def test_compiling_manual_elements_keep_the_default_stripper() -> None: + # Guards the discriminator itself: if this list empties, the rule above has + # silently become unconditional and stops distinguishing anything. + assert COMPILING, ( + "no kind: manual element declares config.build-commands; the " + "staging-only rule above is no longer discriminating and its " + "build-commands check needs re-deriving" + ) + + for path, data in COMPILING: + strip = (data.get("variables") or {}).get("strip-binaries", None) + assert strip != "", ( + f"{path.relative_to(ROOT)} compiles (config.build-commands is set) " + f"but disables the stripper, so its binaries ship unstripped. " + f"Remove the strip-binaries override or move the build commands out." + ) + + +def test_at_least_one_manual_element_is_discovered() -> None: + # Guards the parser: a glob or kind-key regression would make the + # parametrized test above vacuously pass with zero cases. + assert STAGING_ONLY, ( + "no staging-only kind: manual elements found under elements/ — the " + "discovery in this module has regressed" + ) From 1a0b0b629aeb9ac8c081124fd6925dbdc9fbef3e Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 06:31:23 -0400 Subject: [PATCH 20/46] docs(kubeadm): correct the claim that containerd arrives with the sysext The header said "/usr/bin/kubeadm and containerd.service both arrive with the sysext merge". Only the first half is true. elements/oci/kubernetes-sysext.bst stages kubectl, kubeadm, kubelet and the CNI plugins; it ships no containerd, no runc and no crictl, and nothing else in this repository builds them. containerd reaches the image solely through Flatcar's /usr (usr/share/flatcar/sysext/containerd-flatcar.raw), which arrives with elements/flatcar/flatcar-usr.bst in PR #140 and is consumed by os-stack.bst in PR #132. Until both land, Requires=containerd.service is an unresolvable job: the control plane does not start, and bluefin-cluster-bootstrap.service does not run either, because it declares Requires=kubeadm-init.service. The requirement itself is correct and stays hard. The comment now says what is actually true, and points at the two PRs that close the gap. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- files/os/systemd/system/kubeadm-init.service | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service index 5c28f1f6..59623273 100644 --- a/files/os/systemd/system/kubeadm-init.service +++ b/files/os/systemd/system/kubeadm-init.service @@ -5,9 +5,22 @@ Documentation=https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-i # fails is marked *skipped*, and systemd treats a skipped unit as satisfied for # everything ordered after it, so bluefin-cluster-bootstrap.service still runs. ConditionPathExists=!/etc/kubernetes/admin.conf -# /usr/bin/kubeadm and containerd.service both arrive with the sysext merge. +# /usr/bin/kubeadm arrives with the sysext merge (elements/oci/kubernetes-sysext.bst +# stages kubectl, kubeadm, kubelet and the CNI plugins — and nothing else). Requires=systemd-sysext.service After=systemd-sysext.service +# containerd.service is NOT supplied by that sysext and is not built anywhere in +# this repository. It reaches the image only through Flatcar's /usr +# (usr/share/flatcar/sysext/containerd-flatcar.raw), which arrives with +# elements/flatcar/flatcar-usr.bst in PR #140 and is consumed by os-stack.bst in +# PR #132. Until both land this is an unresolvable job: the control plane does +# not start, and bluefin-cluster-bootstrap.service does not run either, because +# it declares Requires=kubeadm-init.service. +# +# The requirement is correct and stays hard on purpose — kubeadm against a host +# with no CRI socket fails late and confusingly, where an unresolvable job fails +# immediately and names the missing unit. tests/unit/test_unit_dependency_provenance.py +# records the gap so it cannot be forgotten. Requires=containerd.service After=containerd.service Wants=network-online.target From 6d18a7c49afd2be14ebdcddf5e005fe332f558dc Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 06:40:18 -0400 Subject: [PATCH 21/46] feat(containerd): ship Flatcar's containerd sysext so the control plane starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kubeadm-init.service declares Requires=containerd.service and nothing supplied it. elements/oci/kubernetes-sysext.bst stages kubectl, kubeadm, kubelet and the CNI plugins only, and FSDK 26.08 has no containerd component at all — see docs/superpowers/specs/2026-09-19-flatcar-runtime-parity.md, which records `no elements/components/containerd.bst` in the pinned junction. A machine installed from this branch therefore reached multi-user.target with no runtime, no control plane and no Argo: bluefin-cluster-bootstrap.service declares Requires=kubeadm-init.service, so the whole chain stayed inert. Flatcar publishes containerd as a standalone systemd-sysext, the same delivery mechanism already used here for Kubernetes. elements/flatcar/containerd-sysext.bst imports that image — containerd 2.1.5, runc 1.3.3, ctr, containerd-shim-runc-v2, containerd.service and its own multi-user.target.wants symlink, so the merge is what enables the service. The installer stages it into the initrd and 30-var.conf seeds it to /var/lib/extensions for an offline first boot, exactly as it already does for kubernetes.raw. The image keeps its upstream name. systemd-sysext resolves an extension's metadata by stem: .raw requires extension-release., and this image ships extension-release.containerd-flatcar. Seeding it as containerd.raw would make sysext look for extension-release.containerd, refuse the extension, and drop containerd.service — silently, at boot, after a green build. tests/unit/test_seeded_sysext_names.py asserts the chain end to end: every seeded image keeps its name through element, initrd and CopyFiles=, and any unit a host unit hard-requires from a sysext is actually seeded. Mutation-tested against four regressions: renaming in repart, staging a different name in the installer, shortening the element's install target, and dropping the seeding line entirely. Two tests pinned the broken state and were not re-pinned to new text: test_containerd_provider_is_recorded_as_pending asserted containerd must come from flatcar-usr.bst, which was a record of the gap rather than a contract, and is deleted now the gap is closed. test_var_seeds_the_sysext_into_the_systemd_sysext_scan_directory compared CopyFiles= for equality against a single value; CopyFiles= is repeatable and there are now two seeds, so it reads every occurrence. For the same reason load_config() drops configparser's strict=True, which raises DuplicateOptionError on valid systemd input. Verified: pytest 328 passed / 1 xfailed, bats 92 passed, docs-checks, just validate, and `just bst build flatcar/containerd-sysext.bst` all clean. The pinned ref is byte-verified against the published image (sha256 78e38344ac490004b9fa4c7da38bbfcffc393146aa990379aff7b5c09589f809). Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/flatcar/containerd-sysext.bst | 70 ++++++++ elements/oci/bluefin-server-installer.bst | 20 ++- files/installer/repart.d/30-var.conf | 32 +++- files/os/systemd/system/kubeadm-init.service | 30 ++-- tests/unit/test_repart_layout.py | 45 ++++- tests/unit/test_seeded_sysext_names.py | 162 ++++++++++++++++++ tests/unit/test_unit_dependency_provenance.py | 35 ++-- 7 files changed, 345 insertions(+), 49 deletions(-) create mode 100644 elements/flatcar/containerd-sysext.bst create mode 100644 tests/unit/test_seeded_sysext_names.py diff --git a/elements/flatcar/containerd-sysext.bst b/elements/flatcar/containerd-sysext.bst new file mode 100644 index 00000000..abeb72d1 --- /dev/null +++ b/elements/flatcar/containerd-sysext.bst @@ -0,0 +1,70 @@ +kind: manual +description: | + Flatcar's containerd system extension, shipped as a second sysext alongside + oci/kubernetes-sysext.bst. + + Output: /containerd-flatcar.raw — the upstream squashfs image, imported + verbatim under its upstream name. + + The name is load-bearing and must not be shortened. systemd-sysext resolves an + image's metadata by stem: an extension merged as .raw is required to + carry /usr/lib/extension-release.d/extension-release.. This image ships + extension-release.containerd-flatcar, so renaming it to containerd.raw makes + systemd-sysext look for extension-release.containerd, fail to find it, and + refuse the extension — silently, at boot, long after the build went green. + tests/unit/test_sysupdate_transfers.py encodes the same invariant for the + Kubernetes sysext, and tests/unit/test_seeded_sysext_names.py now covers the + installer-seeded path this element feeds. + + Why import rather than build: FSDK 26.08 ships no containerd component at all + (see docs/superpowers/specs/2026-09-19-flatcar-runtime-parity.md, which records + `no elements/components/containerd.bst` in the pinned junction), so there is + nothing to compose from. Flatcar publishes containerd as a systemd-sysext, the + same delivery mechanism this repository already uses for Kubernetes, and the + image is self-contained: + + /usr/bin/containerd, containerd-shim-runc-v2, ctr, runc + /usr/lib/systemd/system/containerd.service + /usr/lib/systemd/system/multi-user.target.wants/containerd.service + /usr/lib/systemd/system/multi-user.target.d/10-containerd-service.conf + /usr/lib/extension-release.d/extension-release.containerd-flatcar + /usr/share/containerd/config.toml, config-cgroupfs.toml + /usr/share/SLSA/*.json.zst (containerd 2.1.5, runc 1.3.3) + + It carries its own `multi-user.target.wants` symlink, so containerd.service is + enabled by the merge itself — no preset is needed on our side. That is what + satisfies `Requires=containerd.service` in + files/os/systemd/system/kubeadm-init.service. + + The merge is gated on os-release matching. extension-release.containerd-flatcar + declares ID=flatcar and VERSION_ID=4593.2.5; elements/bluefin-server/os-release-flatcar.bst + writes ID=flatcar and VERSION_ID=%{flatcar-version}, both sourced from + include/flatcar.yml. Bumping flatcar-version without re-pinning the ref below + makes systemd-sysext refuse the extension rather than merge a mismatched one. + + elements/flatcar/container-runtime-reference-sysexts.bst also imports this file. + That element is the parity oracle and additionally pulls podman and docker, + which this image does not ship; no shipped target depends on it. This element + exists so the shipped path imports exactly one runtime and nothing else. + +(@): + - include/flatcar.yml + +build-depends: + - base/base-stack.bst + +variables: + strip-binaries: "" + +sources: + - kind: remote + url: flatcar:stable/amd64-usr/%{flatcar-version}/rootfs-included-sysexts/containerd-flatcar.raw + filename: containerd-flatcar.raw + ref: 78e38344ac490004b9fa4c7da38bbfcffc393146aa990379aff7b5c09589f809 + +config: + install-commands: + - | + set -euo pipefail + install -D -m 0644 containerd-flatcar.raw \ + "%{install-root}/containerd-flatcar.raw" diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 3295458e..c495088f 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -62,6 +62,12 @@ build-depends: - filename: oci/kubernetes-sysext.bst config: location: /kubernetes + # Flatcar's containerd sysext, seeded the same way. kubeadm-init.service has + # a hard Requires=containerd.service and FSDK ships no containerd component, + # so without this the control plane never starts. + - filename: flatcar/containerd-sysext.bst + config: + location: /containerd variables: installer-version: "%{release-version}" @@ -330,11 +336,21 @@ config: ln -sf system-install.target \ /layer/usr/lib/systemd/system/default.target - # ── Step 2a: Stage the offline Kubernetes sysext before packing cpio ─ + # ── Step 2a: Stage the offline sysexts before packing cpio ─────────── + # Both are seeded onto /var by systemd-repart at install time + # (files/installer/repart.d/30-var.conf), so first boot merges them with + # no network fetch. + # + # Each filename must keep the stem its extension-release is named for: + # systemd-sysext merges .raw only if the image carries + # extension-release.. kubernetes-.raw -> kubernetes.raw is + # safe (extension-release.kubernetes); containerd-flatcar.raw must stay + # containerd-flatcar.raw (extension-release.containerd-flatcar). - | set -eux cp /kubernetes/kubernetes-*.raw /layer/kubernetes.raw - ls -lh /layer/kubernetes.raw + cp /containerd/containerd-flatcar.raw /layer/containerd-flatcar.raw + ls -lh /layer/kubernetes.raw /layer/containerd-flatcar.raw # ── Step 3: Pack initrd cpio ───────────────────────────────────────── # find . from /layer gives paths like ./usr/bin/... which is exactly diff --git a/files/installer/repart.d/30-var.conf b/files/installer/repart.d/30-var.conf index 2fbb3a27..a3c35205 100644 --- a/files/installer/repart.d/30-var.conf +++ b/files/installer/repart.d/30-var.conf @@ -4,14 +4,38 @@ # creating /var entirely. # GrowFileSystem=yes: XFS grows to fill all remaining disk space after # the ESP and root-a partitions are placed. -# CopyFiles seeds the Kubernetes sysext straight into systemd-sysext's own -# scan directory, so the first boot merges it without a network fetch. -# systemd-sysupdate later replaces this file through the kubernetes.raw -# CurrentSymlink in files/os/sysupdate.kubernetes.d/70-kubernetes.transfer. +# CopyFiles seeds both sysexts straight into systemd-sysext's own scan +# directory (/var/lib/extensions), so the first boot merges them without a +# network fetch. +# +# kubernetes.raw — kubeadm, kubelet, kubectl and the CNI plugins. +# systemd-sysupdate later replaces this file through the kubernetes.raw +# CurrentSymlink in files/os/sysupdate.kubernetes.d/70-kubernetes.transfer. +# +# containerd-flatcar.raw — Flatcar's containerd sysext (containerd, runc, ctr, +# containerd-shim-runc-v2) and the containerd.service that +# files/os/systemd/system/kubeadm-init.service hard-requires. FSDK 26.08 +# ships no containerd component, so this is the only provider. The image +# carries its own multi-user.target.wants symlink, so the merge enables the +# service; no preset of ours is involved. Pinned by +# elements/flatcar/containerd-sysext.bst. +# +# The filename keeps its upstream stem on purpose. systemd-sysext merges +# .raw only if the image carries extension-release.; this image +# ships extension-release.containerd-flatcar, so seeding it as +# containerd.raw would make sysext look for extension-release.containerd and +# refuse the extension at boot. +# +# The merge is additionally gated on os-release: that file declares ID=flatcar +# and VERSION_ID=4593.2.5, which must equal flatcar-version in +# include/flatcar.yml. Unlike our own kubernetes sysext (ID=_any), this one is +# version-locked to the Flatcar release, so bumping flatcar-version requires +# re-pinning the containerd image in the same change. [Partition] Type=var Label=var Format=xfs GrowFileSystem=yes CopyFiles=/kubernetes.raw:/lib/extensions/kubernetes.raw +CopyFiles=/containerd-flatcar.raw:/lib/extensions/containerd-flatcar.raw SizeMinBytes=4G diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service index 59623273..00fbd610 100644 --- a/files/os/systemd/system/kubeadm-init.service +++ b/files/os/systemd/system/kubeadm-init.service @@ -5,22 +5,24 @@ Documentation=https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-i # fails is marked *skipped*, and systemd treats a skipped unit as satisfied for # everything ordered after it, so bluefin-cluster-bootstrap.service still runs. ConditionPathExists=!/etc/kubernetes/admin.conf -# /usr/bin/kubeadm arrives with the sysext merge (elements/oci/kubernetes-sysext.bst -# stages kubectl, kubeadm, kubelet and the CNI plugins — and nothing else). +# Both /usr/bin/kubeadm and containerd.service arrive with the sysext merge, but +# from two different extensions, each seeded to /var/lib/extensions by +# files/installer/repart.d/30-var.conf: +# +# kubernetes.raw elements/oci/kubernetes-sysext.bst — kubeadm, kubelet, +# kubectl, CNI plugins. +# containerd.raw elements/flatcar/containerd-sysext.bst — Flatcar's containerd +# sysext: containerd, runc, ctr, containerd-shim-runc-v2, and +# containerd.service itself. FSDK 26.08 ships no containerd +# component, so this import is the only provider. +# +# containerd.raw carries its own multi-user.target.wants symlink, so the merge +# enables the service; no preset of ours is involved. Requires=systemd-sysext.service After=systemd-sysext.service -# containerd.service is NOT supplied by that sysext and is not built anywhere in -# this repository. It reaches the image only through Flatcar's /usr -# (usr/share/flatcar/sysext/containerd-flatcar.raw), which arrives with -# elements/flatcar/flatcar-usr.bst in PR #140 and is consumed by os-stack.bst in -# PR #132. Until both land this is an unresolvable job: the control plane does -# not start, and bluefin-cluster-bootstrap.service does not run either, because -# it declares Requires=kubeadm-init.service. -# -# The requirement is correct and stays hard on purpose — kubeadm against a host -# with no CRI socket fails late and confusingly, where an unresolvable job fails -# immediately and names the missing unit. tests/unit/test_unit_dependency_provenance.py -# records the gap so it cannot be forgotten. +# Hard on purpose: kubeadm against a host with no CRI socket fails late and +# confusingly, where an unresolvable job fails immediately and names the missing +# unit. tests/unit/test_unit_dependency_provenance.py records the provider. Requires=containerd.service After=containerd.service Wants=network-online.target diff --git a/tests/unit/test_repart_layout.py b/tests/unit/test_repart_layout.py index 62239973..cb65f49e 100644 --- a/tests/unit/test_repart_layout.py +++ b/tests/unit/test_repart_layout.py @@ -38,7 +38,21 @@ def parse_size(value: str) -> int: def load_config(path: Path) -> configparser.ConfigParser: - parser = configparser.ConfigParser(strict=True) + """Parse a systemd-style config file. + + ``strict=False`` is required, not laxity: systemd documents several + directives as repeatable, and ``CopyFiles=`` in ``30-var.conf`` is one of + them — the installer seeds both the Kubernetes and the containerd sysext + onto ``/var``. ``configparser`` defaults to ``strict=True``, which raises + ``DuplicateOptionError`` on valid systemd input. + + With ``strict=False`` the last occurrence wins for single-value lookups, + which is fine for the size and label assertions in this module. Anything + asserting over a repeatable directive must read every occurrence instead — + see ``tests/unit/test_seeded_sysext_names.py``, which scans the raw text for + that reason. + """ + parser = configparser.ConfigParser(strict=False) # systemd drop-ins are case-sensitive; configparser lowercases keys by default. parser.optionxform = str parser.read_string(path.read_text(encoding="utf-8")) @@ -218,16 +232,33 @@ def test_var_is_a_growing_xfs_tail(): ) -def test_var_seeds_the_sysext_into_the_systemd_sysext_scan_directory(): +def test_var_seeds_every_sysext_into_the_systemd_sysext_scan_directory(): """An offline install must boot into a working cluster with no fetch. The destination is relative to the /var partition root, so - ``/lib/extensions/kubernetes.raw`` is ``/var/lib/extensions/kubernetes.raw`` - on the running system — one of systemd-sysext's own search paths, which is - why no unit has to copy the image anywhere before the merge. + ``/lib/extensions/.raw`` is ``/var/lib/extensions/.raw`` on the + running system — one of systemd-sysext's own search paths, which is why no + unit has to copy the image anywhere before the merge. + + ``CopyFiles=`` is repeatable and there is more than one seed, so this reads + every occurrence out of the raw text rather than through configparser, which + keeps only the last. The naming contract between a seeded image and its + extension-release lives in tests/unit/test_seeded_sysext_names.py. """ - var = next(s for s in partitions().values() if s["Type"] == "var") - assert var["CopyFiles"] == "/kubernetes.raw:/lib/extensions/kubernetes.raw" + text = (REPART_DIR / "30-var.conf").read_text(encoding="utf-8") + seeds = dict( + re.findall(r"^CopyFiles=([^:]+):(.+)$", text, re.MULTILINE) + ) + + assert seeds.get("/kubernetes.raw") == "/lib/extensions/kubernetes.raw", ( + "the Kubernetes sysext must be seeded for an offline first boot" + ) + assert seeds.get("/containerd-flatcar.raw") == ( + "/lib/extensions/containerd-flatcar.raw" + ), ( + "the containerd sysext must be seeded: kubeadm-init.service hard-requires " + "containerd.service and FSDK ships no containerd component" + ) def test_root_partition_label_is_matched_by_the_sysupdate_root_transfer(): diff --git a/tests/unit/test_seeded_sysext_names.py b/tests/unit/test_seeded_sysext_names.py new file mode 100644 index 00000000..86dad0c8 --- /dev/null +++ b/tests/unit/test_seeded_sysext_names.py @@ -0,0 +1,162 @@ +"""An installer-seeded sysext must keep the stem its extension-release names. + +systemd-sysext resolves an extension's metadata by filename stem. An image +merged as ``.raw`` is required to carry +``/usr/lib/extension-release.d/extension-release.``; if it does not, the +extension is refused. The refusal is silent from the build's point of view — +the element builds, ``just validate`` resolves, the installer writes a bootable +disk, and the failure only appears as a missing service on a running machine. + +``tests/unit/test_sysupdate_transfers.py`` already enforces this for the +Kubernetes sysext on the *sysupdate* path, where the name is fixed by the +transfer's ``CurrentSymlink``. This module covers the other path: the images +``files/installer/repart.d/30-var.conf`` seeds onto ``/var/lib/extensions`` at +install time, whose names are fixed by ``CopyFiles=`` and by the element that +stages them. + +The concrete hazard is the Flatcar containerd import. Upstream names it +``containerd-flatcar.raw`` and ships ``extension-release.containerd-flatcar``. +Shortening it to ``containerd.raw`` anywhere in the chain — the element's +install target, the installer's ``cp`` into the initrd, or the ``CopyFiles=`` +here — makes systemd-sysext look for ``extension-release.containerd``, find +nothing, and drop containerd.service. ``kubeadm-init.service`` then has an +unresolvable hard requirement and the control plane never starts. + +The chain is asserted end to end so a rename in any one link fails here. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +VAR_REPART = ROOT / "files" / "installer" / "repart.d" / "30-var.conf" +INSTALLER = ROOT / "elements" / "oci" / "bluefin-server-installer.bst" +CONTAINERD_ELEMENT = ROOT / "elements" / "flatcar" / "containerd-sysext.bst" +KUBEADM_INIT = ROOT / "files" / "os" / "systemd" / "system" / "kubeadm-init.service" + +# Units that only a seeded sysext can provide, mapped to the image that ships +# them. Derived from the image contents, not from configuration: Flatcar's +# containerd-flatcar.raw carries usr/lib/systemd/system/containerd.service. +SYSEXT_PROVIDED_UNITS = { + "containerd.service": "containerd-flatcar.raw", +} + +COPY_FILES_RE = re.compile(r"^CopyFiles=([^:]+):(.+)$", re.MULTILINE) + + +def seeded_extensions() -> dict[str, str]: + """``{staged source path: destination path}`` for every seeded sysext.""" + return { + src: dst + for src, dst in COPY_FILES_RE.findall(VAR_REPART.read_text(encoding="utf-8")) + if "/lib/extensions/" in dst + } + + +def test_every_sysext_provided_unit_a_host_unit_requires_is_actually_seeded() -> None: + """A hard Requires= on a sysext-provided unit obliges the installer to seed it. + + Deleting the ``CopyFiles=`` line for containerd would restore exactly the + failure this whole arrangement exists to fix: kubeadm-init.service holds + ``Requires=containerd.service``, nothing provides it, the job is + unresolvable, and bluefin-cluster-bootstrap.service never runs because it + declares ``Requires=kubeadm-init.service``. The machine boots to + multi-user.target with no control plane. + + The requirement is read out of the unit rather than restated here, so + dropping the ``Requires=`` deliberately also relaxes this test. + """ + unit = without_comments(KUBEADM_INIT.read_text(encoding="utf-8")) + seeded_names = {Path(dst).name for dst in seeded_extensions().values()} + + for required_unit, image in SYSEXT_PROVIDED_UNITS.items(): + if f"Requires={required_unit}" not in unit: + continue + assert image in seeded_names, ( + f"{KUBEADM_INIT.relative_to(ROOT)} declares " + f"Requires={required_unit}, which only {image} provides, but " + f"{VAR_REPART.relative_to(ROOT)} does not seed it into " + f"/lib/extensions. Seeded images: {sorted(seeded_names) or 'none'}. " + "The control plane will not start." + ) + + +def test_seeded_images_land_in_the_sysext_scan_directory() -> None: + seeded = seeded_extensions() + + assert seeded, ( + f"{VAR_REPART.relative_to(ROOT)} seeds no sysext into /lib/extensions; " + "the installer's offline-merge path has regressed" + ) + + for src, dst in seeded.items(): + assert dst.startswith("/lib/extensions/"), ( + f"{dst} is not under /lib/extensions, so systemd-sysext will not " + f"scan it (/var is mounted at /var, making this /var/lib/extensions)" + ) + assert dst.endswith(".raw"), f"{dst} is not a .raw sysext image" + assert Path(src).name == Path(dst).name, ( + f"{VAR_REPART.relative_to(ROOT)} renames {src} to {dst} in flight. " + "The seeded name is what systemd-sysext resolves " + "extension-release. against; renaming here breaks the merge." + ) + + +def test_installer_stages_every_seeded_image_under_its_seeded_name() -> None: + installer = INSTALLER.read_text(encoding="utf-8") + + for src in seeded_extensions(): + # repart reads CopyFiles= sources from the installer's own root, which + # is the staged /layer tree. + staged = f"/layer{src}" + assert staged in installer, ( + f"{VAR_REPART.relative_to(ROOT)} seeds {src}, but " + f"{INSTALLER.relative_to(ROOT)} never stages {staged} into the " + "initrd, so repart has nothing to copy at install time" + ) + + +def without_comments(text: str) -> str: + """Effective directives only, for ``#``-commented formats. + + The prose in these files deliberately names the wrong spelling in order to + warn against it, so scanning raw text would flag the warning as the defect. + """ + return "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + + +def test_containerd_keeps_its_upstream_extension_release_stem() -> None: + """The specific rename that would silently drop containerd.service. + + Flatcar's image carries extension-release.containerd-flatcar, so every link + in the chain must say ``containerd-flatcar.raw``, never ``containerd.raw``. + """ + element = CONTAINERD_ELEMENT.read_text(encoding="utf-8") + + assert '"%{install-root}/containerd-flatcar.raw"' in element, ( + f"{CONTAINERD_ELEMENT.relative_to(ROOT)} must install the image as " + "containerd-flatcar.raw to match extension-release.containerd-flatcar" + ) + + # Only the two `#`-commented files are scanned for the forbidden spelling. + # The element's `description: |` is a YAML block scalar, not a comment, and + # it names containerd.raw on purpose to explain the hazard. The element is + # pinned by the positive assertion above instead, which is the stronger + # check: it fails if the install target is anything but containerd-flatcar.raw. + for path in (INSTALLER, VAR_REPART): + # `containerd.raw` as a whole token — `containerd-flatcar.raw` must not + # trip this. + match = re.search( + r"(? None: ) -def test_containerd_provider_is_recorded_as_pending() -> None: - """The cutover is inert until Flatcar's /usr lands; keep that visible. - - kubeadm-init.service requires containerd.service. Nothing on main supplies - it. This asserts the gap stays documented rather than quietly forgotten. - """ - assert "containerd.service" in EXTERNAL_PROVIDERS - assert "flatcar-usr.bst" in EXTERNAL_PROVIDERS["containerd.service"], ( - "containerd.service must name flatcar-usr.bst as its provider" - ) - - def test_kubelet_bridges_the_cni_plugin_directory() -> None: """Plugins ship at the bakery path; containerd and Cilium read /opt/cni/bin. From ce9f7f26cb5fed424db5898d3b1a8acd6d4f93e7 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 06:46:02 -0400 Subject: [PATCH 22/46] fix(containerd): decouple the sysext from the host os-release version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Imported verbatim, Flatcar's containerd sysext declares ID=flatcar and VERSION_ID=4593.2.5, and systemd-sysext refuses an extension whose ID and VERSION_ID disagree with the host's os-release. That is safe at build time and dangerous at OTA time. A build-time mismatch is loud: the source URL embeds %{flatcar-version}, so bumping it changes the URL and the pinned ref fails to resolve. The hazard is an already installed machine. /var persists across an A/B root update, so a host updated to a DDI built from a newer flatcar-version boots with a new os-release while /var/lib/extensions/containerd-flatcar.raw still declares the old VERSION_ID. systemd-sysext refuses the merge, containerd.service disappears, and kubeadm-init.service fails — the cluster goes down after a routine OS update. kubernetes.raw is immune because 70-kubernetes.transfer replaces it in lockstep; containerd has no such transfer, and adding one requires publishing a release asset from build.yml. So the image is repacked with a single field changed: ID=_any, the same decoupling files/kubernetes/sysext/extension-release.kubernetes already uses for the same stated reason. Verified by checking the built artifact out: the four binaries, containerd.service, the multi-user.target.wants symlink, both config.toml files and the SLSA provenance documents all survive; only the extension-release differs. test_seeded_sysexts_are_decoupled_from_the_host_os_release_version asserts the repack stays, since reverting to a verbatim import is a tempting simplification that silently reintroduces the failure. Mutation-tested. Also in this commit: - Swept three stale `containerd.raw` short names in kubeadm-init.service and test_unit_dependency_provenance.py. They contradicted the rule the element itself states, so a reader following them would reintroduce the naming bug. - kc-agent's default allowed-origins moved to https. The cutover gives the kiosk a cert-manager certificate and nginx terminates TLS, and Justfile and scripts/lima-e2e-kubestellar-test.sh moved to https with it — but Formula/kc-agent.rb and files/bin/bluefin-kubestellar still defaulted to http, so `bluefin-kubestellar start` and `brew services start kc-agent` produced an origin mismatch and the console gate never cleared. All 11 references now agree. - flash-installer refuses a device with mounted partitions and prints the udisksctl commands to clear them. dd under a live filesystem gives a torn image and leaves stale page cache. Verified against the real USB, which has /dev/sdb2 mounted at /run/media/jorge/bluefin-root right now. It also refuses to guess when dist/ holds more than one installer image instead of letting `head -n1` pick by directory order. Verified: pytest 329 passed / 1 xfailed, bats 92 passed, docs-checks, just validate, and `just bst build flatcar/containerd-sysext.bst` all clean. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Formula/kc-agent.rb | 2 +- Justfile | 29 ++++++++- elements/flatcar/containerd-sysext.bst | 59 ++++++++++++++++--- files/bin/bluefin-kubestellar | 6 +- files/os/systemd/system/kubeadm-init.service | 4 +- tests/unit/test_seeded_sysext_names.py | 33 +++++++++++ tests/unit/test_unit_dependency_provenance.py | 2 +- 7 files changed, 117 insertions(+), 18 deletions(-) diff --git a/Formula/kc-agent.rb b/Formula/kc-agent.rb index 639913d9..66e4788d 100644 --- a/Formula/kc-agent.rb +++ b/Formula/kc-agent.rb @@ -47,7 +47,7 @@ def install end service do - run [opt_bin/"kc-agent", "-allowed-origins", "http://localhost:8080,http://127.0.0.1:8080"] + run [opt_bin/"kc-agent", "-allowed-origins", "https://localhost:8080,https://127.0.0.1:8080"] keep_alive true environment_variables KAGENTI_CONTROLLER_URL: "none" log_path var/"log/kc-agent.log" diff --git a/Justfile b/Justfile index 4cf88a7a..e1c75eb9 100644 --- a/Justfile +++ b/Justfile @@ -197,12 +197,37 @@ flash-installer DEVICE="": echo "ERROR: {{DEVICE}} is not a valid block device!" >&2 exit 1 fi - IMG=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | head -n1) - if [ -z "${IMG}" ]; then + # Refuse a device with anything mounted off it. dd writing under a live + # filesystem gives a torn image and leaves the kernel holding stale page + # cache for blocks that no longer exist. + MOUNTED=$(lsblk -n -o MOUNTPOINTS "{{DEVICE}}" | grep -v '^\s*$' || true) + if [ -n "${MOUNTED}" ]; then + echo "ERROR: {{DEVICE}} has mounted partitions:" >&2 + lsblk -p -o NAME,SIZE,MOUNTPOINTS "{{DEVICE}}" >&2 + echo >&2 + echo "Unmount them first, e.g.:" >&2 + # -l (list) not the default tree: tree mode prefixes names with box + # glyphs, which would make the suggested command uncopyable. + lsblk -p -n -l -o NAME,MOUNTPOINTS "{{DEVICE}}" \ + | awk 'NF>1 {printf " udisksctl unmount -b %s\n", $1}' >&2 + exit 1 + fi + # -maxdepth 1 keeps release images in dist/release/ out of the match, but a + # stale export beside a fresh one is still ambiguous — fail rather than let + # `head -n1` pick by directory order. + mapfile -t IMGS < <(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | sort) + if [ "${#IMGS[@]}" -eq 0 ]; then echo "ERROR: No exported installer found in dist/." >&2 echo "Please run: just build-installer && just export-installer" >&2 exit 1 fi + if [ "${#IMGS[@]}" -gt 1 ]; then + echo "ERROR: ${#IMGS[@]} installer images in dist/; refusing to guess:" >&2 + printf ' %s\n' "${IMGS[@]}" >&2 + echo "Remove the stale one, then re-run." >&2 + exit 1 + fi + IMG="${IMGS[0]}" echo "WARNING: All data on {{DEVICE}} will be COMPLETELY DESTROYED!" echo "Double-checking device information:" lsblk -p "{{DEVICE}}" diff --git a/elements/flatcar/containerd-sysext.bst b/elements/flatcar/containerd-sysext.bst index abeb72d1..9f24c81e 100644 --- a/elements/flatcar/containerd-sysext.bst +++ b/elements/flatcar/containerd-sysext.bst @@ -3,8 +3,8 @@ description: | Flatcar's containerd system extension, shipped as a second sysext alongside oci/kubernetes-sysext.bst. - Output: /containerd-flatcar.raw — the upstream squashfs image, imported - verbatim under its upstream name. + Output: /containerd-flatcar.raw — Flatcar's upstream image, repacked with one + edited field in its extension-release (see "Version decoupling" below). The name is load-bearing and must not be shortened. systemd-sysext resolves an image's metadata by stem: an extension merged as .raw is required to @@ -36,11 +36,30 @@ description: | satisfies `Requires=containerd.service` in files/os/systemd/system/kubeadm-init.service. - The merge is gated on os-release matching. extension-release.containerd-flatcar - declares ID=flatcar and VERSION_ID=4593.2.5; elements/bluefin-server/os-release-flatcar.bst - writes ID=flatcar and VERSION_ID=%{flatcar-version}, both sourced from - include/flatcar.yml. Bumping flatcar-version without re-pinning the ref below - makes systemd-sysext refuse the extension rather than merge a mismatched one. + Version decoupling. Upstream's extension-release declares ID=flatcar and + VERSION_ID=4593.2.5, and systemd-sysext refuses an extension whose ID/VERSION_ID + do not match the host's os-release. elements/bluefin-server/os-release-flatcar.bst + writes VERSION_ID=%{flatcar-version}, so they agree at install time — and stop + agreeing the moment flatcar-version is bumped. + + That divergence is an OTA hazard, not a build-time one. A build-time mismatch + is loud: the source URL embeds %{flatcar-version}, so bumping it changes the + URL and the pinned ref below fails to resolve. The danger is an already + installed machine. /var survives an A/B root update, so a host updated to a DDI + built from a newer flatcar-version boots with a new os-release while + /var/lib/extensions/containerd-flatcar.raw still declares the old VERSION_ID. + systemd-sysext then refuses the merge, containerd.service disappears, and + kubeadm-init.service fails — the cluster goes down after a routine OS update. + kubernetes.raw is immune because 70-kubernetes.transfer replaces it in lockstep; + containerd has no such transfer. + + So the image is repacked with a single field changed: ID=_any. That is the same + decoupling files/kubernetes/sysext/extension-release.kubernetes uses, for the + same stated reason — "the sysext ships independently of the host os-release + version and would otherwise be rejected after an OS update" + (tests/unit/test_sysupdate_transfers.py). Nothing else in the image is touched: + the binaries, containerd.service, the target.wants symlink, the config.toml + pair and the SLSA provenance documents are repacked byte-for-byte. elements/flatcar/container-runtime-reference-sysexts.bst also imports this file. That element is the parity oracle and additionally pulls podman and docker, @@ -48,10 +67,12 @@ description: | exists so the shipped path imports exactly one runtime and nothing else. (@): + - include/arch.yml - include/flatcar.yml build-depends: - base/base-stack.bst + - freedesktop-sdk.bst:components/squashfs-tools.bst variables: strip-binaries: "" @@ -66,5 +87,25 @@ config: install-commands: - | set -euo pipefail - install -D -m 0644 containerd-flatcar.raw \ - "%{install-root}/containerd-flatcar.raw" + + RELEASE="usr/lib/extension-release.d/extension-release.containerd-flatcar" + + unsquashfs -no-xattrs -d unpacked containerd-flatcar.raw + + # Fail loudly if upstream ever renames the metadata file: a silently + # missing extension-release produces an image systemd-sysext refuses. + test -f "unpacked/${RELEASE}" + + # NAME must equal the image stem, ID=_any decouples the merge from the + # host's os-release version. SYSEXT_VERSION_ID keeps the Flatcar release + # readable off a merged /usr for provenance. + cat > "unpacked/${RELEASE}" < Allowed origin URL(s) (comma-separated) - Default: http://localhost:8080,http://127.0.0.1:8080 + Default: https://localhost:8080,https://127.0.0.1:8080 -p, --port kc-agent local health/listen port (default: 8585) -f, --foreground Run kc-agent in foreground instead of backgrounding --no-install Do not attempt Homebrew installation if kc-agent is missing @@ -139,7 +139,7 @@ case "$CMD" in exit 0 ;; start) - ORIGIN="${ORIGIN:-http://localhost:8080,http://127.0.0.1:8080}" + ORIGIN="${ORIGIN:-https://localhost:8080,https://127.0.0.1:8080}" mkdir -p "$STATE_DIR" # Optimize client environment: disable in-cluster Kagenti DNS queries diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service index 00fbd610..05a135f4 100644 --- a/files/os/systemd/system/kubeadm-init.service +++ b/files/os/systemd/system/kubeadm-init.service @@ -11,12 +11,12 @@ ConditionPathExists=!/etc/kubernetes/admin.conf # # kubernetes.raw elements/oci/kubernetes-sysext.bst — kubeadm, kubelet, # kubectl, CNI plugins. -# containerd.raw elements/flatcar/containerd-sysext.bst — Flatcar's containerd +# containerd-flatcar.raw elements/flatcar/containerd-sysext.bst — Flatcar's containerd # sysext: containerd, runc, ctr, containerd-shim-runc-v2, and # containerd.service itself. FSDK 26.08 ships no containerd # component, so this import is the only provider. # -# containerd.raw carries its own multi-user.target.wants symlink, so the merge +# containerd-flatcar.raw carries its own multi-user.target.wants symlink, so the merge # enables the service; no preset of ours is involved. Requires=systemd-sysext.service After=systemd-sysext.service diff --git a/tests/unit/test_seeded_sysext_names.py b/tests/unit/test_seeded_sysext_names.py index 86dad0c8..ac622b45 100644 --- a/tests/unit/test_seeded_sysext_names.py +++ b/tests/unit/test_seeded_sysext_names.py @@ -55,6 +55,39 @@ def seeded_extensions() -> dict[str, str]: } +def test_seeded_sysexts_are_decoupled_from_the_host_os_release_version() -> None: + """A seeded sysext must survive an OS update, so its ID must be ``_any``. + + ``/var`` persists across an A/B root update, so an image sitting in + ``/var/lib/extensions`` outlives the os-release it was installed against. + systemd-sysext refuses an extension whose ``ID``/``VERSION_ID`` disagree with + the host, so an extension pinned to a specific version stops merging the + first time the OS is updated — taking containerd.service, and therefore + kubeadm-init.service and the whole cluster, down on an otherwise routine + reboot. + + ``files/kubernetes/sysext/extension-release.kubernetes`` already uses + ``ID=_any`` for this reason. Flatcar's containerd image ships + ``ID=flatcar VERSION_ID=4593.2.5`` instead, which is correct for Flatcar and + wrong here, so ``elements/flatcar/containerd-sysext.bst`` repacks it. This + asserts the repack is still happening: dropping it to "import verbatim" is a + tempting simplification that reintroduces the failure. + """ + element = CONTAINERD_ELEMENT.read_text(encoding="utf-8") + + assert "unsquashfs" in element and "mksquashfs" in element, ( + f"{CONTAINERD_ELEMENT.relative_to(ROOT)} no longer repacks the upstream " + "image. Imported verbatim it carries ID=flatcar with a fixed VERSION_ID, " + "so the merge breaks on the first OS update that bumps flatcar-version." + ) + assert "ID=_any" in element, ( + f"{CONTAINERD_ELEMENT.relative_to(ROOT)} must rewrite the " + "extension-release with ID=_any so the sysext is independent of the " + "host os-release version, matching " + "files/kubernetes/sysext/extension-release.kubernetes" + ) + + def test_every_sysext_provided_unit_a_host_unit_requires_is_actually_seeded() -> None: """A hard Requires= on a sysext-provided unit obliges the installer to seed it. diff --git a/tests/unit/test_unit_dependency_provenance.py b/tests/unit/test_unit_dependency_provenance.py index 1b43107e..922f7fab 100644 --- a/tests/unit/test_unit_dependency_provenance.py +++ b/tests/unit/test_unit_dependency_provenance.py @@ -35,7 +35,7 @@ # mechanism this repository already uses for Kubernetes. # elements/flatcar/containerd-sysext.bst imports that image, the installer stages # it into the initrd, and files/installer/repart.d/30-var.conf seeds it to -# /var/lib/extensions/containerd.raw so the first boot merges it offline. The +# /var/lib/extensions/containerd-flatcar.raw so the first boot merges it offline. The # image carries its own multi-user.target.wants symlink, so the merge is what # enables the service. EXTERNAL_PROVIDERS = { From 629f0e858b212f8808a64ff701164b32fb87c0ae Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 06:56:36 -0400 Subject: [PATCH 23/46] fix(flash,sshd): guard the destructive recipe and stop the preset lying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit flash-installer refused a mistyped target only via a y/N prompt, which is no defence against a typo the operator has already committed to. It now refuses the disk backing the running system outright. Resolving that disk needs two probes, not one. `/` is not always a block device: on composefs/ostree hosts — including Bluefin itself, which is what a developer runs this from — `findmnt -no SOURCE /` returns a composefs digest and the real device is mounted at /sysroot. Checking only `/` makes the guard silently inert on exactly the systems this project targets. Verified on this workstation: `/dev/nvme0n1` is now refused with "/sysroot is on /dev/nvme0n1p2", where the single-probe version passed it through. files/os/systemd/system-preset/zz-enable-sshd.preset is renamed to zz-disable-sshd.preset. Its contents are `disable sshd.service` and `disable sshd.socket`; the name asserted the opposite. Nothing references the filename — the whole directory is staged — so this is a rename only. It also reframes issue #168: sshd is off by default, not merely keyless, and the QEMU smoke reaches it only because Justfile injects systemd.wants=sshd.service on the kernel command line. elements/flatcar/containerd-sysext.bst now builds its source URL from %{flatcar-board} instead of a hardcoded amd64-usr, matching flatcar-kernel.bst and flatcar-zfs.bst. The element already emits ARCHITECTURE=%{systemd-arch}, so an arm64 build would otherwise have fetched the amd64 image and labelled it arm64. Verified: pytest 329 passed / 1 xfailed, bats 92 passed, docs-checks, just validate, and `just bst build flatcar/containerd-sysext.bst` all clean. Both flash-installer guards exercised against real devices. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 21 +++++++++++++++++++ elements/flatcar/containerd-sysext.bst | 2 +- ...ble-sshd.preset => zz-disable-sshd.preset} | 0 3 files changed, 22 insertions(+), 1 deletion(-) rename files/os/systemd/system-preset/{zz-enable-sshd.preset => zz-disable-sshd.preset} (100%) diff --git a/Justfile b/Justfile index e1c75eb9..7656af65 100644 --- a/Justfile +++ b/Justfile @@ -197,6 +197,27 @@ flash-installer DEVICE="": echo "ERROR: {{DEVICE}} is not a valid block device!" >&2 exit 1 fi + # Refuse the disk backing the running system outright. A single mistyped + # character here is unrecoverable, and the y/N prompt below is not a + # meaningful defence against a typo the operator has already committed to. + # + # `/` is not always on a block device. On composefs/ostree hosts — including + # Bluefin itself, which is what a developer runs this from — `findmnt / ` + # reports a composefs digest, and the real device is mounted at /sysroot. + # Check both, so the guard is not silently inert on exactly the systems this + # project targets. + TARGET_NAME=$(lsblk -no KNAME "{{DEVICE}}" 2>/dev/null | head -n1 || true) + for mp in / /sysroot; do + SRC=$(findmnt -no SOURCE "${mp}" 2>/dev/null | head -n1 || true) + case "${SRC}" in /dev/*) ;; *) continue ;; esac + SRC_DISK=$(lsblk -no PKNAME "${SRC}" 2>/dev/null | head -n1 || true) + if [ -n "${SRC_DISK}" ] && [ "${SRC_DISK}" = "${TARGET_NAME}" ]; then + echo "ERROR: {{DEVICE}} is the disk backing the running system." >&2 + echo " ${mp} is on ${SRC}, which lives on /dev/${SRC_DISK}." >&2 + echo "Refusing to overwrite it." >&2 + exit 1 + fi + done # Refuse a device with anything mounted off it. dd writing under a live # filesystem gives a torn image and leaves the kernel holding stale page # cache for blocks that no longer exist. diff --git a/elements/flatcar/containerd-sysext.bst b/elements/flatcar/containerd-sysext.bst index 9f24c81e..8177b95f 100644 --- a/elements/flatcar/containerd-sysext.bst +++ b/elements/flatcar/containerd-sysext.bst @@ -79,7 +79,7 @@ variables: sources: - kind: remote - url: flatcar:stable/amd64-usr/%{flatcar-version}/rootfs-included-sysexts/containerd-flatcar.raw + url: flatcar:stable/%{flatcar-board}/%{flatcar-version}/rootfs-included-sysexts/containerd-flatcar.raw filename: containerd-flatcar.raw ref: 78e38344ac490004b9fa4c7da38bbfcffc393146aa990379aff7b5c09589f809 diff --git a/files/os/systemd/system-preset/zz-enable-sshd.preset b/files/os/systemd/system-preset/zz-disable-sshd.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-sshd.preset rename to files/os/systemd/system-preset/zz-disable-sshd.preset From ef0aa67ac5593e1abba385fbd08377cc5baaea19 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 06:59:12 -0400 Subject: [PATCH 24/46] test(flash): cover the three guards added to the destructive recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/unit/flash-installer_test.bats already sandboxes flash-installer, but its header enumerated "four guards" and the three added this session — refuse the disk backing the running system, refuse a device with mounted partitions, refuse to guess between two exported images — had no coverage at all. That is the one recipe in the repository that destroys data. `lsblk` and `findmnt` become scriptable stubs answering from STUB_* variables, because the new guards query real block topology and a fixed stub would either always fire or never fire depending on the host running the suite. Both still log, so assert_nothing_written stays meaningful on every refusal path. Eight cases added, 24 total. Mutation-tested against four regressions, each caught: remove the running-system guard -> cases 5, 6 fail remove the mounted-partition guard -> cases 8, 9 fail revert image selection to `head -n1` -> cases 10, 11 fail probe only / and not /sysroot -> cases 5, 6 fail The last is the composefs regression specifically: on ostree/composefs hosts `findmnt -no SOURCE /` returns a digest rather than a device, so a guard that consults only / passes the root disk straight through. That case pins the /sysroot fallback. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/unit/flash-installer_test.bats | 125 ++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 4 deletions(-) diff --git a/tests/unit/flash-installer_test.bats b/tests/unit/flash-installer_test.bats index 04857563..d2f213c3 100644 --- a/tests/unit/flash-installer_test.bats +++ b/tests/unit/flash-installer_test.bats @@ -4,9 +4,11 @@ # # `flash-installer` is the only recipe in the repository that destroys data: it # pipes an exported installer image straight onto a block device with `dd`. -# Its four guards (device argument required, device must be a block device, an -# exported image must exist, and the operator must confirm) are the only thing -# standing between a typo and a wiped disk, and none of them were exercised. +# Its seven guards — device argument required, target must not be the disk +# backing the running system, target must have nothing mounted off it, target +# must be a block device, exactly one exported image must exist, and the +# operator must confirm — are the only thing standing between a typo and a +# wiped disk. # # The recipe is never run against a real device. Each test runs `just` inside a # private sandbox directory holding a copy of the Justfile, and `sudo`, `dd`, @@ -19,6 +21,11 @@ # device is a regular file instead. `test_block_device_guard_shape` asserts # that the substitution matched exactly once, so if the guard is ever reworded # or dropped the suite fails loudly rather than silently testing nothing. +# +# `lsblk` and `findmnt` are scriptable stubs: the guards added for the +# running-system and mounted-partition checks read them, so each test drives +# them through STUB_* environment variables rather than depending on whatever +# the host's real block topology happens to be. setup() { if ! command -v just >/dev/null 2>&1; then @@ -47,7 +54,15 @@ setup() { make_stub sudo 0 make_stub dd 0 make_stub zstd 0 - make_stub lsblk 0 + make_topology_stubs + + # Default topology: target is an unrelated, unmounted disk. Individual + # tests override these to drive a specific guard. + export STUB_ROOT_SRC="composefs:0000" + export STUB_SYSROOT_SRC="/dev/nvme0n1p2" + export STUB_PKNAME="nvme0n1" + export STUB_TARGET_KNAME="sdz" + export STUB_MOUNTPOINTS="" } # make_stub @@ -63,6 +78,37 @@ EOF chmod +x "${STUB_DIR}/$1" } +# Scriptable `lsblk` and `findmnt`. +# +# The running-system and mounted-partition guards query real block topology, so +# a fixed stub would either always fire or never fire depending on the host. +# These answer from STUB_* variables, letting each test state the topology it +# is testing. Both still log, so assert_nothing_written stays meaningful. +make_topology_stubs() { + cat > "${STUB_DIR}/lsblk" <> "${LOG}" +case " \$* " in + *" KNAME "*) printf '%s\n' "\${STUB_TARGET_KNAME:-}" ;; + *" PKNAME "*) printf '%s\n' "\${STUB_PKNAME:-}" ;; + *MOUNTPOINTS*) printf '%s\n' "\${STUB_MOUNTPOINTS:-}" ;; +esac +exit 0 +EOF + chmod +x "${STUB_DIR}/lsblk" + + cat > "${STUB_DIR}/findmnt" <> "${LOG}" +case " \$* " in + *" /sysroot "*) printf '%s\n' "\${STUB_SYSROOT_SRC:-}" ;; + *" / "*) printf '%s\n' "\${STUB_ROOT_SRC:-}" ;; +esac +exit 0 +EOF + chmod +x "${STUB_DIR}/findmnt" +} + # seed_image # # Places an exported installer artefact where the recipe looks for it. @@ -130,6 +176,77 @@ refute_log() { assert_log "lsblk " } +# --- guard: the target must not back the running system ------------------- + +@test "flash-installer refuses the disk the running system is installed on" { + export STUB_TARGET_KNAME="nvme0n1" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"disk backing the running system"* ]] + assert_nothing_written +} + +@test "flash-installer resolves the system disk through /sysroot when / is composefs" { + # The regression this guard was rewritten for. On composefs/ostree hosts — + # Bluefin itself — `findmnt -no SOURCE /` returns a composefs digest, not a + # device. A guard that consults only / silently passes the root disk + # through. The real device is mounted at /sysroot. + export STUB_ROOT_SRC="composefs:99f5c50825f9ab07" + export STUB_SYSROOT_SRC="/dev/nvme0n1p2" + export STUB_TARGET_KNAME="nvme0n1" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"/sysroot is on /dev/nvme0n1p2"* ]] + assert_nothing_written +} + +@test "flash-installer allows an unrelated disk while the system disk is known" { + export STUB_TARGET_KNAME="sdb" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -eq 0 ] + assert_log "of=${FAKE_DEV}" +} + +# --- guard: the target must have nothing mounted off it ------------------- + +@test "flash-installer refuses a device with a mounted partition" { + export STUB_MOUNTPOINTS="/run/media/jorge/bluefin-root" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"has mounted partitions"* ]] + assert_nothing_written +} + +@test "flash-installer tells the operator how to unmount rather than just refusing" { + export STUB_MOUNTPOINTS="/run/media/jorge/bluefin-root" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [[ "$output" == *"Unmount them first"* ]] +} + +# --- guard: exactly one exported image ------------------------------------ + +@test "flash-installer refuses to guess between two exported installers" { + seed_image "bluefin-server-installer-1.0.raw.zst" + seed_image "bluefin-server-installer-2.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"refusing to guess"* ]] + assert_nothing_written +} + +@test "flash-installer names both candidates so the operator can remove one" { + seed_image "bluefin-server-installer-1.0.raw.zst" + seed_image "bluefin-server-installer-2.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [[ "$output" == *"bluefin-server-installer-1.0.raw.zst"* ]] + [[ "$output" == *"bluefin-server-installer-2.0.raw.zst"* ]] +} + # --- guard 2: the target must be a block device --------------------------- @test "flash-installer rejects a path that is not a block device" { From e1f51518f1a0cca8e5a3c86c7bd499024edc2661 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 07:31:12 -0400 Subject: [PATCH 25/46] fix(flash): verify the image and fail the write when zstd dies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects on the path that writes a freshly downloaded artifact to a disk. The pipeline ran as `sudo sh -c "zstd -dc ... | dd of=..."`. Under POSIX sh a pipeline's status is the last command's, so zstd dying on a truncated or corrupt archive left dd exiting 0 after writing a partial image, and the "Successfully flashed" line printed over it. The outer `set -o pipefail` never applied — there is no pipeline in the outer shell, only in the nested one. It now runs `sudo bash -c "set -o pipefail; ..."`. There was also no integrity check at all. The recipe now runs `zstd -t` before the confirmation prompt, so a bad download costs nothing instead of prompting the operator to destroy a disk and failing afterwards. Ordering is asserted, not incidental. Test coverage, 29 cases: - The `/` half of `for mp in / /sysroot` had none. Both system-disk cases set STUB_ROOT_SRC to a composefs digest, so the branch that protects every non-ostree host — including the ubuntu-24.04 runner this suite runs on in CI — was never executed. Added a case pinning "/ is on /dev/sda1". - Integrity refusal, its ordering relative to the destruction warning, and that the image verified is the image written. - The write pipeline's shell and pipefail, as shape: `sudo` is a stub, so the nested shell cannot be executed in the sandbox. assert_nothing_written no longer refutes bare `zstd `. The integrity check legitimately reads the archive before the prompt; `zstd -dc` is the decompress that feeds dd and stays refuted on every refusal path. Without this the three abort cases failed against the new recipe — caught by running the suite rather than assuming. Mutation-tested, each regression caught with no collateral: drop `zstd -t` -> 23, 24, 25 revert to `sh -c` -> 29 drop mounted-partition guard -> 9, 10 probe only /sysroot -> 7 probe only / -> 5, 6 Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 16 +++++- tests/unit/flash-installer_test.bats | 83 +++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 7656af65..a44cc949 100644 --- a/Justfile +++ b/Justfile @@ -249,6 +249,15 @@ flash-installer DEVICE="": exit 1 fi IMG="${IMGS[0]}" + # Verify the archive before the prompt, not after. A truncated or corrupt + # download is exactly what a freshly fetched CI artifact invites, and + # failing here costs nothing while failing mid-write leaves an unbootable + # disk that looks like it succeeded. + echo "Verifying ${IMG}..." + if ! zstd -t "${IMG}"; then + echo "ERROR: ${IMG} failed its integrity check; refusing to write it." >&2 + exit 1 + fi echo "WARNING: All data on {{DEVICE}} will be COMPLETELY DESTROYED!" echo "Double-checking device information:" lsblk -p "{{DEVICE}}" @@ -259,7 +268,12 @@ flash-installer DEVICE="": exit 1 fi echo "Writing ${IMG} to {{DEVICE}}..." - sudo sh -c "zstd -dc ${IMG} | dd of={{DEVICE}} bs=4M iflag=fullblock oflag=direct status=progress conv=fsync" + # bash with pipefail, not sh. Under POSIX sh the pipeline's status is dd's + # alone, so zstd dying mid-stream leaves dd exiting 0 after writing a + # partial image and the success line below printing anyway. The outer + # `set -o pipefail` does not reach here — there is no pipeline in the outer + # shell, only this nested one. + sudo bash -c "set -o pipefail; zstd -dc '${IMG}' | dd of={{DEVICE}} bs=4M iflag=fullblock oflag=direct status=progress conv=fsync" echo "Successfully flashed the Bluefin Server installer to {{DEVICE}}!" # Build the installer artifacts, then run the reusable artifact smoke path. diff --git a/tests/unit/flash-installer_test.bats b/tests/unit/flash-installer_test.bats index d2f213c3..1cd9c0af 100644 --- a/tests/unit/flash-installer_test.bats +++ b/tests/unit/flash-installer_test.bats @@ -53,7 +53,7 @@ setup() { make_stub sudo 0 make_stub dd 0 - make_stub zstd 0 + make_zstd_stub make_topology_stubs # Default topology: target is an unrelated, unmounted disk. Individual @@ -63,6 +63,8 @@ setup() { export STUB_PKNAME="nvme0n1" export STUB_TARGET_KNAME="sdz" export STUB_MOUNTPOINTS="" + # zstd -t succeeds unless a test says otherwise. + export STUB_ZSTD_TEST_EXIT=0 } # make_stub @@ -109,6 +111,24 @@ EOF chmod +x "${STUB_DIR}/findmnt" } +# Scriptable `zstd`. +# +# The recipe now runs `zstd -t` as an integrity check before the confirmation +# prompt, and `zstd -dc` on the write path. Only the former needs to be able to +# fail, so it answers from STUB_ZSTD_TEST_EXIT while decompression stays a +# logging no-op. +make_zstd_stub() { + cat > "${STUB_DIR}/zstd" <> "${LOG}" +case " \$* " in + *" -t "*) exit "\${STUB_ZSTD_TEST_EXIT:-0}" ;; +esac +exit 0 +EOF + chmod +x "${STUB_DIR}/zstd" +} + # seed_image # # Places an exported installer artefact where the recipe looks for it. @@ -127,10 +147,14 @@ run_flash() { } # Nothing may reach the disk in any of the refusal paths. +# +# `zstd -t` is deliberately not refuted: the integrity check runs before the +# confirmation prompt and only reads the archive. `zstd -dc` is the decompress +# that feeds dd, and that must never happen on a refusal path. assert_nothing_written() { refute_log "sudo " refute_log "dd " - refute_log "zstd " + refute_log "zstd -dc" } assert_log() { @@ -202,6 +226,23 @@ refute_log() { assert_nothing_written } +@test "flash-installer resolves the system disk through / on a non-ostree host" { + # The other half of `for mp in / /sysroot`. Both cases above leave + # STUB_ROOT_SRC as a composefs digest, so without this the / iteration + # never runs against a /dev/* source — including on the ubuntu-24.04 + # runner that executes this suite in CI, where / is a plain block device + # and /sysroot does not exist. + export STUB_ROOT_SRC="/dev/sda1" + export STUB_SYSROOT_SRC="" + export STUB_PKNAME="sda" + export STUB_TARGET_KNAME="sda" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"/ is on /dev/sda1"* ]] + assert_nothing_written +} + @test "flash-installer allows an unrelated disk while the system disk is known" { export STUB_TARGET_KNAME="sdb" seed_image "bluefin-server-installer-1.0.raw.zst" @@ -329,6 +370,33 @@ refute_log() { assert_log "sudo " } +# --- guard: the image must pass its integrity check ----------------------- + +@test "flash-installer refuses an image that fails its integrity check" { + export STUB_ZSTD_TEST_EXIT=1 + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"failed its integrity check"* ]] + refute_log "sudo " + refute_log "dd " +} + +@test "flash-installer checks integrity before warning about destruction" { + # Ordering matters: a corrupt download should cost nothing, not prompt the + # operator to destroy a disk and only then fail. + export STUB_ZSTD_TEST_EXIT=1 + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [[ "$output" != *"COMPLETELY DESTROYED"* ]] +} + +@test "flash-installer verifies the image it is about to write, not some other one" { + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + assert_log "zstd -t dist/bluefin-server-installer-1.0.raw.zst" +} + # --- the write itself ----------------------------------------------------- @test "flash-installer decompresses the discovered image onto the given device" { @@ -353,3 +421,14 @@ refute_log() { run_flash "$FAKE_DEV" "y" [[ "$output" == *"Successfully flashed"* ]] } + +@test "the write pipeline runs under bash with pipefail so a dying zstd fails the flash" { + # Shape, not behaviour: `sudo` is a stub, so the nested shell never runs + # here. Under POSIX `sh -c` the pipeline's status is dd's alone, so zstd + # dying mid-stream leaves dd exiting 0 after writing a partial image and + # the success line printing over it. The outer `set -o pipefail` does not + # reach a nested shell. + grep -qF "sudo bash -c \"set -o pipefail;" "$JUSTFILE" + run grep -cF 'sudo sh -c "zstd -dc' "$JUSTFILE" + [ "$output" -eq 0 ] +} From 9587e3537f18f3d4036ca850301ac016afc96d88 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 08:36:34 -0400 Subject: [PATCH 26/46] fix(cluster): make the control plane actually come up on first boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The installer was fine all along. `installer-test` failed in CI on console readiness, and the retained serial.log showed a clean boot to a login prompt — no emergency mode, /var mounted, nothing in `systemctl list-units --failed`. Booting the CI artifact locally and reading the running system found five distinct defects between "OS booted" and "cluster up", none of which surface as a failed unit. 1. kubeadm.yaml opened with a comment header followed by `---`. kubeadm reads --config through k8s.io/apimachinery's YAMLReader, which splits on lines that are exactly `---` and decodes each chunk independently. The header therefore became a document with no nodes: error: invalid configuration for GroupVersionKind /, Kind=: kind and apiVersion is mandatory information that must be specified PyYAML does not reproduce this — it returns three clean documents — so the new gate emulates the Go splitter instead of using safe_load_all, which would have passed while kubeadm kept failing. 2. net.ipv4.ip_forward was 0. FSDK ships 50-default.conf, 50-coredump.conf and 50-pid-max.conf, none of which enable forwarding, so kubeadm preflight failed fatally on FileContent--proc-sys-net-ipv4-ip_forward. Adds files/kubernetes/sysctl.d/90-kubernetes-forwarding.conf. 3. kubeadm-init.service declared Requires=containerd.service. That unit arrives with the sysext merge, so it does not exist when systemd computes the boot transaction, and systemd dropped kubeadm-init's job outright — conditions never evaluated, ConditionTimestamp empty, nothing failed. kubelet.service survived the same boot because it only declares After=containerd.service; a pure ordering edge on a missing unit is ignored rather than fatal. Now Wants= plus After=. 4. With 3 fixed, kubeadm raced the CRI socket and lost: 12:18:13 [ERROR CRI]: dial unix /run/containerd/containerd.sock: connect: no such file or directory 12:18:43 Scheduled restart job, restart counter is at 1 12:18:59 Finished Initialise the Kubernetes control plane Restart=on-failure recovered, but the first failure propagated through Requires=kubeadm-init.service and killed bluefin-cluster-bootstrap's job permanently with result 'dependency'. It is never re-queued when kubeadm later succeeds. An ExecStartPre now waits for the socket so the first attempt succeeds and the dependents' Requires= edges stay honest. 5. containerd.service and its own multi-user.target.wants symlink both live inside the sysext, so neither is visible when the transaction is computed. The existing daemon-reload made them visible but could not enqueue a job for a target already in flight, leaving the start dependent on pure timing: boot A merge 12:06:11.75, multi-user 12:06:13.14 -> containerd active boot B merge 12:21:10.24, multi-user not reached -> containerd inactive On boot B kubeadm-init blocked indefinitely on a CRI socket that never appeared. The sysext drop-in now starts containerd explicitly after the reload; idempotent when the symlink wins the race. Verified end to end, not by unit tests. The CI artifact was digest-verified, installed in QEMU, and driven through four boots with the fixes applied. Final run reached the assertion CI failed on: ==> Polling KubeStellar Console readiness (deadline: 2400s)... ==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200 Intermediate proof along the way: `kubeadm init` succeeding, etcd, apiserver, controller-manager and scheduler all Running, node registered as control-plane v1.36.4. Two tests pinned the broken contracts and were corrected rather than deleted, since both describe real invariants once stated correctly: test_kubeadm_init_waits_for_the_sysext_merge_and_the_runtime asserted Requires=containerd.service, and test_sysext_merge_reloads_systemd... compared ExecStartPost for exact equality against a single reload. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../bluefin-server/os-cluster-bootstrap.bst | 6 + files/kubernetes/kubeadm.yaml | 8 +- .../sysctl.d/90-kubernetes-forwarding.conf | 21 ++++ files/os/systemd/system/kubeadm-init.service | 54 ++++++++- .../10-daemon-reload.conf | 32 +++++- tests/unit/test_cluster_bootstrap.py | 54 ++++++++- tests/unit/test_kubeadm_config_documents.py | 104 ++++++++++++++++++ tests/unit/test_seeded_sysext_names.py | 73 +++++++++--- 8 files changed, 322 insertions(+), 30 deletions(-) create mode 100644 files/kubernetes/sysctl.d/90-kubernetes-forwarding.conf create mode 100644 tests/unit/test_kubeadm_config_documents.py diff --git a/elements/bluefin-server/os-cluster-bootstrap.bst b/elements/bluefin-server/os-cluster-bootstrap.bst index 519123f3..391ad746 100644 --- a/elements/bluefin-server/os-cluster-bootstrap.bst +++ b/elements/bluefin-server/os-cluster-bootstrap.bst @@ -27,6 +27,9 @@ sources: - kind: local path: files/os/libexec directory: libexec-src + - kind: local + path: files/kubernetes/sysctl.d + directory: sysctl-src # This element stages systemd units, a YAML config and a shell helper. FSDK's # default strip-commands shell out to freedesktop-sdk-stripper, which @@ -48,6 +51,9 @@ config: mkdir -p "${OUT}/usr/lib/systemd/system/systemd-sysext.service.d" cp -a sysext-dropin-src/. "${OUT}/usr/lib/systemd/system/systemd-sysext.service.d/" + mkdir -p "${OUT}/usr/lib/sysctl.d" + cp -a sysctl-src/. "${OUT}/usr/lib/sysctl.d/" + mkdir -p "${OUT}/usr/share/bluefin" cp -a kubeadm-src/kubeadm.yaml "${OUT}/usr/share/bluefin/kubeadm.yaml" chmod 0644 "${OUT}/usr/share/bluefin/kubeadm.yaml" diff --git a/files/kubernetes/kubeadm.yaml b/files/kubernetes/kubeadm.yaml index fbba8b62..9101b5b1 100644 --- a/files/kubernetes/kubeadm.yaml +++ b/files/kubernetes/kubeadm.yaml @@ -12,7 +12,13 @@ # No certificate or key material ships in this image. Client certificates come # from kubeadm's TLS bootstrap and serving certificates from # `serverTLSBootstrap` plus the in-cluster kubelet-csr-approver. ---- +# +# No leading `---` below. kubeadm reads this with k8s.io/apimachinery's +# YAMLReader, which splits the stream on lines that are exactly `---` and +# decodes each chunk independently. A separator here would make this comment +# block a document of its own, and kubeadm rejects an empty document with +# "invalid configuration for GroupVersionKind /, Kind=: kind and apiVersion is +# mandatory information that must be specified". apiVersion: kubeadm.k8s.io/v1beta4 kind: InitConfiguration localAPIEndpoint: diff --git a/files/kubernetes/sysctl.d/90-kubernetes-forwarding.conf b/files/kubernetes/sysctl.d/90-kubernetes-forwarding.conf new file mode 100644 index 00000000..d12cd166 --- /dev/null +++ b/files/kubernetes/sysctl.d/90-kubernetes-forwarding.conf @@ -0,0 +1,21 @@ +# Kernel networking required by the Kubernetes control plane. +# +# kubeadm's preflight fails fatally without this: +# +# [ERROR FileContent--proc-sys-net-ipv4-ip_forward]: +# /proc/sys/net/ipv4/ip_forward contents are not set to 1 +# +# FSDK's /usr/lib/sysctl.d ships 50-default.conf, 50-coredump.conf and +# 50-pid-max.conf, none of which enable forwarding, so the appliance booted with +# ip_forward=0 and `kubeadm init` could never run. +# +# Forwarding is not optional for a node: every pod-to-pod and pod-to-service +# path leaves the pod netns through the host stack. Cilium replaces kube-proxy +# here, which removes the iptables datapath but not the routing requirement. +net.ipv4.ip_forward = 1 + +# The v6 counterpart. The cluster is v4-only today (podSubnet 10.244.0.0/16, +# serviceSubnet 10.96.0.0/12 in files/kubernetes/kubeadm.yaml), so this changes +# nothing now; it is here so enabling dual-stack is a kubeadm.yaml edit rather +# than a second debugging session. +net.ipv6.conf.all.forwarding = 1 diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service index 05a135f4..c1bb6d03 100644 --- a/files/os/systemd/system/kubeadm-init.service +++ b/files/os/systemd/system/kubeadm-init.service @@ -20,10 +20,35 @@ ConditionPathExists=!/etc/kubernetes/admin.conf # enables the service; no preset of ours is involved. Requires=systemd-sysext.service After=systemd-sysext.service -# Hard on purpose: kubeadm against a host with no CRI socket fails late and -# confusingly, where an unresolvable job fails immediately and names the missing -# unit. tests/unit/test_unit_dependency_provenance.py records the provider. -Requires=containerd.service +# Wants=, not Requires=, and the distinction is load-bearing. +# +# systemd resolves dependencies when it computes the boot transaction. At that +# moment containerd.service does not exist: it lives inside containerd-flatcar.raw +# and only appears once systemd-sysext merges /usr. Observed ordering on a real +# first boot: +# +# 12:06:10.283 Applying preset policy +# 12:06:11.751 Merged extensions into '/usr' <- containerd.service appears +# 12:06:13.141 Reached target Multi-User System +# +# With Requires=, the unresolvable dependency made systemd drop this unit's job +# from the transaction outright. It was never re-enqueued after the merge, so +# its conditions were never even evaluated (ConditionTimestamp empty), kubeadm +# never ran, and bluefin-cluster-bootstrap.service never ran either because it +# declares Requires=kubeadm-init.service. The machine booted to multi-user with +# no control plane and nothing in `systemctl list-units --failed` to show for it. +# +# kubelet.service survived the same boot precisely because it only declares +# After=containerd.service: a pure ordering edge on a unit that does not yet +# exist is ignored rather than fatal. +# +# Wants= is ignored the same way when unresolvable, and After= still orders +# correctly once the merge makes containerd.service real — it is pulled into +# the transaction by its own multi-user.target.wants symlink inside the sysext. +# If kubeadm does start without a CRI socket, ExecStart fails against +# unix:///run/containerd/containerd.sock and Restart=on-failure retries every +# 30s, which converges instead of silently doing nothing. +Wants=containerd.service After=containerd.service Wants=network-online.target After=network-online.target @@ -37,6 +62,27 @@ RemainAfterExit=yes Restart=on-failure RestartSec=30s TimeoutStartSec=900 +# Wait for the CRI socket rather than assuming After=containerd.service ordered +# us. It cannot: containerd.service arrives with the sysext merge, so it is not +# in the boot transaction when systemd computes ordering, and After= against a +# unit that is not in the transaction orders nothing. Observed without this: +# +# 12:18:13 [ERROR CRI]: dial unix /run/containerd/containerd.sock: +# connect: no such file or directory +# 12:18:43 Scheduled restart job, restart counter is at 1 +# 12:18:59 Finished Initialise the Kubernetes control plane +# +# kubeadm recovered on its own 30s later, but the damage was already done: +# bluefin-cluster-bootstrap.service declares Requires=kubeadm-init.service, and +# that first failure killed its job permanently with result 'dependency'. It is +# never re-queued when kubeadm later succeeds, so the node came up with a +# control plane and no CNI, no Argo, and no console — and nothing in +# `systemctl list-units --failed` to explain it. +# +# Waiting here makes the first attempt succeed, which keeps the Requires= edge +# in the dependents honest. Bounded by TimeoutStartSec above; if containerd +# genuinely never arrives, Restart=on-failure still retries. +ExecStartPre=/usr/bin/bash -c 'until [ -S /run/containerd/containerd.sock ]; do sleep 1; done' # Cilium provides the kube-proxy replacement; installing kube-proxy as well # would leave two datapaths fighting over the same Services. ExecStart=/usr/bin/kubeadm init --config /usr/share/bluefin/kubeadm.yaml --skip-phases=addon/kube-proxy diff --git a/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf b/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf index 3e12ecb3..96c61f37 100644 --- a/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf +++ b/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf @@ -1,6 +1,32 @@ # A sysext merge makes unit files appear under /usr; it does not by itself make -# systemd notice them. Without this reload, ordering edges on containerd.service -# and kubelet.service resolve against units systemd has never read, and the -# failure mode is silent: image merged, nothing running. +# systemd notice them, and noticing them is still not the same as running them. +# +# The reload is necessary: without it, ordering edges on containerd.service and +# kubelet.service resolve against units systemd has never read. +# +# The explicit start is necessary too, and this is the subtle half. Flatcar's +# containerd sysext ships its own enablement symlink at +# /usr/lib/systemd/system/multi-user.target.wants/containerd.service. That +# symlink only becomes visible when the merge completes, which is *after* +# systemd has computed the boot transaction — and systemd does not retroactively +# pull newly-appeared .wants entries into a transaction already in flight. So +# whether containerd starts depends purely on whether the merge happened to land +# before multi-user.target was queued. +# +# That is a race, and it was observed resolving both ways on the same image: +# +# boot A merge 12:06:11.75, multi-user 12:06:13.14 -> containerd active +# boot B merge 12:21:10.24, multi-user not reached -> containerd inactive, +# kubeadm-init blocked +# forever on the CRI +# socket +# +# Starting it here removes the race: the merge and the start are the same +# transaction, in order. `start` is idempotent — if the .wants symlink did win +# the race and containerd is already running, this is a no-op. +# +# --no-block so a slow containerd start cannot stall systemd-sysext.service and +# with it every unit ordered after the merge. [Service] ExecStartPost=/usr/bin/systemctl daemon-reload +ExecStartPost=/usr/bin/systemctl start --no-block containerd.service diff --git a/tests/unit/test_cluster_bootstrap.py b/tests/unit/test_cluster_bootstrap.py index 202b0c6b..76e30052 100644 --- a/tests/unit/test_cluster_bootstrap.py +++ b/tests/unit/test_cluster_bootstrap.py @@ -126,13 +126,28 @@ def test_the_idempotency_guard_lives_on_kubeadm_init_alone() -> None: def test_kubeadm_init_waits_for_the_sysext_merge_and_the_runtime() -> None: init = unit(KUBEADM_INIT)["Unit"] requires = set(init.get("Requires", [])) + wants = set(init.get("Wants", [])) after = set(init.get("After", [])) - # kubeadm, kubelet and containerd.service all arrive with the merge. + # systemd-sysext.service is part of systemd and exists when the boot + # transaction is computed, so requiring it is safe and correct: without the + # merge there is no /usr/bin/kubeadm to run. assert "systemd-sysext.service" in requires assert "systemd-sysext.service" in after - assert "containerd.service" in requires + + # containerd.service is different. It arrives *inside* the merge, so it does + # not exist yet when systemd resolves dependencies. This previously read + # `Requires=containerd.service`, and systemd dropped kubeadm-init's job from + # the boot transaction outright — conditions never evaluated, kubeadm never + # ran, nothing in `systemctl list-units --failed`. Observed on a real boot: + # preset 12:06:10, merge 12:06:11, multi-user 12:06:13. + assert "containerd.service" not in requires, ( + "containerd.service does not exist when the boot transaction is " + "computed; a hard requirement silently drops this unit's job" + ) + assert "containerd.service" in wants assert "containerd.service" in after + assert "network-online.target" in after @@ -153,9 +168,38 @@ def test_kubeadm_init_reads_the_staged_config_and_skips_kube_proxy() -> None: ], "Cilium supplies the kube-proxy replacement; installing both fights" -def test_sysext_merge_reloads_systemd_so_merged_units_become_visible() -> None: - dropin = unit(SYSEXT_DROPIN)["Service"] - assert dropin["ExecStartPost"] == ["/usr/bin/systemctl daemon-reload"] +def test_sysext_merge_reloads_systemd_and_starts_the_merged_runtime() -> None: + """Making a merged unit visible is not the same as running it. + + containerd.service and its own multi-user.target.wants symlink both live + inside the sysext, so neither exists when systemd computes the boot + transaction. The reload makes them visible afterwards, but a reload does not + enqueue jobs for a target that is already active or already in flight — so + whether containerd starts comes down to whether the merge happened to land + before multi-user.target was queued. + + That race was observed resolving both ways on the same image:: + + boot A merge 12:06:11.75, multi-user 12:06:13.14 -> containerd active + boot B merge 12:21:10.24, multi-user not reached -> containerd inactive + + On boot B kubeadm-init blocked forever waiting for a CRI socket that never + appeared. The explicit start removes the race; it is idempotent when the + symlink did win. + """ + post = unit(SYSEXT_DROPIN)["Service"]["ExecStartPost"] + + assert "/usr/bin/systemctl daemon-reload" in post, ( + "without a reload, ordering edges resolve against units systemd has " + "never read" + ) + assert "/usr/bin/systemctl start --no-block containerd.service" in post, ( + "containerd.service must be started explicitly: its enablement symlink " + "arrives with the merge, too late for systemd to act on" + ) + assert post.index("/usr/bin/systemctl daemon-reload") < post.index( + "/usr/bin/systemctl start --no-block containerd.service" + ), "the unit must be read before it can be started" def test_kubelet_is_configured_by_kubeadm_rather_than_by_the_image() -> None: diff --git a/tests/unit/test_kubeadm_config_documents.py b/tests/unit/test_kubeadm_config_documents.py new file mode 100644 index 00000000..ef7c1755 --- /dev/null +++ b/tests/unit/test_kubeadm_config_documents.py @@ -0,0 +1,104 @@ +"""`files/kubernetes/kubeadm.yaml` must parse the way kubeadm parses it. + +kubeadm reads `--config` through `k8s.io/apimachinery/pkg/util/yaml`, whose +YAMLReader splits the stream on lines that are exactly ``---`` and decodes each +chunk independently. Every chunk must carry both ``apiVersion`` and ``kind``; +an empty one is rejected with:: + + error: invalid configuration for GroupVersionKind /, Kind=: kind and + apiVersion is mandatory information that must be specified + +That is not a hypothetical. The file opened with a fourteen-line comment header +followed by a ``---`` separator, which made the header a document of its own. +`kubeadm init` failed on every boot, `kubeadm-init.service` restart-looped, +`bluefin-cluster-bootstrap.service` never ran because it declares +``Requires=kubeadm-init.service``, and the installer smoke gate timed out after +900 s waiting for a console that could never come up. Nothing else caught it: +the element builds, `just validate` resolves, and the unit suite passed. + +**This module deliberately does not use ``yaml.safe_load_all``.** PyYAML is +more forgiving than the Go splitter: on the broken file it returned three clean +documents and no empty one, so a PyYAML-based gate would have passed while +kubeadm kept failing. Emulating the splitter is the whole point. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +KUBEADM_CONFIG = ROOT / "files" / "kubernetes" / "kubeadm.yaml" + +# k8s.io/apimachinery/pkg/util/yaml YAMLReader: a separator is a line that is +# exactly `---`, optionally followed by whitespace. +SEPARATOR = re.compile(r"(?m)^---\s*$") + + +def chunks() -> list[str]: + return SEPARATOR.split(KUBEADM_CONFIG.read_text(encoding="utf-8")) + + +def test_config_exists() -> None: + assert KUBEADM_CONFIG.is_file(), f"{KUBEADM_CONFIG.relative_to(ROOT)} is missing" + + +def test_every_document_kubeadm_sees_has_apiversion_and_kind() -> None: + parts = chunks() + + assert parts, "the splitter produced no chunks at all" + + for index, chunk in enumerate(parts): + document = yaml.safe_load(chunk) if chunk.strip() else None + + assert document is not None, ( + f"chunk {index} of {KUBEADM_CONFIG.relative_to(ROOT)} carries no " + f"YAML nodes — only blank lines or comments. kubeadm decodes each " + f"`---`-separated chunk independently and rejects an empty one with " + f"'invalid configuration for GroupVersionKind /, Kind=: kind and " + f"apiVersion is mandatory'. A comment block above the first `---` " + f"becomes exactly this. Chunk content:\n{chunk[:200]}" + ) + assert document.get("apiVersion"), ( + f"chunk {index} has no apiVersion; kubeadm requires one per document" + ) + assert document.get("kind"), ( + f"chunk {index} has no kind; kubeadm requires one per document" + ) + + +def test_the_file_does_not_open_with_a_separator() -> None: + """The specific shape that produced the empty leading document. + + Stated directly as well as through the splitter, because this is the form + the mistake actually takes: a header comment, then `---`, then the first + real document. + """ + meaningful = [ + line + for line in KUBEADM_CONFIG.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + + assert meaningful, f"{KUBEADM_CONFIG.relative_to(ROOT)} has no content" + assert meaningful[0].strip() != "---", ( + "the first non-comment line is a `---` separator, so everything above " + "it becomes an empty document and kubeadm init fails. Drop the leading " + "separator; the header comment belongs to the first real document." + ) + + +@pytest.mark.parametrize("required", ["InitConfiguration", "ClusterConfiguration"]) +def test_the_kinds_kubeadm_init_needs_are_present(required: str) -> None: + kinds = { + (yaml.safe_load(c) or {}).get("kind") + for c in chunks() + if c.strip() + } + assert required in kinds, ( + f"{required} is missing; `kubeadm init --config` needs it to build the " + f"control plane. Found: {sorted(k for k in kinds if k)}" + ) diff --git a/tests/unit/test_seeded_sysext_names.py b/tests/unit/test_seeded_sysext_names.py index ac622b45..262ce6e2 100644 --- a/tests/unit/test_seeded_sysext_names.py +++ b/tests/unit/test_seeded_sysext_names.py @@ -88,31 +88,70 @@ def test_seeded_sysexts_are_decoupled_from_the_host_os_release_version() -> None ) -def test_every_sysext_provided_unit_a_host_unit_requires_is_actually_seeded() -> None: - """A hard Requires= on a sysext-provided unit obliges the installer to seed it. +def test_every_sysext_provided_unit_a_host_unit_depends_on_is_actually_seeded() -> None: + """Depending on a sysext-provided unit obliges the installer to seed it. Deleting the ``CopyFiles=`` line for containerd would restore exactly the - failure this whole arrangement exists to fix: kubeadm-init.service holds - ``Requires=containerd.service``, nothing provides it, the job is - unresolvable, and bluefin-cluster-bootstrap.service never runs because it - declares ``Requires=kubeadm-init.service``. The machine boots to - multi-user.target with no control plane. - - The requirement is read out of the unit rather than restated here, so - dropping the ``Requires=`` deliberately also relaxes this test. + failure this whole arrangement exists to fix: kubeadm-init.service depends + on ``containerd.service``, nothing provides it, and + bluefin-cluster-bootstrap.service never runs because it declares + ``Requires=kubeadm-init.service``. The machine boots to multi-user.target + with no control plane. + + Both ``Requires=`` and ``Wants=`` count. The unit deliberately uses ``Wants=`` + — see ``test_sysext_provided_units_are_not_hard_required`` — so matching only + ``Requires=`` here would make this gate silently vacuous. """ unit = without_comments(KUBEADM_INIT.read_text(encoding="utf-8")) seeded_names = {Path(dst).name for dst in seeded_extensions().values()} - for required_unit, image in SYSEXT_PROVIDED_UNITS.items(): - if f"Requires={required_unit}" not in unit: + for dep_unit, image in SYSEXT_PROVIDED_UNITS.items(): + if not any(f"{kw}={dep_unit}" in unit for kw in ("Requires", "Wants")): continue assert image in seeded_names, ( - f"{KUBEADM_INIT.relative_to(ROOT)} declares " - f"Requires={required_unit}, which only {image} provides, but " - f"{VAR_REPART.relative_to(ROOT)} does not seed it into " - f"/lib/extensions. Seeded images: {sorted(seeded_names) or 'none'}. " - "The control plane will not start." + f"{KUBEADM_INIT.relative_to(ROOT)} depends on {dep_unit}, which " + f"only {image} provides, but {VAR_REPART.relative_to(ROOT)} does " + f"not seed it into /lib/extensions. Seeded images: " + f"{sorted(seeded_names) or 'none'}. The control plane will not start." + ) + + +def test_sysext_provided_units_are_not_hard_required() -> None: + """A unit that only exists after the merge cannot be a ``Requires=``. + + systemd resolves dependencies when it computes the boot transaction, which + happens before systemd-sysext merges ``/usr``. Observed on a real first boot + of this image:: + + 12:06:10.283 Applying preset policy + 12:06:11.751 Merged extensions into '/usr' <- containerd.service appears + 12:06:13.141 Reached target Multi-User System + + With ``Requires=containerd.service`` the unresolvable dependency made systemd + drop kubeadm-init.service's job from the transaction entirely. It was never + re-enqueued after the merge, so its conditions were never evaluated, kubeadm + never ran, and nothing appeared in ``systemctl list-units --failed``. + kubelet.service survived the same boot because it only declares + ``After=containerd.service``; a pure ordering edge on a missing unit is + ignored rather than fatal. + + ``Wants=`` is ignored the same way, and ``After=`` still orders correctly once + the merge makes the unit real. + """ + unit = without_comments(KUBEADM_INIT.read_text(encoding="utf-8")) + + for dep_unit in SYSEXT_PROVIDED_UNITS: + assert f"Requires={dep_unit}" not in unit, ( + f"{KUBEADM_INIT.relative_to(ROOT)} hard-requires {dep_unit}, which " + f"does not exist when systemd computes the boot transaction — it " + f"arrives with the sysext merge. systemd will drop this unit's job " + f"and never re-enqueue it, so kubeadm never runs and the failure is " + f"invisible to `systemctl list-units --failed`. Use Wants= plus " + f"After=." + ) + assert f"After={dep_unit}" in unit, ( + f"{KUBEADM_INIT.relative_to(ROOT)} must still order itself " + f"After={dep_unit}, or kubeadm can run before the CRI socket exists." ) From 3172acfaf59688bcefa4710550bf109ae6de7c62 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 08:39:16 -0400 Subject: [PATCH 27/46] fix(cluster): stop one transient kubeadm failure stranding the seed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bluefin-cluster-bootstrap.service declared Requires=kubeadm-init.service. A job that dies with result 'dependency' never runs, so its own Restart=on-failure never engages and systemd never re-queues it. One failed kubeadm attempt therefore killed the seed permanently. The ExecStartPre socket wait added in 832e50d removes the known race, but not the class: a slow etcd, an image pull, or clock skew fails the first attempt just as effectively, and kubeadm-init carries Restart=on-failure precisely because that is expected. Observed the failure mode directly — kubeadm failed at 12:18:13, succeeded on retry at 12:18:59, and the seed stayed inactive behind a healthy control plane with no CNI, no Argo and no console. Now Wants= plus After=. A premature start is harmless: every seed phase is a server-side apply, and the unit already has Restart=on-failure, RestartSec=30s and TimeoutStartSec=1800, so it retries against the API server until the control plane answers. bluefin-cluster-repo.service stays Requires= — it is a base-OS unit present at transaction time, and without the mirrored manifests there is nothing to apply. Also corrects an overstatement in the kubeadm-init header. It claimed "After= still orders correctly once the merge makes containerd.service real". It does not: containerd.service is absent from the boot transaction on every boot, because the sysext merges mid-boot each time and the systemd-sysext drop-in starts containerd imperatively rather than as a queued job. An After= against a unit outside the transaction orders nothing. The edge is kept because it is correct whenever containerd does enter one, but the ordering this unit relies on at boot is the ExecStartPre socket wait. test_seed_requires_both_prerequisite_units asserted the old contract and is rewritten as test_seed_is_ordered_after_both_prerequisite_units, which pins the distinction that matters: ordering for both, hard dependency only for the repo. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../system/bluefin-cluster-bootstrap.service | 21 ++++++++++++++- files/os/systemd/system/kubeadm-init.service | 17 +++++++----- tests/unit/test_cluster_bootstrap.py | 26 ++++++++++++++++--- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/files/os/systemd/system/bluefin-cluster-bootstrap.service b/files/os/systemd/system/bluefin-cluster-bootstrap.service index d4cb71c8..78335b5b 100644 --- a/files/os/systemd/system/bluefin-cluster-bootstrap.service +++ b/files/os/systemd/system/bluefin-cluster-bootstrap.service @@ -5,8 +5,27 @@ Documentation=file:///usr/share/bluefin/seed # *skipped* rather than failed, which would silently no-op the seed on first # boot and leave a node with no CNI and no reconciler. Every phase below is a # server-side apply and is idempotent on its own, so re-running is free. -Requires=kubeadm-init.service +# Wants=, not Requires=, on kubeadm-init.service. +# +# A job that dies with result 'dependency' never runs, so its own +# Restart=on-failure never engages and systemd never re-queues it. Under +# Requires= that made this unit permanently dead whenever kubeadm-init failed +# even once — and kubeadm-init has Restart=on-failure precisely because a first +# attempt can lose a race with the CRI socket, a slow etcd, an image pull or +# clock skew. Observed exactly that: kubeadm failed at 12:18:13, succeeded on +# retry at 12:18:59, and this unit stayed inactive forever with a control plane +# up and no CNI, no Argo and no console. +# +# With Wants= plus After=, a premature start is harmless: every phase is a +# server-side apply, the unit has Restart=on-failure/RestartSec=30s and +# TimeoutStartSec=1800, so it retries against the API server until the control +# plane answers. The chain converges instead of depending on kubeadm-init +# succeeding on the very first try. +Wants=kubeadm-init.service After=kubeadm-init.service +# bluefin-cluster-repo.service stays a hard requirement: it is a base-OS unit +# that exists at transaction time, and without the mirrored manifests in +# /var/lib/bluefin/cluster.git there is nothing for the seed phases to apply. Requires=bluefin-cluster-repo.service After=bluefin-cluster-repo.service Wants=network-online.target diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service index c1bb6d03..208c75b0 100644 --- a/files/os/systemd/system/kubeadm-init.service +++ b/files/os/systemd/system/kubeadm-init.service @@ -42,12 +42,17 @@ After=systemd-sysext.service # After=containerd.service: a pure ordering edge on a unit that does not yet # exist is ignored rather than fatal. # -# Wants= is ignored the same way when unresolvable, and After= still orders -# correctly once the merge makes containerd.service real — it is pulled into -# the transaction by its own multi-user.target.wants symlink inside the sysext. -# If kubeadm does start without a CRI socket, ExecStart fails against -# unix:///run/containerd/containerd.sock and Restart=on-failure retries every -# 30s, which converges instead of silently doing nothing. +# Wants= is ignored the same way when unresolvable. Be clear about what After= +# then buys: nothing, at boot. containerd.service is absent from the boot +# transaction on *every* boot, not just the first, because the sysext merges +# mid-boot each time and the drop-in on systemd-sysext.service starts containerd +# imperatively rather than as a queued job. An After= against a unit that is not +# in the transaction orders nothing. +# +# It is kept because it costs nothing and is correct whenever containerd does +# enter a transaction — a later `systemctl restart containerd`, for instance. +# The ordering this unit actually relies on at boot is the ExecStartPre socket +# wait below. Wants=containerd.service After=containerd.service Wants=network-online.target diff --git a/tests/unit/test_cluster_bootstrap.py b/tests/unit/test_cluster_bootstrap.py index 76e30052..8e73ad58 100644 --- a/tests/unit/test_cluster_bootstrap.py +++ b/tests/unit/test_cluster_bootstrap.py @@ -297,11 +297,31 @@ def test_seed_waits_for_the_git_bridge_before_handing_over_to_argo() -> None: ) -def test_seed_requires_both_prerequisite_units() -> None: +def test_seed_is_ordered_after_both_prerequisite_units() -> None: + """Ordering is required for both; hard dependency only for the repo. + + ``bluefin-cluster-repo.service`` is a base-OS unit and the seed has nothing + to apply without the mirrored manifests, so it stays ``Requires=``. + + ``kubeadm-init.service`` must not be, and the distinction is what keeps this + chain self-healing. A job that dies with ``result 'dependency'`` never runs, + so its own ``Restart=on-failure`` never engages and systemd never re-queues + it. Under ``Requires=``, a single failed kubeadm attempt killed this unit + permanently — observed with kubeadm failing at 12:18:13, succeeding on retry + at 12:18:59, and the seed staying inactive forever behind a working control + plane. + """ bootstrap = unit(CLUSTER_BOOTSTRAP)["Unit"] + for name in ("kubeadm-init.service", "bluefin-cluster-repo.service"): - assert name in bootstrap["Requires"] - assert name in bootstrap["After"] + assert name in bootstrap["After"], f"{name} must be ordered before the seed" + + assert "bluefin-cluster-repo.service" in bootstrap["Requires"] + assert "kubeadm-init.service" not in bootstrap.get("Requires", []), ( + "a hard requirement on kubeadm-init makes one transient kubeadm failure " + "strand the seed permanently; use Wants= so Restart=on-failure converges" + ) + assert "kubeadm-init.service" in bootstrap["Wants"] def test_kubeadm_config_bootstraps_every_certificate_it_uses() -> None: From 1b46192deffacd09ea27dcddca2e322c2d85e1e4 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 08:41:43 -0400 Subject: [PATCH 28/46] fix(cluster): probe the API server before seeding, and say so while waiting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relaxing kubeadm-init to Wants= in c81ecc4 bought self-healing at the cost of diagnosability, and that trade should not be silent. Under Requires=, a broken kubeadm-init produced exactly one legible line — "Job ... failed with result 'dependency'" — which is how the stranding bug was found. Under Wants= the same breakage instead retries every 30s until TimeoutStartSec=1800, and in CI the 900s readiness gate fires first, so the operator sees a timeout with no stated cause and up to thirty rounds of kubectl connection errors to read through. An ExecStartPre now polls `kubectl get --raw /readyz` and echoes why it is waiting on each attempt. That collapses the noise into one self-describing line per cycle, and it sharpens what a later failure means: once the probe passes, the API server is answering, so anything failing in the phases below is a genuine apply failure rather than a control plane that has not arrived yet. /readyz is the cheapest call that proves the apiserver is serving and needs no RBAC beyond the admin kubeconfig already in Environment=KUBECONFIG. Gated by test_seed_waits_for_the_api_server_before_it_applies_anything, which asserts the probe retries and explains itself rather than merely existing. Mutation-tested: deleting the ExecStartPre fails it. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../system/bluefin-cluster-bootstrap.service | 16 ++++++++++ tests/unit/test_cluster_bootstrap.py | 32 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/files/os/systemd/system/bluefin-cluster-bootstrap.service b/files/os/systemd/system/bluefin-cluster-bootstrap.service index 78335b5b..c3b758a0 100644 --- a/files/os/systemd/system/bluefin-cluster-bootstrap.service +++ b/files/os/systemd/system/bluefin-cluster-bootstrap.service @@ -41,6 +41,22 @@ Environment=KUBECONFIG=/etc/kubernetes/admin.conf # The seed is append-only and is never pruned: kubectl's allowlist --prune is # alpha and incompatible with --server-side, and --applyset cannot span the # namespaces this seed touches. Argo CD owns pruning for everything downstream. +# Wait for the API server before applying anything, and say so while waiting. +# +# This is the cost of Wants= over Requires= above. Under Requires=, a broken +# kubeadm-init produced exactly one legible line — "Job ... failed with result +# 'dependency'" — which is how the stranding bug was found in the first place. +# Under Wants= this unit instead retries every 30s until TimeoutStartSec=1800, +# and in CI the 900s readiness gate fires first, so the operator would see a +# timeout with no stated cause and up to thirty rounds of kubectl connection +# errors to read through. +# +# Probing here collapses that into one self-describing line per attempt, and it +# sharpens the signal: once this passes, the API server is answering, so any +# failure in the phases below is a genuine apply failure rather than a control +# plane that has not arrived yet. `get --raw /readyz` is the cheapest call that +# proves the apiserver is serving, and needs no RBAC beyond the admin kubeconfig. +ExecStartPre=/usr/bin/bash -c 'until /usr/bin/kubectl --request-timeout=5s get --raw /readyz >/dev/null 2>&1; do echo "waiting for the Kubernetes API server; kubeadm-init has not finished"; sleep 10; done' # Phase 1: CNI. Nothing else can schedule until the node leaves NotReady. ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/00-cilium ExecStart=/usr/bin/kubectl wait --for=condition=Ready node --all --timeout=300s diff --git a/tests/unit/test_cluster_bootstrap.py b/tests/unit/test_cluster_bootstrap.py index 8e73ad58..0a19b96b 100644 --- a/tests/unit/test_cluster_bootstrap.py +++ b/tests/unit/test_cluster_bootstrap.py @@ -266,6 +266,38 @@ def test_seed_applies_server_side_and_never_prunes() -> None: assert "-k " in command, f"{command} must apply a kustomization" +def test_seed_waits_for_the_api_server_before_it_applies_anything() -> None: + """The probe that keeps Wants= diagnosable and the first apply meaningful. + + Relaxing kubeadm-init to ``Wants=`` stopped one transient failure stranding + this unit, but it also removed the single legible + ``Job ... failed with result 'dependency'`` line that made the stranding + visible. Without a probe, a control plane that never arrives shows up as + thirty rounds of kubectl connection errors and then a gate timeout with no + stated cause. + + The probe also sharpens what a failure means: once it passes, the API server + is answering, so anything failing below is a genuine apply failure rather + than a control plane that has not arrived yet. + """ + pre = unit(CLUSTER_BOOTSTRAP)["Service"].get("ExecStartPre", []) + + assert pre, ( + "no ExecStartPre: the seed would start applying against an API server " + "that may not exist yet, and the retries would be unreadable" + ) + probe = " ".join(pre) + assert "/readyz" in probe, ( + "probe the apiserver's /readyz endpoint; it is the cheapest call that " + "proves it is serving and needs no RBAC beyond the admin kubeconfig" + ) + assert "until" in probe, "the probe must retry rather than fail once" + assert "echo" in probe, ( + "the probe must say why it is waiting, or a stalled control plane is " + "indistinguishable from a slow one in the journal" + ) + + def test_seed_waits_for_the_node_to_leave_notready_before_argo() -> None: commands = exec_starts(CLUSTER_BOOTSTRAP) cilium = next(i for i, c in enumerate(commands) if c.endswith("/00-cilium")) From b6401b10f2e7c5d991dfb08985a86158434a4d88 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 08:44:20 -0400 Subject: [PATCH 29/46] fix(cluster): bound the readiness waits and size the start budget to fit them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects in the waits added over the last two commits, both mine. The API-server probe was an open-ended `until`. It shares TimeoutStartSec with the apply phases, so a control plane that never arrives would consume the entire budget before phase 1 ran, and systemd's kill would be attributed to the seed rather than to the missing control plane — precisely the ambiguity the probe was added to remove. It now runs 60 attempts (600s) and exits non-zero with a line naming the cause, handing the failure to Restart=on-failure instead of eating the window. The CRI socket wait in kubeadm-init.service had the same shape against TimeoutStartSec=900. A containerd that never starts would burn the whole budget in silence and read as "kubeadm was slow" rather than "there was no container runtime". Bounded at 300s, leaving kubeadm the larger share. TimeoutStartSec on the seed was also too small for its own contents. The four kubectl waits sum to 2100s (300 + 600 + 600 + 600) against a 1800s budget, so systemd could kill the unit mid-chain and the kill would read as a generic timeout instead of naming the phase that stalled. With the 600s probe the bounded steps total 2700s; the budget is now 3000s, which covers them with headroom for the applies without letting a genuinely stuck seed sit for an hour before Restart=on-failure gets a turn. The arithmetic is written out in the unit rather than left implicit. test_start_timeout_covers_every_bounded_step derives the sum from the unit instead of restating it, so changing any --timeout or the probe bound without adjusting the budget fails. Mutation-tested: reverting the budget to 1800 fails with "2100s of kubectl --timeout plus 600s of probe", and restoring the unbounded probe fails its own gate. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../system/bluefin-cluster-bootstrap.service | 33 ++++++++++++--- files/os/systemd/system/kubeadm-init.service | 14 +++++-- tests/unit/test_cluster_bootstrap.py | 40 ++++++++++++++++++- 3 files changed, 76 insertions(+), 11 deletions(-) diff --git a/files/os/systemd/system/bluefin-cluster-bootstrap.service b/files/os/systemd/system/bluefin-cluster-bootstrap.service index c3b758a0..6374af8c 100644 --- a/files/os/systemd/system/bluefin-cluster-bootstrap.service +++ b/files/os/systemd/system/bluefin-cluster-bootstrap.service @@ -36,7 +36,21 @@ Type=oneshot RemainAfterExit=yes Restart=on-failure RestartSec=30s -TimeoutStartSec=1800 +# TimeoutStartSec must cover every bounded step below, or systemd kills the unit +# mid-chain and the kill reads as a generic timeout instead of naming the phase +# that stalled. The budget is partitioned rather than shared: +# +# ExecStartPre API server probe 600 +# phase 1 kubectl wait node Ready 300 +# phase 2 kubectl wait deployment Available 600 +# phase 2 kubectl rollout status statefulset 600 +# phase 3 kubectl wait deployment Available 600 +# ---- +# 2700 +# +# 3000 leaves headroom for the applies themselves without letting a genuinely +# stuck seed sit for an hour before Restart=on-failure gets a turn. +TimeoutStartSec=3000 Environment=KUBECONFIG=/etc/kubernetes/admin.conf # The seed is append-only and is never pruned: kubectl's allowlist --prune is # alpha and incompatible with --server-side, and --applyset cannot span the @@ -46,17 +60,24 @@ Environment=KUBECONFIG=/etc/kubernetes/admin.conf # This is the cost of Wants= over Requires= above. Under Requires=, a broken # kubeadm-init produced exactly one legible line — "Job ... failed with result # 'dependency'" — which is how the stranding bug was found in the first place. -# Under Wants= this unit instead retries every 30s until TimeoutStartSec=1800, -# and in CI the 900s readiness gate fires first, so the operator would see a -# timeout with no stated cause and up to thirty rounds of kubectl connection -# errors to read through. +# Under Wants= this unit instead retries every 30s until TimeoutStartSec, and in +# CI the 900s readiness gate fires first, so the operator would see a timeout +# with no stated cause and many rounds of kubectl connection errors to read +# through. # # Probing here collapses that into one self-describing line per attempt, and it # sharpens the signal: once this passes, the API server is answering, so any # failure in the phases below is a genuine apply failure rather than a control # plane that has not arrived yet. `get --raw /readyz` is the cheapest call that # proves the apiserver is serving, and needs no RBAC beyond the admin kubeconfig. -ExecStartPre=/usr/bin/bash -c 'until /usr/bin/kubectl --request-timeout=5s get --raw /readyz >/dev/null 2>&1; do echo "waiting for the Kubernetes API server; kubeadm-init has not finished"; sleep 10; done' +# +# The loop is bounded at 60 attempts (600s), not open-ended. An unbounded `until` +# would share this unit's TimeoutStartSec with the apply phases and could consume +# the entire budget before phase 1 ran — the systemd kill would then be +# attributed to the seed rather than to the control plane, which is exactly the +# ambiguity this probe exists to remove. Exiting non-zero instead hands the +# failure to Restart=on-failure with a line that names the cause. +ExecStartPre=/usr/bin/bash -c 'for i in $(seq 60); do /usr/bin/kubectl --request-timeout=5s get --raw /readyz >/dev/null 2>&1 && exit 0; echo "waiting for the Kubernetes API server; kubeadm-init has not finished (attempt ${i}/60)"; sleep 10; done; echo "API server never became ready after 600s; not attempting the seed" >&2; exit 1' # Phase 1: CNI. Nothing else can schedule until the node leaves NotReady. ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/00-cilium ExecStart=/usr/bin/kubectl wait --for=condition=Ready node --all --timeout=300s diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service index 208c75b0..ace397df 100644 --- a/files/os/systemd/system/kubeadm-init.service +++ b/files/os/systemd/system/kubeadm-init.service @@ -84,10 +84,16 @@ TimeoutStartSec=900 # control plane and no CNI, no Argo, and no console — and nothing in # `systemctl list-units --failed` to explain it. # -# Waiting here makes the first attempt succeed, which keeps the Requires= edge -# in the dependents honest. Bounded by TimeoutStartSec above; if containerd -# genuinely never arrives, Restart=on-failure still retries. -ExecStartPre=/usr/bin/bash -c 'until [ -S /run/containerd/containerd.sock ]; do sleep 1; done' +# Waiting here makes the first attempt succeed, which keeps the Wants=/Requires= +# edges in the dependents honest. +# +# Bounded at 300 attempts (300s) rather than open-ended. An unbounded `until` +# shares TimeoutStartSec=900 with kubeadm itself, so a containerd that never +# arrives would burn the whole window in silence and the eventual systemd kill +# would look like "kubeadm was slow" rather than "there was no container +# runtime". Exiting non-zero hands it to Restart=on-failure with a line that +# names the cause, and still leaves kubeadm the larger share of the budget. +ExecStartPre=/usr/bin/bash -c 'for i in $(seq 300); do [ -S /run/containerd/containerd.sock ] && exit 0; sleep 1; done; echo "no CRI socket at /run/containerd/containerd.sock after 300s; containerd.service did not start" >&2; exit 1' # Cilium provides the kube-proxy replacement; installing kube-proxy as well # would leave two datapaths fighting over the same Services. ExecStart=/usr/bin/kubeadm init --config /usr/share/bluefin/kubeadm.yaml --skip-phases=addon/kube-proxy diff --git a/tests/unit/test_cluster_bootstrap.py b/tests/unit/test_cluster_bootstrap.py index 0a19b96b..afa6be12 100644 --- a/tests/unit/test_cluster_bootstrap.py +++ b/tests/unit/test_cluster_bootstrap.py @@ -14,6 +14,7 @@ from __future__ import annotations +import re from pathlib import Path import pytest @@ -291,13 +292,50 @@ def test_seed_waits_for_the_api_server_before_it_applies_anything() -> None: "probe the apiserver's /readyz endpoint; it is the cheapest call that " "proves it is serving and needs no RBAC beyond the admin kubeconfig" ) - assert "until" in probe, "the probe must retry rather than fail once" + assert "exit 1" in probe, ( + "the probe must be bounded and fail loudly. An unbounded loop shares " + "TimeoutStartSec with the apply phases and can consume the whole budget " + "before phase 1 runs, so the systemd kill gets attributed to the seed " + "rather than to the control plane — the exact ambiguity this removes" + ) assert "echo" in probe, ( "the probe must say why it is waiting, or a stalled control plane is " "indistinguishable from a slow one in the journal" ) +def test_start_timeout_covers_every_bounded_step() -> None: + """A budget smaller than its parts kills the unit mid-chain. + + systemd enforces TimeoutStartSec across ExecStartPre plus every ExecStart. + If the bounded waits sum past it, the unit is killed partway through and the + kill reads as a generic timeout instead of naming the phase that stalled — + the same loss of attribution the probe above exists to prevent. + + Previously the four kubectl waits alone summed to 2100s against a 1800s + budget. + """ + service = unit(CLUSTER_BOOTSTRAP)["Service"] + budget = int(service["TimeoutStartSec"][0]) + + steps = service.get("ExecStartPre", []) + service.get("ExecStart", []) + declared = sum( + int(match) for step in steps for match in re.findall(r"--timeout=(\d+)s", step) + ) + # The probe bounds itself by attempt count rather than a --timeout flag. + probe_bound = sum( + int(match) * 10 + for step in service.get("ExecStartPre", []) + for match in re.findall(r"seq (\d+)", step) + ) + + assert budget > declared + probe_bound, ( + f"TimeoutStartSec={budget}s does not cover the bounded steps " + f"({declared}s of kubectl --timeout plus {probe_bound}s of probe), so " + f"systemd can kill the seed mid-chain and the failure loses its cause" + ) + + def test_seed_waits_for_the_node_to_leave_notready_before_argo() -> None: commands = exec_starts(CLUSTER_BOOTSTRAP) cilium = next(i for i, c in enumerate(commands) if c.endswith("/00-cilium")) From 0718b4766522a6a8b7358e052c7548ef1f012fc4 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 08:46:19 -0400 Subject: [PATCH 30/46] test(cluster): require the probe to declare a bound, not just a failure path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `assert "exit 1" in probe` was not sufficient. This passes it: until kubectl get --raw /readyz; do echo waiting; sleep 10; done; exit 1 The `exit 1` is unreachable, the loop runs forever, and the probe consumes the whole TimeoutStartSec before phase 1 — exactly the budget-consumption bug the assertion was added to prevent, wearing the token the assertion looks for. The gate now also requires a finite declared bound, read from the probe itself rather than restated: `seq N` times the loop's `sleep`. A `probe_bound()` helper does that extraction once and is shared with test_start_timeout_covers_every_bounded_step, so the same number feeds the boundedness check and the budget arithmetic and the two cannot drift. Deliberately not asserting on loop keywords. `"until" in probe` was pinning an implementation detail, and swapping it for `"seq" in probe` would repeat the mistake in a new place. The invariants are that the probe terminates, declares how long it may take, and says why it failed. Mutation-tested with the unreachable-exit case above: fails with "the probe must declare a finite bound", where the previous assertion accepted it. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/unit/test_cluster_bootstrap.py | 42 ++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/unit/test_cluster_bootstrap.py b/tests/unit/test_cluster_bootstrap.py index afa6be12..7f1db9a8 100644 --- a/tests/unit/test_cluster_bootstrap.py +++ b/tests/unit/test_cluster_bootstrap.py @@ -293,10 +293,16 @@ def test_seed_waits_for_the_api_server_before_it_applies_anything() -> None: "proves it is serving and needs no RBAC beyond the admin kubeconfig" ) assert "exit 1" in probe, ( - "the probe must be bounded and fail loudly. An unbounded loop shares " - "TimeoutStartSec with the apply phases and can consume the whole budget " - "before phase 1 runs, so the systemd kill gets attributed to the seed " - "rather than to the control plane — the exact ambiguity this removes" + "the probe must fail loudly rather than returning quietly, or a control " + "plane that never arrives is indistinguishable from one that did" + ) + assert probe_bound(unit(CLUSTER_BOOTSTRAP)["Service"]) > 0, ( + "the probe must declare a finite bound. `exit 1` alone is not enough: " + "`until cond; do ...; done; exit 1` has an unreachable failure path and " + "still loops forever, sharing TimeoutStartSec with the apply phases and " + "consuming the whole budget before phase 1 runs — so systemd's kill gets " + "attributed to the seed rather than to the control plane, which is the " + "exact ambiguity this probe exists to remove" ) assert "echo" in probe, ( "the probe must say why it is waiting, or a stalled control plane is " @@ -304,6 +310,23 @@ def test_seed_waits_for_the_api_server_before_it_applies_anything() -> None: ) +def probe_bound(service: dict[str, list[str]]) -> int: + """Seconds the ExecStartPre probe is allowed to spend, from its own bound. + + The probe bounds itself by attempt count rather than a ``--timeout`` flag, + so this reads ``seq N`` and multiplies by the loop's sleep. Returns 0 when + no bound is declared, which the callers treat as a failure rather than as + "costs nothing" — an unbounded probe costs the entire budget. + """ + total = 0 + for step in service.get("ExecStartPre", []): + attempts = re.findall(r"seq (\d+)", step) + sleeps = re.findall(r"sleep (\d+)", step) + if attempts and sleeps: + total += int(attempts[0]) * int(sleeps[0]) + return total + + def test_start_timeout_covers_every_bounded_step() -> None: """A budget smaller than its parts kills the unit mid-chain. @@ -322,16 +345,11 @@ def test_start_timeout_covers_every_bounded_step() -> None: declared = sum( int(match) for step in steps for match in re.findall(r"--timeout=(\d+)s", step) ) - # The probe bounds itself by attempt count rather than a --timeout flag. - probe_bound = sum( - int(match) * 10 - for step in service.get("ExecStartPre", []) - for match in re.findall(r"seq (\d+)", step) - ) + bound = probe_bound(service) - assert budget > declared + probe_bound, ( + assert budget > declared + bound, ( f"TimeoutStartSec={budget}s does not cover the bounded steps " - f"({declared}s of kubectl --timeout plus {probe_bound}s of probe), so " + f"({declared}s of kubectl --timeout plus {bound}s of probe), so " f"systemd can kill the seed mid-chain and the failure loses its cause" ) From a994566d3e843bba75b0a3238eb78106040e1917 Mon Sep 17 00:00:00 2001 From: "sec-check[bot]" Date: Sun, 20 Sep 2026 11:51:38 -0400 Subject: [PATCH 31/46] test(kubestellar): follow the https console-origin default This branch moved the default allowed origin for `bluefin-kubestellar start` from http to https, alongside the kiosk TLS work (cert-manager issuers, kiosk certificate, PKI manifests): ORIGIN="${ORIGIN:-https://localhost:8080,https://127.0.0.1:8080}" The usage text and header comment were updated with it, but the unit test still asserted the old http default, so the `unit` check failed: not ok 11 start defaults to the localhost console origins `[[ "$output" =~ "Allowed origins: http://localhost:8080,http://127.0.0.1:8080" ]]' failed Point the assertion at the new default. Verified against the script's real output with a stubbed kc-agent: the https form matches and the http form does not. The other origin assertions in this file pass explicit --origin values, so they were unaffected. Signed-off-by: sec-check[bot] --- tests/unit/bluefin-kubestellar_test.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/bluefin-kubestellar_test.bats b/tests/unit/bluefin-kubestellar_test.bats index b7cdb299..4d65af18 100755 --- a/tests/unit/bluefin-kubestellar_test.bats +++ b/tests/unit/bluefin-kubestellar_test.bats @@ -165,7 +165,7 @@ run_isolated() { stub_kc_agent_foreground run_isolated --foreground [ "$status" -eq 0 ] - [[ "$output" =~ "Allowed origins: http://localhost:8080,http://127.0.0.1:8080" ]] + [[ "$output" =~ "Allowed origins: https://localhost:8080,https://127.0.0.1:8080" ]] } @test "foreground start execs kc-agent with -allowed-origins" { From 2bc5f6cbb2f2e0eda04a2a8301fd753c004bcc6c Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 11:35:36 -0400 Subject: [PATCH 32/46] fix(flash): relocate the GPT backup header after writing to the device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dd writes the image verbatim, so the GPT backup header lands at the end of the IMAGE. GPT requires it at the device's last LBA, so on any medium larger than the image the table is only half valid: the primary header points at a backup LBA that is not where the device ends. Observed on a 58.6 GB stick carrying the 3.6 GB installer. parted says it outright: Warning: Not all of the space available to /dev/sdb appears to be used, you can fix the GPT to use all of the space (an extra 115791863 blocks) and tools disagree about partition sizes between reads — lsblk reported /dev/sdb2 as 2429186048 bytes and then 2554318848 bytes on the same device. Firmware that validates the backup header can refuse the medium outright. flash-installer now runs `sfdisk --relocate gpt-bak-std` after the write, re-reads the table, and verifies it before printing success. Nothing we had could have caught this. `just test-installer-artifact` and the CI installer-test both attach the installer as a drive sized exactly to the image, so device size always equals image size and the condition cannot arise. A fully green installer-test therefore still produced a stick with a broken table, and the only way to see it is to write to real media larger than the image. Three bats cases: the relocation happens, it happens strictly after the write rather than before, and the repaired table is verified before success is claimed. Mutation-tested — deleting the relocation fails two, moving it ahead of the write fails the ordering one. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 22 +++++++++++++++++ tests/unit/flash-installer_test.bats | 37 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/Justfile b/Justfile index a44cc949..0f185201 100644 --- a/Justfile +++ b/Justfile @@ -274,6 +274,28 @@ flash-installer DEVICE="": # `set -o pipefail` does not reach here — there is no pipeline in the outer # shell, only this nested one. sudo bash -c "set -o pipefail; zstd -dc '${IMG}' | dd of={{DEVICE}} bs=4M iflag=fullblock oflag=direct status=progress conv=fsync" + # Move the GPT backup header to the end of the DEVICE. + # + # dd writes the image verbatim, so the backup header lands at the end of the + # IMAGE. GPT requires it at the device's last LBA, so on any medium larger + # than the image the table is only half valid: the primary header points at + # a backup LBA that is not where the device ends. parted says so directly — + # + # Warning: Not all of the space available to /dev/sdb appears to be used, + # you can fix the GPT to use all of the space (an extra 115791863 blocks) + # + # — and tools disagree about partition sizes between reads. Firmware that + # validates the backup header can refuse to boot the medium outright. + # + # This is invisible to every test we have. `just test-installer-artifact` + # and the CI installer-test attach the image as a drive sized exactly to the + # image, so device size always equals image size and the condition cannot + # arise. Only real media larger than the image exposes it, which is why a + # green installer-test still produced a stick with a broken table. + echo "Relocating the GPT backup header to the end of {{DEVICE}}..." + sudo sfdisk --relocate gpt-bak-std {{DEVICE}} + sudo partprobe {{DEVICE}} || true + sudo sfdisk --verify {{DEVICE}} echo "Successfully flashed the Bluefin Server installer to {{DEVICE}}!" # Build the installer artifacts, then run the reusable artifact smoke path. diff --git a/tests/unit/flash-installer_test.bats b/tests/unit/flash-installer_test.bats index 1cd9c0af..e46fcfb1 100644 --- a/tests/unit/flash-installer_test.bats +++ b/tests/unit/flash-installer_test.bats @@ -53,6 +53,8 @@ setup() { make_stub sudo 0 make_stub dd 0 + make_stub sfdisk 0 + make_stub partprobe 0 make_zstd_stub make_topology_stubs @@ -416,6 +418,41 @@ refute_log() { assert_log "conv=fsync" } +@test "flash-installer relocates the GPT backup header to the end of the device" { + # dd writes the image verbatim, so the backup header lands at the end of the + # IMAGE. On any medium larger than the image the table is half valid: the + # primary header points at a backup LBA that is not the device's last LBA. + # parted reports "Not all of the space available ... appears to be used", + # tools disagree about partition sizes between reads, and firmware that + # validates the backup header can refuse the medium. + # + # No other test can catch this: test-installer-artifact and the CI + # installer-test attach a drive sized exactly to the image, so device size + # always equals image size and the condition cannot arise. + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -eq 0 ] + assert_log "sfdisk --relocate gpt-bak-std ${FAKE_DEV}" +} + +@test "flash-installer repairs the table after writing it, not before" { + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + # dd runs inside `sudo bash -c "... | dd of=..."`, so it never appears as a + # top-level `dd ` line in the call log — match the sudo invocation carrying it. + write_line=$(grep -n "dd of=${FAKE_DEV}" "$LOG" | head -1 | cut -d: -f1) + relocate_line=$(grep -n "relocate gpt-bak-std" "$LOG" | head -1 | cut -d: -f1) + [ -n "$write_line" ] + [ -n "$relocate_line" ] + [ "$relocate_line" -gt "$write_line" ] +} + +@test "flash-installer verifies the repaired table before declaring success" { + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + assert_log "sfdisk --verify ${FAKE_DEV}" +} + @test "flash-installer reports success only after the write is attempted" { seed_image "bluefin-server-installer-1.0.raw.zst" run_flash "$FAKE_DEV" "y" From 457087018349fe3a9ba4b9d044be9647e03c720b Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 12:22:45 -0400 Subject: [PATCH 33/46] fix(installer): boot the medium via systemd-boot, not a 434 MiB UKI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The installer medium does not boot. Firmware loads its UKI and never executes it, so a flashed stick sits at a black screen and the target disk is never touched. Step 4 built one UKI with `ukify build --linux --initrd`. The installer initrd is ~382 MiB because it carries the whole live environment, so the result is a 434 MiB PE at EFI/BOOT/BOOTX64.EFI. OVMF opens it, reports FSOpen: Open '\EFI\BOOT\BOOTX64.EFI' Success Loading driver at 0x0014DF90000 EntryPoint=0x0014DFA03C0 and then goes silent — no kernel output despite console=ttyS0,115200 in the embedded cmdline, and an attached 16 GiB target is byte-identical after seven minutes. Isolated by varying only the image, against one OVMF build, one ESP and one QEMU invocation: SizeOfImage 0x020D1000 ( 34 MiB) kernel boots, initrd runs SizeOfImage 0x03CCA000 ( 64 MiB) kernel boots, initrd runs SizeOfImage 0x19E6D000 (434 MiB) hangs after EntryPoint The 64 MiB case is the target OS UKI, which demonstrably boots on installed systems, so it is a known-good control rather than something I constructed. All three share ImageBase 0x14DF90000, so the base is not the variable; size is. The medium now ships systemd-boot at the fallback path with the kernel and initrd as ordinary files and a type-1 loader entry, which is the layout systemd-boot exists for and has no PE size ceiling. The target OS keeps its UKI: 64 MiB boots, and systemd-sysupdate is built around replacing one. The build now refuses to emit a fallback image larger than 16 MiB, because a 434 MiB one reached real hardware with every gate green. Why nothing caught it: test-installer-artifact and the CI installer-test both boot the installer with -kernel/-initrd and attach installer.raw as a data disk. The ESP is never executed, so no test in this repository has ever booted the installer medium. They prove the installer installs; they have never proven the medium boots. test_installer_medium_does_not_ship_an_unexecutable_fallback_image and test_no_ukify_writes_to_the_media_fallback_path pin the contract. Mutation-tested: restoring the ukify call, dropping the size ceiling, or dropping the loader entry each fail. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-installer.bst | 89 ++++++++++++++++++++--- tests/unit/test_installer_contract.py | 81 +++++++++++++++++++-- 2 files changed, 151 insertions(+), 19 deletions(-) diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index c495088f..5ad6c350 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -385,21 +385,83 @@ config: | gzip -9 -c > /installer.cpio.gz ls -lh /installer.cpio.gz - # ── Step 4: Build the UKI ──────────────────────────────────────────── - # Unsigned UKI — acceptable for USB installer media. Secure Boot - # signing can be layered on by passing --secureboot-private-key / - # --secureboot-certificate if keys are injected at CI time. + # ── Step 4: Lay out the ESP: systemd-boot + a type-1 loader entry ──── + # + # NOT a single UKI. `ukify build --linux --initrd` produced one 434 MiB PE + # at EFI/BOOT/BOOTX64.EFI, and firmware cannot execute it: OVMF opens the + # file, reports + # + # Loading driver at 0x0014DF90000 EntryPoint=0x0014DFA03C0 + # + # and then nothing happens — no kernel output despite console=ttyS0 in the + # embedded cmdline, and an attached target disk is still byte-identical + # after seven minutes. + # + # Measured against the same firmware, same ESP, same QEMU invocation: + # + # SizeOfImage 0x020D1000 ( 34 MiB) boots + # SizeOfImage 0x03CCA000 ( 64 MiB) boots <- the target OS UKI + # SizeOfImage 0x19E6D000 (434 MiB) hangs <- this one + # + # All three share ImageBase 0x14DF90000, so the base is not the variable; + # size is. The installer's initrd is ~382 MiB because it carries the whole + # live environment, so it cannot live inside a PE section. + # + # systemd-boot is a ~100 KiB loader that reads the kernel and initrd as + # ordinary files, which is the layout it is designed for and which has no + # such ceiling. The target OS keeps its UKI: at 64 MiB it boots fine, and + # a UKI is what systemd-sysupdate expects to replace. - | set -eux - mkdir -p /layer/boot/efi/EFI/BOOT INSTALLER_KVER="$(basename "$(find /layer/usr/lib/modules -mindepth 1 -maxdepth 1 -type d | head -n1)")" INSTALLER_KERNEL="/layer/usr/lib/modules/${INSTALLER_KVER}/vmlinuz" - ukify build \ - --linux="${INSTALLER_KERNEL}" \ - --initrd=/installer.cpio.gz \ - --cmdline="systemd.unit=system-install.target console=tty0 console=ttyS0,115200 rw quiet loglevel=3 audit=0" \ - --output=/layer/boot/efi/EFI/BOOT/BOOTX64.EFI - ls -lh /layer/boot/efi/EFI/BOOT/BOOTX64.EFI + + mkdir -p /layer/boot/efi/EFI/BOOT + mkdir -p /layer/boot/efi/EFI/systemd + mkdir -p /layer/boot/efi/loader/entries + mkdir -p /layer/boot/efi/bluefin + + # systemd-boot at both the removable-media fallback path and its own, + # so the medium boots whether or not firmware has an NVRAM entry. + SDBOOT="$(find / -name 'systemd-bootx64.efi' -path '*/boot/efi/*' -print -quit 2>/dev/null || true)" + if [ -z "${SDBOOT}" ]; then + SDBOOT="$(find / -name 'systemd-bootx64.efi' -print -quit)" + fi + test -n "${SDBOOT}" + install -D -m 0644 "${SDBOOT}" /layer/boot/efi/EFI/BOOT/BOOTX64.EFI + install -D -m 0644 "${SDBOOT}" /layer/boot/efi/EFI/systemd/systemd-bootx64.efi + + # Kernel and initrd as plain files — no PE size ceiling applies. + install -D -m 0644 "${INSTALLER_KERNEL}" /layer/boot/efi/bluefin/vmlinuz + install -D -m 0644 /installer.cpio.gz /layer/boot/efi/bluefin/initrd + + # printf, not a heredoc: this block is itself indented inside the element, + # and a heredoc would write those leading spaces into the config verbatim. + printf '%s\n' \ + 'title Bluefin Server Installer' \ + 'linux /bluefin/vmlinuz' \ + 'initrd /bluefin/initrd' \ + 'options systemd.unit=system-install.target console=tty0 console=ttyS0,115200 rw quiet loglevel=3 audit=0' \ + > /layer/boot/efi/loader/entries/bluefin-installer.conf + + # Boot it without waiting for a keypress; the medium has one entry. + printf '%s\n' \ + 'default bluefin-installer.conf' \ + 'timeout 0' \ + 'console-mode keep' \ + > /layer/boot/efi/loader/loader.conf + + cat /layer/boot/efi/loader/entries/bluefin-installer.conf + + ls -lh /layer/boot/efi/EFI/BOOT/BOOTX64.EFI \ + /layer/boot/efi/bluefin/vmlinuz \ + /layer/boot/efi/bluefin/initrd + # Fail the build rather than ship another unbootable medium. + BOOT_BYTES="$(stat -c %s /layer/boot/efi/EFI/BOOT/BOOTX64.EFI)" + if [ "${BOOT_BYTES}" -gt 16777216 ]; then + echo "ERROR: EFI/BOOT/BOOTX64.EFI is ${BOOT_BYTES} bytes; firmware could not execute a 434 MiB image." >&2 + exit 1 + fi # ── Step 5: Decompress DDI payload for embedding ───────────────────── # This runs AFTER the initrd cpio is packed (Step 3), so the DDI is NOT @@ -413,7 +475,10 @@ config: # ── Step 6: Assemble installer GPT disk image ──────────────────────── # Two-partition layout: - # 10-esp.conf: ESP (1G) — UKI lives here; boots the installer + # 10-esp.conf: ESP (1G) — systemd-boot, the kernel, the initrd and a + # type-1 loader entry; boots the installer. Sized for + # the ~382 MiB initrd plus the kernel, which is why it + # is 1G rather than the few MiB a UKI-only ESP needs. # 20-ddi-data.conf: Data (XFS) — bluefin-server-ddi.raw copied here block-for-block # # At install time, systemd-sysinstall mounts the data partition to diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index e9b0a4a1..34e67823 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -16,14 +16,21 @@ DDI_ELEMENT = REPO_ROOT / "elements" / "oci" / "bluefin-server-ddi.bst" -def _published_uki_cmdline(installer_element: str) -> str: +def _installer_boot_cmdline(installer_element: str) -> str: + """The cmdline the installer medium boots with. + + It lives in a systemd-boot type-1 loader entry, not in a UKI. A single + `ukify build --linux --initrd` produced a 434 MiB PE that firmware loads and + then never executes; measured on the same firmware and ESP, 34 MiB and + 64 MiB images boot and 434 MiB does not. The installer's initrd is ~382 MiB + because it carries the live environment, so it cannot live in a PE section. + """ match = re.search( - r'ukify build\s+.*?--cmdline="([^"]+)"\s+' - r"[ \t\\\r\n]+--output=/layer/boot/efi/EFI/BOOT/BOOTX64\.EFI", + r"^\s*'options ([^']+)'\s*\\?\s*$", installer_element, - flags=re.DOTALL, + flags=re.MULTILINE, ) - assert match, "published Installer UKI ukify command must be present" + assert match, "installer loader entry must declare an options= line" return match.group(1) @@ -42,12 +49,12 @@ def test_installer_runtime_and_boot_contracts() -> None: installer_stack = INSTALLER_STACK.read_text(encoding="utf-8") installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") justfile = JUSTFILE.read_text(encoding="utf-8") - published_uki_cmdline = _published_uki_cmdline(installer_element) + installer_boot_cmdline = _installer_boot_cmdline(installer_element) target_uki_cmdline = _target_uki_cmdline(installer_element) assert "freedesktop-sdk.bst:bootstrap/bash.bst" in installer_stack assert "console=ttyS0,115200 rw" in installer_element - assert "unattended" not in published_uki_cmdline + assert "unattended" not in installer_boot_cmdline assert target_uki_cmdline == "rw console=ttyS0,115200 console=tty0 quiet loglevel=3 audit=0" assert ( '-append "systemd.unit=system-install.target ' @@ -146,3 +153,63 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: assert "find /layer -type f -name '*.a' -delete" in installer_element assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element assert "find /layer -type f -name '*.a' -delete" in ddi_element + + +def test_installer_medium_does_not_ship_an_unexecutable_fallback_image() -> None: + """EFI/BOOT/BOOTX64.EFI must be a loader, not a whole live environment. + + The installer medium previously shipped a single UKI built with + ``ukify build --linux --initrd``. With a ~382 MiB initrd that produced a + 434 MiB PE, and firmware loads it but never executes it. Measured against + one OVMF build, one ESP and one QEMU invocation, varying only the image: + + SizeOfImage 0x020D1000 ( 34 MiB) kernel boots + SizeOfImage 0x03CCA000 ( 64 MiB) kernel boots + SizeOfImage 0x19E6D000 (434 MiB) hangs after EntryPoint + + All three share ImageBase 0x14DF90000, so size is the variable. OVMF logs + ``Loading driver at 0x0014DF90000 EntryPoint=0x0014DFA03C0`` and then goes + silent; an attached target disk stays byte-identical. + + No other test can catch this. test-installer-artifact and the CI + installer-test both boot the installer with ``-kernel``/``-initrd``, so the + ESP is never executed and the medium's boot path is never exercised. + """ + installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") + + assert "systemd-bootx64.efi" in installer_element, ( + "the medium must boot via systemd-boot, which reads the kernel and " + "initrd as ordinary files rather than embedding them in a PE" + ) + assert "/layer/boot/efi/loader/entries/bluefin-installer.conf" in installer_element, ( + "systemd-boot needs a type-1 loader entry naming the kernel and initrd" + ) + assert "/layer/boot/efi/bluefin/initrd" in installer_element, ( + "the initrd must be a plain file on the ESP, not a PE section" + ) + + # The build must refuse to ship an oversized fallback image rather than + # producing another medium that loads and never runs. + assert "BOOT_BYTES" in installer_element and "16777216" in installer_element, ( + "the build must fail when EFI/BOOT/BOOTX64.EFI exceeds a loader-sized " + "ceiling; a 434 MiB image reached real hardware because nothing checked" + ) + + +def test_no_ukify_writes_to_the_media_fallback_path() -> None: + """The regression, stated directly. + + The target OS keeps its UKI — at 64 MiB it boots, and systemd-sysupdate is + built around replacing one. Only the *medium's* fallback path must not be a + UKI carrying the live environment. + """ + installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") + + assert not re.search( + r"ukify build[^#]*?--output=/layer/boot/efi/EFI/BOOT/BOOTX64\.EFI", + installer_element, + flags=re.DOTALL, + ), ( + "ukify must not write the medium's fallback image: with the installer " + "initrd embedded it produces a 434 MiB PE that firmware cannot execute" + ) From f979d885f314dff2df6b14879ecdee6b3f09dcf2 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 12:25:15 -0400 Subject: [PATCH 34/46] fix(installer): pick systemd-boot deterministically, assert the boot target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects in the previous commit, both mine. `find / -name systemd-bootx64.efi -print -quit` matches either the installer rootfs at /layer or the target rootfs staged at /target-root, and -quit takes whichever the traversal reaches first. Which loader shipped on the medium would therefore vary between builds for no visible reason. Now named explicitly, with a fallback to the sandbox copy and a loud failure if neither exists. Moving the cmdline from the UKI's --cmdline into the loader entry's options= also quietly reduced its coverage to a single negative assertion. Only "unattended" not being present survived; nothing checked that the medium boots into system-install.target at all. A medium that boots and lands in a shell is as useless as one that does not boot. The positive contract is asserted again: system-install.target, console=ttyS0,115200 for the QEMU gate and headless hardware, console=tty0 for a physical operator, and rw for the live root. Mutation-tested, each caught: dropping system-install.target from the entry, dropping the serial console, and restoring the unbounded find. The find assertion scans effective lines only. The element's own comment names the wrong spelling to warn against it, and matching raw text flagged that warning as the defect — the same false positive pattern as the containerd naming gate. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-installer.bst | 21 ++++++++---- tests/unit/test_installer_contract.py | 39 +++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 5ad6c350..bc360431 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -421,13 +421,22 @@ config: mkdir -p /layer/boot/efi/loader/entries mkdir -p /layer/boot/efi/bluefin - # systemd-boot at both the removable-media fallback path and its own, - # so the medium boots whether or not firmware has an NVRAM entry. - SDBOOT="$(find / -name 'systemd-bootx64.efi' -path '*/boot/efi/*' -print -quit 2>/dev/null || true)" - if [ -z "${SDBOOT}" ]; then - SDBOOT="$(find / -name 'systemd-bootx64.efi' -print -quit)" + # systemd-boot at both the removable-media fallback path and its own, so + # the medium boots whether or not firmware has an NVRAM entry. + # + # Explicit paths, not a search. `find / -name systemd-bootx64.efi` can + # match either the installer rootfs at /layer or the target rootfs staged + # at /target-root, and -quit takes whichever the traversal reaches first — + # so which loader ships would vary between builds for no visible reason. + # The installer medium must boot the installer's own copy. + SDBOOT=/layer/usr/lib/systemd/boot/efi/systemd-bootx64.efi + if [ ! -f "${SDBOOT}" ]; then + SDBOOT=/usr/lib/systemd/boot/efi/systemd-bootx64.efi + fi + if [ ! -f "${SDBOOT}" ]; then + echo "ERROR: systemd-bootx64.efi not found; the medium would have no bootloader." >&2 + exit 1 fi - test -n "${SDBOOT}" install -D -m 0644 "${SDBOOT}" /layer/boot/efi/EFI/BOOT/BOOTX64.EFI install -D -m 0644 "${SDBOOT}" /layer/boot/efi/EFI/systemd/systemd-bootx64.efi diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 34e67823..03f02d0c 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -54,6 +54,24 @@ def test_installer_runtime_and_boot_contracts() -> None: assert "freedesktop-sdk.bst:bootstrap/bash.bst" in installer_stack assert "console=ttyS0,115200 rw" in installer_element + # The contract that used to live on the UKI's --cmdline and now lives in the + # loader entry's options=. Assert it positively: a medium that boots but + # lands in a shell instead of system-install.target is just as useless as + # one that does not boot, and only the negative "unattended" check survived + # the move. + assert "systemd.unit=system-install.target" in installer_boot_cmdline, ( + "the medium must boot straight into the installer" + ) + assert "console=ttyS0,115200" in installer_boot_cmdline, ( + "serial console is how the QEMU gate and headless hardware observe the " + "install; without it a failure is silent" + ) + assert "console=tty0" in installer_boot_cmdline, ( + "a physical operator watches tty0" + ) + assert "rw" in installer_boot_cmdline.split(), ( + "the live environment needs a writable root" + ) assert "unattended" not in installer_boot_cmdline assert target_uki_cmdline == "rw console=ttyS0,115200 console=tty0 quiet loglevel=3 audit=0" assert ( @@ -190,6 +208,27 @@ def test_installer_medium_does_not_ship_an_unexecutable_fallback_image() -> None # The build must refuse to ship an oversized fallback image rather than # producing another medium that loads and never runs. + # Which systemd-boot ships must be deterministic. `find / -name + # systemd-bootx64.efi -print -quit` matches either the installer rootfs at + # /layer or the target rootfs staged at /target-root, and takes whichever + # the traversal reaches first — so the medium's bootloader would vary + # between builds for no visible reason. + assert "/layer/usr/lib/systemd/boot/efi/systemd-bootx64.efi" in installer_element, ( + "name the installer's own systemd-boot explicitly rather than searching" + ) + # Scan effective lines only: the element's own comment names the wrong + # spelling in order to warn against it, and matching that would flag the + # warning as the defect. + effective = "\n".join( + line + for line in installer_element.splitlines() + if not line.lstrip().startswith("#") + ) + assert not re.search(r"find / -name '?systemd-bootx64", effective), ( + "an unbounded find can pick the target rootfs copy instead of the " + "installer's, making the shipped bootloader nondeterministic" + ) + assert "BOOT_BYTES" in installer_element and "16777216" in installer_element, ( "the build must fail when EFI/BOOT/BOOTX64.EFI exceeds a loader-sized " "ceiling; a 434 MiB image reached real hardware because nothing checked" From 6e5ab295b7428146e7a991e984b449d6fa8eb8f8 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 12:45:16 -0400 Subject: [PATCH 35/46] Revert "fix(installer): boot the medium via systemd-boot, not a 434 MiB UKI" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts 4570870 and f979d88. The premise was wrong: the 434 MiB UKI boots. I concluded it did not because the serial log showed two BdsDxe lines and then apparent silence, and a target disk stayed byte-identical. Both observations were real and both were misread. Dumping the post-handoff bytes instead of cat-ing them shows they are not silence. They are ANSI and OSC escapes, and among them: ]3008;start=0f6066edf2484778bef4fd9e4d5bdab7;user=root;hostname=localhost; machineid=6797d0d01c474e2fb53d50be267cd5b3;bootid=...;pid=1;comm=systemd;type=boot A machineid and a bootid cannot come from firmware. systemd PID 1 was running. Re-running with loglevel=7 via an SMBIOS cmdline addendum produced 50 KB of ordinary boot log — mounting filesystems, starting journald, coldplugging udev. The installer live environment comes up. The apparent silence is `quiet loglevel=3` in the medium's cmdline. The untouched target is the deliberate absence of `unattended`: the medium waits for an operator. Both are by design. My 34 MiB and 64 MiB "controls" printed only because they FAILED — dracut emergency mode and "no mountable /usr" — and error paths print through `quiet`. I read failure output as proof of booting and correct silence as proof of hanging, which is exactly backwards. Reverting restores a boot path that works. Replacing it with systemd-boot plus a type-1 entry would have swapped a validated design for an unvalidated one to fix nothing. Kept: 2bc5f6c, the GPT backup-header relocation in flash-installer. That defect is real and independently evidenced — sfdisk reports "MyLBA mismatch with real position at backup header" and parted reports ~59 GB unaddressed on any medium larger than the image. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-installer.bst | 98 +++--------------- tests/unit/test_installer_contract.py | 120 ++-------------------- 2 files changed, 19 insertions(+), 199 deletions(-) diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index bc360431..c495088f 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -385,92 +385,21 @@ config: | gzip -9 -c > /installer.cpio.gz ls -lh /installer.cpio.gz - # ── Step 4: Lay out the ESP: systemd-boot + a type-1 loader entry ──── - # - # NOT a single UKI. `ukify build --linux --initrd` produced one 434 MiB PE - # at EFI/BOOT/BOOTX64.EFI, and firmware cannot execute it: OVMF opens the - # file, reports - # - # Loading driver at 0x0014DF90000 EntryPoint=0x0014DFA03C0 - # - # and then nothing happens — no kernel output despite console=ttyS0 in the - # embedded cmdline, and an attached target disk is still byte-identical - # after seven minutes. - # - # Measured against the same firmware, same ESP, same QEMU invocation: - # - # SizeOfImage 0x020D1000 ( 34 MiB) boots - # SizeOfImage 0x03CCA000 ( 64 MiB) boots <- the target OS UKI - # SizeOfImage 0x19E6D000 (434 MiB) hangs <- this one - # - # All three share ImageBase 0x14DF90000, so the base is not the variable; - # size is. The installer's initrd is ~382 MiB because it carries the whole - # live environment, so it cannot live inside a PE section. - # - # systemd-boot is a ~100 KiB loader that reads the kernel and initrd as - # ordinary files, which is the layout it is designed for and which has no - # such ceiling. The target OS keeps its UKI: at 64 MiB it boots fine, and - # a UKI is what systemd-sysupdate expects to replace. + # ── Step 4: Build the UKI ──────────────────────────────────────────── + # Unsigned UKI — acceptable for USB installer media. Secure Boot + # signing can be layered on by passing --secureboot-private-key / + # --secureboot-certificate if keys are injected at CI time. - | set -eux + mkdir -p /layer/boot/efi/EFI/BOOT INSTALLER_KVER="$(basename "$(find /layer/usr/lib/modules -mindepth 1 -maxdepth 1 -type d | head -n1)")" INSTALLER_KERNEL="/layer/usr/lib/modules/${INSTALLER_KVER}/vmlinuz" - - mkdir -p /layer/boot/efi/EFI/BOOT - mkdir -p /layer/boot/efi/EFI/systemd - mkdir -p /layer/boot/efi/loader/entries - mkdir -p /layer/boot/efi/bluefin - - # systemd-boot at both the removable-media fallback path and its own, so - # the medium boots whether or not firmware has an NVRAM entry. - # - # Explicit paths, not a search. `find / -name systemd-bootx64.efi` can - # match either the installer rootfs at /layer or the target rootfs staged - # at /target-root, and -quit takes whichever the traversal reaches first — - # so which loader ships would vary between builds for no visible reason. - # The installer medium must boot the installer's own copy. - SDBOOT=/layer/usr/lib/systemd/boot/efi/systemd-bootx64.efi - if [ ! -f "${SDBOOT}" ]; then - SDBOOT=/usr/lib/systemd/boot/efi/systemd-bootx64.efi - fi - if [ ! -f "${SDBOOT}" ]; then - echo "ERROR: systemd-bootx64.efi not found; the medium would have no bootloader." >&2 - exit 1 - fi - install -D -m 0644 "${SDBOOT}" /layer/boot/efi/EFI/BOOT/BOOTX64.EFI - install -D -m 0644 "${SDBOOT}" /layer/boot/efi/EFI/systemd/systemd-bootx64.efi - - # Kernel and initrd as plain files — no PE size ceiling applies. - install -D -m 0644 "${INSTALLER_KERNEL}" /layer/boot/efi/bluefin/vmlinuz - install -D -m 0644 /installer.cpio.gz /layer/boot/efi/bluefin/initrd - - # printf, not a heredoc: this block is itself indented inside the element, - # and a heredoc would write those leading spaces into the config verbatim. - printf '%s\n' \ - 'title Bluefin Server Installer' \ - 'linux /bluefin/vmlinuz' \ - 'initrd /bluefin/initrd' \ - 'options systemd.unit=system-install.target console=tty0 console=ttyS0,115200 rw quiet loglevel=3 audit=0' \ - > /layer/boot/efi/loader/entries/bluefin-installer.conf - - # Boot it without waiting for a keypress; the medium has one entry. - printf '%s\n' \ - 'default bluefin-installer.conf' \ - 'timeout 0' \ - 'console-mode keep' \ - > /layer/boot/efi/loader/loader.conf - - cat /layer/boot/efi/loader/entries/bluefin-installer.conf - - ls -lh /layer/boot/efi/EFI/BOOT/BOOTX64.EFI \ - /layer/boot/efi/bluefin/vmlinuz \ - /layer/boot/efi/bluefin/initrd - # Fail the build rather than ship another unbootable medium. - BOOT_BYTES="$(stat -c %s /layer/boot/efi/EFI/BOOT/BOOTX64.EFI)" - if [ "${BOOT_BYTES}" -gt 16777216 ]; then - echo "ERROR: EFI/BOOT/BOOTX64.EFI is ${BOOT_BYTES} bytes; firmware could not execute a 434 MiB image." >&2 - exit 1 - fi + ukify build \ + --linux="${INSTALLER_KERNEL}" \ + --initrd=/installer.cpio.gz \ + --cmdline="systemd.unit=system-install.target console=tty0 console=ttyS0,115200 rw quiet loglevel=3 audit=0" \ + --output=/layer/boot/efi/EFI/BOOT/BOOTX64.EFI + ls -lh /layer/boot/efi/EFI/BOOT/BOOTX64.EFI # ── Step 5: Decompress DDI payload for embedding ───────────────────── # This runs AFTER the initrd cpio is packed (Step 3), so the DDI is NOT @@ -484,10 +413,7 @@ config: # ── Step 6: Assemble installer GPT disk image ──────────────────────── # Two-partition layout: - # 10-esp.conf: ESP (1G) — systemd-boot, the kernel, the initrd and a - # type-1 loader entry; boots the installer. Sized for - # the ~382 MiB initrd plus the kernel, which is why it - # is 1G rather than the few MiB a UKI-only ESP needs. + # 10-esp.conf: ESP (1G) — UKI lives here; boots the installer # 20-ddi-data.conf: Data (XFS) — bluefin-server-ddi.raw copied here block-for-block # # At install time, systemd-sysinstall mounts the data partition to diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 03f02d0c..e9b0a4a1 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -16,21 +16,14 @@ DDI_ELEMENT = REPO_ROOT / "elements" / "oci" / "bluefin-server-ddi.bst" -def _installer_boot_cmdline(installer_element: str) -> str: - """The cmdline the installer medium boots with. - - It lives in a systemd-boot type-1 loader entry, not in a UKI. A single - `ukify build --linux --initrd` produced a 434 MiB PE that firmware loads and - then never executes; measured on the same firmware and ESP, 34 MiB and - 64 MiB images boot and 434 MiB does not. The installer's initrd is ~382 MiB - because it carries the live environment, so it cannot live in a PE section. - """ +def _published_uki_cmdline(installer_element: str) -> str: match = re.search( - r"^\s*'options ([^']+)'\s*\\?\s*$", + r'ukify build\s+.*?--cmdline="([^"]+)"\s+' + r"[ \t\\\r\n]+--output=/layer/boot/efi/EFI/BOOT/BOOTX64\.EFI", installer_element, - flags=re.MULTILINE, + flags=re.DOTALL, ) - assert match, "installer loader entry must declare an options= line" + assert match, "published Installer UKI ukify command must be present" return match.group(1) @@ -49,30 +42,12 @@ def test_installer_runtime_and_boot_contracts() -> None: installer_stack = INSTALLER_STACK.read_text(encoding="utf-8") installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") justfile = JUSTFILE.read_text(encoding="utf-8") - installer_boot_cmdline = _installer_boot_cmdline(installer_element) + published_uki_cmdline = _published_uki_cmdline(installer_element) target_uki_cmdline = _target_uki_cmdline(installer_element) assert "freedesktop-sdk.bst:bootstrap/bash.bst" in installer_stack assert "console=ttyS0,115200 rw" in installer_element - # The contract that used to live on the UKI's --cmdline and now lives in the - # loader entry's options=. Assert it positively: a medium that boots but - # lands in a shell instead of system-install.target is just as useless as - # one that does not boot, and only the negative "unattended" check survived - # the move. - assert "systemd.unit=system-install.target" in installer_boot_cmdline, ( - "the medium must boot straight into the installer" - ) - assert "console=ttyS0,115200" in installer_boot_cmdline, ( - "serial console is how the QEMU gate and headless hardware observe the " - "install; without it a failure is silent" - ) - assert "console=tty0" in installer_boot_cmdline, ( - "a physical operator watches tty0" - ) - assert "rw" in installer_boot_cmdline.split(), ( - "the live environment needs a writable root" - ) - assert "unattended" not in installer_boot_cmdline + assert "unattended" not in published_uki_cmdline assert target_uki_cmdline == "rw console=ttyS0,115200 console=tty0 quiet loglevel=3 audit=0" assert ( '-append "systemd.unit=system-install.target ' @@ -171,84 +146,3 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: assert "find /layer -type f -name '*.a' -delete" in installer_element assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element assert "find /layer -type f -name '*.a' -delete" in ddi_element - - -def test_installer_medium_does_not_ship_an_unexecutable_fallback_image() -> None: - """EFI/BOOT/BOOTX64.EFI must be a loader, not a whole live environment. - - The installer medium previously shipped a single UKI built with - ``ukify build --linux --initrd``. With a ~382 MiB initrd that produced a - 434 MiB PE, and firmware loads it but never executes it. Measured against - one OVMF build, one ESP and one QEMU invocation, varying only the image: - - SizeOfImage 0x020D1000 ( 34 MiB) kernel boots - SizeOfImage 0x03CCA000 ( 64 MiB) kernel boots - SizeOfImage 0x19E6D000 (434 MiB) hangs after EntryPoint - - All three share ImageBase 0x14DF90000, so size is the variable. OVMF logs - ``Loading driver at 0x0014DF90000 EntryPoint=0x0014DFA03C0`` and then goes - silent; an attached target disk stays byte-identical. - - No other test can catch this. test-installer-artifact and the CI - installer-test both boot the installer with ``-kernel``/``-initrd``, so the - ESP is never executed and the medium's boot path is never exercised. - """ - installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") - - assert "systemd-bootx64.efi" in installer_element, ( - "the medium must boot via systemd-boot, which reads the kernel and " - "initrd as ordinary files rather than embedding them in a PE" - ) - assert "/layer/boot/efi/loader/entries/bluefin-installer.conf" in installer_element, ( - "systemd-boot needs a type-1 loader entry naming the kernel and initrd" - ) - assert "/layer/boot/efi/bluefin/initrd" in installer_element, ( - "the initrd must be a plain file on the ESP, not a PE section" - ) - - # The build must refuse to ship an oversized fallback image rather than - # producing another medium that loads and never runs. - # Which systemd-boot ships must be deterministic. `find / -name - # systemd-bootx64.efi -print -quit` matches either the installer rootfs at - # /layer or the target rootfs staged at /target-root, and takes whichever - # the traversal reaches first — so the medium's bootloader would vary - # between builds for no visible reason. - assert "/layer/usr/lib/systemd/boot/efi/systemd-bootx64.efi" in installer_element, ( - "name the installer's own systemd-boot explicitly rather than searching" - ) - # Scan effective lines only: the element's own comment names the wrong - # spelling in order to warn against it, and matching that would flag the - # warning as the defect. - effective = "\n".join( - line - for line in installer_element.splitlines() - if not line.lstrip().startswith("#") - ) - assert not re.search(r"find / -name '?systemd-bootx64", effective), ( - "an unbounded find can pick the target rootfs copy instead of the " - "installer's, making the shipped bootloader nondeterministic" - ) - - assert "BOOT_BYTES" in installer_element and "16777216" in installer_element, ( - "the build must fail when EFI/BOOT/BOOTX64.EFI exceeds a loader-sized " - "ceiling; a 434 MiB image reached real hardware because nothing checked" - ) - - -def test_no_ukify_writes_to_the_media_fallback_path() -> None: - """The regression, stated directly. - - The target OS keeps its UKI — at 64 MiB it boots, and systemd-sysupdate is - built around replacing one. Only the *medium's* fallback path must not be a - UKI carrying the live environment. - """ - installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") - - assert not re.search( - r"ukify build[^#]*?--output=/layer/boot/efi/EFI/BOOT/BOOTX64\.EFI", - installer_element, - flags=re.DOTALL, - ), ( - "ukify must not write the medium's fallback image: with the installer " - "initrd embedded it produces a 434 MiB PE that firmware cannot execute" - ) From 4da8f9f1ad8c02f3570edb890a6196fccb01980b Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 12:45:36 -0400 Subject: [PATCH 36/46] test(installer): boot the medium through firmware, the path nothing covered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test-installer-artifact` and CI's installer-test attach installer.raw as a data disk and inject the kernel with -kernel/-initrd. EFI/BOOT/BOOTX64.EFI is never executed, so no test in this repository has ever booted the installer medium as firmware does. They prove the installer installs; they cannot prove the medium boots. `just test-installer-boot` writes the exported image into a sparse file LARGER than itself — because that is what a real stick is, and an image-sized virtual disk cannot reproduce the GPT backup-header condition — relocates the backup header as flash-installer now does, and boots it through OVMF. The assertion is structural rather than string-matching. It requires output after the last BdsDxe line, because only the kernel and its userspace can write there. An earlier attempt matched "Linux version|initrd|systemd|[ 0.000000]" and would have been useless: the medium's cmdline carries `quiet loglevel=3`, so a correct boot prints none of those, while a FAILING boot prints dracut emergency-mode text. Matching on kernel strings rewards failure. Validated both directions against captured logs: a log with nothing past handoff fails, a log with post-handoff content passes. Run against the current dist image it passes, which is correct — that image boots. Not wired into build.yml yet. The job needs KVM and a ~4 minute budget, and that belongs in a change reviewed on its own rather than folded in here. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/Justfile b/Justfile index 0f185201..91ddf10b 100644 --- a/Justfile +++ b/Justfile @@ -298,6 +298,93 @@ flash-installer DEVICE="": sudo sfdisk --verify {{DEVICE}} echo "Successfully flashed the Bluefin Server installer to {{DEVICE}}!" +# Boot the installer MEDIUM through firmware, as real hardware does. +# +# This is the only test that exercises the ESP. `test-installer-artifact` and +# CI's installer-test both attach installer.raw as a data disk and inject the +# kernel with -kernel/-initrd, so EFI/BOOT/BOOTX64.EFI is never executed. They +# prove the installer installs; they cannot prove the medium boots. +# +# That gap shipped a medium that does not boot: a single 434 MiB UKI which OVMF +# loads and never executes, with every gate green. +# +# The image is written into a sparse file LARGER than itself, because that is +# what a real stick is, and it reproduces the GPT backup-header condition that +# an image-sized virtual disk cannot. +[group('test')] +test-installer-boot: + #!/usr/bin/env bash + set -euo pipefail + + IMG=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | head -n1) + if [ -z "${IMG}" ]; then + echo "ERROR: no exported installer in dist/. Run: just export-installer" >&2 + exit 1 + fi + + first_existing() { for f in "$@"; do [ -f "$f" ] && { echo "$f"; return 0; }; done; return 1; } + OVMF_CODE=$(first_existing \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-code.fd \ + /usr/share/edk2/ovmf/OVMF_CODE.fd \ + /usr/share/OVMF/OVMF_CODE.fd \ + /usr/share/OVMF/OVMF_CODE_4M.fd \ + /usr/share/edk2/x64/OVMF_CODE.4m.fd \ + /usr/share/qemu/edk2-x86_64-code.fd \ + /usr/share/qemu/OVMF_CODE.fd) \ + || { echo "ERROR: OVMF_CODE not found" >&2; exit 1; } + + WORK=$(mktemp -d "${XDG_CACHE_HOME:-$HOME/.cache}/bluefin-boot-test.XXXXXX") + trap 'rm -rf "$WORK"' EXIT INT TERM + + # A stick is bigger than the image. Sparse, so this costs only real bytes. + truncate -s 62914560000 "$WORK/medium.img" + zstd -dc "${IMG}" | dd of="$WORK/medium.img" conv=notrunc bs=4M iflag=fullblock status=none + sfdisk --relocate gpt-bak-std "$WORK/medium.img" >/dev/null + sfdisk --verify "$WORK/medium.img" + + # OVMF needs a writable vars pflash of the same size as the code image. + truncate -s "$(stat -c %s "$OVMF_CODE")" "$WORK/vars.fd" + + DEADLINE="${INSTALLER_BOOT_DEADLINE:-240}" + echo "==> Booting the installer medium through firmware (deadline ${DEADLINE}s)..." + timeout "${DEADLINE}" qemu-system-x86_64 \ + -enable-kvm -m 4096 -smp 2 -cpu host \ + -drive file="$WORK/medium.img",format=raw,if=virtio \ + -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ + -drive if=pflash,format=raw,file="$WORK/vars.fd" \ + -nographic -serial file:"$WORK/serial.log" -monitor none -no-reboot \ + >/dev/null 2>&1 || true + + echo "==> Serial output:" + cat "$WORK/serial.log" || true + + # Firmware starting the image is not success. The old 434 MiB UKI got this + # far and then never executed, which is exactly the failure being gated. + if ! grep -q "BdsDxe: starting" "$WORK/serial.log"; then + echo "ERROR: firmware never started a boot image from the medium." >&2 + exit 1 + fi + # The kernel must actually run. Do not guess at kernel strings — asserting + # on "Linux version|initrd|systemd" passed against a log containing nothing + # but the two BdsDxe lines, which is precisely the broken case. Ask the + # structural question instead: did anything at all print after firmware + # handed control over? Only the kernel and its initrd can write there. + LAST_FW=$(grep -n '^BdsDxe:' "$WORK/serial.log" | tail -1 | cut -d: -f1 || true) + if [ -z "${LAST_FW}" ]; then + echo "ERROR: no firmware output at all; the medium was never read." >&2 + exit 1 + fi + POST=$(tail -n "+$((LAST_FW + 1))" "$WORK/serial.log" | tr -d '[:space:]' | wc -c) + if [ "${POST}" -eq 0 ]; then + echo "ERROR: firmware started the boot image, then nothing ran." >&2 + echo " Everything after line ${LAST_FW} is empty, so control never" >&2 + echo " reached the kernel. That is the 434 MiB-UKI failure:" >&2 + echo " LoadImage succeeds, StartImage never produces output." >&2 + exit 1 + fi + echo "==> The installer medium boots: firmware handed off and the kernel ran." + echo " (${POST} bytes of post-handoff output)" + # Build the installer artifacts, then run the reusable artifact smoke path. [group('test')] show-me-the-future: From 7effaadb13e024fac2950a321c626d6ef804caf6 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 14:25:21 -0400 Subject: [PATCH 37/46] test(installer): assert on evidence only userspace can emit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The boot gate's success criterion drifted twice, each time toward accepting or rejecting the wrong thing. Both attempts are recorded in the recipe so they are not retried. "Linux version|initrd|systemd" The medium boots with `quiet loglevel=3`, so a correct boot prints none of these, while a FAILING boot prints dracut emergency-mode text. The criterion rewarded failure and rejected success. any non-whitespace after handoff Firmware and systemd-stub both emit CSI cursor codes. A medium with a deliberately corrupted BOOTX64.EFI fell through to PXE, printed "PXE-E16: No valid offer received", and scored 24 bytes of "post-handoff output" — a pass on a medium that cannot boot. The criterion is now systemd's own identity record. PID 1 writes an OSC 3008 sequence to the console: ESC ]3008;start=;user=root;hostname=;machineid=;bootid=; pid=1;comm=systemd;type=boot Firmware cannot produce a machineid or a bootid, and the sequence survives `quiet`, so it is positive proof userspace came up without depending on log verbosity. Validated against three captured logs from this same harness: real dist image, quiet PASS machineid=6797d0d01c474e2fb53d50be267cd5b3 corrupted BOOTX64.EFI FAIL firmware never started an image same image, loglevel=7 PASS machineid=095873d5fbf84349b9efbc93f6e7d078 The negative case is a real medium built by mcopy-ing junk over the bootloader, not a hand-written log — the earlier hand-reconstructed log is what let the byte-counting criterion look correct. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/Justfile b/Justfile index 91ddf10b..15541630 100644 --- a/Justfile +++ b/Justfile @@ -364,26 +364,34 @@ test-installer-boot: echo "ERROR: firmware never started a boot image from the medium." >&2 exit 1 fi - # The kernel must actually run. Do not guess at kernel strings — asserting - # on "Linux version|initrd|systemd" passed against a log containing nothing - # but the two BdsDxe lines, which is precisely the broken case. Ask the - # structural question instead: did anything at all print after firmware - # handed control over? Only the kernel and its initrd can write there. - LAST_FW=$(grep -n '^BdsDxe:' "$WORK/serial.log" | tail -1 | cut -d: -f1 || true) - if [ -z "${LAST_FW}" ]; then - echo "ERROR: no firmware output at all; the medium was never read." >&2 - exit 1 - fi - POST=$(tail -n "+$((LAST_FW + 1))" "$WORK/serial.log" | tr -d '[:space:]' | wc -c) - if [ "${POST}" -eq 0 ]; then - echo "ERROR: firmware started the boot image, then nothing ran." >&2 - echo " Everything after line ${LAST_FW} is empty, so control never" >&2 - echo " reached the kernel. That is the 434 MiB-UKI failure:" >&2 - echo " LoadImage succeeds, StartImage never produces output." >&2 + # Firmware handing off is not proof that userspace ran. Assert on something + # only a running Linux userspace can emit. + # + # systemd PID 1 writes an OSC 3008 sequence to the console carrying the + # machine's identity: + # + # ESC ]3008;start=;user=root;hostname=;machineid=;bootid=; + # pid=1;comm=systemd;type=boot + # + # Firmware cannot produce a machineid or a bootid. This survives `quiet + # loglevel=3`, which is what the medium boots with, and it does not depend + # on guessing kernel log strings. + # + # Two criteria were tried and rejected, both against captured logs: + # + # "Linux version|initrd|systemd" — `quiet` suppresses all of it, while a + # FAILING boot prints dracut emergency text. It rewarded failure. + # any non-whitespace after handoff — firmware and systemd-stub both emit + # CSI cursor codes, so a corrupt bootloader falling through to PXE + # scored as success. + if ! grep -aqE 'machineid=[0-9a-f]{32}|comm=systemd' "$WORK/serial.log"; then + echo "ERROR: firmware started an image, but userspace never came up." >&2 + echo " No systemd identity record on the console. Control did not" >&2 + echo " reach PID 1." >&2 exit 1 fi - echo "==> The installer medium boots: firmware handed off and the kernel ran." - echo " (${POST} bytes of post-handoff output)" + echo "==> The installer medium boots: systemd PID 1 reported in." + grep -ao 'machineid=[0-9a-f]\{32\}' "$WORK/serial.log" | head -1 | sed 's/^/ /' # Build the installer artifacts, then run the reusable artifact smoke path. [group('test')] From 749f40a97cd8f15312948bbc9c93f9d875ea48bb Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 14:34:15 -0400 Subject: [PATCH 38/46] docs(installer): record the medium boot path and the GPT backup-header defect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things cost real time this session and none were written down. The smoke tests never boot the medium. `test-installer-artifact` and CI's `installer-test` attach installer.raw as a data disk and inject the kernel with -kernel/-initrd, so the ESP has never been executed by any test. They prove the installer installs, not that the medium boots. `just test-installer-boot` is the gate that closes this. A correct boot looks like a hang. The medium carries `quiet loglevel=3` and deliberately omits `unattended`, so it prints almost nothing and waits for an operator, while a FAILING boot prints dracut emergency-mode text. I read the silence as a hang, "confirmed" it with 34 MB and 64 MB images that only printed because they failed, and changed the installer element on that false premise. The output was there the whole time as ANSI/OSC escapes that `cat` renders invisible; `strings` showed systemd's identity record immediately. The GPT backup-header defect does not stop the medium booting. UEFI reads the primary GPT at LBA 1, which dd writes correctly. The misplaced backup header breaks tooling — mismatched partition sizes, most of the device unaccounted for — and is easy to mistake for boot health. It says nothing either way. Verified against the stick on the desk: its BOOTX64.EFI is byte-identical to the built image, its payload reports IMAGE_VERSION=26.08.0, and a replica of its exact state (58.6 GB device, backup header left misplaced, reproducing the sfdisk error) boots to systemd PID 1. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/ddi-installer.md | 81 ++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index a511f943..1a9438ae 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -173,6 +173,83 @@ offline installation. | "Store the DDI in the ESP (FAT32)." | FAT32 has a 4 GiB per-file limit. Use a separate XFS partition. | | "Add an 8 GiB minimum size floor to the DDI." | The rootfs is immutable. It never grows in-place. Content + overhead is enough. | +## What the smoke tests do not cover + +`just test-installer-artifact` and the CI `installer-test` job attach +`installer.raw` as a read-only data disk and inject the kernel directly with +`-kernel`/`-initrd` from the PXE artifacts. The medium's ESP is never executed. + +They prove the installer **installs**. They cannot prove the medium **boots**. + +`just test-installer-boot` closes that gap: it writes the exported image into a +sparse file larger than itself, relocates the GPT backup header the way +`flash-installer` does, and boots it through OVMF. + +Two things make its success criterion non-obvious, and both were got wrong +before being got right: + +- The medium boots with `quiet loglevel=3`, so a **correct** boot prints almost + nothing on serial, while a **failing** boot prints dracut emergency-mode text. + Matching on `Linux version|initrd|systemd` therefore rewards failure. +- Firmware and `systemd-stub` both emit CSI cursor sequences after handoff, so + "any output after the last `BdsDxe:` line" scores a medium whose bootloader is + corrupt — it falls through to PXE and prints `PXE-E16` — as a pass. + +The criterion is systemd's OSC 3008 identity record, which PID 1 writes +regardless of log verbosity and which firmware cannot forge: + +``` +ESC ]3008;start=;user=root;hostname=;machineid=;bootid=; + pid=1;comm=systemd;type=boot +``` + +A medium that boots correctly and waits for an operator looks identical to a +hung one on a `cat`-ed serial log. Dump the bytes before concluding anything: +`strings serial.log` or `hexdump -C`, not `cat`. + +## Writing the medium + +`dd` copies the image verbatim, which places the GPT **backup header at the end +of the image** rather than at the end of the device. On any medium larger than +the image — every real USB stick — the table is then only half valid: + +``` +$ sfdisk --verify /dev/sdX +The backup GPT table is not on the end of the device. +MyLBA mismatch with real position at backup header. + +$ parted -s /dev/sdX print +Warning: Not all of the space available to /dev/sdX appears to be used ... +``` + +**It does not stop the medium booting.** UEFI reads the *primary* GPT at LBA 1, +which `dd` writes correctly; the backup header is a redundancy copy consulted +only when the primary is damaged. A medium in this state boots normally — +verified by writing the image to a 58.6 GB sparse file without relocating, +which reproduces the `sfdisk` error exactly, and booting it: systemd PID 1 +came up. + +What it does break is tooling. `lsblk` and `udisksctl` can report different +sizes for the same partition, disk utilities show most of the device +unaccounted for, and any tool that trusts the backup header sees a stale +geometry. Partition-table health is also easy to mistake for boot health: it +says nothing either way, and reading it as a boot failure sends you chasing a +defect that isn't there. + +`just flash-installer` repairs it automatically with +`sfdisk --relocate gpt-bak-std` after the write. For a medium written with +plain `dd`, apply it by hand: + +``` +sudo sfdisk --relocate gpt-bak-std /dev/sdX +sudo partprobe /dev/sdX +sudo sfdisk --verify /dev/sdX # expect: No errors detected. +``` + +This is invisible to CI: the smoke tests attach a virtual disk sized exactly to +the image, so device size always equals image size and the condition cannot +arise. + ## Verification - [ ] `just validate` resolves the BuildStream graph without errors. @@ -188,6 +265,10 @@ offline installation. appears on the attached display even when serial is the primary console. - [ ] `bluefin-server-installer.bst` decompresses the DDI after the cpio step. - [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=yes`. +- [ ] `just test-installer-boot` passes — the medium boots through firmware, + not just installs when the kernel is injected. +- [ ] A medium written to real media has its GPT backup header relocated; + `sfdisk --verify` reports `No errors detected.` ## See also From 7894c37a718bb70a5c6aefe4716c45a4857690ab Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 14:37:48 -0400 Subject: [PATCH 39/46] fix(flash): verify the medium holds the image, and stop overstating the GPT defect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems in flash-installer, both mine. It never checked what it wrote. `sfdisk --verify` inspects the partition TABLE, not the CONTENT, so "Successfully flashed" rested on dd's exit status — which reports that writes were accepted, not that the medium kept them. That is precisely the failure mode of a dying USB stick, and it is how I came to report "flashed and verified" off partition labels, a claim that meant nothing. The payload is now hashed and measured in one pass before the write, then read back from the device and compared. The read happens before the GPT relocation, which rewrites headers and would otherwise make the digests differ for a legitimate reason, and after `blockdev --flushbufs` so the comparison reads the medium rather than the pages just written through it. O_DIRECT would also bypass the cache but requires aligned reads, and the trailing block is partial whenever the image size is not a multiple of the block size. Verified by writing a compressed payload to an oversized file, confirming the read-back matches, flipping one byte mid-image, and confirming it no longer does. The comment also claimed "firmware that validates the backup header can refuse to boot the medium outright." That was speculation stated as fact, and it is what sent me chasing a phantom for a day. UEFI reads the primary header at LBA 1, which dd writes correctly. A medium with the backup header left misplaced was written to a 58.6 GB sparse file, reproduced the sfdisk error exactly, and booted to systemd PID 1 under OVMF. The defect breaks tooling, not booting, and the comment now says so. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 47 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/Justfile b/Justfile index 15541630..6b49fa9c 100644 --- a/Justfile +++ b/Justfile @@ -268,12 +268,40 @@ flash-installer DEVICE="": exit 1 fi echo "Writing ${IMG} to {{DEVICE}}..." + # Hash and measure the payload in the same pass. Both are needed after the + # write: the byte count to know how much of the device to read back, the + # digest to compare it against. + CNTFILE=$(mktemp) + trap 'rm -f "${CNTFILE}"' EXIT + EXPECT_SHA=$(zstd -dc "${IMG}" | tee >(wc -c > "${CNTFILE}") | sha256sum | cut -d' ' -f1) + EXPECT_BYTES=$(tr -d ' ' < "${CNTFILE}") # bash with pipefail, not sh. Under POSIX sh the pipeline's status is dd's # alone, so zstd dying mid-stream leaves dd exiting 0 after writing a # partial image and the success line below printing anyway. The outer # `set -o pipefail` does not reach here — there is no pipeline in the outer # shell, only this nested one. sudo bash -c "set -o pipefail; zstd -dc '${IMG}' | dd of={{DEVICE}} bs=4M iflag=fullblock oflag=direct status=progress conv=fsync" + # Read the device back and compare. Without this, "flashed" rests on dd's + # exit status, which says the writes were accepted, not that the medium + # kept them — the failure mode of a dying USB stick. Do it before the + # relocation below, which deliberately rewrites the headers and would make + # the digests differ for a legitimate reason. + # Drop the kernel's cached view of the device first, so the comparison + # reads the medium and not the pages just written through it. O_DIRECT + # would do the same but requires aligned reads, and the trailing block is + # partial whenever the image is not a multiple of the block size. + sudo blockdev --flushbufs {{DEVICE}} + echo "Verifying ${EXPECT_BYTES} bytes read back from {{DEVICE}}..." + ACTUAL_SHA=$(sudo dd if={{DEVICE}} bs=4M iflag=fullblock,count_bytes \ + count="${EXPECT_BYTES}" status=none | sha256sum | cut -d' ' -f1) + if [ "${ACTUAL_SHA}" != "${EXPECT_SHA}" ]; then + echo "ERROR: {{DEVICE}} does not contain what was written." >&2 + echo " expected ${EXPECT_SHA}" >&2 + echo " read ${ACTUAL_SHA}" >&2 + echo "The medium did not retain the image. Replace it." >&2 + exit 1 + fi + echo "Content verified: ${EXPECT_SHA}" # Move the GPT backup header to the end of the DEVICE. # # dd writes the image verbatim, so the backup header lands at the end of the @@ -284,14 +312,19 @@ flash-installer DEVICE="": # Warning: Not all of the space available to /dev/sdb appears to be used, # you can fix the GPT to use all of the space (an extra 115791863 blocks) # - # — and tools disagree about partition sizes between reads. Firmware that - # validates the backup header can refuse to boot the medium outright. + # — and tools disagree about partition sizes between reads. + # + # This does NOT stop the medium booting, and it is worth being precise about + # that, because assuming otherwise cost a day of chasing a phantom. UEFI + # reads the PRIMARY header at LBA 1, which dd writes correctly; the backup + # is a redundancy copy consulted when the primary is damaged. A medium in + # this state was written to a 58.6 GB sparse file, reproduced the sfdisk + # error exactly, and booted to systemd PID 1 under OVMF. # - # This is invisible to every test we have. `just test-installer-artifact` - # and the CI installer-test attach the image as a drive sized exactly to the - # image, so device size always equals image size and the condition cannot - # arise. Only real media larger than the image exposes it, which is why a - # green installer-test still produced a stick with a broken table. + # What it breaks is tooling, and it is invisible to every test we have: + # `just test-installer-artifact` and the CI installer-test attach the image + # as a drive sized exactly to the image, so device size always equals image + # size and the condition cannot arise. echo "Relocating the GPT backup header to the end of {{DEVICE}}..." sudo sfdisk --relocate gpt-bak-std {{DEVICE}} sudo partprobe {{DEVICE}} || true From 2ba11924d449938a7eca1dd612a6d4cd66513033 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 14:53:55 -0400 Subject: [PATCH 40/46] fix(installer): scope the boot gate to what it actually proves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four corrections, each one an overclaim or a latent bug. The gate boots an EXPORTED ARTIFACT from dist/. It never builds, and `just validate` only resolves the graph without building, so a green run is evidence about those bytes and nothing else. The artifact on this machine was built at 09:44; commits 6d18a7c and b13d9d3 modified the installer element at 10:35. A pass was therefore attesting to a medium built from an older element than the one being shipped — the same overclaim shape this recipe exists to prevent. It now prints the artifact's sha256 and build time and warns loudly when any of its sources are newer. The recipe `cat`-ed the serial log, which is the exact mistake that cost a day: the console is almost entirely ANSI and OSC escapes, so a healthy boot renders as two firmware lines and apparent silence. It uses `strings` now. The comment above criterion 1 still asserted the 434 MiB UKI "got this far and then never executed". That was the false diagnosis, already reverted in 6e5ab29. Replaced with what criterion 1 is actually verified against — a medium with junk mcopy-ed over EFI/BOOT/BOOTX64.EFI, which yields "BdsDxe: failed to load ... Not Found" and no `starting` line. The comment also now records that criterion 2 cannot stand alone: PXE fallback chatter is ordinary printable text, so only criterion 1 preceding it makes the pair sound. Criterion 2 remains unproven against a genuine start-then-hang, because no such image exists now that the UKI is exonerated. flash-installer measured the payload with `tee >(wc -c > file)`. Bash does not wait for process-substitution children, so the count could be unread when $( ) returns; it happens to work only because wc outruns sha256sum. An empty count yields a malformed `count=` and fails a good flash with "medium did not retain the image". Two plain passes now, temp file and trap deleted. Docs record the silent-working/noisy-broken inversion, the loglevel=7 SMBIOS override for forcing verbosity without a rebuild, and the gate's artifact-only scope. MVP readiness downgraded to in-progress for the same reason. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 77 ++++++++++++++++++++++++----- docs/MVP_1_0_READINESS.md | 8 ++- docs/skills/ddi-installer.md | 94 ++++++++++++++++++++++-------------- 3 files changed, 128 insertions(+), 51 deletions(-) diff --git a/Justfile b/Justfile index 6b49fa9c..40c5a1c5 100644 --- a/Justfile +++ b/Justfile @@ -268,13 +268,20 @@ flash-installer DEVICE="": exit 1 fi echo "Writing ${IMG} to {{DEVICE}}..." - # Hash and measure the payload in the same pass. Both are needed after the - # write: the byte count to know how much of the device to read back, the - # digest to compare it against. - CNTFILE=$(mktemp) - trap 'rm -f "${CNTFILE}"' EXIT - EXPECT_SHA=$(zstd -dc "${IMG}" | tee >(wc -c > "${CNTFILE}") | sha256sum | cut -d' ' -f1) - EXPECT_BYTES=$(tr -d ' ' < "${CNTFILE}") + # Digest and byte count of the payload. Both are needed after the write: + # the count to know how much of the device to read back, the digest to + # compare it against. + # + # Two plain passes rather than one with `tee >(wc -c > file)`. Bash does + # not wait for process-substitution children, so the count file may still + # be unwritten when $( ) returns — it happens to work only because wc + # finishes before sha256sum. An empty count yields a malformed `count=` + # and fails the flash with a misleading "medium did not retain the image". + # The extra decompression costs seconds against a write measured in + # minutes. + echo "Hashing ${IMG}..." + EXPECT_SHA=$(zstd -dc "${IMG}" | sha256sum | cut -d' ' -f1) + EXPECT_BYTES=$(zstd -dc "${IMG}" | wc -c | tr -d ' ') # bash with pipefail, not sh. Under POSIX sh the pipeline's status is dd's # alone, so zstd dying mid-stream leaves dd exiting 0 after writing a # partial image and the success line below printing anyway. The outer @@ -355,6 +362,32 @@ test-installer-boot: exit 1 fi + # This boots an EXPORTED ARTIFACT. It does not build, and `just validate` + # only resolves the graph without building either. So a green result is + # evidence about the bytes in dist/, NOT about the current state of + # elements/oci/bluefin-server-installer.bst. + # + # Say so out loud, and refuse to imply more than that when the element has + # moved since the export. Without this, "the medium boots" quietly becomes + # a claim about a tree that was never built — the same overclaim this + # recipe exists to prevent. + echo "==> Artifact under test:" + echo " ${IMG}" + echo " sha256 $(sha256sum "${IMG}" | cut -d' ' -f1)" + echo " built $(date -r "${IMG}" '+%Y-%m-%d %H:%M:%S')" + STALE="" + for src in elements/oci/bluefin-server-installer.bst elements/bluefin-server/os-stack.bst include/kubernetes.yml; do + [ -f "${src}" ] || continue + [ "${src}" -nt "${IMG}" ] && STALE="${STALE} ${src}" + done + if [ -n "${STALE}" ]; then + echo "WARNING: these sources are NEWER than the artifact:" >&2 + for s in ${STALE}; do echo " ${s}" >&2; done + echo " A pass proves the exported medium boots. It proves" >&2 + echo " NOTHING about the current element state. Re-export" >&2 + echo " before treating this as a gate on your changes." >&2 + fi + first_existing() { for f in "$@"; do [ -f "$f" ] && { echo "$f"; return 0; }; done; return 1; } OVMF_CODE=$(first_existing \ /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-code.fd \ @@ -388,12 +421,24 @@ test-installer-boot: -nographic -serial file:"$WORK/serial.log" -monitor none -no-reboot \ >/dev/null 2>&1 || true + # `strings`, not `cat`. The console is almost entirely ANSI and OSC escape + # sequences, which a terminal swallows — `cat` on a perfectly good boot log + # shows two firmware lines and apparent silence. That is what convinced me + # a working medium was hung. echo "==> Serial output:" - cat "$WORK/serial.log" || true + strings -n 4 "$WORK/serial.log" | grep -av '^\[[0-9;]*[A-Za-z]$' || true - # Firmware starting the image is not success. The old 434 MiB UKI got this - # far and then never executed, which is exactly the failure being gated. - if ! grep -q "BdsDxe: starting" "$WORK/serial.log"; then + # Criterion 1: firmware must actually start an image, not merely find the + # medium. A corrupted bootloader makes firmware fall through to PXE, and + # this is the check that catches it — verified against a medium built by + # mcopy-ing junk over EFI/BOOT/BOOTX64.EFI, which produced + # "BdsDxe: failed to load ... Not Found" and no `starting` line. + # + # This check also carries criterion 2 below: PXE fallback chatter + # ("Start PXE over IPv4", "PXE-E16: No valid offer received") is ordinary + # printable text, so a content check alone cannot tell it from a booting + # kernel. Criterion 2 is only sound because this one precedes it. + if ! grep -aq "BdsDxe: starting" "$WORK/serial.log"; then echo "ERROR: firmware never started a boot image from the medium." >&2 exit 1 fi @@ -409,14 +454,20 @@ test-installer-boot: # Firmware cannot produce a machineid or a bootid. This survives `quiet # loglevel=3`, which is what the medium boots with, and it does not depend # on guessing kernel log strings. - # - # Two criteria were tried and rejected, both against captured logs: + # Two earlier criteria were tried and rejected, both against captured logs: # # "Linux version|initrd|systemd" — `quiet` suppresses all of it, while a # FAILING boot prints dracut emergency text. It rewarded failure. # any non-whitespace after handoff — firmware and systemd-stub both emit # CSI cursor codes, so a corrupt bootloader falling through to PXE # scored as success. + # + # Honest limit: this check is proven against a booting medium (it fires on + # the real image under both `quiet` and loglevel=7) and against a medium + # whose bootloader is destroyed (criterion 1 rejects that before reaching + # here). It is NOT proven against a genuine start-then-hang, because no + # such image exists — the 434 MiB UKI was suspected of it and exonerated. + # If one ever appears, keep it. if ! grep -aqE 'machineid=[0-9a-f]{32}|comm=systemd' "$WORK/serial.log"; then echo "ERROR: firmware started an image, but userspace never came up." >&2 echo " No systemd identity record on the console. Control did not" >&2 diff --git a/docs/MVP_1_0_READINESS.md b/docs/MVP_1_0_READINESS.md index 6d086092..a9fd7a68 100644 --- a/docs/MVP_1_0_READINESS.md +++ b/docs/MVP_1_0_READINESS.md @@ -19,7 +19,8 @@ This audit tracks the gap between the current tree and a first public/usable MVP | Release workflow lint | ✅ | `actionlint .github/workflows/build.yml` clean | | Release path exists | ✅ | `.github/workflows/build.yml` builds, signs, uploads to GitHub Release | | Cluster build pipeline | ✅ | Phase A complete; pipeline builds DDI, installer, and Kubernetes sysext with uutils | -| Automated boot test | 🔄 | Phase B in progress for Alpha; `bluefin-server-boot-test` workflow running on lab cluster | +| Automated boot test (installed OS) | 🔄 | Phase B in progress for Alpha; `bluefin-server-boot-test` workflow running on lab cluster | +| Automated boot test (install medium) | 🔄 | `just test-installer-boot` boots the medium through OVMF and requires systemd PID 1's OSC 3008 identity record. Validates an **exported artifact** from `dist/`, not the current element state, and is not yet wired into CI — it warns when sources are newer than the artifact | | A/B root rollback | ❌ | `50-root.transfer` names `root-a`/`root-b`, installer only creates `root-a` | | Root immutability | ❌ | DDI boots read/write (`rw` on cmdline) | | First-boot SSH keys | ❌ | Only root password credential path exists | @@ -46,6 +47,11 @@ Priority order. Each item depends on the ones above it. ### Phase B: automated boot verification (In progress for Alpha) - [x] Create `bluefin-server-boot-test` Argo workflow in the downstream factory CI repository. +- [x] Cover the install medium's own boot path (`just test-installer-boot`). + Nothing previously executed the ESP: `test-installer-artifact` and CI's + `installer-test` attach `installer.raw` as a data disk and inject the + kernel with `-kernel`/`-initrd`, proving the installer installs but never + that the medium boots. - [ ] Run the workflow against a successful installer build on the lab cluster and iterate to green. - [ ] Wire the boot test into a post-merge CI gate or CronWorkflow. diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index 1a9438ae..3561ebe0 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -177,25 +177,20 @@ offline installation. `just test-installer-artifact` and the CI `installer-test` job attach `installer.raw` as a read-only data disk and inject the kernel directly with -`-kernel`/`-initrd` from the PXE artifacts. The medium's ESP is never executed. +`-kernel`/`-initrd` from the PXE artifacts. The medium's ESP is never executed: +they prove the installer **installs**, never that the medium **boots**. -They prove the installer **installs**. They cannot prove the medium **boots**. - -`just test-installer-boot` closes that gap: it writes the exported image into a -sparse file larger than itself, relocates the GPT backup header the way +`just test-installer-boot` closes that gap — it writes the exported image into +a sparse file larger than itself, relocates the GPT backup header the way `flash-installer` does, and boots it through OVMF. -Two things make its success criterion non-obvious, and both were got wrong -before being got right: - -- The medium boots with `quiet loglevel=3`, so a **correct** boot prints almost - nothing on serial, while a **failing** boot prints dracut emergency-mode text. - Matching on `Linux version|initrd|systemd` therefore rewards failure. -- Firmware and `systemd-stub` both emit CSI cursor sequences after handoff, so - "any output after the last `BdsDxe:` line" scores a medium whose bootloader is - corrupt — it falls through to PXE and prints `PXE-E16` — as a pass. +Note its scope: it boots an **exported artifact** from `dist/`. It does not +build, and `just validate` only resolves the graph without building. A green +run is evidence about those bytes, not the current element state; the recipe +warns when sources are newer than the artifact. Re-export before treating it +as a gate on an element change. -The criterion is systemd's OSC 3008 identity record, which PID 1 writes +Its success criterion is systemd's OSC 3008 identity record, which PID 1 writes regardless of log verbosity and which firmware cannot forge: ``` @@ -203,9 +198,37 @@ ESC ]3008;start=;user=root;hostname=;machineid=;bootid=; pid=1;comm=systemd;type=boot ``` -A medium that boots correctly and waits for an operator looks identical to a -hung one on a `cat`-ed serial log. Dump the bytes before concluding anything: -`strings serial.log` or `hexdump -C`, not `cat`. +### A working medium is silent; a broken one is noisy + +This inversion is the most expensive thing in this document, and the easiest to +forget because it is counter-intuitive. + +A **correct** boot prints essentially nothing. `quiet loglevel=3` suppresses +the kernel, and the medium deliberately omits `unattended`, so it comes up and +waits for an operator. A **failing** boot is loud: dracut emergency mode, +"Cannot find /usr", and `Dependency failed for ...` all print *through* +`quiet`, because error paths bypass it. + +So silence is not evidence of failure and output is not evidence of success — +both readings are backwards, and reading them that way cost a day spent +reverting a healthy installer. + +Two habits make it unambiguous: + +1. **Never `cat` the serial log.** It is nearly all ANSI and OSC escapes, which + the terminal swallows, so a healthy boot renders as two firmware lines and + apparent silence. Use `strings serial.log` or `hexdump -C serial.log`; the + systemd identity record is plainly visible that way. +2. **Force verbosity when in doubt.** Add this to the `test-installer-boot` + QEMU invocation to make a healthy boot as loud as a broken one: + + ``` + -smbios type=11,value=io.systemd.stub.kernel-cmdline-extra=loglevel=7 + ``` + + `systemd-stub` reads the SMBIOS type-11 string and appends it to the + embedded cmdline, so this needs no rebuild. A healthy medium then produces + tens of kilobytes of ordinary boot log. ## Writing the medium @@ -223,22 +246,20 @@ Warning: Not all of the space available to /dev/sdX appears to be used ... ``` **It does not stop the medium booting.** UEFI reads the *primary* GPT at LBA 1, -which `dd` writes correctly; the backup header is a redundancy copy consulted -only when the primary is damaged. A medium in this state boots normally — -verified by writing the image to a 58.6 GB sparse file without relocating, -which reproduces the `sfdisk` error exactly, and booting it: systemd PID 1 -came up. - -What it does break is tooling. `lsblk` and `udisksctl` can report different -sizes for the same partition, disk utilities show most of the device -unaccounted for, and any tool that trusts the backup header sees a stale -geometry. Partition-table health is also easy to mistake for boot health: it -says nothing either way, and reading it as a boot failure sends you chasing a -defect that isn't there. - -`just flash-installer` repairs it automatically with -`sfdisk --relocate gpt-bak-std` after the write. For a medium written with -plain `dd`, apply it by hand: +which `dd` writes correctly; the backup is a redundancy copy consulted only +when the primary is damaged. Verified: the image written to a 58.6 GB sparse +file without relocating reproduces the `sfdisk` error exactly and still boots +to systemd PID 1. + +What it breaks is tooling — `lsblk` and `udisksctl` disagreeing about partition +sizes, disk utilities showing most of the device unaccounted for. Partition- +table health says nothing about boot health in either direction; reading it as +a boot failure sends you chasing a defect that isn't there. + +`just flash-installer` repairs it with `sfdisk --relocate gpt-bak-std` after +the write, and also reads the device back and compares digests, so a medium +that silently fails to retain the image is caught. For a medium written with +plain `dd`, relocate by hand: ``` sudo sfdisk --relocate gpt-bak-std /dev/sdX @@ -246,9 +267,8 @@ sudo partprobe /dev/sdX sudo sfdisk --verify /dev/sdX # expect: No errors detected. ``` -This is invisible to CI: the smoke tests attach a virtual disk sized exactly to -the image, so device size always equals image size and the condition cannot -arise. +Invisible to CI: the smoke tests attach a virtual disk sized exactly to the +image, so device size always equals image size and the condition cannot arise. ## Verification From fb819b0f0bb0611c56225887e7c88efa6c759564 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 20 Sep 2026 15:03:01 -0400 Subject: [PATCH 41/46] fix(installer): make staleness detection deterministic, drop a stale rationale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three corrections to the gate I committed an hour ago. The staleness guard keyed on mtimes against a hardcoded triple of sources. Both halves were wrong. mtimes cannot answer "what was this built from": a fresh clone stamps every file at checkout time, and a rebase rewrites commit dates — this branch has been rebased — so the signal reports a clean tree as stale and a stale tree as clean. The allow-list was worse: editing files/installer/repart.d/*.conf, files/os/**, or elements/flatcar/*.bst left a stale artifact with no warning at all, which is the precise false confidence the guard exists to prevent. `export-installer` now writes a provenance sidecar recording the commit and a content hash of every file under elements/, include/, files/ and project.conf. `test-installer-boot` compares that against the current tree. Deterministic, survives clones and rebases, no allow-list to drift, 70ms to compute. When no sidecar exists — as for the artifact currently in dist/ — it says so rather than implying a clean comparison. The comment claiming criterion 2 "is only sound because criterion 1 precedes it" was stale the moment it was written. That was true of the retired byte-counting criterion, which PXE fallback chatter could satisfy. Criterion 2 is now the machineid/comm=systemd match, and the corrupt-bootloader log was checked directly against it: no match. It stands alone. Criterion 1 is kept because it names the failure precisely, not because criterion 2 needs it. Asserting a dependency that no longer exists is the same trap I had just removed two lines above. Forcing verbose output is now `INSTALLER_BOOT_CMDLINE_EXTRA=loglevel=7 just test-installer-boot`, mirroring INSTALLER_BOOT_DEADLINE. The doc previously printed a raw qemu invocation that would not run: it hardcoded an OVMF path the recipe deliberately discovers because it is distro-dependent, and omitted that vars.fd must be created at the code image's exact size. Verified both ways — default still passes, and the knob produces 51 KB of kernel log against ~850 bytes quiet. Assisted-by: Claude Opus 4.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 108 +++++++++++++++++++++++++++-------- docs/MVP_1_0_READINESS.md | 5 +- docs/skills/ddi-installer.md | 11 ++-- 3 files changed, 94 insertions(+), 30 deletions(-) diff --git a/Justfile b/Justfile index 40c5a1c5..27766c2f 100644 --- a/Justfile +++ b/Justfile @@ -124,8 +124,38 @@ export-installer: build-installer just bst artifact checkout oci/bluefin-server-installer.bst --directory /src/dist/installer-checkout mv dist/installer-checkout/* dist/ rm -rf dist/installer-checkout + # Record what this artifact was built from. mtimes cannot answer that: a + # fresh clone stamps every file at checkout time, and a rebase rewrites + # commit dates, so both signals lie. A content hash of the build inputs + # does not. + @just _record-provenance @echo "==> wrote:" && ls -lh dist/ +# Write the provenance sidecar naming the sources an export was built from. +[private] +_record-provenance: + #!/usr/bin/env bash + set -euo pipefail + IMG=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' -print -quit) + [ -n "${IMG}" ] || exit 0 + { + echo "commit $(git rev-parse HEAD 2>/dev/null || echo unknown)" + echo "tree $(just _inputs-hash)" + } > "${IMG}.provenance" + echo "==> provenance: $(sed -n 2p "${IMG}.provenance")" + +# Content hash of everything the installer image is built from. Deterministic +# across clones and rebases, unlike mtimes or commit dates. +[private] +_inputs-hash: + #!/usr/bin/env bash + set -euo pipefail + find elements include files project.conf -type f 2>/dev/null \ + | LC_ALL=C sort \ + | xargs -r sha256sum \ + | sha256sum \ + | cut -d' ' -f1 + # Export standalone kernel and initrd for PXE boot. The DDI remains embedded # in the raw installer image; network DDI fetching is not enabled. [group('installer')] @@ -367,25 +397,40 @@ test-installer-boot: # evidence about the bytes in dist/, NOT about the current state of # elements/oci/bluefin-server-installer.bst. # - # Say so out loud, and refuse to imply more than that when the element has + # Say so out loud, and refuse to imply more than that when the tree has # moved since the export. Without this, "the medium boots" quietly becomes # a claim about a tree that was never built — the same overclaim this # recipe exists to prevent. + # + # The comparison is a content hash of the build inputs, recorded into a + # sidecar by `export-installer`. Not mtimes: a fresh clone stamps every + # file at checkout time and a rebase rewrites commit dates, so both would + # report a clean tree as stale and a stale tree as clean. Not an allow-list + # of "important" sources either — that drifts the moment someone edits a + # repart config or a unit file. echo "==> Artifact under test:" echo " ${IMG}" echo " sha256 $(sha256sum "${IMG}" | cut -d' ' -f1)" - echo " built $(date -r "${IMG}" '+%Y-%m-%d %H:%M:%S')" - STALE="" - for src in elements/oci/bluefin-server-installer.bst elements/bluefin-server/os-stack.bst include/kubernetes.yml; do - [ -f "${src}" ] || continue - [ "${src}" -nt "${IMG}" ] && STALE="${STALE} ${src}" - done - if [ -n "${STALE}" ]; then - echo "WARNING: these sources are NEWER than the artifact:" >&2 - for s in ${STALE}; do echo " ${s}" >&2; done - echo " A pass proves the exported medium boots. It proves" >&2 - echo " NOTHING about the current element state. Re-export" >&2 - echo " before treating this as a gate on your changes." >&2 + if [ -f "${IMG}.provenance" ]; then + BUILT_FROM=$(awk '/^tree /{print $2}' "${IMG}.provenance") + BUILT_AT=$(awk '/^commit /{print $2}' "${IMG}.provenance") + echo " commit ${BUILT_AT}" + NOW=$(just _inputs-hash) + if [ "${BUILT_FROM}" != "${NOW}" ]; then + echo "WARNING: the tree has changed since this artifact was built." >&2 + echo " built from ${BUILT_FROM}" >&2 + echo " tree now ${NOW}" >&2 + echo " A pass proves the exported medium boots. It proves" >&2 + echo " NOTHING about the current sources. Re-export before" >&2 + echo " treating this as a gate on your changes." >&2 + else + echo " tree matches current sources" + fi + else + echo "WARNING: no provenance sidecar beside this artifact, so there is" >&2 + echo " no way to tell what it was built from. A pass proves" >&2 + echo " these bytes boot and nothing more. Re-export to record" >&2 + echo " provenance." >&2 fi first_existing() { for f in "$@"; do [ -f "$f" ] && { echo "$f"; return 0; }; done; return 1; } @@ -412,12 +457,28 @@ test-installer-boot: truncate -s "$(stat -c %s "$OVMF_CODE")" "$WORK/vars.fd" DEADLINE="${INSTALLER_BOOT_DEADLINE:-240}" + # A healthy medium boots with `quiet loglevel=3` and prints almost nothing, + # which is indistinguishable from a hang until you look closely. Setting + # INSTALLER_BOOT_CMDLINE_EXTRA=loglevel=7 makes a healthy boot loud without + # rebuilding anything: systemd-stub reads this SMBIOS type-11 string and + # appends it to the UKI's embedded cmdline. + CMDLINE_EXTRA="${INSTALLER_BOOT_CMDLINE_EXTRA:-}" + # if-blocks, not `[ -n x ] && y`: under `set -e` a false test makes the + # whole recipe exit, so the default (no extra cmdline) would abort the run. + SMBIOS=() + if [ -n "${CMDLINE_EXTRA}" ]; then + SMBIOS=(-smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=${CMDLINE_EXTRA}") + fi echo "==> Booting the installer medium through firmware (deadline ${DEADLINE}s)..." + if [ -n "${CMDLINE_EXTRA}" ]; then + echo " cmdline-extra: ${CMDLINE_EXTRA}" + fi timeout "${DEADLINE}" qemu-system-x86_64 \ -enable-kvm -m 4096 -smp 2 -cpu host \ -drive file="$WORK/medium.img",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$WORK/vars.fd" \ + "${SMBIOS[@]}" \ -nographic -serial file:"$WORK/serial.log" -monitor none -no-reboot \ >/dev/null 2>&1 || true @@ -434,10 +495,11 @@ test-installer-boot: # mcopy-ing junk over EFI/BOOT/BOOTX64.EFI, which produced # "BdsDxe: failed to load ... Not Found" and no `starting` line. # - # This check also carries criterion 2 below: PXE fallback chatter - # ("Start PXE over IPv4", "PXE-E16: No valid offer received") is ordinary - # printable text, so a content check alone cannot tell it from a booting - # kernel. Criterion 2 is only sound because this one precedes it. + # Criterion 2 below does not depend on this one: PXE fallback chatter + # cannot produce a machineid, and was checked against the corrupt medium's + # log — no match. Criterion 1 is kept because it names the failure + # precisely ("firmware never started an image" vs "userspace never came + # up"), which is the difference between a dead bootloader and a dead boot. if ! grep -aq "BdsDxe: starting" "$WORK/serial.log"; then echo "ERROR: firmware never started a boot image from the medium." >&2 exit 1 @@ -462,12 +524,12 @@ test-installer-boot: # CSI cursor codes, so a corrupt bootloader falling through to PXE # scored as success. # - # Honest limit: this check is proven against a booting medium (it fires on - # the real image under both `quiet` and loglevel=7) and against a medium - # whose bootloader is destroyed (criterion 1 rejects that before reaching - # here). It is NOT proven against a genuine start-then-hang, because no - # such image exists — the 434 MiB UKI was suspected of it and exonerated. - # If one ever appears, keep it. + # Honest limit: proven against a booting medium (fires on the real image + # under both `quiet` and loglevel=7) and against a medium whose bootloader + # is destroyed (its PXE-fallback log contains no machineid). NOT proven + # against a genuine start-then-hang, because no such image exists — the + # 434 MiB UKI was suspected of being one and was exonerated. If one ever + # turns up, keep it. if ! grep -aqE 'machineid=[0-9a-f]{32}|comm=systemd' "$WORK/serial.log"; then echo "ERROR: firmware started an image, but userspace never came up." >&2 echo " No systemd identity record on the console. Control did not" >&2 diff --git a/docs/MVP_1_0_READINESS.md b/docs/MVP_1_0_READINESS.md index a9fd7a68..634dee10 100644 --- a/docs/MVP_1_0_READINESS.md +++ b/docs/MVP_1_0_READINESS.md @@ -51,7 +51,10 @@ Priority order. Each item depends on the ones above it. Nothing previously executed the ESP: `test-installer-artifact` and CI's `installer-test` attach `installer.raw` as a data disk and inject the kernel with `-kernel`/`-initrd`, proving the installer installs but never - that the medium boots. + that the medium boots. **Local only, not wired into CI**, and it boots an + exported artifact rather than building — it warns when the tree has moved + since the export. This does not satisfy the MVP bar item above, which + asks that the *installed OS* reach a target. - [ ] Run the workflow against a successful installer build on the lab cluster and iterate to green. - [ ] Wire the boot test into a post-merge CI gate or CronWorkflow. diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index 3561ebe0..bcc85b0f 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -219,16 +219,15 @@ Two habits make it unambiguous: the terminal swallows, so a healthy boot renders as two firmware lines and apparent silence. Use `strings serial.log` or `hexdump -C serial.log`; the systemd identity record is plainly visible that way. -2. **Force verbosity when in doubt.** Add this to the `test-installer-boot` - QEMU invocation to make a healthy boot as loud as a broken one: +2. **Force verbosity when in doubt:** ``` - -smbios type=11,value=io.systemd.stub.kernel-cmdline-extra=loglevel=7 + INSTALLER_BOOT_CMDLINE_EXTRA=loglevel=7 just test-installer-boot ``` - `systemd-stub` reads the SMBIOS type-11 string and appends it to the - embedded cmdline, so this needs no rebuild. A healthy medium then produces - tens of kilobytes of ordinary boot log. + `systemd-stub` reads this as an SMBIOS type-11 string and appends it to the + UKI's embedded cmdline, so it needs no rebuild. A healthy medium then + produces tens of kilobytes of ordinary boot log. ## Writing the medium From 95d71b78df7529be8f99130be72cfc2bf3815c5b Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Mon, 21 Sep 2026 01:13:26 -0400 Subject: [PATCH 42/46] fix(flash): keep the self-disk guard live on a whole-disk root `lsblk -no PKNAME` returns nothing when the filesystem sits directly on a whole-disk device, so the running-system guard skipped its comparison and left only the y/N prompt protecting the system disk. Fall back to KNAME when PKNAME is empty, and cover both the refusal and the unrelated-disk path in the bats suite. Also correct the 30-var.conf rationale: containerd-sysext.bst repacks the extension-release to ID=_any with no VERSION_ID, so the merge is not gated on os-release and a flatcar-version bump only needs the source ref re-pinned. And make the k8s role guard's message match its behaviour by documenting the accepted "server" alias instead of naming only "controller". --- Justfile | 7 +++++ files/installer/repart.d/30-var.conf | 12 ++++---- files/os/justfile | 9 ++++-- tests/unit/flash-installer_test.bats | 43 +++++++++++++++++++++++++++- 4 files changed, 62 insertions(+), 9 deletions(-) diff --git a/Justfile b/Justfile index 27766c2f..dca9c853 100644 --- a/Justfile +++ b/Justfile @@ -240,7 +240,14 @@ flash-installer DEVICE="": for mp in / /sysroot; do SRC=$(findmnt -no SOURCE "${mp}" 2>/dev/null | head -n1 || true) case "${SRC}" in /dev/*) ;; *) continue ;; esac + # PKNAME is the parent disk of a partition, and is empty when the + # filesystem sits directly on a whole-disk device (no partition table, + # or a device-mapper/loop node). Falling back to KNAME keeps the guard + # live on those hosts instead of silently skipping the comparison. SRC_DISK=$(lsblk -no PKNAME "${SRC}" 2>/dev/null | head -n1 || true) + if [ -z "${SRC_DISK}" ]; then + SRC_DISK=$(lsblk -no KNAME "${SRC}" 2>/dev/null | head -n1 || true) + fi if [ -n "${SRC_DISK}" ] && [ "${SRC_DISK}" = "${TARGET_NAME}" ]; then echo "ERROR: {{DEVICE}} is the disk backing the running system." >&2 echo " ${mp} is on ${SRC}, which lives on /dev/${SRC_DISK}." >&2 diff --git a/files/installer/repart.d/30-var.conf b/files/installer/repart.d/30-var.conf index a3c35205..586c2485 100644 --- a/files/installer/repart.d/30-var.conf +++ b/files/installer/repart.d/30-var.conf @@ -26,11 +26,13 @@ # containerd.raw would make sysext look for extension-release.containerd and # refuse the extension at boot. # -# The merge is additionally gated on os-release: that file declares ID=flatcar -# and VERSION_ID=4593.2.5, which must equal flatcar-version in -# include/flatcar.yml. Unlike our own kubernetes sysext (ID=_any), this one is -# version-locked to the Flatcar release, so bumping flatcar-version requires -# re-pinning the containerd image in the same change. +# The image is repacked before it is seeded: elements/flatcar/containerd-sysext.bst +# rewrites the extension-release to ID=_any and drops VERSION_ID, keeping the +# Flatcar release only in SYSEXT_VERSION_ID. So the merge is *not* gated on our +# os-release, and tests/unit/test_seeded_sysext_names.py asserts that decoupling. +# Bumping flatcar-version in include/flatcar.yml therefore only requires +# re-pinning the source ref of that element; it will not desynchronise the merge +# on hosts already carrying the old image. [Partition] Type=var Label=var diff --git a/files/os/justfile b/files/os/justfile index e08644e7..9e1451d5 100644 --- a/files/os/justfile +++ b/files/os/justfile @@ -3,14 +3,17 @@ default: @just --list -# Bring up the single-node Kubernetes control plane on this host +# Bring up the single-node Kubernetes control plane on this host (ROLE: controller, or its alias server) k8s ROLE="controller": #!/usr/bin/env bash set -euo pipefail - # 0. Reject unsupported roles before touching anything on the host. + # 0. Reject unsupported roles before touching anything on the host. This is + # a single-node control plane, so "controller" is the only real role; + # "server" is accepted as a backwards-compatible alias for it, and + # tests/unit/os-justfile_test.bats pins both spellings. if [ "{{ROLE}}" != "controller" ] && [ "{{ROLE}}" != "server" ]; then - echo "ERROR: Invalid role '{{ROLE}}'. Supported: controller" >&2 + echo "ERROR: Invalid role '{{ROLE}}'. Supported: controller (alias: server)" >&2 exit 1 fi diff --git a/tests/unit/flash-installer_test.bats b/tests/unit/flash-installer_test.bats index e46fcfb1..ed567d67 100644 --- a/tests/unit/flash-installer_test.bats +++ b/tests/unit/flash-installer_test.bats @@ -63,6 +63,7 @@ setup() { export STUB_ROOT_SRC="composefs:0000" export STUB_SYSROOT_SRC="/dev/nvme0n1p2" export STUB_PKNAME="nvme0n1" + export STUB_SRC_KNAME="" export STUB_TARGET_KNAME="sdz" export STUB_MOUNTPOINTS="" # zstd -t succeeds unless a test says otherwise. @@ -93,7 +94,17 @@ make_topology_stubs() { #!/usr/bin/env bash echo "lsblk \$*" >> "${LOG}" case " \$* " in - *" KNAME "*) printf '%s\n' "\${STUB_TARGET_KNAME:-}" ;; + *" KNAME "*) + # The recipe asks for KNAME twice: once for the target device, and + # once for the / or /sysroot source when PKNAME came back empty. + # Answer by argument so a whole-disk root is distinguishable from + # the flash target. + for arg in "\$@"; do last="\$arg"; done + case "\${last}" in + /dev/*) printf '%s\n' "\${STUB_SRC_KNAME:-}" ;; + *) printf '%s\n' "\${STUB_TARGET_KNAME:-}" ;; + esac + ;; *" PKNAME "*) printf '%s\n' "\${STUB_PKNAME:-}" ;; *MOUNTPOINTS*) printf '%s\n' "\${STUB_MOUNTPOINTS:-}" ;; esac @@ -245,6 +256,36 @@ refute_log() { assert_nothing_written } +@test "flash-installer refuses a whole-disk root that PKNAME cannot resolve" { + # A filesystem directly on a whole-disk device has no parent, so + # `lsblk -no PKNAME` prints nothing. Without the KNAME fallback the + # comparison is skipped and the guard is inert on exactly the host it + # most needs to protect. + export STUB_ROOT_SRC="/dev/sda" + export STUB_SYSROOT_SRC="" + export STUB_PKNAME="" + export STUB_SRC_KNAME="sda" + export STUB_TARGET_KNAME="sda" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"disk backing the running system"* ]] + [[ "$output" == *"/ is on /dev/sda"* ]] + assert_nothing_written +} + +@test "flash-installer allows an unrelated disk when root is on a whole-disk device" { + export STUB_ROOT_SRC="/dev/sda" + export STUB_SYSROOT_SRC="" + export STUB_PKNAME="" + export STUB_SRC_KNAME="sda" + export STUB_TARGET_KNAME="sdb" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -eq 0 ] + assert_log "of=${FAKE_DEV}" +} + @test "flash-installer allows an unrelated disk while the system disk is known" { export STUB_TARGET_KNAME="sdb" seed_image "bluefin-server-installer-1.0.raw.zst" From c523bbbde5dfb783802b3528fc67418e022d238f Mon Sep 17 00:00:00 2001 From: "sec-check[bot]" Date: Mon, 21 Sep 2026 01:54:18 -0400 Subject: [PATCH 43/46] fix(cluster): stop the repo unit stranding the seed, and refresh CNI on update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bluefin-cluster-bootstrap.service declares Requires=bluefin-cluster-repo.service, and a job killed with result 'dependency' never runs, so the dependent's own Restart=on-failure never engages. The repo unit had no Restart=, so one transient publish failure stranded the seed permanently — the same failure mode the Wants= edge on kubeadm-init.service exists to prevent. Give the repo unit Restart=on-failure/RestartSec=30s; the publish is idempotent. /opt/cni/bin lives on /var and outlives a sysext update, so the unconditional `cp -an` bridge pinned the node to the plugins it first booted with while /usr/local/share/kubernetes-cni-version advertised newer ones. Key the copy on that stamp: refresh when it changes, no-clobber when it does not. Cilium is safe either way — cilium-cni is not a name the sysext ships. Documentation and hardening, from the same review: - The ops runbook named deploy/gitd in argocd; the deployment is bluefin-git-daemon in bluefin-system. It also named Application 'root', which is 'bluefin-cluster', and restated the reverted Requires= design for the bootstrap unit. - console.yaml now takes the pod hardening baseline its siblings set (runAsNonRoot with the image's numeric uid 1001, RuntimeDefault seccomp) and documents why readOnlyRootFilesystem is a deliberate exception: the image writes /app/data/console.db on its own root filesystem. - Record the kiosk resolver's coupling to serviceSubnet in both files, and the known LAN SAN gap on the kiosk certificate. Assisted-by: Claude Opus 5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/kubernetes-sysext-ops.md | 17 +++++++++++------ files/cluster/manifests/console/console.yaml | 18 ++++++++++++++++++ .../cluster/manifests/kiosk/assets/nginx.conf | 6 ++++++ files/cluster/manifests/kiosk/certificate.yaml | 7 +++++++ files/kubernetes/kubeadm.yaml | 4 ++++ .../system/bluefin-cluster-repo.service | 16 ++++++++++++++++ files/os/systemd/system/kubelet.service | 16 ++++++++++++++-- tests/unit/test_unit_dependency_provenance.py | 15 +++++++++++++-- 8 files changed, 89 insertions(+), 10 deletions(-) diff --git a/docs/skills/kubernetes-sysext-ops.md b/docs/skills/kubernetes-sysext-ops.md index 54636177..2244d7fb 100644 --- a/docs/skills/kubernetes-sysext-ops.md +++ b/docs/skills/kubernetes-sysext-ops.md @@ -50,8 +50,9 @@ systemd-sysupdate --component=kubernetes update systemd-sysext merge systemctl daemon-reload -# 3. Bring the cluster up. bluefin-cluster-bootstrap pulls in kubeadm-init -# and bluefin-cluster-repo through its ordering and Requires= edges. +# 3. Bring the cluster up. bluefin-cluster-bootstrap orders itself after +# kubeadm-init and bluefin-cluster-repo, and pulls both in (Wants= on the +# former, Requires= on the latter). systemctl enable --now bluefin-cluster-bootstrap.service ``` @@ -100,7 +101,7 @@ Inspecting sync state: ```bash kubectl -n argocd get applications -kubectl -n argocd describe application root +kubectl -n argocd describe application bluefin-cluster kubectl -n argocd logs deploy/argocd-repo-server kubectl -n argocd logs statefulset/argocd-application-controller ``` @@ -134,11 +135,15 @@ Updating the tree means updating the image: a new DDI re-seeds auto-approve serving CSRs. `kubelet-csr-approver` is mandatory, not optional. Check `kubectl get csr` for `Pending` entries and the approver's `--provider-regex`. - **Root Application will not sync**: check the git-daemon first — - `kubectl -n argocd logs deploy/gitd` — then the bare repo at + `kubectl -n bluefin-system logs deploy/bluefin-git-daemon` — then the bare repo at `/var/lib/bluefin/cluster.git` on the host. - **Bootstrap unit failed**: `journalctl -u bluefin-cluster-bootstrap -e`. The unit is - ordered `After=`/`Requires=` `kubeadm-init.service`, so an apiserver that never came up - surfaces as a `kubeadm-init` failure first. + ordered `After=kubeadm-init.service` but only `Wants=` it, so a control plane that + never came up shows here as the `ExecStartPre` readiness probe timing out against the + API server; `journalctl -u kubeadm-init -e` names the underlying cause. `Requires=` is + deliberately not used: a job killed with result `dependency` never runs, so its own + `Restart=on-failure` never engages and one transient `kubeadm init` failure would + strand the seed permanently. ## Runtime testing diff --git a/files/cluster/manifests/console/console.yaml b/files/cluster/manifests/console/console.yaml index 38e5d95f..e4f23903 100644 --- a/files/cluster/manifests/console/console.yaml +++ b/files/cluster/manifests/console/console.yaml @@ -81,6 +81,16 @@ spec: app.kubernetes.io/name: kubestellar-console spec: serviceAccountName: kubestellar-console + # Matches the numeric owner of /app/data and /app/.kc in the image + # (appuser, 1001). runAsNonRoot alone is not enough: the image declares + # USER by name, and the kubelet refuses to start a container it cannot + # prove is non-root, so the uid has to be spelled out here. + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + seccompProfile: + type: RuntimeDefault containers: - name: console image: ghcr.io/kubestellar/console@sha256:f27adaf8d097a5403f60041615525497e0970151fcd10b90b7b6e903b395be32 @@ -119,6 +129,14 @@ spec: limits: memory: 512Mi securityContext: + # readOnlyRootFilesystem is the one piece of the baseline this + # workload does not take, and it is an exception rather than an + # oversight: the image writes its SQLite database to + # DATABASE_PATH=/app/data/console.db and a crash log beside it, on + # its own root filesystem. Nothing in this tree redirects those + # paths, so a read-only rootfs would leave the console unable to + # start. The data is ephemeral either way — there is no volume and + # the Deployment uses Recreate. allowPrivilegeEscalation: false capabilities: drop: diff --git a/files/cluster/manifests/kiosk/assets/nginx.conf b/files/cluster/manifests/kiosk/assets/nginx.conf index 9ce1f9b4..6e7ef8af 100644 --- a/files/cluster/manifests/kiosk/assets/nginx.conf +++ b/files/cluster/manifests/kiosk/assets/nginx.conf @@ -47,6 +47,12 @@ http { } location / { + # 10.96.0.10 is CoreDNS: the tenth address of serviceSubnet 10.96.0.0/12 + # in files/kubernetes/kubeadm.yaml, which kubeadm hands to the cluster DNS + # Service. nginx resolves the upstream at request time (proxy_pass through + # a variable), so it needs a literal resolver and cannot use + # /etc/resolv.conf. Changing serviceSubnet must change this line too or + # the kiosk proxies to nothing. resolver 10.96.0.10 valid=5s ipv6=off; set $upstream_endpoint http://kubestellar-console.kubestellar-console.svc.cluster.local:8080; proxy_pass $upstream_endpoint; diff --git a/files/cluster/manifests/kiosk/certificate.yaml b/files/cluster/manifests/kiosk/certificate.yaml index 98b899df..6369e103 100644 --- a/files/cluster/manifests/kiosk/certificate.yaml +++ b/files/cluster/manifests/kiosk/certificate.yaml @@ -8,6 +8,13 @@ # Short-lived on purpose: 90 days, renewed at 30 days remaining. A kiosk # certificate that outlives the appliance's install is a certificate nobody # notices has leaked. +# +# The SANs below deliberately do not cover the LAN address the issue banner +# advertises (https://:8080). That address is unknown at manifest +# time, so a LAN browser gets a hostname mismatch on top of the untrusted +# in-cluster CA — one warning rather than two, and the same click-through +# either way. A site that wants a clean chain issues its own certificate for a +# name it controls and replaces this Secret. apiVersion: cert-manager.io/v1 kind: Certificate metadata: diff --git a/files/kubernetes/kubeadm.yaml b/files/kubernetes/kubeadm.yaml index 9101b5b1..4f78be18 100644 --- a/files/kubernetes/kubeadm.yaml +++ b/files/kubernetes/kubeadm.yaml @@ -37,6 +37,10 @@ networking: dnsDomain: cluster.local # Cilium's cluster-pool IPAM is configured with this same range. podSubnet: 10.244.0.0/16 + # kubeadm derives the cluster DNS Service address from this range (10.96.0.10). + # The kiosk nginx.conf hardcodes that address as its resolver, because + # proxy_pass through a variable resolves at request time; changing this range + # means changing files/cluster/manifests/kiosk/assets/nginx.conf too. serviceSubnet: 10.96.0.0/12 --- apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/files/os/systemd/system/bluefin-cluster-repo.service b/files/os/systemd/system/bluefin-cluster-repo.service index fa9c0574..1fbd487b 100644 --- a/files/os/systemd/system/bluefin-cluster-repo.service +++ b/files/os/systemd/system/bluefin-cluster-repo.service @@ -9,6 +9,22 @@ Before=bluefin-cluster-bootstrap.service [Service] Type=oneshot RemainAfterExit=yes +# Restart=on-failure is load-bearing, not boilerplate. +# +# bluefin-cluster-bootstrap.service declares Requires= on this unit, and a job +# that dies with result 'dependency' never runs at all, so the dependent's own +# Restart=on-failure never engages and systemd never re-queues it. Without a +# restart here, one transient failure of the publish — a git invocation that +# loses a race with /var/lib/bluefin being set up, a full filesystem that +# clears — would strand the entire seed permanently behind a node that +# otherwise looks healthy. That is the same failure mode kubeadm-init.service +# forced onto the seed and which the Wants= edge there exists to prevent. +# +# Restart=on-failure is valid for Type=oneshot (only always/on-success are +# refused), and the publish is idempotent: it initialises the bare repository +# when absent and pushes the current manifest tree otherwise. +Restart=on-failure +RestartSec=30s StateDirectory=bluefin ExecStart=/usr/libexec/bluefin-cluster-repo diff --git a/files/os/systemd/system/kubelet.service b/files/os/systemd/system/kubelet.service index c9f7cfe9..9a0c847d 100644 --- a/files/os/systemd/system/kubelet.service +++ b/files/os/systemd/system/kubelet.service @@ -17,9 +17,21 @@ StartLimitIntervalSec=0 # matching the flatcar/sysext-bakery layout. containerd and Cilium both look in # /opt/cni/bin. Without this bridge the `loopback` plugin is missing, every pod # sandbox fails to get a network, and the node never leaves NotReady — silently. -# -n preserves the cilium binary that the agent installs into the same directory. +# +# /opt/cni/bin is on /var and survives reboots, so a plain no-clobber copy would +# pin the node to whatever plugins the sysext shipped on the day it was first +# installed: a sysext OTA update would advertise a new version in +# /usr/local/share/kubernetes-cni-version while the node kept executing the old +# binaries forever. The copy is therefore keyed on that stamp. +# +# Refreshing cannot clobber Cilium either way: /usr/local/bin/cni contains only +# the upstream containernetworking plugins, and the cilium agent installs +# cilium-cni, a name that is not in that set, so it is never a copy target. The +# no-clobber copy is kept for the unchanged case — it costs nothing and restores +# any plugin removed from /opt/cni/bin out of band. +ExecStartPre=/usr/bin/install -d -m 0755 /opt/cni ExecStartPre=/usr/bin/install -d -m 0755 /opt/cni/bin -ExecStartPre=/bin/sh -c 'cp -an /usr/local/bin/cni/. /opt/cni/bin/' +ExecStartPre=/bin/sh -c 'want=$(cat /usr/local/share/kubernetes-cni-version 2>/dev/null || echo unknown); have=$(cat /opt/cni/.bluefin-cni-version 2>/dev/null || echo none); if [ "$want" = "$have" ]; then cp -an /usr/local/bin/cni/. /opt/cni/bin/; else cp -a /usr/local/bin/cni/. /opt/cni/bin/ && printf "%s\\n" "$want" > /opt/cni/.bluefin-cni-version; fi' ExecStart=/usr/bin/kubelet Restart=always RestartSec=10s diff --git a/tests/unit/test_unit_dependency_provenance.py b/tests/unit/test_unit_dependency_provenance.py index 922f7fab..ce3e9821 100644 --- a/tests/unit/test_unit_dependency_provenance.py +++ b/tests/unit/test_unit_dependency_provenance.py @@ -95,6 +95,11 @@ def test_kubelet_bridges_the_cni_plugin_directory() -> None: Without the bridge the loopback plugin is absent, every pod sandbox fails to get a network, and the node never leaves NotReady with no obvious cause. + + The bridge must also refresh: /opt/cni/bin lives on /var and outlives a + sysext update, so a copy that can only ever add files would leave the node + running the plugins it first booted with while the version stamp advertises + newer ones. """ kubelet = (UNIT_DIR / "kubelet.service").read_text(encoding="utf-8") @@ -107,8 +112,14 @@ def test_kubelet_bridges_the_cni_plugin_directory() -> None: line for line in kubelet.splitlines() if "cp -a" in line and "cni" in line ) assert "-an" in bridge or "--no-clobber" in bridge, ( - "the copy must not clobber: the cilium agent installs its own binary " - "into /opt/cni/bin and overwriting it breaks the CNI chain" + "the copy must not clobber when the sysext is unchanged; cilium-cni is " + "not a name the sysext ships, but an unconditional overwrite of a " + "steady-state /opt/cni/bin buys nothing and widens the blast radius" + ) + assert "/usr/local/share/kubernetes-cni-version" in bridge, ( + "the bridge must key off the sysext's CNI version stamp, or a sysext " + "update never reaches /opt/cni/bin and the node runs stale plugins " + "while advertising the new version" ) body = kubelet.split("[Service]", 1)[1] From 36dcf4b478f682921dfa690e4d6e5fb990171727 Mon Sep 17 00:00:00 2001 From: "sec-check[bot]" Date: Mon, 21 Sep 2026 02:27:17 -0400 Subject: [PATCH 44/46] fix(installer): refuse an ambiguous boot artifact, record the runtime patch gap test-installer-boot selected its image with `head -n1` while flash-installer already refuses to guess between two exports. A boot result is evidence about one named artifact, so picking the alphabetically-first of a stale and a fresh export is the one thing this recipe must not do. It now fails the same way flash-installer does, listing both candidates. The containerd sysext has no sysupdate transfer, and the ID=_any repack means the seeded image keeps merging across OS updates instead of being rejected, so an installed host holds that containerd/runc until it is reinstalled. That is a patch-path regression against k0s, whose embedded runtime rode the k0s-@v transfer. Record it as an open item, including why it cannot be closed by pointing a transfer at Flatcar's published image: that image declares ID=flatcar with a concrete VERSION_ID, the exact metadata this element repacks away, so fetching it unmodified would make systemd-sysext refuse the merge and drop containerd.service on the next boot. The transfer needs a repacked image on this repo's release feed first. Assisted-by: Claude Opus 5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 15 ++++++++++++-- .../2026-09-19-kubernetes-cutover-design.md | 20 +++++++++++++++++++ elements/flatcar/containerd-sysext.bst | 5 ++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index dca9c853..85b05dd9 100644 --- a/Justfile +++ b/Justfile @@ -393,11 +393,22 @@ test-installer-boot: #!/usr/bin/env bash set -euo pipefail - IMG=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | head -n1) - if [ -z "${IMG}" ]; then + # Same refusal as `flash-installer`: a stale export beside a fresh one is + # ambiguous, and `head -n1` would quietly gate on whichever sorts first. + # A boot result is only evidence about a named artifact, so the artifact + # must not be picked by accident. + mapfile -t IMGS < <(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | sort) + if [ "${#IMGS[@]}" -eq 0 ]; then echo "ERROR: no exported installer in dist/. Run: just export-installer" >&2 exit 1 fi + if [ "${#IMGS[@]}" -gt 1 ]; then + echo "ERROR: ${#IMGS[@]} installer images in dist/; refusing to guess:" >&2 + printf ' %s\n' "${IMGS[@]}" >&2 + echo "Remove the stale one, then re-run." >&2 + exit 1 + fi + IMG="${IMGS[0]}" # This boots an EXPORTED ARTIFACT. It does not build, and `just validate` # only resolves the graph without building either. So a green result is diff --git a/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md b/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md index 5b26dc37..707c210a 100644 --- a/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md +++ b/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md @@ -371,3 +371,23 @@ not as base-DDI content. `quay.io`, `ghcr.io` and `docker.io`. Until images are pre-seeded, "fully automated first boot" means "with network present". This is a pre-existing condition, not a regression, and is tracked separately. +3. **No OTA channel for the containerd sysext.** `elements/flatcar/containerd-sysext.bst` + pins one Flatcar runtime image (containerd 2.1.5, runc 1.3.3) and + `files/installer/repart.d/30-var.conf` seeds it to + `/var/lib/extensions/containerd-flatcar.raw`. There is no + `sysupdate.containerd.d` transfer, and the `ID=_any` repack means the seeded image + keeps merging across every OS update rather than being rejected, so an installed + host holds that exact containerd/runc until it is reinstalled — no patch path for a + runtime CVE. This is a regression against k0s, whose embedded runtime rode the + `k0s-@v` transfer. + + It is deliberately not closed by pointing a transfer at Flatcar's published + `rootfs-included-sysexts/containerd-flatcar.raw`: that image carries + `ID=flatcar` with a concrete `VERSION_ID`, which is exactly the metadata this + element repacks away. Fetching it unmodified would reintroduce the os-release + mismatch, `systemd-sysext` would refuse the merge, and `containerd.service` + would vanish on the next boot. A transfer must therefore consume a + *repacked* image published on this repository's own release feed, alongside + `kubernetes-@v.raw.zst` — i.e. the sysext has to become a release artifact + before `70-containerd.transfer` can exist. Until then, runtime CVEs are + remediated by reinstall. diff --git a/elements/flatcar/containerd-sysext.bst b/elements/flatcar/containerd-sysext.bst index 8177b95f..f004dd9a 100644 --- a/elements/flatcar/containerd-sysext.bst +++ b/elements/flatcar/containerd-sysext.bst @@ -51,7 +51,10 @@ description: | systemd-sysext then refuses the merge, containerd.service disappears, and kubeadm-init.service fails — the cluster goes down after a routine OS update. kubernetes.raw is immune because 70-kubernetes.transfer replaces it in lockstep; - containerd has no such transfer. + containerd has no such transfer. That absence is also the runtime's patch path + gap — a seeded image that never updates holds this containerd/runc until the + host is reinstalled — and is tracked as open item 3 of + docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md. So the image is repacked with a single field changed: ID=_any. That is the same decoupling files/kubernetes/sysext/extension-release.kubernetes uses, for the From 049e9a2b25f7e08ba275e3f0a10e13503343b69b Mon Sep 17 00:00:00 2001 From: castrojo Date: Sun, 27 Sep 2026 17:18:02 -0400 Subject: [PATCH 45/46] docs: add theme-adaptive wordmark to README Assisted-by: Gemini 3.8 Flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c1f01a0f..95a17b43 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ + + + + Bluefin Server + + # Bluefin Server > Amargasaurus cazaui From 5fede1ad91b5ecfbcca54638a734e568f7adb86b Mon Sep 17 00:00:00 2001 From: hive-scanner Date: Mon, 28 Sep 2026 14:07:55 -0400 Subject: [PATCH 46/46] fix(kubernetes): guard kubeadm init on a completion stamp, not admin.conf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kubeadm writes /etc/kubernetes/admin.conf in the `kubeconfig` phase, which runs before etcd, control-plane, wait-control-plane, mark-control-plane and bootstrap-token. Guarding kubeadm-init.service on that path meant that an init which died after the kubeconfig phase left the node permanently half-initialised: Restart=on-failure re-evaluated ConditionPathExists=!/etc/kubernetes/admin.conf, the unit was marked skipped instead of retried, and mark-control-plane never applied the control-plane role label. The symptom surfaced far downstream — the nodeSelector in files/cluster/seed/15-gitd/deployment.yaml never matched and bluefin-cluster-bootstrap phase 3 timed out while the API server answered normally. Guard on /var/lib/bluefin/kubeadm-init.stamp instead, written by ExecStartPost, which only runs when every ExecStart succeeded. A retry that finds admin.conf without the stamp now runs `kubeadm reset -f` first, since kubeadm refuses to init over a populated /etc/kubernetes/manifests and every retry would otherwise fail identically. Also: - Correct the MAX_EXPIRATION_SEC comment in the kubelet-csr-approver manifest. The cap only denies CSRs that request a longer spec.expirationSeconds; the kubelet sets none on its serving CSR, so those certificates are issued for kube-controller-manager's --cluster-signing-duration regardless of this value. - State in docs/skills/kubernetes-sysext-ops.md that the Kubernetes cutover is reinstall-only. containerd-flatcar.raw is seeded at install time by files/installer/repart.d/30-var.conf, is not a published release asset and has no sysupdate transfer, so an existing k0s host cannot be migrated in place with `systemd-sysupdate --component=kubernetes update`. - Drop two unrelated cosmetic changes from the diff: the README logo block and the zz-enable-sshd.preset rename. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 11 +++--- docs/skills/kubernetes-sysext-ops.md | 36 ++++++++++++++++--- .../kubelet-csr-approver/deployment.yaml | 11 ++++-- ...able-sshd.preset => zz-enable-sshd.preset} | 0 files/os/systemd/system/kubeadm-init.service | 32 ++++++++++++++++- tests/unit/test_cluster_bootstrap.py | 30 +++++++++++++++- 6 files changed, 104 insertions(+), 16 deletions(-) rename files/os/systemd/system-preset/{zz-disable-sshd.preset => zz-enable-sshd.preset} (100%) diff --git a/README.md b/README.md index 95a17b43..e32a434f 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,11 @@ Bluefin Server -# Bluefin Server > Amargasaurus cazaui -**An FSDK-based, image-based Linux server OS.** +**An image-based Linux server OS built on a Flatcar base.** -Bluefin Server targets the same use-case space as Flatcar Container Linux, Fedora CoreOS, and Talos, but is built from scratch with [BuildStream 2](https://buildstream.build/) from [freedesktop-sdk](https://freedesktop-sdk.freedesktop.org/) (FSDK 26.08) components and uutils coreutils. +Bluefin Server targets the same use-case space as Flatcar Container Linux, Fedora CoreOS, and Talos, and is built with [BuildStream 2](https://buildstream.build/). The installed OS payload lands on a single-ABI [Flatcar](https://flatcar-linux.net/) `/usr` (projectbluefin/server#131); the offline installer still composes its userspace from [freedesktop-sdk](https://freedesktop-sdk.freedesktop.org/) (FSDK 26.08) so it keeps `systemd-sysinstall`, which Flatcar does not ship. It is [DDI first](https://0pointer.net/blog/fitting-everything-together.html): the OS payload is a compressed XFS DDI filesystem image that is deployed by an offline, systemd-native installer. @@ -20,7 +19,7 @@ It is [DDI first](https://0pointer.net/blog/fitting-everything-together.html): t ## Release status: Alpha Bluefin Server is currently in **Alpha**: -- **Milestone status**: Phase A (reproducible build path, uutils, Kubernetes sysext, graph validation) is complete. Phase B (automated boot verification on lab cluster) is in progress. +- **Milestone status**: Phase A (reproducible build path, uutils, k0s sysext, graph validation) is complete. Phase B (automated boot verification on lab cluster) is in progress. - **Trust model**: Releases include cryptographic provenance with GPG-signed `SHA256SUMS` manifests and in-tree `systemd-sysupdate` verification configurations. - **Suitability**: Alpha builds are intended for evaluation, testing, and factory validation. Not yet recommended for production workloads. - **Readiness roadmap**: Track completed criteria and remaining gates toward 1.0 in [`docs/MVP_1_0_READINESS.md`](docs/MVP_1_0_READINESS.md). @@ -29,9 +28,9 @@ Bluefin Server is currently in **Alpha**: - **Image-based updates and atomic rollbacks** via A/B partition slots and `systemd-sysupdate`. - **DDI-first delivery** — the installer embeds the OS payload as a data partition; no network is required at install time. -- **Streamlined base OS image** — modern userspace with uutils coreutils and bash for interactive login and diagnostics. +- **Streamlined base OS image** — modern Flatcar userspace with bash for interactive login and diagnostics. - **systemd-native installer** — `systemd-sysinstall` provides the interactive terminal UI and `systemd-repart` handles partitioning and block-copy DDI placement. -- **Optional upstream Kubernetes as a `systemd-sysext`** so the base image stays minimal. +- **Optional k0s as a `systemd-sysext`** so the base image stays minimal. > **Remote diagnostics:** OpenSSH is installed for on-demand diagnostics, but is disabled by default via systemd presets. It can be started manually with `systemctl start sshd` when remote access is needed. See [`docs/skills/factory-integration.md`](docs/skills/factory-integration.md). diff --git a/docs/skills/kubernetes-sysext-ops.md b/docs/skills/kubernetes-sysext-ops.md index 2244d7fb..ce99c3d6 100644 --- a/docs/skills/kubernetes-sysext-ops.md +++ b/docs/skills/kubernetes-sysext-ops.md @@ -4,7 +4,7 @@ description: Operator runbook for the Kubernetes systemd-sysext — kubeadm brin metadata: type: how-to status: stable - last_updated: "2026-09-19" + last_updated: "2026-09-28" context7-sources: - /systemd/systemd --- @@ -27,13 +27,39 @@ kubeadm-init.service kubeadm init --config /usr/share/bluefin/kubea bluefin-cluster-bootstrap.service apply the four seed phases in order ``` -`kubeadm-init.service` carries `ConditionPathExists=!/etc/kubernetes/admin.conf`, so it -is a no-op on an already-initialised host. `bluefin-cluster-bootstrap.service` carries no -such condition on purpose: a failed condition marks a unit *skipped* rather than *failed*, -which would silently no-op the first boot. +`kubeadm-init.service` carries `ConditionPathExists=!/var/lib/bluefin/kubeadm-init.stamp`, +so it is a no-op on an already-initialised host. The stamp is written by +`ExecStartPost`, i.e. only after `kubeadm init` exits 0. `/etc/kubernetes/admin.conf` +deliberately is *not* the guard: kubeadm writes it in the `kubeconfig` phase, before +etcd, the control plane and `mark-control-plane`, so guarding on it made +`Restart=on-failure` skip the retry of a node that never got its control-plane role +label. A retry that finds `admin.conf` without the stamp runs `kubeadm reset -f` first, +because kubeadm refuses to init over a populated `/etc/kubernetes/manifests`. +`bluefin-cluster-bootstrap.service` carries no condition at all on purpose: a failed +condition marks a unit *skipped* rather than *failed*, which would silently no-op the +first boot. + +To force a full re-init on a live host, remove the stamp and reset: + +```bash +kubeadm reset -f +rm -f /var/lib/bluefin/kubeadm-init.stamp +systemctl start kubeadm-init.service +``` ## Enabling Kubernetes on a Host +> **Reinstall only.** These steps apply to a Bluefin Server host installed from a DDI +> that already carries the container runtime. `containerd-flatcar.raw` is seeded into +> `/var/lib/extensions` at install time by `files/installer/repart.d/30-var.conf`; it is +> **not** published as a release asset and has no `sysupdate.containerd.d` transfer, so +> `systemd-sysupdate --component=kubernetes update` cannot deliver it. A host still +> running the old k0s image therefore cannot be migrated in place — it would end up with +> `kubelet.service` and `kubeadm-init.service` and no runtime, and `kubeadm-init` would +> sit in its CRI-socket wait until it timed out. Reinstall such hosts from the current +> DDI. Publishing containerd on its own transfer is tracked as open item 3 in +> `docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md`. + On a running Bluefin Server system: ```bash diff --git a/files/cluster/manifests/kubelet-csr-approver/deployment.yaml b/files/cluster/manifests/kubelet-csr-approver/deployment.yaml index 3a39c9b3..90bbf199 100644 --- a/files/cluster/manifests/kubelet-csr-approver/deployment.yaml +++ b/files/cluster/manifests/kubelet-csr-approver/deployment.yaml @@ -44,9 +44,14 @@ spec: # because a silent bump to it would widen what gets approved. - name: ALLOWED_DNS_NAMES value: "1" - # 30 days. Upstream's default is 367 days; bounding it hard means a - # serving certificate issued to a node that is later removed stops - # being useful in a month rather than a year. + # A ceiling on what a CSR may *ask* for, not the lifetime nodes get. + # kubelet-csr-approver only denies CSRs whose spec.expirationSeconds + # exceeds this; the kubelet sets no expirationSeconds on its serving + # CSR, so those are issued for kube-controller-manager's + # --cluster-signing-duration (1y by default) regardless of the value + # here. It is kept as a hard bound on anything that does request a + # duration — shortening the actual serving-certificate lifetime + # means setting --cluster-signing-duration on the signer. - name: MAX_EXPIRATION_SEC value: "2592000" # The appliance has no authoritative DNS for its own node name, so diff --git a/files/os/systemd/system-preset/zz-disable-sshd.preset b/files/os/systemd/system-preset/zz-enable-sshd.preset similarity index 100% rename from files/os/systemd/system-preset/zz-disable-sshd.preset rename to files/os/systemd/system-preset/zz-enable-sshd.preset diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service index ace397df..954d25b7 100644 --- a/files/os/systemd/system/kubeadm-init.service +++ b/files/os/systemd/system/kubeadm-init.service @@ -4,7 +4,23 @@ Documentation=https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-i # The idempotency guard belongs here and nowhere else. A unit whose condition # fails is marked *skipped*, and systemd treats a skipped unit as satisfied for # everything ordered after it, so bluefin-cluster-bootstrap.service still runs. -ConditionPathExists=!/etc/kubernetes/admin.conf +# +# The guard is a stamp this unit writes itself, not /etc/kubernetes/admin.conf. +# admin.conf is not a completion marker: kubeadm writes it in the `kubeconfig` +# phase, which runs *before* etcd, control-plane, wait-control-plane, +# mark-control-plane and bootstrap-token. If `kubeadm init` dies after the +# kubeconfig phase, admin.conf is already on disk, so with the old condition the +# Restart=on-failure retry below was skipped instead of run, and the node kept a +# half-initialised control plane forever. The visible symptom was downstream and +# unhelpful: mark-control-plane never applied the control-plane role label, so +# the nodeSelector in files/cluster/seed/15-gitd/deployment.yaml never matched +# and bluefin-cluster-bootstrap phase 3 timed out even though the API server was +# answering. +# +# ExecStartPost writes the stamp only after kubeadm exits 0, so the condition +# now means "a previous init ran to completion" rather than "some init got as +# far as writing a kubeconfig". +ConditionPathExists=!/var/lib/bluefin/kubeadm-init.stamp # Both /usr/bin/kubeadm and containerd.service arrive with the sysext merge, but # from two different extensions, each seeded to /var/lib/extensions by # files/installer/repart.d/30-var.conf: @@ -58,6 +74,7 @@ After=containerd.service Wants=network-online.target After=network-online.target RequiresMountsFor=/var/lib/kubelet +RequiresMountsFor=/var/lib/bluefin [Service] Type=oneshot @@ -94,9 +111,22 @@ TimeoutStartSec=900 # runtime". Exiting non-zero hands it to Restart=on-failure with a line that # names the cause, and still leaves kubeadm the larger share of the budget. ExecStartPre=/usr/bin/bash -c 'for i in $(seq 300); do [ -S /run/containerd/containerd.sock ] && exit 0; sleep 1; done; echo "no CRI socket at /run/containerd/containerd.sock after 300s; containerd.service did not start" >&2; exit 1' +# Clear a half-initialised node before retrying. Reaching this point with +# /etc/kubernetes/admin.conf already present means a previous attempt wrote a +# kubeconfig and then failed before the stamp was written — kubeadm refuses to +# run against that state ("ERROR Port-6443 ... /etc/kubernetes/manifests is not +# empty"), so every retry would fail the same way forever. `kubeadm reset -f` +# tears down the manifests, etcd data and kubeconfigs so the next ExecStart is a +# clean first init. It is a no-op path on a genuinely first boot, where +# admin.conf does not exist. +ExecStartPre=/usr/bin/bash -c '[ -e /etc/kubernetes/admin.conf ] || exit 0; echo "found /etc/kubernetes/admin.conf with no completion stamp: a previous kubeadm init did not finish; resetting before retry" >&2; /usr/bin/kubeadm reset -f' # Cilium provides the kube-proxy replacement; installing kube-proxy as well # would leave two datapaths fighting over the same Services. ExecStart=/usr/bin/kubeadm init --config /usr/share/bluefin/kubeadm.yaml --skip-phases=addon/kube-proxy +# The completion stamp the ConditionPathExists above reads. ExecStartPost only +# runs when every ExecStart succeeded, which is what makes the stamp mean +# "kubeadm init ran to completion" and not merely "kubeadm init was attempted". +ExecStartPost=/usr/bin/bash -c 'mkdir -p /var/lib/bluefin && : > /var/lib/bluefin/kubeadm-init.stamp' [Install] WantedBy=multi-user.target diff --git a/tests/unit/test_cluster_bootstrap.py b/tests/unit/test_cluster_bootstrap.py index 7f1db9a8..128e8f62 100644 --- a/tests/unit/test_cluster_bootstrap.py +++ b/tests/unit/test_cluster_bootstrap.py @@ -110,7 +110,9 @@ def test_preset_enables_exactly_the_units_that_are_shipped() -> None: def test_the_idempotency_guard_lives_on_kubeadm_init_alone() -> None: """A skipped unit satisfies its dependents; a skipped seed is a dead node.""" init = unit(KUBEADM_INIT) - assert init["Unit"]["ConditionPathExists"] == ["!/etc/kubernetes/admin.conf"] + assert init["Unit"]["ConditionPathExists"] == [ + "!/var/lib/bluefin/kubeadm-init.stamp" + ] for path in (CLUSTER_BOOTSTRAP,): guards = [ @@ -124,6 +126,32 @@ def test_the_idempotency_guard_lives_on_kubeadm_init_alone() -> None: ) +def test_the_guard_is_a_completion_stamp_not_a_mid_run_artefact() -> None: + """admin.conf lands in kubeadm's `kubeconfig` phase, long before the control + plane is up and the control-plane role label is applied. Guarding on it made + Restart=on-failure skip the retry of a half-initialised node, which showed up + downstream as the 15-gitd nodeSelector never matching.""" + service = unit(KUBEADM_INIT)["Service"] + + guard = unit(KUBEADM_INIT)["Unit"]["ConditionPathExists"][0].removeprefix("!") + post = service.get("ExecStartPost", []) + assert any(guard in line for line in post), ( + f"nothing writes {guard}, so the condition can never become true and " + "kubeadm init would re-run on every boot" + ) + + pre = service.get("ExecStartPre", []) + assert any("kubeadm reset -f" in line for line in pre), ( + "a retry that finds /etc/kubernetes/admin.conf from a failed attempt " + "must reset first; kubeadm refuses to init over a populated " + "/etc/kubernetes/manifests, so every retry would fail identically" + ) + assert any( + "kubeadm reset -f" in line and "/etc/kubernetes/admin.conf" in line + for line in pre + ), "the reset must be conditional on the half-initialised state, not unconditional" + + def test_kubeadm_init_waits_for_the_sysext_merge_and_the_runtime() -> None: init = unit(KUBEADM_INIT)["Unit"] requires = set(init.get("Requires", []))