From 55156689a1e89e27361344cacb818f56902b85e4 Mon Sep 17 00:00:00 2001 From: "sec-check[bot]" Date: Thu, 24 Sep 2026 02:15:24 -0400 Subject: [PATCH 1/4] fix(boot): rename systemd presets to sort before Flatcar's 99-default 'disable *' Flatcar's /usr ships /usr/lib/systemd/system-preset/99-default.preset containing 'disable *'. systemd reads preset files in lexicographic filename order and the first matching line wins, so every Bluefin preset named zz-enable-*.preset sorted after it and was ignored at first-boot preset-all. k0s-first-boot.service was therefore never enabled: the QEMU smoke test sat at 'kiosk-ready: waiting k0s=inactive' for the full 900s window on every run since the Flatcar payload switch. Rename the presets to 20-bluefin-*.preset so they sort before 99-default.preset, and add a unit test asserting no staged preset can regress behind the Flatcar catch-all again. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check[bot] --- ...credentials.preset => 20-bluefin-firstboot-credentials.preset} | 0 ...ble-k0s-first-boot.preset => 20-bluefin-k0s-first-boot.preset} | 0 .../{zz-enable-networkd.preset => 20-bluefin-networkd.preset} | 0 .../{zz-enable-sshd.preset => 20-bluefin-sshd.preset} | 0 ...nerator.preset => 20-bluefin-systemd-network-generator.preset} | 0 .../{zz-enable-var-mount.preset => 20-bluefin-var-mount.preset} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename files/os/systemd/system-preset/{zz-enable-bluefin-firstboot-credentials.preset => 20-bluefin-firstboot-credentials.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-k0s-first-boot.preset => 20-bluefin-k0s-first-boot.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-networkd.preset => 20-bluefin-networkd.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-sshd.preset => 20-bluefin-sshd.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-systemd-network-generator.preset => 20-bluefin-systemd-network-generator.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-var-mount.preset => 20-bluefin-var-mount.preset} (100%) diff --git a/files/os/systemd/system-preset/zz-enable-bluefin-firstboot-credentials.preset b/files/os/systemd/system-preset/20-bluefin-firstboot-credentials.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-bluefin-firstboot-credentials.preset rename to files/os/systemd/system-preset/20-bluefin-firstboot-credentials.preset diff --git a/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset b/files/os/systemd/system-preset/20-bluefin-k0s-first-boot.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset rename to files/os/systemd/system-preset/20-bluefin-k0s-first-boot.preset diff --git a/files/os/systemd/system-preset/zz-enable-networkd.preset b/files/os/systemd/system-preset/20-bluefin-networkd.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-networkd.preset rename to files/os/systemd/system-preset/20-bluefin-networkd.preset diff --git a/files/os/systemd/system-preset/zz-enable-sshd.preset b/files/os/systemd/system-preset/20-bluefin-sshd.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-sshd.preset rename to files/os/systemd/system-preset/20-bluefin-sshd.preset diff --git a/files/os/systemd/system-preset/zz-enable-systemd-network-generator.preset b/files/os/systemd/system-preset/20-bluefin-systemd-network-generator.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-systemd-network-generator.preset rename to files/os/systemd/system-preset/20-bluefin-systemd-network-generator.preset diff --git a/files/os/systemd/system-preset/zz-enable-var-mount.preset b/files/os/systemd/system-preset/20-bluefin-var-mount.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-var-mount.preset rename to files/os/systemd/system-preset/20-bluefin-var-mount.preset From b40d6f6a6f7ee3f8de841d7fa22748d4b69e3201 Mon Sep 17 00:00:00 2001 From: sec-check Date: Thu, 24 Sep 2026 06:26:35 -0400 Subject: [PATCH 2/4] test: update preset paths to 20-bluefin-*, add sort-order guard, log pod states in smoke probe - Update the two unit tests that still referenced the removed zz-enable-*.preset filenames (the rename commit shipped without them). - Add test_presets_sort_before_flatcar_disable_all so no future preset can sort behind Flatcar's 99-default.preset 'disable *' catch-all. - Teach the installer smoke readiness unit to print the pod table and the last Warning events alongside the existing counters, so a run that reaches k0s=active but never gets a pod to Running names the blocker in the serial log instead of only counting it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check --- Justfile | 2 +- tests/unit/test_credentials_provisioning.py | 4 ++-- tests/unit/test_k0s_first_boot.py | 20 ++++++++++++++++++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Justfile b/Justfile index c3cf9bb3..e163c421 100644 --- a/Justfile +++ b/Justfile @@ -466,7 +466,7 @@ test-installer-artifact: Type=oneshot TimeoutStartSec=infinity StandardOutput=journal+console - ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' + ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; k0s kubectl get pods -A --no-headers 2>/dev/null | sed "s/^/<4>kiosk-pods: /"; k0s kubectl get events -A --field-selector type=Warning --no-headers 2>/dev/null | tail -n 5 | sed "s/^/<4>kiosk-events: /"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' UNIT ) diff --git a/tests/unit/test_credentials_provisioning.py b/tests/unit/test_credentials_provisioning.py index f9c36654..852207e1 100644 --- a/tests/unit/test_credentials_provisioning.py +++ b/tests/unit/test_credentials_provisioning.py @@ -24,7 +24,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-bluefin-firstboot-credentials.preset" + / "20-bluefin-firstboot-credentials.preset" ) NETWORK_GENERATOR_PRESET = ( REPO_ROOT @@ -32,7 +32,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-systemd-network-generator.preset" + / "20-bluefin-systemd-network-generator.preset" ) NETWORK_GENERATOR_DROPIN = ( REPO_ROOT diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py index a185a611..72e4a124 100644 --- a/tests/unit/test_k0s_first_boot.py +++ b/tests/unit/test_k0s_first_boot.py @@ -20,7 +20,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-k0s-first-boot.preset" + / "20-bluefin-k0s-first-boot.preset" ) NETWORK = ROOT / "files" / "os" / "systemd" / "network" / "20-wired.network" NETWORK_PRESET = ( @@ -29,7 +29,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-networkd.preset" + / "20-bluefin-networkd.preset" ) ELEMENT = ROOT / "elements" / "bluefin-server" / "os-k0s-first-boot.bst" NETWORK_ELEMENT = ROOT / "elements" / "bluefin-server" / "os-networkd.bst" @@ -146,3 +146,19 @@ def test_k0s_sysupdate_transfer_is_packaged_as_a_component() -> None: "bluefin-server/os-k0s-sysupdate.bst" in STACK.read_text(encoding="utf-8") ) + + +def test_presets_sort_before_flatcar_disable_all() -> None: + # Flatcar's /usr ships /usr/lib/systemd/system-preset/99-default.preset + # containing a catch-all "disable *". systemd applies preset files in + # lexicographic filename order and the first matching line wins, so any + # Bluefin preset named after 99-default.preset is silently ignored at + # first-boot preset-all. + preset_dir = PRESET.parent + presets = sorted(p.name for p in preset_dir.glob("*.preset")) + assert presets, f"no presets found in {preset_dir}" + for name in presets: + assert name < "99-default.preset", ( + f"{name} sorts after Flatcar's 99-default.preset; its " + "'disable *' catch-all would win and the preset would be ignored" + ) From 2b9339efd25387d3ba31439ccc46e18e4d7577a5 Mon Sep 17 00:00:00 2001 From: sec-check Date: Thu, 24 Sep 2026 10:37:01 -0400 Subject: [PATCH 3/4] test(installer): surface node, pull, and k0s runtime errors in the kiosk waiter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The preset rename in this PR gets k0scontroller active, but the smoke gate still times out: every pod sits Pending/ContainerCreating for the full 900s window (kube-proxy ContainerCreating, kube-router Init:0/2, node tainted not-ready). The current 30s debug line only shows Warning events, and the FailedScheduling flood from ten Pending pods pushes the informative events (image pulls, sandbox creation, CNI) out of the tail -n 5 window. Extend the in-guest waiter's periodic dump so the serial log carries enough to diagnose the next layer: - node status line (kiosk-node) — shows Ready/NotReady and the CRI version - all event kinds minus the FailedScheduling flood (kiosk-events) - error/fail/taint lines from the k0scontroller journal (kiosk-k0s), where the embedded kubelet and containerd log Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index e163c421..788423f4 100644 --- a/Justfile +++ b/Justfile @@ -466,7 +466,7 @@ test-installer-artifact: Type=oneshot TimeoutStartSec=infinity StandardOutput=journal+console - ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; k0s kubectl get pods -A --no-headers 2>/dev/null | sed "s/^/<4>kiosk-pods: /"; k0s kubectl get events -A --field-selector type=Warning --no-headers 2>/dev/null | tail -n 5 | sed "s/^/<4>kiosk-events: /"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' + ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; k0s kubectl get pods -A --no-headers 2>/dev/null | sed "s/^/<4>kiosk-pods: /"; k0s kubectl get nodes -o wide --no-headers 2>/dev/null | sed "s/^/<4>kiosk-node: /"; k0s kubectl get events -A --no-headers 2>/dev/null | grep -v FailedScheduling | tail -n 8 | sed "s/^/<4>kiosk-events: /"; journalctl -u k0scontroller --no-pager -n 800 2>/dev/null | grep -iE "error|fail|taint" | grep -vE "FailedScheduling|apply.*retrying" | tail -n 8 | sed "s/^/<4>kiosk-k0s: /"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' UNIT ) From f1a52e5f56e1146c2418742fcbf69d963755e151 Mon Sep 17 00:00:00 2001 From: sec-check Date: Thu, 24 Sep 2026 14:50:29 -0400 Subject: [PATCH 4/4] fix(boot): seed /etc/resolv.conf symlink via tmpfiles.d so kubelet can create pod sandboxes The preset rename got k0s and kubelet running, but every pod sandbox then failed with 'open /etc/resolv.conf: no such file or directory' and the console readiness gate timed out (run 36014129412). Flatcar ships the /etc/resolv.conf symlink inside the image /etc and strips /etc-populating lines from /usr/lib/tmpfiles.d at image build; this DDI imports only Flatcar's /usr and first-boots with an empty /etc, so the symlink never exists. Recreate Flatcar's stock symlink (/etc/resolv.conf -> ../run/systemd/resolve/resolv.conf, the full resolver list, matching Flatcar >= 3510.3.0) with a tmpfiles.d rule installed from a new os-resolv-conf.bst element, and guard it with a unit test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check --- elements/bluefin-server/os-resolv-conf.bst | 14 ++++++++++ elements/bluefin-server/os-stack.bst | 3 +++ files/os/tmpfiles.d/20-bluefin-resolv.conf | 11 ++++++++ tests/unit/test_k0s_first_boot.py | 30 ++++++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 elements/bluefin-server/os-resolv-conf.bst create mode 100644 files/os/tmpfiles.d/20-bluefin-resolv.conf diff --git a/elements/bluefin-server/os-resolv-conf.bst b/elements/bluefin-server/os-resolv-conf.bst new file mode 100644 index 00000000..79361cc1 --- /dev/null +++ b/elements/bluefin-server/os-resolv-conf.bst @@ -0,0 +1,14 @@ +kind: import +description: | + Install the tmpfiles.d rule that recreates Flatcar's /etc/resolv.conf + symlink to systemd-resolved's resolv.conf. The Flatcar /usr payload does + not carry it (Flatcar ships the symlink in the image /etc, which this DDI + does not import), and kubelet requires /etc/resolv.conf to create any pod + sandbox. + +sources: + - kind: local + path: files/os/tmpfiles.d + +config: + target: /usr/lib/tmpfiles.d diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst index 44f80576..5aa65158 100644 --- a/elements/bluefin-server/os-stack.bst +++ b/elements/bluefin-server/os-stack.bst @@ -44,6 +44,9 @@ depends: # Wired DHCP configuration for the installed OS - bluefin-server/os-networkd.bst + # /etc/resolv.conf symlink to systemd-resolved (tmpfiles.d) + - bluefin-server/os-resolv-conf.bst + # Activate the optional k0s sysext after installation - bluefin-server/os-k0s-first-boot.bst diff --git a/files/os/tmpfiles.d/20-bluefin-resolv.conf b/files/os/tmpfiles.d/20-bluefin-resolv.conf new file mode 100644 index 00000000..085ac2ab --- /dev/null +++ b/files/os/tmpfiles.d/20-bluefin-resolv.conf @@ -0,0 +1,11 @@ +# Flatcar's image build populates /etc/resolv.conf in the image's /etc and +# strips /etc-populating lines from /usr/lib/tmpfiles.d (see +# flatcar/scripts build_library/build_image_util.sh). This DDI imports only +# Flatcar's /usr and boots with an empty /etc, so nothing ever creates the +# symlink and kubelet fails every pod sandbox with +# "open /etc/resolv.conf: no such file or directory". +# +# Recreate Flatcar's stock symlink: /etc/resolv.conf points at +# systemd-resolved's full resolver list (not the 127.0.0.53 stub), matching +# Flatcar >= 3510.3.0 behaviour. +L /etc/resolv.conf - - - - ../run/systemd/resolve/resolv.conf diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py index 72e4a124..1f89861e 100644 --- a/tests/unit/test_k0s_first_boot.py +++ b/tests/unit/test_k0s_first_boot.py @@ -162,3 +162,33 @@ def test_presets_sort_before_flatcar_disable_all() -> None: f"{name} sorts after Flatcar's 99-default.preset; its " "'disable *' catch-all would win and the preset would be ignored" ) + + +def test_etc_resolv_conf_symlink_is_seeded_for_kubelet() -> None: + # Flatcar ships /etc/resolv.conf inside the image /etc and strips + # /etc-populating lines from /usr/lib/tmpfiles.d at image build. This DDI + # imports only Flatcar's /usr and first-boots with an empty /etc, so + # without an explicit tmpfiles.d rule the symlink never exists and + # kubelet fails every pod sandbox with + # "open /etc/resolv.conf: no such file or directory". + conf = ( + ROOT / "files" / "os" / "tmpfiles.d" / "20-bluefin-resolv.conf" + ) + assert conf.is_file(), "resolv.conf tmpfiles rule is missing" + lines = [ + line + for line in conf.read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") + ] + assert lines == [ + "L /etc/resolv.conf - - - - ../run/systemd/resolve/resolv.conf" + ] + + element = ( + ROOT / "elements" / "bluefin-server" / "os-resolv-conf.bst" + ).read_text(encoding="utf-8") + assert "path: files/os/tmpfiles.d" in element + assert "target: /usr/lib/tmpfiles.d" in element + assert ( + "bluefin-server/os-resolv-conf.bst" in STACK.read_text(encoding="utf-8") + )