diff --git a/.github/scripts/check-k0s-version.py b/.github/scripts/check-k0s-version.py deleted file mode 100644 index 3c05acf5..00000000 --- a/.github/scripts/check-k0s-version.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -"""Enforce the k0s version invariant for Bluefin Server. - -k0s is an independently-pinned third-party payload, not an OS asset, so it is -versioned on its own axis. ``include/k0s.yml`` is the single source of truth: - - variables: - k0s-k8s-version: "1.36.4" - k0s-patch: "0" - k0s-upstream-tag: "v%{k0s-k8s-version}%2Bk0s.%{k0s-patch}" - k0s-version: "%{k0s-k8s-version}-k0s.%{k0s-patch}" - -Three consumers restate that version in three different spellings, and all -three must move together or the sysext silently stops updating: - - * ``elements/k0s/k0s-bin.bst`` — the upstream download URL. - * ``elements/oci/k0s-sysext.bst`` — the release asset filename, which - ``files/os/sysupdate.d/70-k0s.transfer`` reads as the version oracle - through its ``@v`` wildcard. - * ``files/k0s/sysext/extension-release.k0s`` — the ``VERSION_ID=`` reported - by ``systemd-sysext status``. - -This script fails closed if any consumer reintroduces a literal k0s version or -drops back onto the OS release axis, and cross-checks that the derived asset -filename still matches the sysupdate transfer's MatchPattern. -""" - -import re -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -K0S_INCLUDE = ROOT / "include" / "k0s.yml" -K0S_BIN = ROOT / "elements" / "k0s" / "k0s-bin.bst" -K0S_SYSEXT = ROOT / "elements" / "oci" / "k0s-sysext.bst" -EXTENSION_RELEASE = ROOT / "files" / "k0s" / "sysext" / "extension-release.k0s" -TRANSFER = ROOT / "files" / "os" / "sysupdate.k0s.d" / "70-k0s.transfer" - -SAFE_VERSION_RE = re.compile(r"^[A-Za-z0-9._~^+-]+$") -LITERAL_VERSION_RE = re.compile(r"v?[0-9]+\.[0-9]+\.[0-9]+(?:\+|%2B|-)k0s\.[0-9]+") - - -def read(path): - if not path.is_file(): - sys.exit(f"ERROR: expected file not found: {path.relative_to(ROOT)}") - return path.read_text(encoding="utf-8") - - -def scalar(text, name, where): - """Read a ``name: "value"`` scalar out of a BuildStream YAML fragment.""" - match = re.search( - rf"^\s*{re.escape(name)}:\s*[\"']([^\"']+)[\"']\s*$", text, re.MULTILINE - ) - if not match: - sys.exit(f"ERROR: {where} does not declare a '{name}:' variable.") - return match.group(1) - - -def expand(value, variables, where): - """Resolve ``%{...}`` references the way BuildStream would.""" - for _ in range(len(variables) + 1): - refs = re.findall(r"%\{([a-zA-Z][a-zA-Z0-9_-]*)\}", value) - if not refs: - return value - for ref in refs: - if ref not in variables: - sys.exit( - f"ERROR: {where} references undefined variable '%{{{ref}}}'." - ) - value = value.replace("%{" + ref + "}", variables[ref]) - sys.exit(f"ERROR: {where} has a circular variable reference: {value}") - - -def fail(problem, fix): - sys.exit(f"ERROR: {problem}\n\nFix: {fix}") - - -def main(): - include = read(K0S_INCLUDE) - atoms = { - "k0s-k8s-version": scalar(include, "k0s-k8s-version", "include/k0s.yml"), - "k0s-patch": scalar(include, "k0s-patch", "include/k0s.yml"), - } - upstream_tag = expand( - scalar(include, "k0s-upstream-tag", "include/k0s.yml"), - atoms, - "include/k0s.yml k0s-upstream-tag", - ) - version = expand( - scalar(include, "k0s-version", "include/k0s.yml"), - atoms, - "include/k0s.yml k0s-version", - ) - - if not SAFE_VERSION_RE.match(version): - fail( - f"derived k0s-version {version!r} is not filename-safe.\n" - " It becomes a GitHub release asset name and is parsed back out by\n" - " the '@v' wildcard in files/os/sysupdate.d/70-k0s.transfer.", - "restrict k0s-version to [A-Za-z0-9._~^+-] in include/k0s.yml.", - ) - - # The upstream download URL must be derived, not restated. - bin_text = read(K0S_BIN) - url_match = re.search(r"^\s*url:\s*(\S+)\s*$", bin_text, re.MULTILINE) - if not url_match: - fail( - "elements/k0s/k0s-bin.bst declares no source 'url:'.", - "restore the pinned upstream k0s release URL.", - ) - url = url_match.group(1) - if "%{k0s-upstream-tag}" not in url: - fail( - "elements/k0s/k0s-bin.bst hardcodes the upstream k0s release tag:\n" - f" url: {url}\n" - " A hardcoded tag drifts from include/k0s.yml, so the binary and\n" - " the version stamped into the sysext can disagree.", - "use 'url: github:k0sproject/k0s/releases/download/" - "%{k0s-upstream-tag}/k0s-%{k0s-upstream-tag}-amd64'.", - ) - - # The asset filename must be on the k0s axis, not the OS release axis. - sysext_text = read(K0S_SYSEXT) - fname_match = re.search(r'^\s*FNAME="([^"]+)"\s*$', sysext_text, re.MULTILINE) - if not fname_match: - fail( - 'elements/oci/k0s-sysext.bst declares no FNAME="..." asset name.', - "restore the sysext asset filename assignment.", - ) - fname_expr = fname_match.group(1) - if "%{release-version}" in fname_expr: - fail( - "elements/oci/k0s-sysext.bst names the k0s sysext on the OS release\n" - f" axis: FNAME=\"{fname_expr}\"\n" - " 70-k0s.transfer reads this filename as the version of the k0s it\n" - " delivers.", - 'use FNAME="k0s-%{k0s-version}.raw".', - ) - if "%{k0s-version}" not in fname_expr: - fail( - "elements/oci/k0s-sysext.bst does not derive the sysext asset name\n" - f" from the pinned k0s version: FNAME=\"{fname_expr}\"", - 'use FNAME="k0s-%{k0s-version}.raw".', - ) - - # VERSION_ID must be generated, so sysext status matches the asset. - if 'echo "VERSION_ID=%{k0s-version}"' not in sysext_text: - fail( - "elements/oci/k0s-sysext.bst does not generate VERSION_ID= from\n" - " %{k0s-version}, so `systemd-sysext status` can disagree with the\n" - " asset filename an operator sees on disk.", - 'append \'VERSION_ID=%{k0s-version}\' to the staged ' - "extension-release.k0s.", - ) - - ext_text = read(EXTENSION_RELEASE) - stale = [ - line - for line in ext_text.splitlines() - if line.strip().startswith("VERSION_ID=") - ] - if stale: - fail( - "files/k0s/sysext/extension-release.k0s hardcodes a version:\n" - f" {stale[0]}\n" - " It is appended at build time from %{k0s-version}; a literal here\n" - " is a second, silently-winning source of truth.", - "delete the VERSION_ID= line from " - "files/k0s/sysext/extension-release.k0s.", - ) - - for path in (K0S_BIN, K0S_SYSEXT, EXTENSION_RELEASE): - for line in read(path).splitlines(): - if line.lstrip().startswith("#"): - continue - found = LITERAL_VERSION_RE.search(line) - if found: - fail( - f"{path.relative_to(ROOT)} restates a literal k0s version:\n" - f" {line.strip()}\n" - f" Literal: {found.group(0)}", - "read the version from include/k0s.yml via " - "%{k0s-upstream-tag} or %{k0s-version}.", - ) - - # The produced asset name must satisfy the sysupdate transfer pattern. - asset = fname_expr.replace("%{k0s-version}", version) + ".zst" - transfer_text = read(TRANSFER) - pattern_match = re.search( - r"^\s*MatchPattern=(\S*@v\S*\.zst)\s*$", transfer_text, re.MULTILINE - ) - if not pattern_match: - fail( - "files/os/sysupdate.d/70-k0s.transfer declares no [Source]\n" - " MatchPattern=...@v....zst.", - "restore the sysupdate source MatchPattern.", - ) - pattern = pattern_match.group(1) - prefix, _, suffix = pattern.partition("@v") - captured = None - if ( - asset.startswith(prefix) - and asset.endswith(suffix) - and len(asset) >= len(prefix) + len(suffix) - ): - captured = asset[len(prefix) : len(asset) - len(suffix)] - if captured != version: - fail( - "the version systemd-sysupdate would read from the k0s sysext asset\n" - " name is not the pinned k0s version:\n" - f" built asset : {asset}\n" - f" MatchPattern : {pattern}\n" - f" '@v' captures : {captured!r}\n" - f" k0s-version : {version!r}\n" - " Hosts compare the captured string against the version they have\n" - " installed, so a mismatch means k0s updates silently stop.", - "align FNAME in elements/oci/k0s-sysext.bst with MatchPattern in " - "files/os/sysupdate.d/70-k0s.transfer.", - ) - - print( - f"OK: k0s {upstream_tag} pinned in include/k0s.yml; " - f"asset {asset} matches {pattern} and VERSION_ID={version}." - ) - - -if __name__ == "__main__": - main() diff --git a/.github/scripts/check-kubernetes-version.py b/.github/scripts/check-kubernetes-version.py new file mode 100755 index 00000000..63519446 --- /dev/null +++ b/.github/scripts/check-kubernetes-version.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Enforce the Kubernetes version invariant for Bluefin Server. + +Kubernetes is an independently-pinned third-party payload, not an OS asset, so +it is versioned on its own axis. ``include/kubernetes.yml`` is the single +source of truth: + + variables: + k8s-version: "1.36.4" + cni-version: "1.9.1" + k8s-upstream-tag: "v%{k8s-version}" + cni-upstream-tag: "v%{cni-version}" + +Four consumers restate that version in four different spellings, and all four +must move together or the sysext silently stops updating: + + * ``elements/kubernetes/kubernetes-bin.bst`` — the dl.k8s.io download URLs + for kubectl, kubeadm and kubelet. + * ``elements/kubernetes/cni-plugins.bst`` — the CNI plugins release URL. + * ``elements/oci/kubernetes-sysext.bst`` — the release asset filename, + which ``files/os/sysupdate.kubernetes.d/70-kubernetes.transfer`` reads as + the version oracle through its ``@v`` wildcard, plus the two + ``/usr/local/share/kubernetes-*version`` stamps. + * ``files/kubernetes/sysext/extension-release.kubernetes`` — the + ``VERSION_ID=`` reported by ``systemd-sysext status``. + +This script fails closed if any consumer reintroduces a literal Kubernetes or +CNI version or drops back onto the OS release axis, and cross-checks that the +derived asset filename still matches the sysupdate transfer's MatchPattern. +""" + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +K8S_INCLUDE = ROOT / "include" / "kubernetes.yml" +K8S_BIN = ROOT / "elements" / "kubernetes" / "kubernetes-bin.bst" +CNI_BIN = ROOT / "elements" / "kubernetes" / "cni-plugins.bst" +K8S_SYSEXT = ROOT / "elements" / "oci" / "kubernetes-sysext.bst" +EXTENSION_RELEASE = ( + ROOT / "files" / "kubernetes" / "sysext" / "extension-release.kubernetes" +) +TRANSFER = ( + ROOT / "files" / "os" / "sysupdate.kubernetes.d" / "70-kubernetes.transfer" +) + +SAFE_VERSION_RE = re.compile(r"^[A-Za-z0-9._~^+-]+$") +LITERAL_VERSION_RE = re.compile(r"(? sysext/usr/local/share/{stamp}' not in sysext_text: + fail( + f"elements/oci/kubernetes-sysext.bst does not generate\n" + f" /usr/local/share/{stamp} from {atom}.\n" + " That file is what a merged /usr reports as the delivered\n" + " version; a stale or absent stamp misreports the running node.", + f'write the stamp with \'echo "{atom}" > ' + f"sysext/usr/local/share/{stamp}'.", + ) + + ext_text = read(EXTENSION_RELEASE) + stale = [ + line + for line in ext_text.splitlines() + if line.strip().startswith("VERSION_ID=") + ] + if stale: + fail( + "files/kubernetes/sysext/extension-release.kubernetes hardcodes a\n" + f" version:\n {stale[0]}\n" + " It is appended at build time from %{k8s-version}; a literal here\n" + " is a second, silently-winning source of truth.", + "delete the VERSION_ID= line from " + "files/kubernetes/sysext/extension-release.kubernetes.", + ) + + for path in (K8S_BIN, CNI_BIN, K8S_SYSEXT, EXTENSION_RELEASE): + for line in read(path).splitlines(): + if line.lstrip().startswith("#"): + continue + found = LITERAL_VERSION_RE.search(line) + if found: + fail( + f"{path.relative_to(ROOT)} restates a literal Kubernetes or CNI\n" + f" version:\n {line.strip()}\n" + f" Literal: {found.group(0)}", + "read the version from include/kubernetes.yml via " + "%{k8s-upstream-tag}, %{cni-upstream-tag}, %{k8s-version} or " + "%{cni-version}.", + ) + + # The produced asset name must satisfy the sysupdate transfer pattern. + asset = fname_expr.replace("%{k8s-version}", version) + ".zst" + transfer_text = read(TRANSFER) + pattern_match = re.search( + r"^\s*MatchPattern=(\S*@v\S*\.zst)\s*$", transfer_text, re.MULTILINE + ) + if not pattern_match: + fail( + "files/os/sysupdate.kubernetes.d/70-kubernetes.transfer declares no\n" + " [Source] MatchPattern=...@v....zst.", + "restore the sysupdate source MatchPattern.", + ) + pattern = pattern_match.group(1) + prefix, _, suffix = pattern.partition("@v") + captured = None + if ( + asset.startswith(prefix) + and asset.endswith(suffix) + and len(asset) >= len(prefix) + len(suffix) + ): + captured = asset[len(prefix) : len(asset) - len(suffix)] + if captured != version: + fail( + "the version systemd-sysupdate would read from the Kubernetes sysext\n" + " asset name is not the pinned Kubernetes version:\n" + f" built asset : {asset}\n" + f" MatchPattern : {pattern}\n" + f" '@v' captures : {captured!r}\n" + f" k8s-version : {version!r}\n" + " Hosts compare the captured string against the version they have\n" + " installed, so a mismatch means Kubernetes updates silently stop.", + "align FNAME in elements/oci/kubernetes-sysext.bst with MatchPattern " + "in files/os/sysupdate.kubernetes.d/70-kubernetes.transfer.", + ) + + print( + f"OK: Kubernetes {upstream_tag} and CNI plugins {cni_tag} pinned in " + f"include/kubernetes.yml; asset {asset} matches {pattern} and " + f"VERSION_ID={version}." + ) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/check-release-version.py b/.github/scripts/check-release-version.py index dd9ed259..a84152b7 100755 --- a/.github/scripts/check-release-version.py +++ b/.github/scripts/check-release-version.py @@ -11,9 +11,10 @@ `bluefin-server-installer-.raw.zst`) and is the version systemd-sysupdate extracts from those filenames via `@v`. -The k0s sysext is deliberately *not* on this axis: it is an -independently-pinned third-party payload versioned from `include/k0s.yml` -and enforced separately by `.github/scripts/check-k0s-version.py`. +The Kubernetes sysext is deliberately *not* on this axis: it is an +independently-pinned third-party payload versioned from +`include/kubernetes.yml` and enforced separately by +`.github/scripts/check-kubernetes-version.py`. The release *tag* is derived independently by the Justfile (`fsdk_version`), which greps the point release out of the pinned diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a00ed52b..4d7fd1b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,7 @@ jobs: just build-installer just export-pxe - - name: Build and export k0s systemd-sysext + - name: Build and export Kubernetes systemd-sysext run: | just build-sysext just export-sysext @@ -102,7 +102,7 @@ jobs: cp dist/bluefin-server-installer-*.raw.zst dist/bluefin-server-*.efi dist/release/ cp dist/bluefin-server-pxe-vmlinuz-* dist/bluefin-server-pxe-initrd-*.cpio.gz dist/release/ cp dist/ddi/bluefin-server-ddi-*.raw.zst dist/release/ - cp dist/sysext/k0s-*.raw.zst dist/release/ + cp dist/sysext/kubernetes-*.raw.zst dist/release/ if [ -d dist/kernel ]; then cp dist/kernel/usr/lib/sysexts/*.raw dist/release/ 2>/dev/null || true fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a9f61ab..35a10f42 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,9 +24,9 @@ repos: language: system pass_filenames: false files: ^(renovate\.json|elements/(freedesktop-sdk|gnome-build-meta)\.bst|\.github/scripts/check-renovate-series\.py)$ - - id: check-k0s-version - name: k0s version derived from include/k0s.yml, not restated - entry: python .github/scripts/check-k0s-version.py + - id: check-kubernetes-version + name: Kubernetes version derived from include/kubernetes.yml, not restated + entry: python .github/scripts/check-kubernetes-version.py language: system pass_filenames: false - files: ^(include/k0s\.yml|elements/k0s/k0s-bin\.bst|elements/oci/k0s-sysext\.bst|files/k0s/sysext/extension-release\.k0s|files/os/sysupdate\.k0s\.d/70-k0s\.transfer|\.github/scripts/check-k0s-version\.py)$ + files: ^(include/kubernetes\.yml|elements/kubernetes/(kubernetes-bin|cni-plugins)\.bst|elements/oci/kubernetes-sysext\.bst|files/kubernetes/sysext/extension-release\.kubernetes|files/os/sysupdate\.kubernetes\.d/70-kubernetes\.transfer|\.github/scripts/check-kubernetes-version\.py)$ diff --git a/AGENTS.md b/AGENTS.md index a79d0de0..a6ec7517 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ Bluefin Server is an FSDK-based, image-based Linux server OS. It produces: - an immutable XFS DDI OS payload (`oci/bluefin-server-ddi.bst`) - an offline, systemd-native installer raw disk (`oci/bluefin-server-installer.bst`) -- an optional k0s `systemd-sysext` (`oci/k0s-sysext.bst`) +- an optional Kubernetes `systemd-sysext` (`oci/kubernetes-sysext.bst`) ## What agents should know first @@ -21,7 +21,7 @@ Bluefin Server is an FSDK-based, image-based Linux server OS. It produces: 1. Compose from FSDK 26.08 `components/*`. Never use `platform.bst`. 2. Keep the CPU baseline broad: no `x86_64_v3`. 3. Installer must stay `systemd-sysinstall`-native and `systemd-repart`-based; no custom installer scripts or non-native installers. -4. Deliver k0s as an optional `systemd-sysext`; never bundle Kubernetes or container runtimes into the base OS DDI. +4. Deliver Kubernetes as an optional `systemd-sysext`; never bundle Kubernetes or container runtimes into the base OS DDI. 5. Boot entries use GPT `PARTUUID`; never hardcode device paths. 6. One canonical source per fact; do not duplicate content across docs. @@ -46,7 +46,7 @@ All local `just` targets run BuildStream inside the FSDK `bst2` container via `j | `just export-ddi` | Export DDI artifacts to `dist/ddi/`. | | `just build-installer` | Local full installer build. | | `just export-installer` | Export installer + UKI to `dist/`. | -| `just build-sysext` | Build the k0s `systemd-sysext`. | +| `just build-sysext` | Build the Kubernetes `systemd-sysext`. | | `just export-sysext` | Export sysext artifacts to `dist/sysext/`. | | `just show-me-the-future` | Local QEMU installer smoke test. | @@ -55,9 +55,9 @@ All local `just` targets run BuildStream inside the FSDK `bst2` container via `j | Task | Skill | |---|---| | Build or debug the installer / DDI | [`docs/skills/ddi-installer.md`](docs/skills/ddi-installer.md), [`docs/skills/ddi-installer-build.md`](docs/skills/ddi-installer-build.md) | -| Factory role, k0s sysext rationale, lab integration | [`docs/skills/factory-integration.md`](docs/skills/factory-integration.md) | +| Factory role, Kubernetes sysext rationale, lab integration | [`docs/skills/factory-integration.md`](docs/skills/factory-integration.md) | | Work with `systemd-sysext` / `systemd-confext` | [`docs/skills/systemd-sysext-extensions.md`](docs/skills/systemd-sysext-extensions.md) | -| Build or ship the k0s sysext | [`docs/skills/k0s-sysext.md`](docs/skills/k0s-sysext.md), [`docs/skills/k0s-sysext-ops.md`](docs/skills/k0s-sysext-ops.md) | +| Build or ship the Kubernetes sysext | [`docs/skills/kubernetes-sysext.md`](docs/skills/kubernetes-sysext.md), [`docs/skills/kubernetes-sysext-ops.md`](docs/skills/kubernetes-sysext-ops.md) | | Update the FSDK pin / versioning | [`docs/skills/bump-fsdk-version.md`](docs/skills/bump-fsdk-version.md) | | Flatcar version parity matrix / audit | [`docs/skills/flatcar-parity-matrix.md`](docs/skills/flatcar-parity-matrix.md) | | CI workflows, action SHA pinning | [`docs/skills/ci-tooling.md`](docs/skills/ci-tooling.md) | diff --git a/CONTEXT.md b/CONTEXT.md index c2fff25e..d38e710a 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -13,7 +13,7 @@ A bootable GPT raw disk image executing systemd-repart to partition and install _Avoid_: live ISO, installation media, setup script **Sysext**: -A systemd-sysext extension raw image merged into `/usr` to deliver decoupled server runtimes such as k0s. +A systemd-sysext extension raw image merged into `/usr` to deliver decoupled server runtimes such as Kubernetes and containerd. _Avoid_: addon, plugin, package, sidecar **Transfer**: diff --git a/Formula/kc-agent.rb b/Formula/kc-agent.rb index 639913d9..66e4788d 100644 --- a/Formula/kc-agent.rb +++ b/Formula/kc-agent.rb @@ -47,7 +47,7 @@ def install end service do - run [opt_bin/"kc-agent", "-allowed-origins", "http://localhost:8080,http://127.0.0.1:8080"] + run [opt_bin/"kc-agent", "-allowed-origins", "https://localhost:8080,https://127.0.0.1:8080"] keep_alive true environment_variables KAGENTI_CONTROLLER_URL: "none" log_path var/"log/kc-agent.log" diff --git a/Justfile b/Justfile index 1bae6d3f..85b05dd9 100644 --- a/Justfile +++ b/Justfile @@ -54,11 +54,11 @@ tags: [group('dev')] validate: python3 .github/scripts/check-release-version.py - python3 .github/scripts/check-k0s-version.py + python3 .github/scripts/check-kubernetes-version.py python3 .github/scripts/check-renovate-series.py just bst show --deps all oci/bluefin-server-ddi.bst just bst show --deps all oci/bluefin-server-installer.bst - just bst show --deps all oci/k0s-sysext.bst + just bst show --deps all oci/kubernetes-sysext.bst # Run the unit test suite (pytest + bats). [group('dev')] @@ -124,8 +124,38 @@ export-installer: build-installer just bst artifact checkout oci/bluefin-server-installer.bst --directory /src/dist/installer-checkout mv dist/installer-checkout/* dist/ rm -rf dist/installer-checkout + # Record what this artifact was built from. mtimes cannot answer that: a + # fresh clone stamps every file at checkout time, and a rebase rewrites + # commit dates, so both signals lie. A content hash of the build inputs + # does not. + @just _record-provenance @echo "==> wrote:" && ls -lh dist/ +# Write the provenance sidecar naming the sources an export was built from. +[private] +_record-provenance: + #!/usr/bin/env bash + set -euo pipefail + IMG=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' -print -quit) + [ -n "${IMG}" ] || exit 0 + { + echo "commit $(git rev-parse HEAD 2>/dev/null || echo unknown)" + echo "tree $(just _inputs-hash)" + } > "${IMG}.provenance" + echo "==> provenance: $(sed -n 2p "${IMG}.provenance")" + +# Content hash of everything the installer image is built from. Deterministic +# across clones and rebases, unlike mtimes or commit dates. +[private] +_inputs-hash: + #!/usr/bin/env bash + set -euo pipefail + find elements include files project.conf -type f 2>/dev/null \ + | LC_ALL=C sort \ + | xargs -r sha256sum \ + | sha256sum \ + | cut -d' ' -f1 + # Export standalone kernel and initrd for PXE boot. The DDI remains embedded # in the raw installer image; network DDI fetching is not enabled. [group('installer')] @@ -134,26 +164,27 @@ export-pxe: export-installer @test -n "$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-pxe-initrd-*.cpio.gz' -print -quit)" || { echo "ERROR: PXE initrd was not exported." >&2; exit 1; } @echo "==> wrote PXE artifacts:" && ls -lh dist/bluefin-server-pxe-* -# -- k0s systemd-sysext ------------------------------------------------------- -# Produces a systemd-sysext extension image for k0s. +# -- Kubernetes systemd-sysext ------------------------------------------------ +# Produces a systemd-sysext extension image carrying kubeadm, kubelet, kubectl +# and the CNI plugins. -# Build the k0s systemd-sysext image. +# Build the Kubernetes systemd-sysext image. [group('sysext')] build-sysext: - just bst build oci/k0s-sysext.bst + just bst build oci/kubernetes-sysext.bst -# Export the k0s systemd-sysext image + SHA256SUMS to dist/sysext/. +# Export the Kubernetes systemd-sysext image + SHA256SUMS to dist/sysext/. # The artifact checkout also emits an uncompressed .raw; only the # versioned .raw.zst release asset and its SHA256SUMS are published. [group('sysext')] export-sysext: build-sysext rm -rf dist/sysext dist/sysext-checkout mkdir -p dist/sysext-checkout dist/sysext - just bst artifact checkout oci/k0s-sysext.bst --directory /src/dist/sysext-checkout - cp dist/sysext-checkout/k0s-*.raw.zst dist/sysext/ + just bst artifact checkout oci/kubernetes-sysext.bst --directory /src/dist/sysext-checkout + cp dist/sysext-checkout/kubernetes-*.raw.zst dist/sysext/ cp dist/sysext-checkout/SHA256SUMS dist/sysext/ rm -rf dist/sysext-checkout - @echo "==> wrote k0s sysext:" && ls -lh dist/sysext/ + @echo "==> wrote kubernetes sysext:" && ls -lh dist/sysext/ # -- Flatcar LTS kernel & ZFS -------------------------------------------------- # Build the Flatcar LTS kernel and ZFS sysext. @@ -196,12 +227,74 @@ flash-installer DEVICE="": echo "ERROR: {{DEVICE}} is not a valid block device!" >&2 exit 1 fi - IMG=$(find dist/ -type f -name 'bluefin-server-installer-*.raw.zst' | head -n1) - if [ -z "${IMG}" ]; then + # Refuse the disk backing the running system outright. A single mistyped + # character here is unrecoverable, and the y/N prompt below is not a + # meaningful defence against a typo the operator has already committed to. + # + # `/` is not always on a block device. On composefs/ostree hosts — including + # Bluefin itself, which is what a developer runs this from — `findmnt / ` + # reports a composefs digest, and the real device is mounted at /sysroot. + # Check both, so the guard is not silently inert on exactly the systems this + # project targets. + TARGET_NAME=$(lsblk -no KNAME "{{DEVICE}}" 2>/dev/null | head -n1 || true) + for mp in / /sysroot; do + SRC=$(findmnt -no SOURCE "${mp}" 2>/dev/null | head -n1 || true) + case "${SRC}" in /dev/*) ;; *) continue ;; esac + # PKNAME is the parent disk of a partition, and is empty when the + # filesystem sits directly on a whole-disk device (no partition table, + # or a device-mapper/loop node). Falling back to KNAME keeps the guard + # live on those hosts instead of silently skipping the comparison. + SRC_DISK=$(lsblk -no PKNAME "${SRC}" 2>/dev/null | head -n1 || true) + if [ -z "${SRC_DISK}" ]; then + SRC_DISK=$(lsblk -no KNAME "${SRC}" 2>/dev/null | head -n1 || true) + fi + if [ -n "${SRC_DISK}" ] && [ "${SRC_DISK}" = "${TARGET_NAME}" ]; then + echo "ERROR: {{DEVICE}} is the disk backing the running system." >&2 + echo " ${mp} is on ${SRC}, which lives on /dev/${SRC_DISK}." >&2 + echo "Refusing to overwrite it." >&2 + exit 1 + fi + done + # Refuse a device with anything mounted off it. dd writing under a live + # filesystem gives a torn image and leaves the kernel holding stale page + # cache for blocks that no longer exist. + MOUNTED=$(lsblk -n -o MOUNTPOINTS "{{DEVICE}}" | grep -v '^\s*$' || true) + if [ -n "${MOUNTED}" ]; then + echo "ERROR: {{DEVICE}} has mounted partitions:" >&2 + lsblk -p -o NAME,SIZE,MOUNTPOINTS "{{DEVICE}}" >&2 + echo >&2 + echo "Unmount them first, e.g.:" >&2 + # -l (list) not the default tree: tree mode prefixes names with box + # glyphs, which would make the suggested command uncopyable. + lsblk -p -n -l -o NAME,MOUNTPOINTS "{{DEVICE}}" \ + | awk 'NF>1 {printf " udisksctl unmount -b %s\n", $1}' >&2 + exit 1 + fi + # -maxdepth 1 keeps release images in dist/release/ out of the match, but a + # stale export beside a fresh one is still ambiguous — fail rather than let + # `head -n1` pick by directory order. + mapfile -t IMGS < <(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | sort) + if [ "${#IMGS[@]}" -eq 0 ]; then echo "ERROR: No exported installer found in dist/." >&2 echo "Please run: just build-installer && just export-installer" >&2 exit 1 fi + if [ "${#IMGS[@]}" -gt 1 ]; then + echo "ERROR: ${#IMGS[@]} installer images in dist/; refusing to guess:" >&2 + printf ' %s\n' "${IMGS[@]}" >&2 + echo "Remove the stale one, then re-run." >&2 + exit 1 + fi + IMG="${IMGS[0]}" + # Verify the archive before the prompt, not after. A truncated or corrupt + # download is exactly what a freshly fetched CI artifact invites, and + # failing here costs nothing while failing mid-write leaves an unbootable + # disk that looks like it succeeded. + echo "Verifying ${IMG}..." + if ! zstd -t "${IMG}"; then + echo "ERROR: ${IMG} failed its integrity check; refusing to write it." >&2 + exit 1 + fi echo "WARNING: All data on {{DEVICE}} will be COMPLETELY DESTROYED!" echo "Double-checking device information:" lsblk -p "{{DEVICE}}" @@ -212,9 +305,258 @@ flash-installer DEVICE="": exit 1 fi echo "Writing ${IMG} to {{DEVICE}}..." - sudo sh -c "zstd -dc ${IMG} | dd of={{DEVICE}} bs=4M iflag=fullblock oflag=direct status=progress conv=fsync" + # Digest and byte count of the payload. Both are needed after the write: + # the count to know how much of the device to read back, the digest to + # compare it against. + # + # Two plain passes rather than one with `tee >(wc -c > file)`. Bash does + # not wait for process-substitution children, so the count file may still + # be unwritten when $( ) returns — it happens to work only because wc + # finishes before sha256sum. An empty count yields a malformed `count=` + # and fails the flash with a misleading "medium did not retain the image". + # The extra decompression costs seconds against a write measured in + # minutes. + echo "Hashing ${IMG}..." + EXPECT_SHA=$(zstd -dc "${IMG}" | sha256sum | cut -d' ' -f1) + EXPECT_BYTES=$(zstd -dc "${IMG}" | wc -c | tr -d ' ') + # bash with pipefail, not sh. Under POSIX sh the pipeline's status is dd's + # alone, so zstd dying mid-stream leaves dd exiting 0 after writing a + # partial image and the success line below printing anyway. The outer + # `set -o pipefail` does not reach here — there is no pipeline in the outer + # shell, only this nested one. + sudo bash -c "set -o pipefail; zstd -dc '${IMG}' | dd of={{DEVICE}} bs=4M iflag=fullblock oflag=direct status=progress conv=fsync" + # Read the device back and compare. Without this, "flashed" rests on dd's + # exit status, which says the writes were accepted, not that the medium + # kept them — the failure mode of a dying USB stick. Do it before the + # relocation below, which deliberately rewrites the headers and would make + # the digests differ for a legitimate reason. + # Drop the kernel's cached view of the device first, so the comparison + # reads the medium and not the pages just written through it. O_DIRECT + # would do the same but requires aligned reads, and the trailing block is + # partial whenever the image is not a multiple of the block size. + sudo blockdev --flushbufs {{DEVICE}} + echo "Verifying ${EXPECT_BYTES} bytes read back from {{DEVICE}}..." + ACTUAL_SHA=$(sudo dd if={{DEVICE}} bs=4M iflag=fullblock,count_bytes \ + count="${EXPECT_BYTES}" status=none | sha256sum | cut -d' ' -f1) + if [ "${ACTUAL_SHA}" != "${EXPECT_SHA}" ]; then + echo "ERROR: {{DEVICE}} does not contain what was written." >&2 + echo " expected ${EXPECT_SHA}" >&2 + echo " read ${ACTUAL_SHA}" >&2 + echo "The medium did not retain the image. Replace it." >&2 + exit 1 + fi + echo "Content verified: ${EXPECT_SHA}" + # Move the GPT backup header to the end of the DEVICE. + # + # dd writes the image verbatim, so the backup header lands at the end of the + # IMAGE. GPT requires it at the device's last LBA, so on any medium larger + # than the image the table is only half valid: the primary header points at + # a backup LBA that is not where the device ends. parted says so directly — + # + # Warning: Not all of the space available to /dev/sdb appears to be used, + # you can fix the GPT to use all of the space (an extra 115791863 blocks) + # + # — and tools disagree about partition sizes between reads. + # + # This does NOT stop the medium booting, and it is worth being precise about + # that, because assuming otherwise cost a day of chasing a phantom. UEFI + # reads the PRIMARY header at LBA 1, which dd writes correctly; the backup + # is a redundancy copy consulted when the primary is damaged. A medium in + # this state was written to a 58.6 GB sparse file, reproduced the sfdisk + # error exactly, and booted to systemd PID 1 under OVMF. + # + # What it breaks is tooling, and it is invisible to every test we have: + # `just test-installer-artifact` and the CI installer-test attach the image + # as a drive sized exactly to the image, so device size always equals image + # size and the condition cannot arise. + echo "Relocating the GPT backup header to the end of {{DEVICE}}..." + sudo sfdisk --relocate gpt-bak-std {{DEVICE}} + sudo partprobe {{DEVICE}} || true + sudo sfdisk --verify {{DEVICE}} echo "Successfully flashed the Bluefin Server installer to {{DEVICE}}!" +# Boot the installer MEDIUM through firmware, as real hardware does. +# +# This is the only test that exercises the ESP. `test-installer-artifact` and +# CI's installer-test both attach installer.raw as a data disk and inject the +# kernel with -kernel/-initrd, so EFI/BOOT/BOOTX64.EFI is never executed. They +# prove the installer installs; they cannot prove the medium boots. +# +# That gap shipped a medium that does not boot: a single 434 MiB UKI which OVMF +# loads and never executes, with every gate green. +# +# The image is written into a sparse file LARGER than itself, because that is +# what a real stick is, and it reproduces the GPT backup-header condition that +# an image-sized virtual disk cannot. +[group('test')] +test-installer-boot: + #!/usr/bin/env bash + set -euo pipefail + + # Same refusal as `flash-installer`: a stale export beside a fresh one is + # ambiguous, and `head -n1` would quietly gate on whichever sorts first. + # A boot result is only evidence about a named artifact, so the artifact + # must not be picked by accident. + mapfile -t IMGS < <(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | sort) + if [ "${#IMGS[@]}" -eq 0 ]; then + echo "ERROR: no exported installer in dist/. Run: just export-installer" >&2 + exit 1 + fi + if [ "${#IMGS[@]}" -gt 1 ]; then + echo "ERROR: ${#IMGS[@]} installer images in dist/; refusing to guess:" >&2 + printf ' %s\n' "${IMGS[@]}" >&2 + echo "Remove the stale one, then re-run." >&2 + exit 1 + fi + IMG="${IMGS[0]}" + + # This boots an EXPORTED ARTIFACT. It does not build, and `just validate` + # only resolves the graph without building either. So a green result is + # evidence about the bytes in dist/, NOT about the current state of + # elements/oci/bluefin-server-installer.bst. + # + # Say so out loud, and refuse to imply more than that when the tree has + # moved since the export. Without this, "the medium boots" quietly becomes + # a claim about a tree that was never built — the same overclaim this + # recipe exists to prevent. + # + # The comparison is a content hash of the build inputs, recorded into a + # sidecar by `export-installer`. Not mtimes: a fresh clone stamps every + # file at checkout time and a rebase rewrites commit dates, so both would + # report a clean tree as stale and a stale tree as clean. Not an allow-list + # of "important" sources either — that drifts the moment someone edits a + # repart config or a unit file. + echo "==> Artifact under test:" + echo " ${IMG}" + echo " sha256 $(sha256sum "${IMG}" | cut -d' ' -f1)" + if [ -f "${IMG}.provenance" ]; then + BUILT_FROM=$(awk '/^tree /{print $2}' "${IMG}.provenance") + BUILT_AT=$(awk '/^commit /{print $2}' "${IMG}.provenance") + echo " commit ${BUILT_AT}" + NOW=$(just _inputs-hash) + if [ "${BUILT_FROM}" != "${NOW}" ]; then + echo "WARNING: the tree has changed since this artifact was built." >&2 + echo " built from ${BUILT_FROM}" >&2 + echo " tree now ${NOW}" >&2 + echo " A pass proves the exported medium boots. It proves" >&2 + echo " NOTHING about the current sources. Re-export before" >&2 + echo " treating this as a gate on your changes." >&2 + else + echo " tree matches current sources" + fi + else + echo "WARNING: no provenance sidecar beside this artifact, so there is" >&2 + echo " no way to tell what it was built from. A pass proves" >&2 + echo " these bytes boot and nothing more. Re-export to record" >&2 + echo " provenance." >&2 + fi + + first_existing() { for f in "$@"; do [ -f "$f" ] && { echo "$f"; return 0; }; done; return 1; } + OVMF_CODE=$(first_existing \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-code.fd \ + /usr/share/edk2/ovmf/OVMF_CODE.fd \ + /usr/share/OVMF/OVMF_CODE.fd \ + /usr/share/OVMF/OVMF_CODE_4M.fd \ + /usr/share/edk2/x64/OVMF_CODE.4m.fd \ + /usr/share/qemu/edk2-x86_64-code.fd \ + /usr/share/qemu/OVMF_CODE.fd) \ + || { echo "ERROR: OVMF_CODE not found" >&2; exit 1; } + + WORK=$(mktemp -d "${XDG_CACHE_HOME:-$HOME/.cache}/bluefin-boot-test.XXXXXX") + trap 'rm -rf "$WORK"' EXIT INT TERM + + # A stick is bigger than the image. Sparse, so this costs only real bytes. + truncate -s 62914560000 "$WORK/medium.img" + zstd -dc "${IMG}" | dd of="$WORK/medium.img" conv=notrunc bs=4M iflag=fullblock status=none + sfdisk --relocate gpt-bak-std "$WORK/medium.img" >/dev/null + sfdisk --verify "$WORK/medium.img" + + # OVMF needs a writable vars pflash of the same size as the code image. + truncate -s "$(stat -c %s "$OVMF_CODE")" "$WORK/vars.fd" + + DEADLINE="${INSTALLER_BOOT_DEADLINE:-240}" + # A healthy medium boots with `quiet loglevel=3` and prints almost nothing, + # which is indistinguishable from a hang until you look closely. Setting + # INSTALLER_BOOT_CMDLINE_EXTRA=loglevel=7 makes a healthy boot loud without + # rebuilding anything: systemd-stub reads this SMBIOS type-11 string and + # appends it to the UKI's embedded cmdline. + CMDLINE_EXTRA="${INSTALLER_BOOT_CMDLINE_EXTRA:-}" + # if-blocks, not `[ -n x ] && y`: under `set -e` a false test makes the + # whole recipe exit, so the default (no extra cmdline) would abort the run. + SMBIOS=() + if [ -n "${CMDLINE_EXTRA}" ]; then + SMBIOS=(-smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=${CMDLINE_EXTRA}") + fi + echo "==> Booting the installer medium through firmware (deadline ${DEADLINE}s)..." + if [ -n "${CMDLINE_EXTRA}" ]; then + echo " cmdline-extra: ${CMDLINE_EXTRA}" + fi + timeout "${DEADLINE}" qemu-system-x86_64 \ + -enable-kvm -m 4096 -smp 2 -cpu host \ + -drive file="$WORK/medium.img",format=raw,if=virtio \ + -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ + -drive if=pflash,format=raw,file="$WORK/vars.fd" \ + "${SMBIOS[@]}" \ + -nographic -serial file:"$WORK/serial.log" -monitor none -no-reboot \ + >/dev/null 2>&1 || true + + # `strings`, not `cat`. The console is almost entirely ANSI and OSC escape + # sequences, which a terminal swallows — `cat` on a perfectly good boot log + # shows two firmware lines and apparent silence. That is what convinced me + # a working medium was hung. + echo "==> Serial output:" + strings -n 4 "$WORK/serial.log" | grep -av '^\[[0-9;]*[A-Za-z]$' || true + + # Criterion 1: firmware must actually start an image, not merely find the + # medium. A corrupted bootloader makes firmware fall through to PXE, and + # this is the check that catches it — verified against a medium built by + # mcopy-ing junk over EFI/BOOT/BOOTX64.EFI, which produced + # "BdsDxe: failed to load ... Not Found" and no `starting` line. + # + # Criterion 2 below does not depend on this one: PXE fallback chatter + # cannot produce a machineid, and was checked against the corrupt medium's + # log — no match. Criterion 1 is kept because it names the failure + # precisely ("firmware never started an image" vs "userspace never came + # up"), which is the difference between a dead bootloader and a dead boot. + if ! grep -aq "BdsDxe: starting" "$WORK/serial.log"; then + echo "ERROR: firmware never started a boot image from the medium." >&2 + exit 1 + fi + # Firmware handing off is not proof that userspace ran. Assert on something + # only a running Linux userspace can emit. + # + # systemd PID 1 writes an OSC 3008 sequence to the console carrying the + # machine's identity: + # + # ESC ]3008;start=;user=root;hostname=;machineid=;bootid=; + # pid=1;comm=systemd;type=boot + # + # Firmware cannot produce a machineid or a bootid. This survives `quiet + # loglevel=3`, which is what the medium boots with, and it does not depend + # on guessing kernel log strings. + # Two earlier criteria were tried and rejected, both against captured logs: + # + # "Linux version|initrd|systemd" — `quiet` suppresses all of it, while a + # FAILING boot prints dracut emergency text. It rewarded failure. + # any non-whitespace after handoff — firmware and systemd-stub both emit + # CSI cursor codes, so a corrupt bootloader falling through to PXE + # scored as success. + # + # Honest limit: proven against a booting medium (fires on the real image + # under both `quiet` and loglevel=7) and against a medium whose bootloader + # is destroyed (its PXE-fallback log contains no machineid). NOT proven + # against a genuine start-then-hang, because no such image exists — the + # 434 MiB UKI was suspected of being one and was exonerated. If one ever + # turns up, keep it. + if ! grep -aqE 'machineid=[0-9a-f]{32}|comm=systemd' "$WORK/serial.log"; then + echo "ERROR: firmware started an image, but userspace never came up." >&2 + echo " No systemd identity record on the console. Control did not" >&2 + echo " reach PID 1." >&2 + exit 1 + fi + echo "==> The installer medium boots: systemd PID 1 reported in." + grep -ao 'machineid=[0-9a-f]\{32\}' "$WORK/serial.log" | head -1 | sed 's/^/ /' + # Build the installer artifacts, then run the reusable artifact smoke path. [group('test')] show-me-the-future: @@ -318,6 +660,11 @@ test-installer-artifact: } trap cleanup EXIT INT TERM + SSH_KEY="$WORKDIR/test_ssh_key" + rm -f "$SSH_KEY" "$SSH_KEY.pub" + ssh-keygen -t ed25519 -N "" -f "$SSH_KEY" >/dev/null 2>&1 + SSH_PUB_B64=$(cat "$SSH_KEY.pub" | base64 -w0) + echo "==> Booting the installed server in QEMU (background)..." qemu-system-x86_64 \ -enable-kvm \ @@ -327,9 +674,10 @@ test-installer-artifact: -drive file="$WORKDIR/target.raw",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080 \ + -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080,hostfwd=tcp:127.0.0.1:2222-:22 \ -smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \ - -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service" \ + -smbios "type=11,value=io.systemd.credential.binary:ssh.authorized_keys.root=${SSH_PUB_B64}" \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service systemd.mask=systemd-homed-firstboot.service systemd.wants=sshd.service" \ -nographic \ -serial file:"$SERIAL_LOG" \ -monitor none & @@ -337,7 +685,7 @@ test-installer-artifact: DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}" START_TIME=$(date +%s) - echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..." + echo "==> Polling KubeStellar Console readiness (deadline: ${DEADLINE_SECS}s)..." while true; do if ! kill -0 "$TARGET_QEMU_PID" 2>/dev/null; then @@ -349,15 +697,20 @@ test-installer-artifact: exit 1 fi - HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) - if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then - ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) + # Probe guest directly over SSH tunnel or in-guest curl to 127.0.0.1:8080. + # The kiosk serves TLS with an in-cluster CA, hence --insecure. Plain HTTP + # is not probed: nginx answers it with a 302 to https, so a fallback would + # report 302 rather than the real status. + HEALTHZ_RESP=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 https://127.0.0.1:8080/healthz 2>/dev/null || true" 2>/dev/null || true) + if [ -n "$HEALTHZ_RESP" ]; then + ROOT_CODE=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p 2222 root@127.0.0.1 "curl --silent --insecure --max-time 2 --output /dev/null --write-out '%{http_code}' https://127.0.0.1:8080/ 2>/dev/null || true" 2>/dev/null || true) if [ "$ROOT_CODE" = "200" ]; then - echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" - break + if echo "$HEALTHZ_RESP" | jq -e '.status == "ok"' >/dev/null 2>&1; then + echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" + break + fi fi fi - NOW=$(date +%s) ELAPSED=$((NOW - START_TIME)) if [ "$ELAPSED" -ge "$DEADLINE_SECS" ]; then @@ -466,7 +819,7 @@ install-vm: } trap cleanup INT TERM - until curl --silent --show-error --max-time 2 --output /dev/null http://127.0.0.1:8080/; do + until curl --silent --show-error --insecure --max-time 2 --output /dev/null https://127.0.0.1:8080/; do if ! kill -0 "$QEMU_PID" 2>/dev/null; then wait "$QEMU_PID" exit 1 @@ -476,14 +829,14 @@ install-vm: HOST_IP="$(ip -4 -o addr show scope global | awk '{print $4}' | cut -d/ -f1 | head -n1)" echo "==> KubeStellar Console is ready!" - echo "==> Access URL (LAN): http://${HOST_IP:-localhost}:8080/" - echo "==> Access URL (Local): http://localhost:8080/" - xdg-open "http://${HOST_IP:-localhost}:8080/" || xdg-open http://localhost:8080/ || true + echo "==> Access URL (LAN): https://${HOST_IP:-localhost}:8080/" + echo "==> Access URL (Local): https://localhost:8080/" + xdg-open "https://${HOST_IP:-localhost}:8080/" || xdg-open https://localhost:8080/ || true wait "$QEMU_PID" # Set up KubeStellar kc-agent for the user in ONE command. [group('test')] -setup-kubestellar ORIGIN="http://localhost:8080,http://127.0.0.1:8080": +setup-kubestellar ORIGIN="https://localhost:8080,https://127.0.0.1:8080": #!/usr/bin/env bash set -euo pipefail if [ -x "files/bin/bluefin-kubestellar" ]; then @@ -500,7 +853,7 @@ setup-kubestellar ORIGIN="http://localhost:8080,http://127.0.0.1:8080": # Run fully automated headless browser test against the KubeStellar console. [group('test')] -test-e2e-browser CONSOLE_URL="http://127.0.0.1:8080": +test-e2e-browser CONSOLE_URL="https://127.0.0.1:8080": python3 tests/e2e/test_kubestellar_browser_login.py --console-url "{{CONSOLE_URL}}" # Complete end-to-end Lima VM orchestration test. diff --git a/NOTES.md b/NOTES.md index 4dd1c417..942a3984 100644 --- a/NOTES.md +++ b/NOTES.md @@ -2,7 +2,7 @@ ## Domain & Repository - **Repository**: `projectbluefin/server` (upstream of Bluefin Server OS). -- **Core Architecture**: FSDK 26.08 (`components/*`), immutable XFS DDI OS (`oci/bluefin-server-ddi.bst`), systemd-sysinstall offline raw disk installer (`oci/bluefin-server-installer.bst`), and optional k0s `systemd-sysext` (`oci/k0s-sysext.bst`). +- **Core Architecture**: FSDK 26.08 (`components/*`), immutable XFS DDI OS (`oci/bluefin-server-ddi.bst`), systemd-sysinstall offline raw disk installer (`oci/bluefin-server-installer.bst`), and optional Kubernetes `systemd-sysext` (`oci/kubernetes-sysext.bst`). - **Release Automation**: Driven by Renovate dependency updates and merges to `main`. Pushes trigger `.github/workflows/build.yml`. ## Tooling Stack @@ -14,9 +14,9 @@ ## Release & Sysupdate Lifecycle - **Version Source**: Pinned FSDK point release in `elements/freedesktop-sdk.bst` (e.g. `26.08.0`). -- **Sysext Migration**: Migrated from legacy k3s to k0s (`oci/k0s-sysext.bst`). Emits `k0s-@v.raw.zst`. -- **Sysupdate Contracts** (`files/os/sysupdate.d/` plus the k0s component directory): +- **Kubernetes Sysext**: Upstream `kubeadm`/`kubelet`/`kubectl` plus CNI plugins (`oci/kubernetes-sysext.bst`). Emits `kubernetes-@v.raw.zst`. Version axis lives in `include/kubernetes.yml`. +- **Sysupdate Contracts** (`files/os/sysupdate.d/` plus the `kubernetes` component directory): - `50-root.transfer`: OS rootfs DDI (`bluefin-server-ddi-@v.raw.zst`). - `60-uki.transfer`: Boot UKI (`bluefin-server-@v.efi`). - - `files/os/sysupdate.k0s.d/70-k0s.transfer`: k0s sysext (`k0s-@v.raw.zst`). + - `files/os/sysupdate.kubernetes.d/70-kubernetes.transfer`: Kubernetes sysext (`kubernetes-@v.raw.zst`), on a minor-pinned track. - **Signing & Manifest**: `dist/release/SHA256SUMS` signed via detached GPG (`SHA256SUMS.gpg`) using secret `SYSUPDATE_SIGNING_KEY`. diff --git a/README.md b/README.md index 51aebccb..e32a434f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,14 @@ -# Bluefin Server + + + + Bluefin Server + + > Amargasaurus cazaui -**An FSDK-based, image-based Linux server OS.** +**An image-based Linux server OS built on a Flatcar base.** -Bluefin Server targets the same use-case space as Flatcar Container Linux, Fedora CoreOS, and Talos, but is built from scratch with [BuildStream 2](https://buildstream.build/) from [freedesktop-sdk](https://freedesktop-sdk.freedesktop.org/) (FSDK 26.08) components and uutils coreutils. +Bluefin Server targets the same use-case space as Flatcar Container Linux, Fedora CoreOS, and Talos, and is built with [BuildStream 2](https://buildstream.build/). The installed OS payload lands on a single-ABI [Flatcar](https://flatcar-linux.net/) `/usr` (projectbluefin/server#131); the offline installer still composes its userspace from [freedesktop-sdk](https://freedesktop-sdk.freedesktop.org/) (FSDK 26.08) so it keeps `systemd-sysinstall`, which Flatcar does not ship. It is [DDI first](https://0pointer.net/blog/fitting-everything-together.html): the OS payload is a compressed XFS DDI filesystem image that is deployed by an offline, systemd-native installer. @@ -23,7 +28,7 @@ Bluefin Server is currently in **Alpha**: - **Image-based updates and atomic rollbacks** via A/B partition slots and `systemd-sysupdate`. - **DDI-first delivery** — the installer embeds the OS payload as a data partition; no network is required at install time. -- **Streamlined base OS image** — modern userspace with uutils coreutils and bash for interactive login and diagnostics. +- **Streamlined base OS image** — modern Flatcar userspace with bash for interactive login and diagnostics. - **systemd-native installer** — `systemd-sysinstall` provides the interactive terminal UI and `systemd-repart` handles partitioning and block-copy DDI placement. - **Optional k0s as a `systemd-sysext`** so the base image stays minimal. diff --git a/SECURITY.md b/SECURITY.md index f6c6bac3..82f503bf 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -61,13 +61,13 @@ For the full trust model, key rotation, and transfer configuration, see This policy covers everything this repository produces and its build pipeline: -- The OS DDI payload, the offline installer disk image, and the k0s +- The OS DDI payload, the offline installer disk image, and the Kubernetes `systemd-sysext`. - BuildStream elements, build tooling, and GitHub Actions workflows. - Release signing and the `systemd-sysupdate` update/verification flow. **Out of scope:** vulnerabilities in upstream components such as -freedesktop-sdk, GNOME build metadata, systemd, or k0s itself. Report those to +freedesktop-sdk, GNOME build metadata, systemd, or Kubernetes itself. Report those to the respective upstream project; report to this repository only when the issue is introduced by our integration, build, signing, or packaging of the component. diff --git a/docs/DOCUMENTATION_OVERHAUL_PLAN.md b/docs/DOCUMENTATION_OVERHAUL_PLAN.md index c28d4e24..53aebf28 100644 --- a/docs/DOCUMENTATION_OVERHAUL_PLAN.md +++ b/docs/DOCUMENTATION_OVERHAUL_PLAN.md @@ -5,6 +5,16 @@ This plan is the executable artifact. It inventories the current docs, prescribes the target structure, defines lazy-loading and token-efficiency rules, drafts the canonical `AGENTS.md`, and gives a file-by-file cleanup sequence. +> **Historical record — completed.** This plan captures the documentation +> inventory and `AGENTS.md` text as they stood when the overhaul was executed. +> Its file listings, line counts, and the embedded `AGENTS.md` block are a +> point-in-time snapshot and are deliberately **not** updated as the tree moves +> on. The k0s skills it inventories were replaced by +> [`docs/skills/kubernetes-sysext.md`](skills/kubernetes-sysext.md) and +> [`docs/skills/kubernetes-sysext-ops.md`](skills/kubernetes-sysext-ops.md) in +> the upstream Kubernetes cutover; read those, and the live `AGENTS.md`, for +> current state. The rules and conventions in sections 2-5 remain in force. + --- ## 1. Research basis & sources diff --git a/docs/MVP_1_0_READINESS.md b/docs/MVP_1_0_READINESS.md index 3f8bef80..634dee10 100644 --- a/docs/MVP_1_0_READINESS.md +++ b/docs/MVP_1_0_READINESS.md @@ -4,7 +4,7 @@ This audit tracks the gap between the current tree and a first public/usable MVP ## MVP 1.0 bar -1. **Reproducible build path** — documented command or CI job that produces the DDI, live installer, and k0s sysext. +1. **Reproducible build path** — documented command or CI job that produces the DDI, live installer, and Kubernetes sysext. 2. **Signed release artifacts** — combined `SHA256SUMS` + detached GPG signature published to GitHub Releases. 3. **Automated boot verification** — at least one non-human test that proves the installer writes a bootable disk and the installed OS reaches a target. 4. **Functional update path** — host can pull the signed manifest and apply an OS update without manual intervention. @@ -15,11 +15,12 @@ This audit tracks the gap between the current tree and a first public/usable MVP | Check | Status | Evidence | |---|---|---| -| Element graph resolves | ✅ | `just validate` succeeds for DDI, installer, and k0s sysext | +| Element graph resolves | ✅ | `just validate` succeeds for DDI, installer, and Kubernetes sysext | | Release workflow lint | ✅ | `actionlint .github/workflows/build.yml` clean | | Release path exists | ✅ | `.github/workflows/build.yml` builds, signs, uploads to GitHub Release | -| Cluster build pipeline | ✅ | Phase A complete; pipeline builds DDI, installer, and k0s sysext with uutils | -| Automated boot test | 🔄 | Phase B in progress for Alpha; `bluefin-server-boot-test` workflow running on lab cluster | +| Cluster build pipeline | ✅ | Phase A complete; pipeline builds DDI, installer, and Kubernetes sysext with uutils | +| Automated boot test (installed OS) | 🔄 | Phase B in progress for Alpha; `bluefin-server-boot-test` workflow running on lab cluster | +| Automated boot test (install medium) | 🔄 | `just test-installer-boot` boots the medium through OVMF and requires systemd PID 1's OSC 3008 identity record. Validates an **exported artifact** from `dist/`, not the current element state, and is not yet wired into CI — it warns when sources are newer than the artifact | | A/B root rollback | ❌ | `50-root.transfer` names `root-a`/`root-b`, installer only creates `root-a` | | Root immutability | ❌ | DDI boots read/write (`rw` on cmdline) | | First-boot SSH keys | ❌ | Only root password credential path exists | @@ -28,7 +29,7 @@ Competitor context: [gap-analysis-distros.md](skills/gap-analysis-distros.md) ## Verdict -**Alpha state — in progress for MVP 1.0.** Phase A (build path, uutils, k0s sysext, validate) is complete. Phase B (automated boot test on lab cluster) is actively in progress for the Alpha milestone. Full MVP 1.0 release requires concluding Phase B boot verification and Phase C runtime hardening (automated rollback, read-only `/usr`, credential delivery). +**Alpha state — in progress for MVP 1.0.** Phase A (build path, uutils, Kubernetes sysext, validate) is complete. Phase B (automated boot test on lab cluster) is actively in progress for the Alpha milestone. Full MVP 1.0 release requires concluding Phase B boot verification and Phase C runtime hardening (automated rollback, read-only `/usr`, credential delivery). ## Roadmap @@ -39,13 +40,21 @@ Priority order. Each item depends on the ones above it. - [x] Extend cluster build deadlines so the pipeline can finish. - [x] Fix `build.yml` actionlint warnings. - [x] Confirm build path succeeds and publishes `bluefin-server-installer:latest` to `:30500`. -- [x] Add `oci/k0s-sysext.bst` to the build and validation pipeline. +- [x] Add `oci/kubernetes-sysext.bst` to the build and validation pipeline. - [x] Integrate uutils coreutils across OS elements. - [x] Merge-contract graph validation (`just validate`) passes clean. ### Phase B: automated boot verification (In progress for Alpha) - [x] Create `bluefin-server-boot-test` Argo workflow in the downstream factory CI repository. +- [x] Cover the install medium's own boot path (`just test-installer-boot`). + Nothing previously executed the ESP: `test-installer-artifact` and CI's + `installer-test` attach `installer.raw` as a data disk and inject the + kernel with `-kernel`/`-initrd`, proving the installer installs but never + that the medium boots. **Local only, not wired into CI**, and it boots an + exported artifact rather than building — it warns when the tree has moved + since the export. This does not satisfy the MVP bar item above, which + asks that the *installed OS* reach a target. - [ ] Run the workflow against a successful installer build on the lab cluster and iterate to green. - [ ] Wire the boot test into a post-merge CI gate or CronWorkflow. diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index c4deaf91..91454b14 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -136,7 +136,7 @@ uploaded to a GitHub Release tagged `installer-v`. syncs `release-version` to the tracked FSDK point release, and proposes the result as its own pull request against `main`. 3. **Full Compilation:** Builds the standalone DDI OS image, live installer, and - k0s systemd-sysext on every pull request and push to `main`. + Kubernetes systemd-sysext on every pull request and push to `main`. 4. **Installer boot test:** Uploads the already-built installer and PXE artifacts and calls `projectbluefin/actions/.github/workflows/server-installer-test.yml`. The shared workflow prepares QEMU/KVM and runs this repository's `just test-installer-artifact`; Lima remains local-only. 5. **Version Derivation:** The release tag is derived with `just version`, which parses the pinned FSDK point release from `elements/freedesktop-sdk.bst`. diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 0916d10f..e6863230 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -26,7 +26,7 @@ just export-installer # export installer + UKI + SHA256SUMS to dist/ just export-pxe # export standalone PXE vmlinuz/initrd to dist/ just build-ddi # build the OS DDI payload just export-ddi # export DDI + SHA256SUMS to dist/ddi/ -just build-sysext # build the k0s sysext +just build-sysext # build the Kubernetes sysext just export-sysext # export sysext artifacts to dist/sysext/ just flash-installer # write the installer image to a USB device just show-me-the-future # end-to-end QEMU installer smoke test @@ -88,7 +88,7 @@ Use direct I/O and full-block reads to avoid dirtying the page cache. The release process is driven by `.github/workflows/build.yml`: - Renovate point-release updates or direct pushes to `main` trigger a full build. -- CI builds the DDI payload, installer, target UKI, k0s sysext, and standalone +- CI builds the DDI payload, installer, target UKI, Kubernetes sysext, and standalone PXE boot inputs (`bluefin-server-pxe-vmlinuz-*`, `bluefin-server-pxe-initrd-*.cpio.gz`). - CI uploads the versioned release assets to the corresponding `installer-v` GitHub Release. diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index a511f943..bcc85b0f 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -173,6 +173,102 @@ offline installation. | "Store the DDI in the ESP (FAT32)." | FAT32 has a 4 GiB per-file limit. Use a separate XFS partition. | | "Add an 8 GiB minimum size floor to the DDI." | The rootfs is immutable. It never grows in-place. Content + overhead is enough. | +## What the smoke tests do not cover + +`just test-installer-artifact` and the CI `installer-test` job attach +`installer.raw` as a read-only data disk and inject the kernel directly with +`-kernel`/`-initrd` from the PXE artifacts. The medium's ESP is never executed: +they prove the installer **installs**, never that the medium **boots**. + +`just test-installer-boot` closes that gap — it writes the exported image into +a sparse file larger than itself, relocates the GPT backup header the way +`flash-installer` does, and boots it through OVMF. + +Note its scope: it boots an **exported artifact** from `dist/`. It does not +build, and `just validate` only resolves the graph without building. A green +run is evidence about those bytes, not the current element state; the recipe +warns when sources are newer than the artifact. Re-export before treating it +as a gate on an element change. + +Its success criterion is systemd's OSC 3008 identity record, which PID 1 writes +regardless of log verbosity and which firmware cannot forge: + +``` +ESC ]3008;start=;user=root;hostname=;machineid=;bootid=; + pid=1;comm=systemd;type=boot +``` + +### A working medium is silent; a broken one is noisy + +This inversion is the most expensive thing in this document, and the easiest to +forget because it is counter-intuitive. + +A **correct** boot prints essentially nothing. `quiet loglevel=3` suppresses +the kernel, and the medium deliberately omits `unattended`, so it comes up and +waits for an operator. A **failing** boot is loud: dracut emergency mode, +"Cannot find /usr", and `Dependency failed for ...` all print *through* +`quiet`, because error paths bypass it. + +So silence is not evidence of failure and output is not evidence of success — +both readings are backwards, and reading them that way cost a day spent +reverting a healthy installer. + +Two habits make it unambiguous: + +1. **Never `cat` the serial log.** It is nearly all ANSI and OSC escapes, which + the terminal swallows, so a healthy boot renders as two firmware lines and + apparent silence. Use `strings serial.log` or `hexdump -C serial.log`; the + systemd identity record is plainly visible that way. +2. **Force verbosity when in doubt:** + + ``` + INSTALLER_BOOT_CMDLINE_EXTRA=loglevel=7 just test-installer-boot + ``` + + `systemd-stub` reads this as an SMBIOS type-11 string and appends it to the + UKI's embedded cmdline, so it needs no rebuild. A healthy medium then + produces tens of kilobytes of ordinary boot log. + +## Writing the medium + +`dd` copies the image verbatim, which places the GPT **backup header at the end +of the image** rather than at the end of the device. On any medium larger than +the image — every real USB stick — the table is then only half valid: + +``` +$ sfdisk --verify /dev/sdX +The backup GPT table is not on the end of the device. +MyLBA mismatch with real position at backup header. + +$ parted -s /dev/sdX print +Warning: Not all of the space available to /dev/sdX appears to be used ... +``` + +**It does not stop the medium booting.** UEFI reads the *primary* GPT at LBA 1, +which `dd` writes correctly; the backup is a redundancy copy consulted only +when the primary is damaged. Verified: the image written to a 58.6 GB sparse +file without relocating reproduces the `sfdisk` error exactly and still boots +to systemd PID 1. + +What it breaks is tooling — `lsblk` and `udisksctl` disagreeing about partition +sizes, disk utilities showing most of the device unaccounted for. Partition- +table health says nothing about boot health in either direction; reading it as +a boot failure sends you chasing a defect that isn't there. + +`just flash-installer` repairs it with `sfdisk --relocate gpt-bak-std` after +the write, and also reads the device back and compares digests, so a medium +that silently fails to retain the image is caught. For a medium written with +plain `dd`, relocate by hand: + +``` +sudo sfdisk --relocate gpt-bak-std /dev/sdX +sudo partprobe /dev/sdX +sudo sfdisk --verify /dev/sdX # expect: No errors detected. +``` + +Invisible to CI: the smoke tests attach a virtual disk sized exactly to the +image, so device size always equals image size and the condition cannot arise. + ## Verification - [ ] `just validate` resolves the BuildStream graph without errors. @@ -188,6 +284,10 @@ offline installation. appears on the attached display even when serial is the primary console. - [ ] `bluefin-server-installer.bst` decompresses the DDI after the cpio step. - [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=yes`. +- [ ] `just test-installer-boot` passes — the medium boots through firmware, + not just installs when the kernel is injected. +- [ ] A medium written to real media has its GPT backup header relocated; + `sfdisk --verify` reports `No errors detected.` ## See also diff --git a/docs/skills/factory-integration.md b/docs/skills/factory-integration.md index 75124ca4..e2f724d2 100644 --- a/docs/skills/factory-integration.md +++ b/docs/skills/factory-integration.md @@ -18,7 +18,7 @@ The factory pattern is broader than a single host: a downstream CI lab or OS fac ┌─────────────────────────────────────────────────────────────┐ │ Downstream CI lab / OS factory │ │ GitOps-style testing and automation for image-based OSes │ -│ • k0s control plane / workload orchestration │ +│ • Kubernetes control plane / workload orchestration │ │ • VM or container workloads │ │ • OCI/bootc image pipelines / release automation │ └─────────────────────────────────────────────────────────────┘ @@ -28,21 +28,23 @@ The factory pattern is broader than a single host: a downstream CI lab or OS fac │ Bluefin Server (this repo) │ │ Core server OS: DDI-first, image-updated │ │ • systemd-sysupdate for atomic A/B updates │ -│ • systemd-sysext for optional layers (k0s, extensions) │ +│ • systemd-sysext for optional layers (Kubernetes, runtime) │ │ • podman for container workloads │ └─────────────────────────────────────────────────────────────┘ ``` -## k0s is a sysext, not base image bloat +## Kubernetes is a sysext, not base image bloat -Kubernetes is not baked into the OS DDI. The base image stays small and stateless; k0s is delivered as a `systemd-sysext` EROFS image that overlays `/usr/` at runtime. +Kubernetes is not baked into the OS DDI. The base image stays small and stateless; `kubeadm`, `kubelet`, `kubectl` and the CNI plugins are delivered as a `systemd-sysext` EROFS image that overlays `/usr/` at runtime. containerd arrives the same way, as Flatcar's own `containerd-flatcar.raw` sysext. -- `elements/oci/k0s-sysext.bst` builds the sysext. -- `files/os/sysupdate.k0s.d/70-k0s.transfer` enables component-scoped OTA - updates of the sysext. +- `elements/oci/kubernetes-sysext.bst` builds the sysext. +- `files/os/sysupdate.kubernetes.d/70-kubernetes.transfer` enables component-scoped OTA + updates of the sysext, on a minor-pinned track. - `files/os/justfile` provides the `just k8s` entrypoint. +- Cluster workload is reconciled by Argo CD Core from an in-image manifest tree, not by + an OS-level addon manager. -See [k0s-sysext.md](k0s-sysext.md) for details. +See [kubernetes-sysext.md](kubernetes-sysext.md) for details. ## Workloads are containers @@ -57,7 +59,7 @@ The workloads the factory tests and ships live in other repositories or image pi | Fully automated, unattended installs | Offline DDI installer (`systemd-sysinstall`) | | Atomic, rollback-capable updates | Image-based A/B updates via `systemd-sysupdate` | | Minimal attack surface / lean base OS | Streamlined DDI with bash; optional tools as sysexts | -| Kubernetes control plane on every node | k0s delivered as `systemd-sysext` | +| Kubernetes control plane on every node | Upstream `kubeadm` delivered as `systemd-sysext` | | Container workloads | `podman` in the base OS stack | | Signed, verifiable release artifacts | GPG-signed `SHA256SUMS` + `import-pubring.gpg` | @@ -82,7 +84,7 @@ The workloads the factory tests and ships live in other repositories or image pi | Rationalization | Reality | |---|---| -| “k0s should be in the base image.” | Keep the OS DDI minimal. k0s is optional and delivered OTA as a sysext. | +| “Kubernetes should be in the base image.” | Keep the OS DDI minimal. The Kubernetes sysext is optional and delivered OTA. | | “We can pull the DDI at install time.” | Unattended installs must survive network loss; the DDI is embedded in the installer media. | | “Let’s add heavy debug tools.” | Base OS includes bash for login; heavy developer/debug tools belong in sysexts or system containers. | | “Package updates are small patches.” | Image-based updates are whole-OS replacements; the rollback unit is the OS image, not a package delta. | @@ -91,18 +93,18 @@ The workloads the factory tests and ships live in other repositories or image pi - Adding a workload dependency to `elements/bluefin-server/os-stack.bst` that could ship as a `systemd-sysext`. - Treating Bluefin Server as a generic Fedora/RHEL replacement rather than the factory core OS. -- Putting Kubernetes tooling in the base DDI instead of the k0s sysext. +- Putting Kubernetes tooling in the base DDI instead of the Kubernetes sysext. - Designing install/update paths that require interactive human steps in the factory. ## Verification - [ ] Any new base-DDI dependency can be justified by the factory core-OS role. - [ ] Optional capabilities are modeled as sysexts or system containers. -- [ ] The k0s sysext still builds and updates independently of the DDI. -- [ ] `systemd-sysupdate` transfer files are present for every OTA-delivered artifact (DDI, UKI, k0s sysext). +- [ ] The Kubernetes sysext still builds and updates independently of the DDI. +- [ ] `systemd-sysupdate` transfer files are present for every OTA-delivered artifact (DDI, UKI, Kubernetes sysext). ## See also -- [k0s-sysext.md](k0s-sysext.md) — building and delivering the k0s sysext. +- [kubernetes-sysext.md](kubernetes-sysext.md) — building and delivering the Kubernetes sysext. - [ddi-installer.md](ddi-installer.md) — offline installer architecture. - [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary. diff --git a/docs/skills/gap-analysis-distros.md b/docs/skills/gap-analysis-distros.md index 12b44132..05619ea0 100644 --- a/docs/skills/gap-analysis-distros.md +++ b/docs/skills/gap-analysis-distros.md @@ -96,9 +96,9 @@ Bluefin Server is a BuildStream 2-based, image-based Linux server OS built from |------|---------------------------------| | **Philosophy** | Systemd-native, minimal, image-based server OS appliance; base DDI includes bash for login and bring-up while heavy developer/debug tools live in sysexts or system containers; intended to run container workloads and Kubernetes via optional sysexts. Sources: [AGENTS.md](../../AGENTS.md), [factory-integration.md](factory-integration.md). | | **State model** | Target OS DDI is an XFS filesystem image. A separate persistent `/var` partition is created by the installer. There is no second root slot provisioned today, and the UKI cmdline currently uses `rw`, so the root is not mounted read-only at runtime. Sources: [bluefin-server-ddi.bst](../../elements/oci/bluefin-server-ddi.bst), [20-root-a.conf](../../files/installer/repart.d/20-root-a.conf), [bluefin-server-installer.bst](../../elements/oci/bluefin-server-installer.bst). | -| **Updates** | `systemd-sysupdate` reads root/UKI transfers from `files/os/sysupdate.d/` and the optional k0s transfer from the `k0s` component directory. Assets are published to GitHub Releases, and the combined `SHA256SUMS` manifest is signed in CI with a GPG key. `Verify=yes` is the default. Sources: [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md), [50-root.transfer](../../files/os/sysupdate.d/50-root.transfer), [60-uki.transfer](../../files/os/sysupdate.d/60-uki.transfer), [70-k0s.transfer](../../files/os/sysupdate.k0s.d/70-k0s.transfer), also `systemd-sysupdate(8)`. | +| **Updates** | `systemd-sysupdate` reads root/UKI transfers from `files/os/sysupdate.d/` and the optional Kubernetes transfer from the `kubernetes` component directory. Assets are published to GitHub Releases, and the combined `SHA256SUMS` manifest is signed in CI with a GPG key. `Verify=yes` is the default. Sources: [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md), [50-root.transfer](../../files/os/sysupdate.d/50-root.transfer), [60-uki.transfer](../../files/os/sysupdate.d/60-uki.transfer), [70-kubernetes.transfer](../../files/os/sysupdate.kubernetes.d/70-kubernetes.transfer), also `systemd-sysupdate(8)`. | | **Provisioning** | The installer is an offline `systemd-sysinstall` image that embeds the DDI as a data partition. First-boot configuration is intended to be delivered via `systemd-creds` through the ESP or hypervisor metadata. Today only `passwd.hashed-password.root` is consumed via `systemd-sysusers.d`; the documented `tmpfiles.extra` path for SSH keys and similar files is not implemented. Sources: [bluefin-server-installer.bst](../../elements/oci/bluefin-server-installer.bst), [10-root-creds.conf](../../files/os/sysusers.d/10-root-creds.conf), [os-creds-prov.bst](../../elements/bluefin-server/os-creds-prov.bst), [systemd-creds(1)](https://www.freedesktop.org/software/systemd/man/latest/systemd-creds.html). | -| **Customization** | Adds software through `systemd-sysext` (overlay `/usr`) and `systemd-confext` (overlay `/etc`) images. The base OS `os-release` advertises `ID=flatcar` and a matching `VERSION_ID` so pre-built Flatcar Bakery extensions load. k0s is shipped as a separately built, optionally enabled sysext. Sources: [systemd-sysext-extensions.md](systemd-sysext-extensions.md), [k0s-sysext.md](k0s-sysext.md), [os-release-flatcar.bst](../../elements/bluefin-server/os-release-flatcar.bst), [systemd-sysext(8)](https://www.freedesktop.org/software/systemd/man/latest/systemd-sysext.html). | +| **Customization** | Adds software through `systemd-sysext` (overlay `/usr`) and `systemd-confext` (overlay `/etc`) images. The base OS `os-release` advertises `ID=flatcar` and a matching `VERSION_ID` so pre-built Flatcar Bakery extensions load. Upstream Kubernetes is shipped as a separately built, optionally enabled sysext. Sources: [systemd-sysext-extensions.md](systemd-sysext-extensions.md), [kubernetes-sysext.md](kubernetes-sysext.md), [os-release-flatcar.bst](../../elements/bluefin-server/os-release-flatcar.bst), [systemd-sysext(8)](https://www.freedesktop.org/software/systemd/man/latest/systemd-sysext.html). | | **Reboot coordination** | `systemd-sysupdate.service` has an `ExecStartPost` that touches `/run/reboot-required`. Rolling reboots across Kubernetes nodes rely on Kured reading that file. Sources: [os-kured-hook.bst](../../elements/bluefin-server/os-kured-hook.bst), [kured-hook.conf](../../files/os/systemd/systemd-sysupdate.service.d/kured-hook.conf), [Kured project](https://github.com/weaveworks/kured). | ## 4. Factual Gaps @@ -181,7 +181,7 @@ These gaps drive the priorities in [architecture-roadmap.md](architecture-roadma - [../../CONTEXT.md](../../CONTEXT.md) - [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md) - [systemd-sysext-extensions.md](systemd-sysext-extensions.md) -- [k0s-sysext.md](k0s-sysext.md) +- [kubernetes-sysext.md](kubernetes-sysext.md) - [factory-integration.md](factory-integration.md) - [tpm2-credential-sealing.md](tpm2-credential-sealing.md) - [elements/oci/bluefin-server-ddi.bst](../../elements/oci/bluefin-server-ddi.bst) @@ -192,6 +192,6 @@ These gaps drive the priorities in [architecture-roadmap.md](architecture-roadma - [files/installer/repart.d/20-root-a.conf](../../files/installer/repart.d/20-root-a.conf) - [files/os/sysupdate.d/50-root.transfer](../../files/os/sysupdate.d/50-root.transfer) - [files/os/sysupdate.d/60-uki.transfer](../../files/os/sysupdate.d/60-uki.transfer) -- [files/os/sysupdate.k0s.d/70-k0s.transfer](../../files/os/sysupdate.k0s.d/70-k0s.transfer) +- [files/os/sysupdate.kubernetes.d/70-kubernetes.transfer](../../files/os/sysupdate.kubernetes.d/70-kubernetes.transfer) - [files/os/sysusers.d/10-root-creds.conf](../../files/os/sysusers.d/10-root-creds.conf) - [files/os/systemd/systemd-sysupdate.service.d/kured-hook.conf](../../files/os/systemd/systemd-sysupdate.service.d/kured-hook.conf) diff --git a/docs/skills/index.md b/docs/skills/index.md index c9322b90..1b192e3c 100644 --- a/docs/skills/index.md +++ b/docs/skills/index.md @@ -31,8 +31,8 @@ This is the lazy-load routing table for agent skills. Keep this file in memory w | [`factory-integration.md`](factory-integration.md) | Lab integration, boot-test workflow, factory role | How Bluefin Server is consumed by the CI lab. | | [`flatcar-parity-matrix.md`](flatcar-parity-matrix.md) | Flatcar version audit, component parity, substitution order | Version parity matrix and gap audit against Flatcar 4593.2.5. | | [`gap-analysis-distros.md`](gap-analysis-distros.md) | Comparing Bluefin Server to other server OSes | Source-verified comparison to Ubuntu, Talos, Flatcar, FCOS. | -| [`k0s-sysext-ops.md`](k0s-sysext-ops.md) | Operating k0s on Bluefin Server | Runtime operation and reboot coordination for k0s. | -| [`k0s-sysext.md`](k0s-sysext.md) | Building the k0s sysext | BuildStream element and publish steps for the k0s sysext. | +| [`kubernetes-sysext-ops.md`](kubernetes-sysext-ops.md) | Operating Kubernetes on Bluefin Server | kubeadm bring-up, the seed phases, and Argo CD Core troubleshooting. | +| [`kubernetes-sysext.md`](kubernetes-sysext.md) | Building the Kubernetes sysext | BuildStream element, version axis, and publish steps for the Kubernetes sysext. | | [`skill-improvement.md`](skill-improvement.md) | Adding, splitting, or refactoring skills | Meta-skill that owns the documentation loop. | | [`system-containers.md`](system-containers.md) | Running `systemd-nspawn` toolboxes | System container operation with `machinectl`. | | [`systemd-sysext-extensions.md`](systemd-sysext-extensions.md) | Optional layers via `systemd-sysext` / `systemd-confext` | Extension identity, loading, and Flatcar compatibility. | diff --git a/docs/skills/k0s-sysext-ops.md b/docs/skills/k0s-sysext-ops.md deleted file mode 100644 index 7b806f80..00000000 --- a/docs/skills/k0s-sysext-ops.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -name: k0s-sysext-ops -description: Operator runbook for the k0s systemd-sysext extension — provisioning, runtime testing, and troubleshooting. -metadata: - type: how-to - status: stable - last_updated: "2026-09-08" - context7-sources: - - /systemd/systemd ---- -# k0s systemd-sysext Operations - -Use this skill when running, testing, or debugging the k0s systemd-sysext on a live Bluefin Server host. - -## Enabling k0s on a Host - -On a running Bluefin Server system: - -```bash -just k8s -``` - -Or manually: - -```bash -# 1. Fetch the extension image into persistent k0s staging. -systemd-sysupdate --component=k0s update - -# 2. Copy it to the ephemeral sysext scan directory and merge into /usr. -install -D -m 0644 /var/lib/k0s/k0s.raw /run/extensions/k0s.raw -systemd-sysext merge - -# 3. Seed declarative manifest stacks into /var/lib/k0s/manifests/ -systemd-tmpfiles --create /usr/lib/tmpfiles.d/k0s-manifests.conf - -# 4. Start the controller service -systemctl enable --now k0scontroller.service -``` - -## Verifying Manifest Reconciliation - -k0s automatically applies all `.yaml` files under `/var/lib/k0s/manifests/argocd/` and `/var/lib/k0s/manifests/kubestellar/`. - -```bash -# Verify pods in argocd and kubestellar namespaces -k0s kubectl get pods -A -``` - -## Troubleshooting - -- **Extension not merged**: Check `systemd-sysext status`. Verify the persistent image is `/var/lib/k0s/k0s.raw`; the boot activation unit copies it into `/run/extensions/k0s.raw`. -- **Manifests not applied**: Check `/var/lib/k0s/manifests/`. Ensure files end in `.yaml` (not `.yml`). -- **Service failed**: Check `journalctl -u k0scontroller -e`. - -## See also - -- [k0s-sysext.md](k0s-sysext.md) -- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). -- `systemd-sysext(8)` diff --git a/docs/skills/k0s-sysext.md b/docs/skills/k0s-sysext.md deleted file mode 100644 index 1e38f94c..00000000 --- a/docs/skills/k0s-sysext.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: k0s-sysext -description: Build, ship, and enable the k0s systemd-sysext extension with native Argo CD & KubeStellar for Bluefin Server. -metadata: - type: how-to - status: stable - last_updated: "2026-09-08" - context7-sources: - - /systemd/systemd ---- -# k0s systemd-sysext - -Use this skill when working on the k0s systemd-sysext extension shipped as an -optional overlay for Bluefin Server. - -## When to Use - -- Bumping the pinned k0s binary version or its SHA256. -- Modifying the sysext image contents (`files/k0s/sysext/`). -- Managing raw YAML manifest stacks (`files/k0s/manifests/argocd/` or `files/k0s/manifests/kubestellar/`). -- Changing manifest seeding rules in `files/k0s/sysext/k0s-manifests.conf`. -- Adding or changing the sysupdate transfer definition - (`files/os/sysupdate.k0s.d/70-k0s.transfer`). -- Debugging why a host cannot pull, merge, or start k0s. - -## When NOT to Use - -- General OS image composition questions (use `ddi-installer.md` or `avoid-over-engineering.md`). -- systemd-sysupdate signature verification (use `systemd-sysupdate-verification.md`). - -## Architecture - -Bluefin Server's base DDI image includes bash for login and bring-up, while -heavy developer and debug tools live in sysexts or system containers. k0s is not -baked into the OS stack. Instead it is delivered as a `systemd-sysext` EROFS -image that overlays `/usr` at runtime. - -Design choices: - -- **Single static binary.** `elements/k0s/k0s-bin.bst` fetches the upstream statically-linked - release binary directly from GitHub. -- **Pure declarative manifests (No Helm).** Declarative stacks for Argo CD and KubeStellar - are shipped in `/usr/share/k0s/manifests/` and seeded into `/var/lib/k0s/manifests/` via - `systemd-tmpfiles`. k0s's internal manifest deployer automatically reconciles them. -- **Helm and autopilot disabled.** `--disable-components=helm,autopilot` ensures zero runtime Helm dependencies and prevents single-node controlnode timeout loops. -- **Flatcar sysext pattern.** The extension uses `ID=_any` in its release metadata so it merges on any host image. -- **First-boot background activation.** `k0s-first-boot.service` copies `/var/lib/k0s/k0s.raw` to `/run/extensions/k0s.raw`, merges the sysext, seeds declarative manifest stacks via `systemd-tmpfiles`, and enables `k0scontroller.service` on boot without holding up system startup. -- **OTA delivery.** A k0s component `systemd-sysupdate` transfer file - (`70-k0s.transfer`) is installed in the base OS so hosts can pull new k0s - sysext releases from GitHub Releases without updating the root or UKI. - -## Repository Layout - -| Path | Purpose | -|------|---------| -| `include/k0s.yml` | **Single source of truth for the k0s version axis** (`%{k0s-upstream-tag}`, `%{k0s-version}`). | -| `elements/k0s/k0s-bin.bst` | Pins the upstream `k0s` binary SHA256; the release URL is derived from `include/k0s.yml`. | -| `elements/oci/k0s-sysext.bst` | Builds the EROFS sysext image (`k0s-.raw`). | -| `files/k0s/sysext/k0scontroller.service` | systemd unit for the k0s single-node controller/worker. Not enabled by default. | -| `files/k0s/sysext/extension-release.k0s` | Static sysext identity (`ID=_any`); `VERSION_ID=`/`ARCHITECTURE=` are appended at build time. | -| `files/k0s/sysext/k0s-manifests.conf` | tmpfiles rule that copies declarative stacks to `/var/lib/k0s/manifests/`. | -| `files/k0s/manifests/argocd/` | Raw YAML manifests for Argo CD. | -| `files/k0s/manifests/kubestellar/` | Raw YAML manifests for KubeFlex, Postgres, KubeStellar core, and Console. | -| `files/os/sysupdate.k0s.d/70-k0s.transfer` | sysupdate transfer track for the k0s sysext component. | -| `Justfile` | `build-sysext` / `export-sysext` targets. | -| `.github/workflows/build.yml` | Builds, signs, and publishes sysext assets. | -| `.github/scripts/check-k0s-version.py` | Fails closed if any consumer restates the k0s version instead of deriving it. | - -## Build Outputs - -`elements/oci/k0s-sysext.bst` produces: - -- `k0s-.raw` — uncompressed EROFS sysext image. -- `k0s-.raw.zst` — zstd-compressed release asset. -- `SHA256SUMS` — checksum manifest for the compressed asset. - -## Justfile Commands - -```bash -just validate # resolve the element graph -just build-sysext # build oci/k0s-sysext.bst -just export-sysext # export sysext artifacts to dist/sysext/ -``` - -## Operations and runtime testing - -For enabling the sysext on a host, common gotchas, and runtime testing guidance, -see [k0s-sysext-ops.md](k0s-sysext-ops.md). - -## See also - -- [k0s-sysext-ops.md](k0s-sysext-ops.md) -- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). -- `systemd-sysext(8)`, `systemd-confext(8)` diff --git a/docs/skills/kubernetes-sysext-ops.md b/docs/skills/kubernetes-sysext-ops.md new file mode 100644 index 00000000..ce99c3d6 --- /dev/null +++ b/docs/skills/kubernetes-sysext-ops.md @@ -0,0 +1,192 @@ +--- +name: kubernetes-sysext-ops +description: Operator runbook for the Kubernetes systemd-sysext — kubeadm bring-up, the seed phases, Argo CD Core troubleshooting, and runtime testing. +metadata: + type: how-to + status: stable + last_updated: "2026-09-28" + context7-sources: + - /systemd/systemd +--- +# Kubernetes systemd-sysext Operations + +Use this skill when running, testing, or debugging Kubernetes on a live Bluefin Server +host. For building and versioning the image itself, see +[kubernetes-sysext.md](kubernetes-sysext.md). + +## Bring-up model + +First boot is fully unit-driven. Nothing here is a manual step on a correctly +provisioned host; the commands below are for inspection and recovery. + +``` +systemd-sysext.service merge containerd + kubernetes sysexts into /usr + -> daemon-reload a merge makes unit files appear; systemd must be told +bluefin-cluster-repo.service init /var/lib/bluefin/cluster.git from /usr/share/bluefin/cluster +kubeadm-init.service kubeadm init --config /usr/share/bluefin/kubeadm.yaml +bluefin-cluster-bootstrap.service apply the four seed phases in order +``` + +`kubeadm-init.service` carries `ConditionPathExists=!/var/lib/bluefin/kubeadm-init.stamp`, +so it is a no-op on an already-initialised host. The stamp is written by +`ExecStartPost`, i.e. only after `kubeadm init` exits 0. `/etc/kubernetes/admin.conf` +deliberately is *not* the guard: kubeadm writes it in the `kubeconfig` phase, before +etcd, the control plane and `mark-control-plane`, so guarding on it made +`Restart=on-failure` skip the retry of a node that never got its control-plane role +label. A retry that finds `admin.conf` without the stamp runs `kubeadm reset -f` first, +because kubeadm refuses to init over a populated `/etc/kubernetes/manifests`. +`bluefin-cluster-bootstrap.service` carries no condition at all on purpose: a failed +condition marks a unit *skipped* rather than *failed*, which would silently no-op the +first boot. + +To force a full re-init on a live host, remove the stamp and reset: + +```bash +kubeadm reset -f +rm -f /var/lib/bluefin/kubeadm-init.stamp +systemctl start kubeadm-init.service +``` + +## Enabling Kubernetes on a Host + +> **Reinstall only.** These steps apply to a Bluefin Server host installed from a DDI +> that already carries the container runtime. `containerd-flatcar.raw` is seeded into +> `/var/lib/extensions` at install time by `files/installer/repart.d/30-var.conf`; it is +> **not** published as a release asset and has no `sysupdate.containerd.d` transfer, so +> `systemd-sysupdate --component=kubernetes update` cannot deliver it. A host still +> running the old k0s image therefore cannot be migrated in place — it would end up with +> `kubelet.service` and `kubeadm-init.service` and no runtime, and `kubeadm-init` would +> sit in its CRI-socket wait until it timed out. Reinstall such hosts from the current +> DDI. Publishing containerd on its own transfer is tracked as open item 3 in +> `docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md`. + +On a running Bluefin Server system: + +```bash +just k8s +``` + +Or manually: + +```bash +# 1. Fetch the extension image into /var/lib/extensions/. +systemd-sysupdate --component=kubernetes update + +# 2. Merge into /usr, then let systemd see the new unit files. +systemd-sysext merge +systemctl daemon-reload + +# 3. Bring the cluster up. bluefin-cluster-bootstrap orders itself after +# kubeadm-init and bluefin-cluster-repo, and pulls both in (Wants= on the +# former, Requires= on the latter). +systemctl enable --now bluefin-cluster-bootstrap.service +``` + +`kubeadm init` runs with `--skip-phases=addon/kube-proxy`; Cilium supplies the +kube-proxy replacement. + +## The seed phases + +The seed is applied by `bluefin-cluster-bootstrap.service` from +`/usr/share/bluefin/seed/`, with `kubectl apply --server-side --force-conflicts`, in +directory order: + +| Phase | Contents | Why it is in the seed and not in Argo | +|---|---|---| +| `00-cilium/` | CNI, kube-proxy replacement | Argo's own pods need pod networking; a node without CNI stays `NotReady` with a `NoSchedule` taint. | +| `10-argocd-core/` | Argo CRDs, RBAC, redis, repo-server, application-controller | Argo cannot install Argo. | +| `15-gitd/` | git-daemon serving `/var/lib/bluefin/cluster.git` | The root Application needs the repository served before Argo can read anything. | +| `20-root-app/` | `AppProject` + root `Application` | Hands ownership of everything downstream to Argo. | + +The bootstrap unit waits for the node to report `Ready` between `00-cilium/` and +`10-argocd-core/`. + +The seed is **append-only by construction and is never pruned**. Argo owns pruning for +all downstream workload via `prune: true`; both upstream `kubectl` pruning modes are +alpha and unsuitable. Do not reopen this. + +## Argo CD Core + +Core runs exactly three pods: `argocd-application-controller`, `argocd-repo-server`, and +`argocd-redis`. There is no `argocd-server`, no `argocd-dex-server`, and no +`argocd-notifications-controller`. + +Consequences for operators: + +- **There is no web UI and the `argocd` CLI does not work.** The CLI speaks gRPC to + `argocd-server`, which Core does not ship. Manage Applications as plain Kubernetes + custom resources with `kubectl`. +- `argocd-applicationset-controller` is patched out; this design has exactly one root + Application. Its CRD is left in place, harmlessly. +- Redis is an in-memory cache with no PersistentVolumeClaim. There is no supported + redis-free Argo CD, so a missing redis is a fault, not a configuration choice. +- Two residual `argocd-server` NetworkPolicy `podSelector` rules admit traffic from a pod + that never exists. They are inert and deliberately left unmodified. + +Inspecting sync state: + +```bash +kubectl -n argocd get applications +kubectl -n argocd describe application bluefin-cluster +kubectl -n argocd logs deploy/argocd-repo-server +kubectl -n argocd logs statefulset/argocd-application-controller +``` + +## Manifest source bridge + +Argo's `Application.source` accepts git, Helm, or OCI — there is no filesystem source, so +Argo cannot read `/usr/share/bluefin/cluster/` directly. `bluefin-cluster-repo.service` +initialises a bare repository at `/var/lib/bluefin/cluster.git` from that tree on first +boot, and the `15-gitd/` git-daemon serves it over the cluster network. No external git +host is contacted, so the manifest tree stays available with the network unplugged. + +Updating the tree means updating the image: a new DDI re-seeds +`/usr/share/bluefin/cluster/`, and `bluefin-cluster-repo.service` refreshes the bare repo. + +## Troubleshooting + +- **Extension not merged**: `systemd-sysext status`. Verify a + `/var/lib/extensions/kubernetes-.raw` exists; if not, the sysupdate fetch is + the failure, not the merge. +- **Merged but nothing runs**: the merge makes unit files appear under `/usr` but does not + by itself make systemd notice them. Run `systemctl daemon-reload` and re-check. +- **`kubeadm-init.service` skipped, cluster absent**: `/etc/kubernetes/admin.conf` already + exists from a previous partial run. Inspect it before removing it; `kubeadm reset` is + the supported way back to a clean state. +- **Node stays `NotReady`**: CNI is not up. `kubectl -n kube-system get pods -l k8s-app=cilium` + and `journalctl -u bluefin-cluster-bootstrap -e`. +- **Pods `Pending` with a `NoSchedule` taint**: same cause; the taint clears when Cilium + reports the node `Ready`. +- **kubelet serving certificate never issued**: kube-controller-manager does not + auto-approve serving CSRs. `kubelet-csr-approver` is mandatory, not optional. Check + `kubectl get csr` for `Pending` entries and the approver's `--provider-regex`. +- **Root Application will not sync**: check the git-daemon first — + `kubectl -n bluefin-system logs deploy/bluefin-git-daemon` — then the bare repo at + `/var/lib/bluefin/cluster.git` on the host. +- **Bootstrap unit failed**: `journalctl -u bluefin-cluster-bootstrap -e`. The unit is + ordered `After=kubeadm-init.service` but only `Wants=` it, so a control plane that + never came up shows here as the `ExecStartPre` readiness probe timing out against the + API server; `journalctl -u kubeadm-init -e` names the underlying cause. `Requires=` is + deliberately not used: a job killed with result `dependency` never runs, so its own + `Restart=on-failure` never engages and one transient `kubeadm init` failure would + strand the seed permanently. + +## Runtime testing + +```bash +just show-me-the-future # QEMU installer smoke test through to multi-user +``` + +Acceptance on a booted host: containerd active, `kubectl get nodes` reports `Ready`, the +three Argo Core pods reach `Running`, and the root Application syncs. + +Workload images are still pulled from public registries, so "fully automated first boot" +currently means "with network present". Image pre-seeding into containerd is tracked +separately. + +## See also + +- [kubernetes-sysext.md](kubernetes-sysext.md) +- [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md) — OTA trust model. +- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). +- `systemd-sysext(8)` diff --git a/docs/skills/kubernetes-sysext.md b/docs/skills/kubernetes-sysext.md new file mode 100644 index 00000000..153b4a4f --- /dev/null +++ b/docs/skills/kubernetes-sysext.md @@ -0,0 +1,124 @@ +--- +name: kubernetes-sysext +description: Build, ship, and version the upstream Kubernetes systemd-sysext (kubeadm, kubelet, kubectl, CNI plugins) for Bluefin Server. +metadata: + type: how-to + status: stable + last_updated: "2026-09-19" + context7-sources: + - /systemd/systemd +--- +# Kubernetes systemd-sysext + +Use this skill when working on the Kubernetes `systemd-sysext` extension shipped as an +optional overlay for Bluefin Server. + +## When to Use + +- Bumping the pinned Kubernetes or CNI plugin version and its SHA256 pins. +- Modifying the sysext image contents (`elements/kubernetes/`, `files/kubernetes/sysext/`). +- Changing the `kubeadm` cluster configuration (`files/kubernetes/kubeadm.yaml`). +- Adding or changing the sysupdate transfer definition + (`files/os/sysupdate.kubernetes.d/70-kubernetes.transfer`). +- Debugging why a host cannot pull or merge the Kubernetes sysext. + +## When NOT to Use + +- Bringing up, troubleshooting, or operating a live cluster — use + [kubernetes-sysext-ops.md](kubernetes-sysext-ops.md). +- Seed and Argo-owned manifest content — that tree lives under `files/cluster/`. +- General OS image composition questions (use `ddi-installer.md` or `avoid-over-engineering.md`). +- systemd-sysupdate signature verification (use `systemd-sysupdate-verification.md`). + +## Architecture + +Bluefin Server's base DDI image includes bash for login and bring-up, while heavy +developer and debug tools live in sysexts or system containers. Kubernetes is not baked +into the OS stack. It is delivered as a `systemd-sysext` EROFS image that overlays `/usr` +at runtime. + +Design choices: + +- **Upstream binaries, no distribution.** `elements/kubernetes/` imports `kubeadm`, + `kubelet`, and `kubectl` from `dl.k8s.io`, each pinned to a published SHA256. There is + no embedded control plane and no vendor-specific control-plane wrapper. +- **Re-baked in-repo.** The element re-bakes the `flatcar/sysext-bakery` `kubernetes.sysext` + recipe under BuildStream rather than consuming the published artifact, so release signing + and provenance stay in this repository. The output layout is wire-compatible with + `extensions.flatcar.org`, so an operator can repoint the sysupdate source upstream + without changing anything else. +- **containerd is a separate sysext.** The runtime ships as Flatcar's + `containerd-flatcar.raw`; it is never part of this image and never part of the base DDI. +- **Flatcar sysext pattern.** The extension uses `ID=_any` in its release metadata so it + merges on any host image. +- **Minor-pinned OTA track.** `70-kubernetes.transfer` tracks a minor-pinned track because + Kubernetes does not support unattended cross-minor in-place upgrades. Crossing a minor + is a deliberate operator action, not an automatic one. + +## Repository Layout + +| Path | Purpose | +|------|---------| +| `include/kubernetes.yml` | **Single source of truth for the version axis** (`%{k8s-version}`, `%{cni-version}`). | +| `elements/kubernetes/` | Pins the upstream `kubeadm`, `kubelet`, `kubectl` and CNI plugin SHA256s; release URLs are derived from `include/kubernetes.yml`. | +| `elements/oci/kubernetes-sysext.bst` | Builds the EROFS sysext image (`kubernetes-.raw`). | +| `files/kubernetes/sysext/extension-release.kubernetes` | Static sysext identity (`NAME=kubernetes`, `ID=_any`); `VERSION_ID=`/`ARCHITECTURE=` are appended at build time. | +| `files/kubernetes/kubeadm.yaml` | `kubeadm` cluster configuration, staged to `/usr/share/bluefin/kubeadm.yaml`. | +| `files/cluster/seed/` | Seed phases applied before Argo owns the cluster; staged to `/usr/share/bluefin/seed/`. | +| `files/cluster/manifests/` | Argo-owned workload tree; staged to `/usr/share/bluefin/cluster/`. | +| `files/os/sysupdate.kubernetes.d/70-kubernetes.transfer` | sysupdate transfer track for the Kubernetes sysext component. | +| `Justfile` | `build-sysext` / `export-sysext` targets. | +| `.github/workflows/build.yml` | Builds, signs, and publishes sysext assets. | +| `.github/scripts/check-kubernetes-version.py` | Fails closed if any consumer restates the version instead of deriving it. | + +## Image Contents + +Mirroring the upstream bakery recipe: + +- `/usr/bin/{kubectl,kubeadm,kubelet}` +- `/usr/local/bin/cni/` from the `containernetworking/plugins` release tarball +- `/usr/local/share/kubernetes-version`, `/usr/local/share/kubernetes-cni-version` +- `/usr/lib/extension-release.d/extension-release.kubernetes` + +No certificate or key material ships in any image. kubelet client certificates come from +`kubeadm`'s native TLS bootstrap; kubelet serving certificates come from +`serverTLSBootstrap` plus `kubelet-csr-approver`; everything else is issued by +cert-manager in-cluster. + +## Build Outputs + +`elements/oci/kubernetes-sysext.bst` produces: + +- `kubernetes-.raw` — uncompressed EROFS sysext image. +- `kubernetes-.raw.zst` — zstd-compressed release asset. +- `SHA256SUMS` — checksum manifest for the compressed asset. + +## Bumping the version + +1. Edit `include/kubernetes.yml` — it is the only place a version literal may appear. +2. Replace every affected SHA256 in `elements/kubernetes/` with the value published + alongside the upstream artifact. Never invent a checksum. +3. If the Kubernetes **minor** changed, update the minor-pinned track name in + `files/os/sysupdate.kubernetes.d/70-kubernetes.transfer`. +4. Run `just validate`; `check-kubernetes-version.py` fails closed when a consumer + restates the version instead of deriving it from `include/kubernetes.yml`. + +## Justfile Commands + +```bash +just validate # resolve the element graph and check the version axis +just build-sysext # build oci/kubernetes-sysext.bst +just export-sysext # export sysext artifacts to dist/sysext/ +``` + +## Operations and runtime testing + +For bringing a cluster up on a host, the seed phases, and troubleshooting guidance, +see [kubernetes-sysext-ops.md](kubernetes-sysext-ops.md). + +## See also + +- [kubernetes-sysext-ops.md](kubernetes-sysext-ops.md) +- [systemd-sysext-extensions.md](systemd-sysext-extensions.md) — extension identity and loading. +- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). +- `systemd-sysext(8)`, `systemd-confext(8)` diff --git a/docs/skills/system-containers.md b/docs/skills/system-containers.md index 7de08566..e61207d1 100644 --- a/docs/skills/system-containers.md +++ b/docs/skills/system-containers.md @@ -31,7 +31,7 @@ runtime UI. ## When NOT to Use - Adding a distroless OCI runtime image — the server repo does not build those. -- Kubernetes workload or cluster questions — use `k0s-sysext.md`. +- Kubernetes workload or cluster questions — use `kubernetes-sysext.md`. ## Lifecycle diff --git a/docs/skills/systemd-sysext-extensions.md b/docs/skills/systemd-sysext-extensions.md index c65ba5ac..6cd1b0af 100644 --- a/docs/skills/systemd-sysext-extensions.md +++ b/docs/skills/systemd-sysext-extensions.md @@ -52,7 +52,7 @@ must match the value baked into `elements/bluefin-server/os-release-flatcar.bst` ## Adding an extension from the Flatcar Bakery -The k0s sysext is the built-in example, but any Flatcar-compatible extension can +The Kubernetes sysext is the built-in example, but any Flatcar-compatible extension can be layered the same way. ```bash @@ -108,6 +108,6 @@ systemd-sysext merge --force ## See also -- [k0s-sysext.md](k0s-sysext.md) for the built-in Kubernetes extension +- [kubernetes-sysext.md](kubernetes-sysext.md) for the built-in Kubernetes extension - [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary (Sysext definition). - `systemd-sysext(8)`, `systemd-confext(8)` diff --git a/docs/skills/systemd-sysupdate-verification.md b/docs/skills/systemd-sysupdate-verification.md index a7048efe..6145c6ff 100644 --- a/docs/skills/systemd-sysupdate-verification.md +++ b/docs/skills/systemd-sysupdate-verification.md @@ -17,7 +17,7 @@ the OS image. ## When to Use - Modifying `files/os/sysupdate.d/*.transfer` or - `files/os/sysupdate.k0s.d/*.transfer` update definitions. + `files/os/sysupdate.kubernetes.d/*.transfer` update definitions. - Rotating or replacing the release signing key. - Debugging `systemd-sysupdate` failures related to `SHA256SUMS.gpg` verification. @@ -50,8 +50,8 @@ Key facts from `sysupdate.d(5)`: The current tree uses a single root/ESP slot and a single signed manifest flow for OTA delivery. Root and UKI transfers live in `sysupdate.d`; the optional -k0s sysext lives in the `k0s` component directory and is selected with -`systemd-sysupdate --component=k0s update`. Future work on dual-slot root +Kubernetes sysext lives in the `kubernetes` component directory and is selected +with `systemd-sysupdate --component=kubernetes update`. Future work on dual-slot root partitions, dual UKIs, and broader rollback strategies is tracked in [architecture-roadmap.md](architecture-roadmap.md). @@ -66,7 +66,7 @@ partitions, dual UKIs, and broader rollback strategies is tracked in `dist/release/SHA256SUMS.gpg`, and uploads `dist/release/*` to the GitHub Release. - `files/os/sysupdate.d/*.transfer` and - `files/os/sysupdate.k0s.d/*.transfer` — each transfer points its static + `files/os/sysupdate.kubernetes.d/*.transfer` — each transfer points its static `Path=` at `https://github.com/projectbluefin/server/releases/latest/download/` so all transfers share the same signed manifest. @@ -127,7 +127,7 @@ partitions, dual UKIs, and broader rollback strategies is tracked in ## Verification - [ ] `files/os/sysupdate.d/*.transfer` and - `files/os/sysupdate.k0s.d/*.transfer` do not contain `Verify=no`. + `files/os/sysupdate.kubernetes.d/*.transfer` do not contain `Verify=no`. - [ ] `elements/bluefin-server/os-stack.bst` includes `bluefin-server/os-sysupdate-keys.bst`. - [ ] `files/os/sysupdate-keys/import-pubring.gpg` exists and contains the @@ -137,7 +137,7 @@ partitions, dual UKIs, and broader rollback strategies is tracked in manifest, producing `dist/release/SHA256SUMS.gpg`. - [ ] CI uploads `dist/release/*` to the GitHub Release. - [ ] Every transfer in `files/os/sysupdate.d/*.transfer` and - `files/os/sysupdate.k0s.d/*.transfer` uses a static `Path=` with no + `files/os/sysupdate.kubernetes.d/*.transfer` uses a static `Path=` with no `@v` placeholder. - [ ] Every transfer uses `@v` only inside `MatchPattern=`. diff --git a/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md b/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md new file mode 100644 index 00000000..707c210a --- /dev/null +++ b/docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md @@ -0,0 +1,393 @@ +# Design Spec: Upstream Kubernetes Cutover with Argo CD Core GitOps + +## Context & Purpose + +Bluefin Server currently delivers Kubernetes as k0s: a single static binary carrying an +embedded control plane, an embedded container runtime, and a proprietary manifest deployer +that reconciles `/var/lib/k0s/manifests/`. + +This specification defines the complete removal of k0s and its replacement with upstream +Kubernetes (`kubeadm`, `kubelet`, `kubectl`) delivered as a `systemd-sysext`, Cilium as CNI, +containerd sourced from Flatcar's `/usr`, and **Argo CD Core** as the reconciler for all +in-cluster workload. + +The decision to run upstream Kubernetes is a product decision and is not re-litigated here. +This document records how the cutover is executed and which properties must be rebuilt. + +--- + +## What k0s Provided That Must Be Rebuilt + +k0s's stack applier supplied four properties for free. Upstream Kubernetes ships no addon +manager, so each must be replaced explicitly: + +| Property | k0s mechanism | Replacement | +|---|---|---| +| Apiserver readiness | applier runs inside the controller process | `kubeadm-init.service` ordering edge | +| Ordering | alphabetical directory scan with retry | explicit seed phases, then Argo sync waves | +| Drift correction | continuous reconcile loop | Argo `selfHeal: true` | +| Pruning | label-tracked stack membership | Argo `prune: true` | + +Upstream's only in-tree mechanisms are static pods (Pod kind only, node-local), +`kubeadm init phase addon` (hardcoded to CoreDNS and kube-proxy), and `kubectl` with +kustomize. The historical `kube-addon-manager` was removed. Every distro that offers a +manifest directory implements it privately; leaving k0s means leaving that layer, not +losing a k0s-specific feature. + +--- + +## Architecture + +### Delivery layers + +| Layer | Mechanism | Contents | +|---|---|---| +| Base DDI | BuildStream, Flatcar `/usr` | systemd, networkd, sysupdate, `crictl` | +| Runtime sysext | Flatcar `containerd-flatcar.raw` | containerd, runc, containerd shims | +| Kubernetes sysext | BuildStream re-bake of the bakery recipe | `kubeadm`, `kubelet`, `kubectl`, CNI plugins | +| Seed | `kubectl apply -k`, four directories | Cilium, Argo CD Core, git-daemon, root Application | +| Workload | Argo CD Core | everything else, including Argo itself | + +### First-boot unit sequence + +``` +systemd-sysext.service merge containerd + kubernetes sysexts into /usr + -> daemon-reload (see "Sysext activation" below) +bluefin-cluster-repo.service init /var/lib/bluefin/cluster.git from /usr/share/bluefin/cluster +kubeadm-init.service kubeadm init --config, skipping the kube-proxy addon +bluefin-cluster-bootstrap.service + apply 00-cilium/ CNI; nothing schedules before this + wait node Ready + apply 10-argocd-core/ Argo CRDs, RBAC, redis, repo-server, application-controller + apply 15-gitd/ git-daemon serving /var/lib/bluefin/cluster.git + apply 20-root-app/ one Application; Argo owns everything downstream +``` + +### Seed contents + +``` +/usr/share/bluefin/seed/ + 00-cilium/ CNI, kube-proxy replacement + 10-argocd-core/ upstream core-install.yaml, applicationset-controller patched out + 15-gitd/ git-daemon Deployment + Service, hostPath /var/lib/bluefin/cluster.git + 20-root-app/ AppProject + root Application +``` + +The seed is **append-only by construction**. It is never pruned. See "Closed decision: +pruning" below for why this matters. + +### Argo-owned tree + +``` +/usr/share/bluefin/cluster/ + cert-manager + SelfSigned -> CA Issuer + kubelet-csr-approver + KubeFlex CRDs + operator + PostgreSQL + KubeStellar core controller-manager + KubeStellar console + kiosk proxy + ControlPlane CR + local-cluster registration + argocd-core self-management +``` + +--- + +## Detailed Component Specifications + +### 1. Kubernetes sysext (`elements/oci/kubernetes-sysext.bst`) + +Re-bakes the `flatcar/sysext-bakery` `kubernetes.sysext` recipe under BuildStream rather +than consuming the published artifact, so release signing and provenance remain in-repo. +The output layout stays wire-compatible with `extensions.flatcar.org`, so an operator can +repoint the sysupdate source at upstream without changing anything else. + +Contents, mirroring the upstream recipe: + +- `/usr/bin/{kubectl,kubeadm,kubelet}` from `dl.k8s.io//bin/linux//`, each + pinned and verified against its published `.sha256`. +- `/usr/local/bin/cni/` from the `containernetworking/plugins` release tarball. +- `/usr/local/share/kubernetes-version`, `/usr/local/share/kubernetes-cni-version`. +- `/usr/lib/extension-release.d/extension-release.kubernetes`. + +`include/kubernetes.yml` replaces `include/k0s.yml` as the single version source. +`.github/scripts/check-kubernetes-version.py` replaces the k0s checker and fails closed +when a consumer restates the version instead of deriving it. + +The sysupdate transfer tracks a **minor-pinned** track (`kubernetes-v1.NN.conf`) because +Kubernetes does not support unattended cross-minor in-place upgrades. + +### 2. containerd + +Flatcar's `/usr` tarball already contains the runtime as a sysext blob. Verified against +the Flatcar 4593.2.5 image contents listing: + +``` +./usr/share/flatcar/sysext/containerd-flatcar.raw 26,001,408 +./usr/share/flatcar/sysext/docker-flatcar.raw 57,118,720 +./usr/bin/crictl 31,488,432 +``` + +No `containerd`, `dockerd`, `runc` or `ctr` binaries exist in `/usr/bin`; only `crictl`. +Acquisition is therefore free and `elements/flatcar/container-runtime-reference-sysexts.bst` +stays reference-only. + +`elements/flatcar/flatcar-usr.bst` gains explicit removals in the existing no-wildcard +style: + +- `usr/share/flatcar/sysext/docker-flatcar.raw` +- `usr/share/flatcar/etc/extensions/docker-flatcar.raw` + +Docker is not shipped: it is a second unused runtime and 57 MiB charged against the 1 GiB +verity budget that the verity `/usr` work introduces. + +### 3. Sysext activation + +`flatcar-usr.bst` removes Flatcar's `ensure-sysext.service`, `download_sysext`, and the +`sysinit.target.wants` enablement symlink. Bluefin does not run Flatcar's `/etc` seeding +either, so the `usr/share/flatcar/etc/extensions/` symlinks are inert. + +Activation therefore requires an explicit replacement. `systemd-sysext.service` is present +and enabled in Flatcar's `/usr` and scans the standard extension search paths. + +**Must verify before implementation:** whether `systemd-sysext.service` performs a +`daemon-reload` after merging at the FSDK systemd version. A merge makes unit files appear +under `/usr`; it does not by itself make systemd notice a new `containerd.service`. Both +the bakery (`RELOAD_SERVICES_ON_MERGE="true"`) and this repo's own `k0s-first-boot.service` +(merge, then `daemon-reload`, then `enable --now`) reload explicitly. If the FSDK unit does +not, carry a drop-in that does. Failing to confirm this reproduces a silent failure mode: +image merged, no runtime running. + +`kubeadm-init.service` carries an explicit ordering edge on the merge completing. + +### 4. Cilium + +Cilium is applied by the seed, not by Argo: Argo's pods require pod networking, and a node +without CNI stays `NotReady` and carries a `NoSchedule` taint. + +`kubeadm init` runs with `--skip-phases=addon/kube-proxy`; Cilium provides kube-proxy +replacement. + +### 5. Argo CD Core (`10-argocd-core/`) + +Upstream `manifests/core-install.yaml`, not a hand-vendored subset. Verified workload +delta against `manifests/install.yaml`: + +| Workload | `install.yaml` | `core-install.yaml` | +|---|---|---| +| `argocd-application-controller` | yes | yes | +| `argocd-repo-server` | yes | yes | +| `argocd-redis` | yes | yes | +| `argocd-applicationset-controller` | yes | yes | +| `argocd-server` | yes | no | +| `argocd-dex-server` | yes | no | +| `argocd-notifications-controller` | yes | no | + +Core additionally drops one of the three `redis.server` consumers along with +`argocd-server`. Two remain: `repo-server` and `application-controller`. Redis is an +in-memory cache with no PersistentVolumeClaim anywhere in the manifest; it caches rendered +manifests and the live resource tree. There is no supported redis-free Argo CD, so redis is +the floor, not a choice. + +`argocd-applicationset-controller` is removed by kustomize patch: the deployment generates +Applications from templates and this design has exactly one root Application. Its CRD is +left in place, harmlessly. + +Two residual `argocd-server` references survive in Core as NetworkPolicy `podSelector` +rules admitting traffic from a pod that never exists. Inert; left untouched to keep the +upstream manifest unmodified. + +Resulting footprint: three pods — `application-controller`, `repo-server`, `redis`. + +No web UI and no `argocd` CLI, which speaks gRPC to `argocd-server`. Applications are +managed as plain Kubernetes custom resources. This removes a second authenticated web +surface from the appliance; the KubeStellar console remains the only dashboard. + +### 6. Manifest source bridge (`15-gitd/`) + +Argo's `Application.source` accepts git, Helm, or OCI. There is no filesystem source, so +Argo cannot read `/usr/share/bluefin/cluster/` directly. This is an impedance mismatch +between how an image-based OS delivers files and how Argo consumes them. + +`bluefin-cluster-repo.service` initialises a bare repository at +`/var/lib/bluefin/cluster.git` from the seed tree on first boot. A minimal git-daemon +Deployment serves it over the cluster network with a hostPath mount, and the root +Application targets it. + +The git-daemon **must** live in the seed and **cannot** be an Argo manifest: the root +Application needs the repository to exist and be served before Argo can read anything. +Same class of circular dependency as Cilium. + +This is preferred over `repoURL: file://` against a hostPath-mounted bare repo. That +pattern is undocumented, repo-server runs non-root with its own volume layout, and the +failure would surface mid-implementation. + +Offline behaviour is preserved: no external git host is contacted. Manifests remain +available with the network unplugged, satisfying the offline-install constraint. + +### 7. Certificates + +No certificate or key material ships in any image. + +- kubelet **client** certificates: `kubeadm`'s native TLS bootstrap, with + `rotateCertificates: true`. +- kubelet **serving** certificates: `serverTLSBootstrap: true` plus `kubelet-csr-approver` + with a strict `--provider-regex` and bounded `--max-expiration-sec`. + kube-controller-manager does not auto-approve serving CSRs, so this component is + mandatory rather than optional. +- Console, kiosk and in-cluster TLS: cert-manager, SelfSigned bootstrap into a CA Issuer, + auto-renewed leaf certificates. + +This permanently replaces the build-time self-signed key currently baked into the k0s +sysext. + +--- + +## Closed decision: pruning + +The seed is applied with `kubectl apply --server-side --force-conflicts` and is **never +pruned**. Argo owns pruning for all downstream workload via `prune: true`. + +Both upstream kubectl pruning modes are alpha and unsuitable: + +- Allowlist `--prune`: "Alpha since Kubernetes v1.5", and per upstream documentation "still + in alpha due to usability, correctness and performance issues with its design". +- ApplySet `--prune --applyset`: "Alpha since Kubernetes v1.27", gated behind + `KUBECTL_APPLYSET=true`, carrying an explicit warning that "backwards incompatible + changes might be introduced in subsequent releases". + +The decisive constraint is ApplySet's namespace rule: "ApplySets spanning multiple +namespaces must use a cluster-scoped custom resource as the parent object." The seed spans +`kube-system`, `argocd`, and later `cert-manager` and the KubeStellar namespaces, so +pruning it would require defining a custom CRD purely to act as an ApplySet parent. + +`--prune` is additionally incompatible with `--server-side`; only the ApplySet path is +server-side-apply compatible. + +Append-only seed plus Argo-owned reconciliation avoids the entire question. This decision +is closed and should not be reopened without upstream promoting ApplySet to beta. + +--- + +## Installer + +`systemd-sysinstall` and `systemd-repart` have no Kubernetes awareness and will not gain +any; AGENTS.md scopes the installer to `systemd-sysinstall`-native, `systemd-repart`-based +behaviour. Their native capability is file placement via `CopyFiles=`, which is how the k0s +sysext is seeded today. + +Manifests therefore ship inside the DDI at `/usr/share/bluefin/`, with the OEM partition +available for per-site overrides once the Flatcar partition layout lands. Defaults stay +immutable inside the verity-sealed image; site-specific material lives on OEM alongside +`systemd-creds`. + +--- + +## Purge & Replacement Matrix + +| k0s component | Action | Replacement | +|---|---|---| +| `include/k0s.yml` | Deleted | `include/kubernetes.yml` | +| `elements/k0s/k0s-bin.bst` | Deleted | `elements/kubernetes/` binary imports | +| `elements/oci/k0s-sysext.bst` | Deleted | `elements/oci/kubernetes-sysext.bst` | +| `elements/bluefin-server/os-k0s-first-boot.bst` | Deleted | `os-cluster-bootstrap.bst` | +| `elements/bluefin-server/os-k0s-sysupdate.bst` | Deleted | `os-kubernetes-sysupdate.bst` | +| `files/k0s/sysext/k0scontroller.service` | Deleted | `kubeadm-init.service` + kubelet unit | +| `files/k0s/sysext/k0s-manifests.conf` | Deleted | none; manifests applied from `/usr` | +| `files/k0s/k0s.yaml` | Deleted | `/usr/share/bluefin/kubeadm.yaml` | +| `files/k0s/manifests/argocd/install.yaml` | Deleted | upstream `core-install.yaml` | +| `files/k0s/manifests/kubestellar/01-*-github-oauth.yaml` | Deleted | none; no baked secrets | +| `files/k0s/manifests/kubestellar/*` | Moved | `/usr/share/bluefin/cluster/`, digest-pinned | +| `files/k0s/kubeflex/generate-postgres-secret.sh` | Deleted | in-cluster generated Secret | +| `files/k0s/kiosk/{cert,key}.pem` | Deleted | cert-manager issued | +| `files/k0s/kiosk/{nginx.conf,kiosk-gate.js,kiosk-gate.css}` | Moved | ConfigMap; hostPath removed | +| `files/os/systemd/system/k0s-first-boot.service` | Deleted | `bluefin-cluster-bootstrap.service` | +| `files/os/systemd/system/k0s-first-boot-fetch.service` | Deleted | sysupdate handles fetch | +| `files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset` | Deleted | equivalent preset | +| `files/os/sysupdate.k0s.d/70-k0s.transfer` | Deleted | `70-kubernetes.transfer` | +| `files/installer/repart.d/30-var.conf` k0s `CopyFiles` | Deleted | sysext seeding | +| `.github/scripts/check-k0s-version.py` | Deleted | `check-kubernetes-version.py` | +| `tests/unit/test_k0s_first_boot.py` | Deleted | `test_cluster_bootstrap.py` | +| `tests/unit/test_k0s_manifests.py` | Deleted | `test_cluster_manifests.py` | +| `tests/unit/test_k0s_version.py` | Deleted | `test_kubernetes_version.py` | +| `docs/skills/k0s-sysext.md` | Deleted | `docs/skills/kubernetes-sysext.md` | +| `docs/skills/k0s-sysext-ops.md` | Deleted | `docs/skills/kubernetes-sysext-ops.md` | +| `Justfile` `k8s` target | Updated | targets `kubeadm`/`kubectl` | +| `AGENTS.md` hard rule 4 | Updated | see ADR scope below | +| `docs/skills/index.md`, `factory-integration.md`, `CONTEXT.md`, `NOTES.md` | Updated | k0s references purged | + +Acceptance gate: `grep -ri k0s` matches only the ADR and the changelog. + +### ADR scope + +Hard rule 4 currently reads: *"Deliver k0s as an optional `systemd-sysext`; never bundle +Kubernetes or container runtimes into the base OS DDI."* + +Only the k0s mandate changes, to deliver **Kubernetes** as an optional sysext. The +container-runtime clause is untouched and remains satisfied: containerd ships as a sysext, +not as base-DDI content. + +### Superseded pull requests + +| PR | Disposition | Reason | +|---|---|---| +| #178 | Close as superseded | hand-vendors 422 lines into a deleted directory; ships no `argoproj.io` CRDs; dead `ARGOCD_GIT_SHARED_DIR` emptyDir; port 443 named `https` fronting plaintext | +| #81 | Close as superseded | pins an EOL `argocd-server` image that Core does not ship | +| #169 | Close as superseded | superseded by #80 | +| #177 | Close against `core-install.yaml` | Core supplies the complete, correctly wired stack | + +--- + +## Verification & Testing Strategy + +1. **Element graph** — `just validate` resolves with zero dangling k0s nodes. +2. **Unit tests** + - `test_kubernetes_version.py`: version single-source invariants. + - `test_flatcar_usr.py`: extended to assert both Docker paths removed and containerd + present exactly once. + - `test_cluster_manifests.py`: every seed and cluster overlay builds under + `kubectl kustomize` offline, and every image carries a digest. + - Secret-material scan across `files/` and `elements/` returns nothing loadable. +3. **Sysext build** — `just build-sysext && just export-sysext` produce the Kubernetes + sysext and `SHA256SUMS`. +4. **Boot proof** — `just show-me-the-future` reaches multi-user; containerd active; + `kubectl get nodes` reports `Ready`; the three Argo Core pods reach `Running`; the root + Application syncs; the KubeStellar console serves live cluster state behind a + cert-manager certificate of under 100 days validity. +5. **Manifest-source proof** — with no external git host reachable, the root Application + syncs from `/var/lib/bluefin/cluster.git` via the git-daemon bridge. This proves the + manifest tree needs no external host. It does **not** prove a fully offline boot: + workload images are still pulled from public registries, so a complete + network-unavailable boot is gated on open item 2 below and is not an acceptance + criterion for this spec. + +--- + +## Open items requiring verification before implementation + +1. `systemd-sysext.service` post-merge `daemon-reload` behaviour at the FSDK systemd + version; carry a reload drop-in if absent. +2. Image pre-seeding into containerd. Every workload manifest currently pulls from + `quay.io`, `ghcr.io` and `docker.io`. Until images are pre-seeded, "fully automated + first boot" means "with network present". This is a pre-existing condition, not a + regression, and is tracked separately. +3. **No OTA channel for the containerd sysext.** `elements/flatcar/containerd-sysext.bst` + pins one Flatcar runtime image (containerd 2.1.5, runc 1.3.3) and + `files/installer/repart.d/30-var.conf` seeds it to + `/var/lib/extensions/containerd-flatcar.raw`. There is no + `sysupdate.containerd.d` transfer, and the `ID=_any` repack means the seeded image + keeps merging across every OS update rather than being rejected, so an installed + host holds that exact containerd/runc until it is reinstalled — no patch path for a + runtime CVE. This is a regression against k0s, whose embedded runtime rode the + `k0s-@v` transfer. + + It is deliberately not closed by pointing a transfer at Flatcar's published + `rootfs-included-sysexts/containerd-flatcar.raw`: that image carries + `ID=flatcar` with a concrete `VERSION_ID`, which is exactly the metadata this + element repacks away. Fetching it unmodified would reintroduce the os-release + mismatch, `systemd-sysext` would refuse the merge, and `containerd.service` + would vanish on the next boot. A transfer must therefore consume a + *repacked* image published on this repository's own release feed, alongside + `kubernetes-@v.raw.zst` — i.e. the sysext has to become a release artifact + before `70-containerd.transfer` can exist. Until then, runtime CVEs are + remediated by reinstall. diff --git a/elements/bluefin-server/os-cluster-bootstrap.bst b/elements/bluefin-server/os-cluster-bootstrap.bst new file mode 100644 index 00000000..391ad746 --- /dev/null +++ b/elements/bluefin-server/os-cluster-bootstrap.bst @@ -0,0 +1,63 @@ +kind: manual +description: | + Install the host units that bring up the Kubernetes control plane and seed + the cluster, plus the configuration they read. + + Units: kubelet.service (+ the kubeadm drop-in), kubeadm-init.service, + bluefin-cluster-repo.service and bluefin-cluster-bootstrap.service, together + with the systemd-sysext.service drop-in that reloads systemd after a merge so + sysext-delivered units become visible. + + Also stages /usr/share/bluefin/kubeadm.yaml and the + /usr/libexec/bluefin-cluster-repo helper. + +build-depends: + - base/base-stack.bst + +sources: + - kind: local + path: files/os/systemd/system + directory: units-src + - kind: local + path: files/os/systemd/systemd-sysext.service.d + directory: sysext-dropin-src + - kind: local + path: files/kubernetes/kubeadm.yaml + directory: kubeadm-src + - kind: local + path: files/os/libexec + directory: libexec-src + - kind: local + path: files/kubernetes/sysctl.d + directory: sysctl-src + +# This element stages systemd units, a YAML config and a shell helper. FSDK's +# default strip-commands shell out to freedesktop-sdk-stripper, which +# base/base-stack.bst does not provide, so the default turns a no-op into +# `command not found` (exit 127). Same contract as os-countme.bst, +# os-image-info.bst and os-release-flatcar.bst. +variables: + strip-binaries: "" + +config: + install-commands: + - | + set -euo pipefail + OUT="%{install-root}" + + mkdir -p "${OUT}/usr/lib/systemd/system" + cp -a units-src/. "${OUT}/usr/lib/systemd/system/" + + mkdir -p "${OUT}/usr/lib/systemd/system/systemd-sysext.service.d" + cp -a sysext-dropin-src/. "${OUT}/usr/lib/systemd/system/systemd-sysext.service.d/" + + mkdir -p "${OUT}/usr/lib/sysctl.d" + cp -a sysctl-src/. "${OUT}/usr/lib/sysctl.d/" + + mkdir -p "${OUT}/usr/share/bluefin" + cp -a kubeadm-src/kubeadm.yaml "${OUT}/usr/share/bluefin/kubeadm.yaml" + chmod 0644 "${OUT}/usr/share/bluefin/kubeadm.yaml" + + mkdir -p "${OUT}/usr/libexec" + cp -a libexec-src/bluefin-cluster-repo "${OUT}/usr/libexec/bluefin-cluster-repo" + chmod 0755 "${OUT}/usr/libexec/bluefin-cluster-repo" diff --git a/elements/bluefin-server/os-cluster-manifests.bst b/elements/bluefin-server/os-cluster-manifests.bst new file mode 100644 index 00000000..f0200f8a --- /dev/null +++ b/elements/bluefin-server/os-cluster-manifests.bst @@ -0,0 +1,72 @@ +kind: manual +description: | + Stage the cluster bootstrap seed and the Argo CD-owned manifest tree. + + Output: + /usr/share/bluefin/seed/ four kustomize phases applied by + bluefin-cluster-bootstrap.service + /usr/share/bluefin/cluster/ the tree bluefin-cluster-repo.service commits + to /var/lib/bluefin/cluster.git, served to Argo + by the in-cluster git-daemon + + Two upstream manifests are fetched and sha256-pinned here rather than vendored + into the repository. Both are large machine-generated files that no reviewer + reads on a bump, and a stale hand-copy silently drops CRDs and RBAC. Their + kustomization.yaml lives in files/cluster/, so the customisation is reviewable + even though the payload is not. + +build-depends: + # Base runtime providing /bin/sh, install and cp in the sandbox. + - base/base-stack.bst + +sources: + - kind: local + path: files/cluster/seed + directory: seed-src + - kind: local + path: files/cluster/manifests + directory: cluster-src + + # Argo CD Core. core-install.yaml, not install.yaml: Core drops argocd-server, + # argocd-dex-server and argocd-notifications-controller, which removes a second + # authenticated web surface from the appliance. + - kind: remote + url: githubraw:argoproj/argo-cd/v3.5.3/manifests/core-install.yaml + filename: core-install.yaml + directory: argocd-src + ref: 1a87025d8eb2eae621653fd312fb9ca51df1b4b3b6992a030e3a9ef38e45c448 + + # cert-manager. Issues the appliance's certificates in-cluster, replacing the + # build-time self-signed key that used to be generated into the sysext. + - kind: remote + url: github:cert-manager/cert-manager/releases/download/v1.21.2/cert-manager.yaml + filename: cert-manager.yaml + directory: certmanager-src + ref: e03b668ec8675214af6b0a671699d088f2601fa3878e0dbe1b41d3feafd1879f + +# This element stages YAML only. FSDK's default strip-commands shell out to +# freedesktop-sdk-stripper, which base/base-stack.bst does not provide, so the +# default turns a no-op into `command not found` (exit 127). Same contract as +# os-countme.bst, os-image-info.bst and os-release-flatcar.bst. +variables: + strip-binaries: "" + +config: + install-commands: + - | + set -euo pipefail + SEED="%{install-root}/usr/share/bluefin/seed" + CLUSTER="%{install-root}/usr/share/bluefin/cluster" + + mkdir -p "${SEED}" "${CLUSTER}" + + # Seed phases, applied in name order by bluefin-cluster-bootstrap.service. + cp -a seed-src/. "${SEED}/" + + # Argo CD-owned tree, committed to the bare repo on first boot. + cp -a cluster-src/. "${CLUSTER}/" + + # Drop the pinned upstream manifests next to the kustomization.yaml that + # patches them. Each is named by its kustomization's resources: entry. + install -m 0644 argocd-src/core-install.yaml "${SEED}/10-argocd-core/core-install.yaml" + install -m 0644 certmanager-src/cert-manager.yaml "${CLUSTER}/cert-manager/cert-manager.yaml" diff --git a/elements/bluefin-server/os-k0s-first-boot.bst b/elements/bluefin-server/os-k0s-first-boot.bst deleted file mode 100644 index 038a1d99..00000000 --- a/elements/bluefin-server/os-k0s-first-boot.bst +++ /dev/null @@ -1,9 +0,0 @@ -kind: import -description: Install the first-boot k0s sysext activation unit. - -sources: - - kind: local - path: files/os/systemd/system - -config: - target: /usr/lib/systemd/system diff --git a/elements/bluefin-server/os-k0s-sysupdate.bst b/elements/bluefin-server/os-k0s-sysupdate.bst deleted file mode 100644 index 45d11926..00000000 --- a/elements/bluefin-server/os-k0s-sysupdate.bst +++ /dev/null @@ -1,9 +0,0 @@ -kind: import -description: Install the k0s component systemd-sysupdate transfer definition. - -sources: - - kind: local - path: files/os/sysupdate.k0s.d - -config: - target: /usr/lib/sysupdate.k0s.d diff --git a/elements/bluefin-server/os-kubernetes-sysupdate.bst b/elements/bluefin-server/os-kubernetes-sysupdate.bst new file mode 100644 index 00000000..444b4b3d --- /dev/null +++ b/elements/bluefin-server/os-kubernetes-sysupdate.bst @@ -0,0 +1,9 @@ +kind: import +description: Install the Kubernetes component systemd-sysupdate transfer definition. + +sources: + - kind: local + path: files/os/sysupdate.kubernetes.d + +config: + target: /usr/lib/sysupdate.kubernetes.d diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst index 074634d0..df032c21 100644 --- a/elements/bluefin-server/os-stack.bst +++ b/elements/bluefin-server/os-stack.bst @@ -43,14 +43,21 @@ depends: # A/B System Update Configurations - bluefin-server/os-sysupdate.bst - # k0s component system update configuration - - bluefin-server/os-k0s-sysupdate.bst + # Kubernetes sysext component system update configuration + - bluefin-server/os-kubernetes-sysupdate.bst # Wired DHCP configuration for the installed OS - bluefin-server/os-networkd.bst - # Activate the optional k0s sysext after installation - - bluefin-server/os-k0s-first-boot.bst + # Control-plane bring-up and cluster seeding host units + - bluefin-server/os-cluster-bootstrap.bst + + # Seed phases and the Argo-owned manifest tree under /usr/share/bluefin + - bluefin-server/os-cluster-manifests.bst + + # git plumbing for bluefin-cluster-repo.service, which republishes + # /usr/share/bluefin/cluster as the bare repo Argo CD syncs from + - freedesktop-sdk.bst:components/git.bst # Public keyring for verifying SHA256SUMS manifests from GitHub Releases - bluefin-server/os-sysupdate-keys.bst diff --git a/elements/flatcar/containerd-sysext.bst b/elements/flatcar/containerd-sysext.bst new file mode 100644 index 00000000..f004dd9a --- /dev/null +++ b/elements/flatcar/containerd-sysext.bst @@ -0,0 +1,114 @@ +kind: manual +description: | + Flatcar's containerd system extension, shipped as a second sysext alongside + oci/kubernetes-sysext.bst. + + Output: /containerd-flatcar.raw — Flatcar's upstream image, repacked with one + edited field in its extension-release (see "Version decoupling" below). + + The name is load-bearing and must not be shortened. systemd-sysext resolves an + image's metadata by stem: an extension merged as .raw is required to + carry /usr/lib/extension-release.d/extension-release.. This image ships + extension-release.containerd-flatcar, so renaming it to containerd.raw makes + systemd-sysext look for extension-release.containerd, fail to find it, and + refuse the extension — silently, at boot, long after the build went green. + tests/unit/test_sysupdate_transfers.py encodes the same invariant for the + Kubernetes sysext, and tests/unit/test_seeded_sysext_names.py now covers the + installer-seeded path this element feeds. + + Why import rather than build: FSDK 26.08 ships no containerd component at all + (see docs/superpowers/specs/2026-09-19-flatcar-runtime-parity.md, which records + `no elements/components/containerd.bst` in the pinned junction), so there is + nothing to compose from. Flatcar publishes containerd as a systemd-sysext, the + same delivery mechanism this repository already uses for Kubernetes, and the + image is self-contained: + + /usr/bin/containerd, containerd-shim-runc-v2, ctr, runc + /usr/lib/systemd/system/containerd.service + /usr/lib/systemd/system/multi-user.target.wants/containerd.service + /usr/lib/systemd/system/multi-user.target.d/10-containerd-service.conf + /usr/lib/extension-release.d/extension-release.containerd-flatcar + /usr/share/containerd/config.toml, config-cgroupfs.toml + /usr/share/SLSA/*.json.zst (containerd 2.1.5, runc 1.3.3) + + It carries its own `multi-user.target.wants` symlink, so containerd.service is + enabled by the merge itself — no preset is needed on our side. That is what + satisfies `Requires=containerd.service` in + files/os/systemd/system/kubeadm-init.service. + + Version decoupling. Upstream's extension-release declares ID=flatcar and + VERSION_ID=4593.2.5, and systemd-sysext refuses an extension whose ID/VERSION_ID + do not match the host's os-release. elements/bluefin-server/os-release-flatcar.bst + writes VERSION_ID=%{flatcar-version}, so they agree at install time — and stop + agreeing the moment flatcar-version is bumped. + + That divergence is an OTA hazard, not a build-time one. A build-time mismatch + is loud: the source URL embeds %{flatcar-version}, so bumping it changes the + URL and the pinned ref below fails to resolve. The danger is an already + installed machine. /var survives an A/B root update, so a host updated to a DDI + built from a newer flatcar-version boots with a new os-release while + /var/lib/extensions/containerd-flatcar.raw still declares the old VERSION_ID. + systemd-sysext then refuses the merge, containerd.service disappears, and + kubeadm-init.service fails — the cluster goes down after a routine OS update. + kubernetes.raw is immune because 70-kubernetes.transfer replaces it in lockstep; + containerd has no such transfer. That absence is also the runtime's patch path + gap — a seeded image that never updates holds this containerd/runc until the + host is reinstalled — and is tracked as open item 3 of + docs/superpowers/specs/2026-09-19-kubernetes-cutover-design.md. + + So the image is repacked with a single field changed: ID=_any. That is the same + decoupling files/kubernetes/sysext/extension-release.kubernetes uses, for the + same stated reason — "the sysext ships independently of the host os-release + version and would otherwise be rejected after an OS update" + (tests/unit/test_sysupdate_transfers.py). Nothing else in the image is touched: + the binaries, containerd.service, the target.wants symlink, the config.toml + pair and the SLSA provenance documents are repacked byte-for-byte. + + elements/flatcar/container-runtime-reference-sysexts.bst also imports this file. + That element is the parity oracle and additionally pulls podman and docker, + which this image does not ship; no shipped target depends on it. This element + exists so the shipped path imports exactly one runtime and nothing else. + +(@): + - include/arch.yml + - include/flatcar.yml + +build-depends: + - base/base-stack.bst + - freedesktop-sdk.bst:components/squashfs-tools.bst + +variables: + strip-binaries: "" + +sources: + - kind: remote + url: flatcar:stable/%{flatcar-board}/%{flatcar-version}/rootfs-included-sysexts/containerd-flatcar.raw + filename: containerd-flatcar.raw + ref: 78e38344ac490004b9fa4c7da38bbfcffc393146aa990379aff7b5c09589f809 + +config: + install-commands: + - | + set -euo pipefail + + RELEASE="usr/lib/extension-release.d/extension-release.containerd-flatcar" + + unsquashfs -no-xattrs -d unpacked containerd-flatcar.raw + + # Fail loudly if upstream ever renames the metadata file: a silently + # missing extension-release produces an image systemd-sysext refuses. + test -f "unpacked/${RELEASE}" + + # NAME must equal the image stem, ID=_any decouples the merge from the + # host's os-release version. SYSEXT_VERSION_ID keeps the Flatcar release + # readable off a merged /usr for provenance. + cat > "unpacked/${RELEASE}" <&2 - echo " arch=%{arch} has no pinned source. Add an arch-conditional" >&2 - echo " source before building k0s for %{arch}." >&2 - exit 1 - fi - - install -D -m 0755 k0s-%{k0s-upstream-tag}-amd64 "%{install-root}/usr/bin/k0s" diff --git a/elements/kubernetes/cni-plugins.bst b/elements/kubernetes/cni-plugins.bst new file mode 100644 index 00000000..524da179 --- /dev/null +++ b/elements/kubernetes/cni-plugins.bst @@ -0,0 +1,59 @@ +kind: manual +description: | + Import the upstream containernetworking/plugins CNI binaries. + + The linux/amd64 release tarball for the tag named by include/kubernetes.yml + (%{cni-upstream-tag}); only the SHA256 below is restated here, taken from + that release's sha256 asset. + + Each plugin is installed by name rather than by wildcard, so an upstream + rename or removal fails the build instead of silently shipping a CNI set + that no longer matches what Cilium and the kubelet expect. + +# %{cni-upstream-tag} — the upstream CNI plugins release tag. Single source of +# truth for the Kubernetes version axis; see include/kubernetes.yml. +(@): include/kubernetes.yml + +build-depends: + # Base runtime providing /bin/sh and install in the sandbox. + - base/base-stack.bst + +variables: + # Do not try to strip the pre-built upstream binaries. + strip-binaries: "" + +sources: + - kind: tar + url: github:containernetworking/plugins/releases/download/%{cni-upstream-tag}/cni-plugins-linux-amd64-%{cni-upstream-tag}.tgz + ref: b98f74a0f8522f0a83867178729c1aa70f2158f90c45a2ca8fa791db1c76b303 + base-dir: "" + directory: cni-src + +config: + install-commands: + - | + if [ "%{arch}" != "x86_64" ]; then + echo "ERROR: elements/kubernetes/cni-plugins.bst pins the upstream" >&2 + echo " linux/amd64 CNI plugins tarball; arch=%{arch} has no" >&2 + echo " pinned source. Add an arch-conditional source before" >&2 + echo " building the CNI plugins for %{arch}." >&2 + exit 1 + fi + - install -D -m 0755 cni-src/bandwidth "%{install-root}/usr/local/bin/cni/bandwidth" + - install -D -m 0755 cni-src/bridge "%{install-root}/usr/local/bin/cni/bridge" + - install -D -m 0755 cni-src/dhcp "%{install-root}/usr/local/bin/cni/dhcp" + - install -D -m 0755 cni-src/dummy "%{install-root}/usr/local/bin/cni/dummy" + - install -D -m 0755 cni-src/firewall "%{install-root}/usr/local/bin/cni/firewall" + - install -D -m 0755 cni-src/host-device "%{install-root}/usr/local/bin/cni/host-device" + - install -D -m 0755 cni-src/host-local "%{install-root}/usr/local/bin/cni/host-local" + - install -D -m 0755 cni-src/ipvlan "%{install-root}/usr/local/bin/cni/ipvlan" + - install -D -m 0755 cni-src/loopback "%{install-root}/usr/local/bin/cni/loopback" + - install -D -m 0755 cni-src/macvlan "%{install-root}/usr/local/bin/cni/macvlan" + - install -D -m 0755 cni-src/portmap "%{install-root}/usr/local/bin/cni/portmap" + - install -D -m 0755 cni-src/ptp "%{install-root}/usr/local/bin/cni/ptp" + - install -D -m 0755 cni-src/sbr "%{install-root}/usr/local/bin/cni/sbr" + - install -D -m 0755 cni-src/static "%{install-root}/usr/local/bin/cni/static" + - install -D -m 0755 cni-src/tap "%{install-root}/usr/local/bin/cni/tap" + - install -D -m 0755 cni-src/tuning "%{install-root}/usr/local/bin/cni/tuning" + - install -D -m 0755 cni-src/vlan "%{install-root}/usr/local/bin/cni/vlan" + - install -D -m 0755 cni-src/vrf "%{install-root}/usr/local/bin/cni/vrf" diff --git a/elements/kubernetes/kubernetes-bin.bst b/elements/kubernetes/kubernetes-bin.bst new file mode 100644 index 00000000..3bb8d75b --- /dev/null +++ b/elements/kubernetes/kubernetes-bin.bst @@ -0,0 +1,46 @@ +kind: manual +description: | + Import the upstream Kubernetes node binaries. + + kubectl, kubeadm and kubelet for x86_64, pinned to the release named by + include/kubernetes.yml (%{k8s-upstream-tag}); only the SHA256 of each + binary is restated here, taken from the `.sha256` asset published beside it + on dl.k8s.io. + +# %{k8s-upstream-tag} — the upstream Kubernetes release directory on +# dl.k8s.io. Single source of truth for the Kubernetes version axis; see +# include/kubernetes.yml. +(@): include/kubernetes.yml + +build-depends: + # Base runtime providing /bin/sh and install in the sandbox. + - base/base-stack.bst + +variables: + # Do not try to strip the pre-built upstream binaries. + strip-binaries: "" + +sources: + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubectl + ref: 8b8f088da2dab964f853b38464033b1be15ede2839eca751482357c45abdd05a + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubeadm + ref: 10d054585907f2c8bb823c84f6a66b3bd2a474c3d795475f04e9a44f78650fe6 + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubelet + ref: cf2562db46c48ab84cc3791540778ee7fb1a0a008529adbb434723dd02ed8b4b + +config: + install-commands: + - | + if [ "%{arch}" != "x86_64" ]; then + echo "ERROR: elements/kubernetes/kubernetes-bin.bst pins the upstream" >&2 + echo " linux/amd64 Kubernetes binaries; arch=%{arch} has no pinned" >&2 + echo " source. Add an arch-conditional source before building" >&2 + echo " Kubernetes for %{arch}." >&2 + exit 1 + fi + - install -D -m 0755 kubectl "%{install-root}/usr/bin/kubectl" + - install -D -m 0755 kubeadm "%{install-root}/usr/bin/kubeadm" + - install -D -m 0755 kubelet "%{install-root}/usr/bin/kubelet" diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index 9f801570..9a57315d 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -38,7 +38,7 @@ config: exit 1 fi rm -f "/layer/usr/lib/modules/${KVER}/vmlinux" "/layer/usr/lib/modules/${KVER}/System.map" - depmod -b /layer "${KVER}" + depmod -b /layer/usr "${KVER}" # Stage extracted CA certificate bundle into target DDI rootfs mkdir -p /layer/etc/pki/ca-trust/extracted /layer/etc/ssl/certs /layer/etc/pki/tls/certs @@ -47,10 +47,10 @@ config: ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/certs/ca-bundle.crt ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/cert.pem - # Mask systemd-firstboot to prevent interactive prompts on headless first boot + # Mask systemd-firstboot and systemd-homed-firstboot to prevent interactive prompts on headless first boot mkdir -p /layer/etc/systemd/system ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service - + ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service # Mask audit-rules to prevent failure on shell-free OS (augenrules requires /bin/sh) ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 44d9f300..c495088f 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -57,10 +57,17 @@ build-depends: - filename: bluefin-server/os-rootfs.bst config: location: /target-root - # Uncompressed k0s sysext staged into the installer initrd for offline install - - filename: oci/k0s-sysext.bst + # Uncompressed Kubernetes sysext staged into the installer initrd for + # offline install; systemd-repart seeds it onto /var at install time. + - filename: oci/kubernetes-sysext.bst config: - location: /k0s + location: /kubernetes + # Flatcar's containerd sysext, seeded the same way. kubeadm-init.service has + # a hard Requires=containerd.service and FSDK ships no containerd component, + # so without this the control plane never starts. + - filename: flatcar/containerd-sysext.bst + config: + location: /containerd variables: installer-version: "%{release-version}" @@ -103,7 +110,7 @@ config: mkdir -p /target-root/boot ln -sf "../usr/lib/modules/${TARGET_KVER}/vmlinuz" /target-root/boot/vmlinuz - depmod -b /target-root "${TARGET_KVER}" || true + depmod -b /target-root/usr "${TARGET_KVER}" || true # Configure dynamic linker search paths so ldconfig indexes FSDK libraries # (e.g. libc.so.6, libgcc_s.so.1, libkmod.so) for dracut-install @@ -329,11 +336,21 @@ config: ln -sf system-install.target \ /layer/usr/lib/systemd/system/default.target - # ── Step 2a: Stage the offline k0s sysext before packing the initrd ─ + # ── Step 2a: Stage the offline sysexts before packing cpio ─────────── + # Both are seeded onto /var by systemd-repart at install time + # (files/installer/repart.d/30-var.conf), so first boot merges them with + # no network fetch. + # + # Each filename must keep the stem its extension-release is named for: + # systemd-sysext merges .raw only if the image carries + # extension-release.. kubernetes-.raw -> kubernetes.raw is + # safe (extension-release.kubernetes); containerd-flatcar.raw must stay + # containerd-flatcar.raw (extension-release.containerd-flatcar). - | set -eux - cp /k0s/k0s-*.raw /layer/k0s.raw - ls -lh /layer/k0s.raw + cp /kubernetes/kubernetes-*.raw /layer/kubernetes.raw + cp /containerd/containerd-flatcar.raw /layer/containerd-flatcar.raw + ls -lh /layer/kubernetes.raw /layer/containerd-flatcar.raw # ── Step 3: Pack initrd cpio ───────────────────────────────────────── # find . from /layer gives paths like ./usr/bin/... which is exactly @@ -360,7 +377,7 @@ config: rm -rf "${MOD_DIR}/drivers/infiniband" fi # Re-run depmod to update module dependencies after stripping unneeded modules - depmod -b /layer "${KVER}" || true + depmod -b /layer/usr "${KVER}" || true fi ( cd /layer && find . -print0 \ diff --git a/elements/oci/k0s-sysext.bst b/elements/oci/k0s-sysext.bst deleted file mode 100644 index dc5bcdbc..00000000 --- a/elements/oci/k0s-sysext.bst +++ /dev/null @@ -1,111 +0,0 @@ -kind: manual -description: | - Produce a systemd-sysext extension image for k0s. - - Output: k0s-.raw, k0s-.raw.zst, and SHA256SUMS. - Contains /usr/bin/k0s, k0scontroller.service, tmpfiles.d/k0s-manifests.conf, - and /usr/share/k0s/manifests/ (Argo CD and KubeStellar). - -(@): - - include/arch.yml - - include/k0s.yml - -build-depends: - - base/base-stack.bst - - freedesktop-sdk.bst:components/erofs-utils.bst - - freedesktop-sdk.bst:components/xz.bst - - freedesktop-sdk.bst:components/zstd.bst - - freedesktop-sdk.bst:components/openssl.bst - - filename: k0s/k0s-bin.bst - config: - location: / - -sources: - - kind: local - path: files/k0s/sysext - directory: sysext-src - - kind: local - path: files/k0s/manifests - directory: manifests-src - - kind: local - path: files/k0s/kiosk - directory: kiosk-src - - kind: local - path: files/k0s/kubeflex - directory: kubeflex-src - - kind: local - path: files/os/issue.d/40-kubestellar.issue - directory: issue-src - -variables: - strip-binaries: "" - -config: - install-commands: - - | - set -euo pipefail - OUT="%{install-root}" - FNAME="k0s-%{k0s-version}.raw" - - if [ "%{arch}" != "x86_64" ]; then - echo "ERROR: elements/oci/k0s-sysext.bst only supports x86_64 (got %{arch})" >&2 - exit 1 - fi - - mkdir -p sysext/usr/bin - mkdir -p sysext/usr/lib/systemd/system - mkdir -p sysext/usr/lib/extension-release.d - mkdir -p sysext/usr/lib/tmpfiles.d - mkdir -p sysext/usr/lib/issue.d - mkdir -p sysext/usr/share/k0s/manifests - mkdir -p sysext/usr/share/k0s/kiosk - mkdir -p "${OUT}" - - # Stage k0s binary - cp -a /usr/bin/k0s sysext/usr/bin/k0s - chmod 0755 sysext/usr/bin/k0s - - # Stage systemd service - cp -a sysext-src/k0scontroller.service sysext/usr/lib/systemd/system/ - - # Stage extension release metadata - cp -a sysext-src/extension-release.k0s sysext/usr/lib/extension-release.d/ - echo "VERSION_ID=%{k0s-version}" >> sysext/usr/lib/extension-release.d/extension-release.k0s - echo "ARCHITECTURE=%{systemd-arch}" >> sysext/usr/lib/extension-release.d/extension-release.k0s - - # Stage tmpfiles seeding configuration - cp -a sysext-src/k0s-manifests.conf sysext/usr/lib/tmpfiles.d/ - - # Stage console issue banner - cp -a issue-src/40-kubestellar.issue sysext/usr/lib/issue.d/ - - # Stage manifest stacks - cp -a manifests-src/* sysext/usr/share/k0s/manifests/ - - # Stage Console kiosk proxy assets. - cp -a kiosk-src/. sysext/usr/share/k0s/kiosk/ - - # Stage KubeStellar first-boot credential helper. - mkdir -p sysext/usr/share/k0s/kubeflex - cp -a kubeflex-src/. sysext/usr/share/k0s/kubeflex/ - chmod 0755 sysext/usr/share/k0s/kubeflex/generate-postgres-secret.sh - - # Generate self-signed TLS certificate for local kiosk proxy - openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ - -keyout sysext/usr/share/k0s/kiosk/key.pem \ - -out sysext/usr/share/k0s/kiosk/cert.pem \ - -subj "/CN=KubeStellar Console" \ - -addext "subjectAltName=DNS:localhost,DNS:*.local,IP:127.0.0.1,IP:10.0.2.15" - chmod 0600 sysext/usr/share/k0s/kiosk/key.pem - chmod 0644 sysext/usr/share/k0s/kiosk/cert.pem - - # Build EROFS image - mkfs.erofs -d0 "${OUT}/${FNAME}" sysext - - # Compress release asset - zstd -T0 -19 -q "${OUT}/${FNAME}" -o "${OUT}/${FNAME}.zst" - - ( - cd "${OUT}" - sha256sum --binary "${FNAME}.zst" > SHA256SUMS - ) diff --git a/elements/oci/kubernetes-sysext.bst b/elements/oci/kubernetes-sysext.bst new file mode 100644 index 00000000..3ca96a91 --- /dev/null +++ b/elements/oci/kubernetes-sysext.bst @@ -0,0 +1,94 @@ +kind: manual +description: | + Produce a systemd-sysext extension image for upstream Kubernetes. + + Output: kubernetes-.raw, kubernetes-.raw.zst, and + SHA256SUMS. Contains /usr/bin/{kubectl,kubeadm,kubelet}, the CNI plugins + under /usr/local/bin/cni/, the two version stamp files the upstream recipe + publishes, and the extension-release metadata. + + This is a re-bake of the flatcar/sysext-bakery `kubernetes.sysext` recipe + under BuildStream, so release signing and provenance stay in-repo. The + layout is deliberately wire-compatible with extensions.flatcar.org: an + operator can repoint the sysupdate source at upstream without changing + anything else. + + No certificate or key material is generated or shipped. Cluster TLS is + issued at runtime — kubelet client certs by kubeadm's TLS bootstrap, + kubelet serving certs by kubelet-csr-approver, everything else by + cert-manager. Nothing here is a manifest either; the seed and Argo-owned + trees are staged by elements/bluefin-server/os-cluster-manifests.bst. + +(@): + - include/arch.yml + - include/kubernetes.yml + +build-depends: + - base/base-stack.bst + - freedesktop-sdk.bst:components/erofs-utils.bst + - freedesktop-sdk.bst:components/xz.bst + - freedesktop-sdk.bst:components/zstd.bst + - filename: kubernetes/kubernetes-bin.bst + config: + location: / + - filename: kubernetes/cni-plugins.bst + config: + location: / + +sources: + - kind: local + path: files/kubernetes/sysext + directory: sysext-src + +variables: + strip-binaries: "" + +config: + install-commands: + - | + set -euo pipefail + OUT="%{install-root}" + FNAME="kubernetes-%{k8s-version}.raw" + + if [ "%{arch}" != "x86_64" ]; then + echo "ERROR: elements/oci/kubernetes-sysext.bst only supports x86_64 (got %{arch})" >&2 + exit 1 + fi + + mkdir -p sysext/usr/bin + mkdir -p sysext/usr/local/bin/cni + mkdir -p sysext/usr/local/share + mkdir -p sysext/usr/lib/extension-release.d + mkdir -p "${OUT}" + + # Stage the Kubernetes node binaries. + cp -a /usr/bin/kubectl sysext/usr/bin/kubectl + cp -a /usr/bin/kubeadm sysext/usr/bin/kubeadm + cp -a /usr/bin/kubelet sysext/usr/bin/kubelet + chmod 0755 sysext/usr/bin/kubectl + chmod 0755 sysext/usr/bin/kubeadm + chmod 0755 sysext/usr/bin/kubelet + + # Stage the CNI plugins where the upstream recipe publishes them. + cp -a /usr/local/bin/cni/. sysext/usr/local/bin/cni/ + + # Stage the version stamps the upstream recipe publishes, so an + # operator can read the delivered versions off a merged /usr. + echo "%{k8s-version}" > sysext/usr/local/share/kubernetes-version + echo "%{cni-version}" > sysext/usr/local/share/kubernetes-cni-version + + # Stage extension release metadata + cp -a sysext-src/extension-release.kubernetes sysext/usr/lib/extension-release.d/ + echo "VERSION_ID=%{k8s-version}" >> sysext/usr/lib/extension-release.d/extension-release.kubernetes + echo "ARCHITECTURE=%{systemd-arch}" >> sysext/usr/lib/extension-release.d/extension-release.kubernetes + + # Build EROFS image + mkfs.erofs -d0 "${OUT}/${FNAME}" sysext + + # Compress release asset + zstd -T0 -19 -q "${OUT}/${FNAME}" -o "${OUT}/${FNAME}.zst" + + ( + cd "${OUT}" + sha256sum --binary "${FNAME}.zst" > SHA256SUMS + ) diff --git a/files/bin/bluefin-kubestellar b/files/bin/bluefin-kubestellar index 4b867384..c3b66589 100755 --- a/files/bin/bluefin-kubestellar +++ b/files/bin/bluefin-kubestellar @@ -4,7 +4,7 @@ # Provides bulletproof setup and launch for connecting client workstations to the # Bluefin Server KubeStellar Console kiosk: # 1. Installs kc-agent via Homebrew (kubestellar/tap) if missing. -# 2. Configures origin flags for the local console (default: http://localhost:8080,http://127.0.0.1:8080). +# 2. Configures origin flags for the local console (default: https://localhost:8080,https://127.0.0.1:8080). # 3. Optimizes client environment by disabling in-cluster Kagenti DNS queries (KAGENTI_CONTROLLER_URL="none"). # 4. Launches kc-agent backgrounded with health verification, or in foreground. # 5. Provides lifecycle management (status, stop, logs). @@ -34,7 +34,7 @@ Commands: Options: -o, --origin Allowed origin URL(s) (comma-separated) - Default: http://localhost:8080,http://127.0.0.1:8080 + Default: https://localhost:8080,https://127.0.0.1:8080 -p, --port kc-agent local health/listen port (default: 8585) -f, --foreground Run kc-agent in foreground instead of backgrounding --no-install Do not attempt Homebrew installation if kc-agent is missing @@ -139,7 +139,7 @@ case "$CMD" in exit 0 ;; start) - ORIGIN="${ORIGIN:-http://localhost:8080,http://127.0.0.1:8080}" + ORIGIN="${ORIGIN:-https://localhost:8080,https://127.0.0.1:8080}" mkdir -p "$STATE_DIR" # Optimize client environment: disable in-cluster Kagenti DNS queries diff --git a/files/cluster/manifests/cert-manager/kustomization.yaml b/files/cluster/manifests/cert-manager/kustomization.yaml new file mode 100644 index 00000000..15b3cec9 --- /dev/null +++ b/files/cluster/manifests/cert-manager/kustomization.yaml @@ -0,0 +1,30 @@ +# cert-manager 1.21.2. +# +# cert-manager.yaml is the upstream release manifest, fetched and sha256-pinned +# by elements/bluefin-server/os-cluster-manifests.bst: +# +# https://github.com/cert-manager/cert-manager/releases/download/v1.21.2/cert-manager.yaml +# sha256: e03b668ec8675214af6b0a671699d088f2601fa3878e0dbe1b41d3feafd1879f +# +# It is not vendored here: it is ~6k lines of CRD schema that no human reviews +# on a bump, and a stale hand-copy is worse than a pinned fetch. +# +# This is the earliest wave in the tree. Everything that asks for a certificate +# needs the webhook Available first, and Argo will not start the next wave until +# these Deployments report Healthy. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cert-manager.yaml + +commonAnnotations: + argocd.argoproj.io/sync-wave: "-10" + +images: + - name: quay.io/jetstack/cert-manager-controller + digest: sha256:70f532fd9cfde0b09d55687200942399d89838bc2d5d5b45152eb799a15912b8 + - name: quay.io/jetstack/cert-manager-webhook + digest: sha256:a60e2dac46dbb8a7f3df95c54ce941012f54c2fe022f0ee55aaa1ab40ed957ae + - name: quay.io/jetstack/cert-manager-cainjector + digest: sha256:c85268c64f2e0e76684bf5fe8906caff34b82523561c6affe0fae3546bd87562 diff --git a/files/cluster/manifests/console/auth-secret.yaml b/files/cluster/manifests/console/auth-secret.yaml new file mode 100644 index 00000000..67d61147 --- /dev/null +++ b/files/cluster/manifests/console/auth-secret.yaml @@ -0,0 +1,113 @@ +# Mints the console's JWT signing key inside the cluster, once. +# +# This is what replaced the shipped Secret that carried +# `jwt-secret: default-jwt-secret-...`. That value was identical on every +# Bluefin Server ever installed, which made every console session forgeable by +# anyone who had read this repository. It does not come back: the key is +# generated here, on this machine, on first sync. +# +# Same idempotence contract as the PostgreSQL password: if the Secret exists the +# Job leaves it alone, so already-issued sessions survive a re-sync. +# +# GitHub OAuth credentials are deliberately absent. They are site-specific, and +# the console reads them with optional: true, so an operator who wants OAuth +# creates kubestellar-console-github-oauth themselves. +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubestellar-console-auth + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubestellar-console-auth + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubestellar-console-auth"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubestellar-console-auth + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubestellar-console-auth +subjects: + - kind: ServiceAccount + name: kubestellar-console-auth + namespace: kubestellar-console +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kubestellar-console-auth + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "3" + argocd.argoproj.io/sync-options: Replace=true +spec: + backoffLimit: 20 + template: + metadata: + labels: + app.kubernetes.io/name: kubestellar-console-auth + spec: + restartPolicy: OnFailure + serviceAccountName: kubestellar-console-auth + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: generate + image: docker.io/alpine/kubectl@sha256:8413f8890d19aa03f63851654f642957e65ba59654b0c9357ddc6ec0b05b63a6 + command: + - /bin/sh + - -c + - | + set -eu + if kubectl -n kubestellar-console get secret kubestellar-console-auth >/dev/null 2>&1; then + echo "kubestellar-console-auth already exists; existing sessions stay valid" + exit 0 + fi + jwt="$(tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 64)" + kubectl -n kubestellar-console create secret generic kubestellar-console-auth \ + --from-literal=jwt-secret="${jwt}" + echo "kubestellar-console-auth created" + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/console/console.yaml b/files/cluster/manifests/console/console.yaml new file mode 100644 index 00000000..e4f23903 --- /dev/null +++ b/files/cluster/manifests/console/console.yaml @@ -0,0 +1,143 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubestellar-console + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +rules: + - apiGroups: [""] + resources: ["nodes", "namespaces", "pods", "services", "events", "configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "statefulsets", "daemonsets", "replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["metrics.k8s.io"] + resources: ["nodes", "pods"] + verbs: ["get", "list"] + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubestellar-console +subjects: + - kind: ServiceAccount + name: kubestellar-console + namespace: kubestellar-console +--- +apiVersion: v1 +kind: Service +metadata: + name: kubestellar-console + namespace: kubestellar-console + labels: + app.kubernetes.io/name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: 8080 + name: http + selector: + app.kubernetes.io/name: kubestellar-console +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubestellar-console + namespace: kubestellar-console + labels: + app.kubernetes.io/name: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "4" +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: kubestellar-console + template: + metadata: + labels: + app.kubernetes.io/name: kubestellar-console + spec: + serviceAccountName: kubestellar-console + # Matches the numeric owner of /app/data and /app/.kc in the image + # (appuser, 1001). runAsNonRoot alone is not enough: the image declares + # USER by name, and the kubelet refuses to start a container it cannot + # prove is non-root, so the uid has to be spelled out here. + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: console + image: ghcr.io/kubestellar/console@sha256:f27adaf8d097a5403f60041615525497e0970151fcd10b90b7b6e903b395be32 + env: + # GitHub OAuth is opt-in and site-specific. Nothing in this tree + # creates kubestellar-console-github-oauth; an operator who wants + # OAuth creates it, and the console starts fine without it. + - name: GITHUB_CLIENT_ID + valueFrom: + secretKeyRef: + name: kubestellar-console-github-oauth + key: client-id + optional: true + - name: GITHUB_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: kubestellar-console-github-oauth + key: client-secret + optional: true + # Generated in-cluster at sync wave 3. Mandatory: a console that + # signs sessions with a key it did not generate is a console whose + # sessions anyone can forge. + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: kubestellar-console-auth + key: jwt-secret + optional: false + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + securityContext: + # readOnlyRootFilesystem is the one piece of the baseline this + # workload does not take, and it is an exception rather than an + # oversight: the image writes its SQLite database to + # DATABASE_PATH=/app/data/console.db and a crash log beside it, on + # its own root filesystem. Nothing in this tree redirects those + # paths, so a read-only rootfs would leave the console unable to + # start. The data is ephemeral either way — there is no volume and + # the Deployment uses Recreate. + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/console/kustomization.yaml b/files/cluster/manifests/console/kustomization.yaml new file mode 100644 index 00000000..302610a6 --- /dev/null +++ b/files/cluster/manifests/console/kustomization.yaml @@ -0,0 +1,12 @@ +# The KubeStellar console: the appliance's only dashboard. +# +# Argo CD Core ships no web UI and no argocd CLI, so this is the single +# authenticated web surface on the machine. DEV_MODE, ALLOW_DEV_MODE_IN_CLUSTER +# and DEV_USER_LOGIN are gone: they turned that surface into an unauthenticated +# one on every installed appliance. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - auth-secret.yaml + - console.yaml diff --git a/files/cluster/manifests/controlplanes/controlplanes.yaml b/files/cluster/manifests/controlplanes/controlplanes.yaml new file mode 100644 index 00000000..fd9cb3ec --- /dev/null +++ b/files/cluster/manifests/controlplanes/controlplanes.yaml @@ -0,0 +1,33 @@ +# KubeFlex control planes. +# +# `local-cluster` registers the appliance's own cluster with KubeFlex so the +# console has something real to show the moment it comes up, rather than an +# empty dashboard that looks broken. type: host means "this is the cluster +# KubeFlex is running on" — no new API server is created for it. +# +# `cp1` is the default tenant control plane: a shared-backend Kubernetes control +# plane whose data lands in the PostgreSQL instance from sync wave 1. It is the +# thing an operator gets to experiment with without touching the host cluster. +--- +apiVersion: tenancy.kflex.kubestellar.org/v1alpha1 +kind: ControlPlane +metadata: + name: local-cluster + labels: + app.kubernetes.io/part-of: bluefin-server + annotations: + argocd.argoproj.io/sync-wave: "6" +spec: + type: host +--- +apiVersion: tenancy.kflex.kubestellar.org/v1alpha1 +kind: ControlPlane +metadata: + name: cp1 + labels: + app.kubernetes.io/part-of: bluefin-server + annotations: + argocd.argoproj.io/sync-wave: "6" +spec: + type: k8s + backend: shared diff --git a/files/cluster/manifests/controlplanes/kustomization.yaml b/files/cluster/manifests/controlplanes/kustomization.yaml new file mode 100644 index 00000000..fa541c5b --- /dev/null +++ b/files/cluster/manifests/controlplanes/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - controlplanes.yaml diff --git a/files/k0s/kiosk/kiosk-gate.css b/files/cluster/manifests/kiosk/assets/kiosk-gate.css similarity index 100% rename from files/k0s/kiosk/kiosk-gate.css rename to files/cluster/manifests/kiosk/assets/kiosk-gate.css diff --git a/files/k0s/kiosk/kiosk-gate.js b/files/cluster/manifests/kiosk/assets/kiosk-gate.js similarity index 100% rename from files/k0s/kiosk/kiosk-gate.js rename to files/cluster/manifests/kiosk/assets/kiosk-gate.js diff --git a/files/cluster/manifests/kiosk/assets/nginx.conf b/files/cluster/manifests/kiosk/assets/nginx.conf new file mode 100644 index 00000000..6e7ef8af --- /dev/null +++ b/files/cluster/manifests/kiosk/assets/nginx.conf @@ -0,0 +1,72 @@ +# Kiosk proxy for the KubeStellar console. +# +# Terminates TLS with a cert-manager-issued certificate and injects the kc-agent +# gate into the console's HTML. Everything here is served from a ConfigMap and a +# Secret; nothing is read from the host filesystem. + +pid /tmp/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # The container root filesystem is read-only, so every scratch path nginx + # might touch has to land on the /tmp emptyDir. + client_body_temp_path /tmp/client_body; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + + server { + listen 8080 ssl; + server_name _; + + # Issued by the in-cluster CA and rotated by cert-manager. No key material + # is baked into any image. + ssl_certificate /etc/kubestellar-kiosk/tls/tls.crt; + ssl_certificate_key /etc/kubestellar-kiosk/tls/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + # Auto-redirect plain HTTP to HTTPS on the same port + error_page 497 https://$http_host$request_uri; + + location = /kiosk-gate.js { + alias /etc/kubestellar-kiosk/assets/kiosk-gate.js; + add_header Cache-Control "no-store"; + } + + location = /kiosk-gate.css { + alias /etc/kubestellar-kiosk/assets/kiosk-gate.css; + add_header Cache-Control "no-store"; + } + + location / { + # 10.96.0.10 is CoreDNS: the tenth address of serviceSubnet 10.96.0.0/12 + # in files/kubernetes/kubeadm.yaml, which kubeadm hands to the cluster DNS + # Service. nginx resolves the upstream at request time (proxy_pass through + # a variable), so it needs a literal resolver and cannot use + # /etc/resolv.conf. Changing serviceSubnet must change this line too or + # the kiosk proxies to nothing. + resolver 10.96.0.10 valid=5s ipv6=off; + set $upstream_endpoint http://kubestellar-console.kubestellar-console.svc.cluster.local:8080; + proxy_pass $upstream_endpoint; + proxy_redirect ~^https?://[^/]+/(.*) /$1; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Accept-Encoding ""; + proxy_set_header Connection ""; + sub_filter_once on; + sub_filter '' ''; + sub_filter '' ''; + } + } +} diff --git a/files/cluster/manifests/kiosk/certificate.yaml b/files/cluster/manifests/kiosk/certificate.yaml new file mode 100644 index 00000000..6369e103 --- /dev/null +++ b/files/cluster/manifests/kiosk/certificate.yaml @@ -0,0 +1,42 @@ +# TLS for the kiosk proxy, issued by the in-cluster CA. +# +# This is what replaced the openssl invocation in the old sysext element, which +# baked a 10-year self-signed key into every published image — the same private +# key on every appliance. cert-manager mints this one on the machine and renews +# it automatically. +# +# Short-lived on purpose: 90 days, renewed at 30 days remaining. A kiosk +# certificate that outlives the appliance's install is a certificate nobody +# notices has leaked. +# +# The SANs below deliberately do not cover the LAN address the issue banner +# advertises (https://:8080). That address is unknown at manifest +# time, so a LAN browser gets a hostname mismatch on top of the untrusted +# in-cluster CA — one warning rather than two, and the same click-through +# either way. A site that wants a clean chain issues its own certificate for a +# name it controls and replaces this Secret. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: kubestellar-kiosk-tls + namespace: kubestellar-console + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + secretName: kubestellar-kiosk-tls + commonName: KubeStellar Console + duration: 2160h # 90 days + renewBefore: 720h # 30 days + privateKey: + algorithm: ECDSA + size: 256 + rotationPolicy: Always + dnsNames: + - localhost + - kubestellar-kiosk-proxy.kubestellar-console.svc.cluster.local + ipAddresses: + - 127.0.0.1 + issuerRef: + name: bluefin-ca + kind: ClusterIssuer + group: cert-manager.io diff --git a/files/cluster/manifests/kiosk/deployment.yaml b/files/cluster/manifests/kiosk/deployment.yaml new file mode 100644 index 00000000..a1bb7257 --- /dev/null +++ b/files/cluster/manifests/kiosk/deployment.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubestellar-kiosk-proxy + namespace: kubestellar-console + labels: + app.kubernetes.io/name: kubestellar-kiosk-proxy + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: kubestellar-kiosk-proxy + template: + metadata: + labels: + app.kubernetes.io/name: kubestellar-kiosk-proxy + spec: + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + seccompProfile: + type: RuntimeDefault + containers: + - name: kiosk-proxy + image: docker.io/library/nginx@sha256:5616878291a2eed594aee8db4dade5878cf7edcb475e59193904b198d9b830de + command: + - nginx + args: + - -g + - daemon off; + - -c + - /etc/kubestellar-kiosk/assets/nginx.conf + ports: + - containerPort: 8080 + # Published on every interface. The previous hostIP: 127.0.0.1 + # binding meant the console was unreachable from the LAN and from + # anything but the one QEMU hostfwd, which is not what a kiosk on + # a server appliance is for. The endpoint is HTTPS-only and the + # console behind it authenticates, so loopback was doing security + # theatre rather than security. + hostPort: 8080 + name: https + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: assets + mountPath: /etc/kubestellar-kiosk/assets + readOnly: true + - name: tls + mountPath: /etc/kubestellar-kiosk/tls + readOnly: true + - name: tmp + mountPath: /tmp + - name: cache + mountPath: /var/cache/nginx + volumes: + # nginx.conf, kiosk-gate.js and kiosk-gate.css. These arrive from the + # cluster, not from a host directory seeded by tmpfiles.d: a workload + # that reads its configuration off the node cannot be reconciled, and + # drifts the moment someone edits the file in place. + - name: assets + configMap: + name: kubestellar-kiosk + - name: tls + secret: + secretName: kubestellar-kiosk-tls + - name: tmp + emptyDir: {} + - name: cache + emptyDir: {} diff --git a/files/cluster/manifests/kiosk/kustomization.yaml b/files/cluster/manifests/kiosk/kustomization.yaml new file mode 100644 index 00000000..16a8d0c9 --- /dev/null +++ b/files/cluster/manifests/kiosk/kustomization.yaml @@ -0,0 +1,25 @@ +# The kiosk proxy: TLS termination and the kc-agent gate in front of the +# console. +# +# The three asset files are a ConfigMap rather than a host directory. Its +# generated name carries a content hash, so editing nginx.conf or the gate +# rolls the Deployment instead of leaving a stale config mounted until someone +# notices. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - certificate.yaml + - deployment.yaml + - service.yaml + +configMapGenerator: + - name: kubestellar-kiosk + namespace: kubestellar-console + options: + annotations: + argocd.argoproj.io/sync-wave: "5" + files: + - assets/nginx.conf + - assets/kiosk-gate.js + - assets/kiosk-gate.css diff --git a/files/cluster/manifests/kiosk/service.yaml b/files/cluster/manifests/kiosk/service.yaml new file mode 100644 index 00000000..595e4de6 --- /dev/null +++ b/files/cluster/manifests/kiosk/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubestellar-kiosk-proxy + namespace: kubestellar-console + labels: + app.kubernetes.io/name: kubestellar-kiosk-proxy + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: https + name: https + selector: + app.kubernetes.io/name: kubestellar-kiosk-proxy diff --git a/files/cluster/manifests/kubeflex/crds.yaml b/files/cluster/manifests/kubeflex/crds.yaml new file mode 100644 index 00000000..d3b62a78 --- /dev/null +++ b/files/cluster/manifests/kubeflex/crds.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: controlplanes.tenancy.kflex.kubestellar.org + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + group: tenancy.kflex.kubestellar.org + names: + kind: ControlPlane + listKind: ControlPlaneList + plural: controlplanes + singular: controlplane + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + subresources: + status: {} + additionalPrinterColumns: + - name: Type + type: string + jsonPath: .spec.type + - name: Age + type: date + jsonPath: .metadata.creationTimestamp diff --git a/files/cluster/manifests/kubeflex/kustomization.yaml b/files/cluster/manifests/kubeflex/kustomization.yaml new file mode 100644 index 00000000..79386e18 --- /dev/null +++ b/files/cluster/manifests/kubeflex/kustomization.yaml @@ -0,0 +1,13 @@ +# KubeFlex: the ControlPlane CRD, the operator that reconciles it, and the +# in-cluster generator for the PostgreSQL superuser password the operator's +# database needs. +# +# Sync waves, not file ordering, decide what lands when: CRD and Secret at wave +# 0, PostgreSQL at wave 1, operator at wave 2. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - crds.yaml + - postgres-secret.yaml + - operator.yaml diff --git a/files/cluster/manifests/kubeflex/operator.yaml b/files/cluster/manifests/kubeflex/operator.yaml new file mode 100644 index 00000000..b6a2c0ac --- /dev/null +++ b/files/cluster/manifests/kubeflex/operator.yaml @@ -0,0 +1,93 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeflex-controller-manager + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "2" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeflex-controller-manager + annotations: + argocd.argoproj.io/sync-wave: "2" +rules: + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes/status", "controlplanes/finalizers"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["namespaces", "services", "secrets", "configmaps", "serviceaccounts"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["apps"] + resources: ["deployments", "statefulsets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeflex-controller-manager + annotations: + argocd.argoproj.io/sync-wave: "2" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeflex-controller-manager +subjects: + - kind: ServiceAccount + name: kubeflex-controller-manager + namespace: kubeflex-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubeflex-controller-manager + namespace: kubeflex-system + labels: + app.kubernetes.io/name: kubeflex + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubeflex + template: + metadata: + labels: + app.kubernetes.io/name: kubeflex + spec: + serviceAccountName: kubeflex-controller-manager + containers: + - name: manager + image: ghcr.io/kubestellar/kubeflex/manager@sha256:ca31299554fa4f4c2f917d4c141761978ede64048914895beaef80f82f6fd177 + command: + - /ko-app/manager + args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/kubeflex/postgres-secret.yaml b/files/cluster/manifests/kubeflex/postgres-secret.yaml new file mode 100644 index 00000000..0db7f71f --- /dev/null +++ b/files/cluster/manifests/kubeflex/postgres-secret.yaml @@ -0,0 +1,105 @@ +# Mints the PostgreSQL superuser password inside the cluster, once. +# +# The password is never committed to git and never appears in a pod spec: this +# Job generates it on first sync and stores it in the kubeflex-postgres Secret, +# which the StatefulSet reads through secretKeyRef (projectbluefin/server#98). +# It replaces the first-boot host script that used to write a Secret manifest +# into the node's manifest directory, where it was readable by anyone with a +# root shell and invisible to the cluster's own RBAC. +# +# Idempotent by construction: if the Secret already exists the Job exits without +# touching it, so the initialised database stays reachable across reboots and +# across re-syncs. Replace=true lets a future change to this Job land at all — +# Job specs are immutable, so an in-place update would otherwise fail the sync. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeflex-postgres-secret + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubeflex-postgres-secret + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubeflex-postgres"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubeflex-postgres-secret + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubeflex-postgres-secret +subjects: + - kind: ServiceAccount + name: kubeflex-postgres-secret + namespace: kubeflex-system +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kubeflex-postgres-secret + namespace: kubeflex-system + annotations: + argocd.argoproj.io/sync-wave: "0" + argocd.argoproj.io/sync-options: Replace=true +spec: + backoffLimit: 20 + template: + metadata: + labels: + app.kubernetes.io/name: kubeflex-postgres-secret + spec: + restartPolicy: OnFailure + serviceAccountName: kubeflex-postgres-secret + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: generate + image: docker.io/alpine/kubectl@sha256:8413f8890d19aa03f63851654f642957e65ba59654b0c9357ddc6ec0b05b63a6 + command: + - /bin/sh + - -c + - | + set -eu + if kubectl -n kubeflex-system get secret kubeflex-postgres >/dev/null 2>&1; then + echo "kubeflex-postgres already exists; leaving the running database's password alone" + exit 0 + fi + password="$(tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 32)" + kubectl -n kubeflex-system create secret generic kubeflex-postgres \ + --from-literal=password="${password}" + echo "kubeflex-postgres created" + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/kubelet-csr-approver/deployment.yaml b/files/cluster/manifests/kubelet-csr-approver/deployment.yaml new file mode 100644 index 00000000..90bbf199 --- /dev/null +++ b/files/cluster/manifests/kubelet-csr-approver/deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubelet-csr-approver + namespace: kube-system + labels: + app.kubernetes.io/name: kubelet-csr-approver + annotations: + argocd.argoproj.io/sync-wave: "-5" +spec: + # Single-node appliance, so leader election buys nothing and a second replica + # would stay Pending. + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubelet-csr-approver + template: + metadata: + labels: + app.kubernetes.io/name: kubelet-csr-approver + spec: + serviceAccountName: kubelet-csr-approver + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: kubelet-csr-approver + image: ghcr.io/postfinance/kubelet-csr-approver@sha256:6469ef517d2b6d3cb108fce8a913483533a48b1cd78b5d4af307fa09e04a7818 + args: + - -metrics-bind-address + - ":8080" + - -health-probe-bind-address + - ":8081" + env: + # Strict by shape rather than by a hardcoded name: the appliance's + # node name is whatever the operator sets as the hostname, but it is + # always a single DNS label. Refusing dotted names refuses the + # actual attack, which is a compromised kubelet asking for a serving + # certificate for some external FQDN. + - name: PROVIDER_REGEX + value: ^[a-z0-9][a-z0-9-]{0,62}$ + # One DNS SAN, the node's own name. Upstream default, restated + # because a silent bump to it would widen what gets approved. + - name: ALLOWED_DNS_NAMES + value: "1" + # A ceiling on what a CSR may *ask* for, not the lifetime nodes get. + # kubelet-csr-approver only denies CSRs whose spec.expirationSeconds + # exceeds this; the kubelet sets no expirationSeconds on its serving + # CSR, so those are issued for kube-controller-manager's + # --cluster-signing-duration (1y by default) regardless of the value + # here. It is kept as a hard bound on anything that does request a + # duration — shortening the actual serving-certificate lifetime + # means setting --cluster-signing-duration on the signer. + - name: MAX_EXPIRATION_SEC + value: "2592000" + # The appliance has no authoritative DNS for its own node name, so + # the forward-lookup check would deny every CSR. The regex above and + # the kubelet's own identity are what gate approval here. + - name: BYPASS_DNS_RESOLUTION + value: "true" + # Refuse CSRs whose requester is not system:node:. + - name: IGNORE_NON_SYSTEM_NODE + value: "false" + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8081 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 128Mi + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/kubelet-csr-approver/kustomization.yaml b/files/cluster/manifests/kubelet-csr-approver/kustomization.yaml new file mode 100644 index 00000000..853006a0 --- /dev/null +++ b/files/cluster/manifests/kubelet-csr-approver/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - rbac.yaml + - deployment.yaml diff --git a/files/cluster/manifests/kubelet-csr-approver/rbac.yaml b/files/cluster/manifests/kubelet-csr-approver/rbac.yaml new file mode 100644 index 00000000..30c68796 --- /dev/null +++ b/files/cluster/manifests/kubelet-csr-approver/rbac.yaml @@ -0,0 +1,73 @@ +# kubelet serving-certificate approver. +# +# kubeadm.yaml sets serverTLSBootstrap: true, so kubelets request their serving +# certificates through the certificates API. kube-controller-manager +# auto-approves kubelet *client* CSRs but deliberately never auto-approves +# *serving* CSRs, because it cannot verify that the requested SANs belong to the +# node. Without this component those CSRs sit Pending forever, the kubelet falls +# back to a self-signed serving certificate, and `kubectl logs` and +# `kubectl exec` fail TLS verification. It is mandatory, not optional. +# +# RBAC below is upstream's, from +# charts/kubelet-csr-approver/templates/clusterrole.yaml at v1.2.15. The approve +# permission is scoped to the kubernetes.io/kubelet-serving signer by +# resourceNames, so this ServiceAccount cannot approve any other kind of CSR. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubelet-csr-approver + namespace: kube-system + labels: + app.kubernetes.io/name: kubelet-csr-approver + annotations: + argocd.argoproj.io/sync-wave: "-5" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubelet-csr-approver + labels: + app.kubernetes.io/name: kubelet-csr-approver + annotations: + argocd.argoproj.io/sync-wave: "-5" +rules: + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + verbs: + - update + - apiGroups: + - certificates.k8s.io + resourceNames: + - kubernetes.io/kubelet-serving + resources: + - signers + verbs: + - approve +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubelet-csr-approver + labels: + app.kubernetes.io/name: kubelet-csr-approver + annotations: + argocd.argoproj.io/sync-wave: "-5" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubelet-csr-approver +subjects: + - kind: ServiceAccount + name: kubelet-csr-approver + namespace: kube-system diff --git a/files/cluster/manifests/kubestellar/core.yaml b/files/cluster/manifests/kubestellar/core.yaml new file mode 100644 index 00000000..cfcf4e56 --- /dev/null +++ b/files/cluster/manifests/kubestellar/core.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kubestellar + annotations: + argocd.argoproj.io/sync-wave: "3" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubestellar-controller-manager + namespace: kubestellar + annotations: + argocd.argoproj.io/sync-wave: "3" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubestellar-controller-manager + annotations: + argocd.argoproj.io/sync-wave: "3" +rules: + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["tenancy.kflex.kubestellar.org"] + resources: ["controlplanes/status"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["namespaces", "secrets", "configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubestellar-controller-manager + annotations: + argocd.argoproj.io/sync-wave: "3" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubestellar-controller-manager +subjects: + - kind: ServiceAccount + name: kubestellar-controller-manager + namespace: kubestellar +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubestellar-controller-manager + namespace: kubestellar + labels: + app.kubernetes.io/name: kubestellar + annotations: + argocd.argoproj.io/sync-wave: "3" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kubestellar + template: + metadata: + labels: + app.kubernetes.io/name: kubestellar + spec: + serviceAccountName: kubestellar-controller-manager + containers: + - name: manager + image: ghcr.io/kubestellar/kubestellar/controller-manager@sha256:53db459bc17af77e644fe34f1469a425cd4c089939567029fe4e96ef5b80a7ab + command: + - /ko-app/controller-manager + args: + - --health-probe-bind-address=:8081 + - --metrics-bind-address=127.0.0.1:8080 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL diff --git a/files/cluster/manifests/kubestellar/kustomization.yaml b/files/cluster/manifests/kubestellar/kustomization.yaml new file mode 100644 index 00000000..9691381b --- /dev/null +++ b/files/cluster/manifests/kubestellar/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - core.yaml diff --git a/files/cluster/manifests/kustomization.yaml b/files/cluster/manifests/kustomization.yaml new file mode 100644 index 00000000..7b4cf1aa --- /dev/null +++ b/files/cluster/manifests/kustomization.yaml @@ -0,0 +1,36 @@ +# The Argo-owned tree. +# +# Staged into the image at /usr/share/bluefin/cluster/, committed to +# /var/lib/bluefin/cluster.git by bluefin-cluster-repo.service on first boot, +# served by the git-daemon from seed phase 15, and reconciled by the single root +# Application from seed phase 20. +# +# This is one Application, not a fan-out: ordering comes from +# argocd.argoproj.io/sync-wave annotations on the resources themselves, which +# Argo honours across the whole tree. +# +# -10 cert-manager +# -5 CA bootstrap chain, kubelet-csr-approver +# 0 KubeFlex CRD, PostgreSQL password generator +# 1 PostgreSQL +# 2 KubeFlex operator +# 3 KubeStellar core, console JWT generator +# 4 KubeStellar console +# 5 kiosk proxy and its certificate +# 6 ControlPlane custom resources +# +# Directory names carry no ordering. The sync waves are the only ordering +# statement, so there is nothing to keep in agreement. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cert-manager + - pki + - kubelet-csr-approver + - kubeflex + - postgres + - kubestellar + - console + - kiosk + - controlplanes diff --git a/files/cluster/manifests/pki/issuers.yaml b/files/cluster/manifests/pki/issuers.yaml new file mode 100644 index 00000000..b4a72c2e --- /dev/null +++ b/files/cluster/manifests/pki/issuers.yaml @@ -0,0 +1,54 @@ +# The appliance's certificate authority. +# +# No certificate or private key ships in any Bluefin Server image. The CA is +# minted in-cluster on first sync and rotated by cert-manager thereafter. This +# permanently replaces the build-time self-signed key that used to be generated +# into the sysext at image build, which meant every installed appliance shared +# one private key. +# +# SelfSigned ClusterIssuer -> CA Certificate -> CA ClusterIssuer is the upstream +# bootstrap chain: the SelfSigned issuer exists only to sign the CA once. +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: bluefin-selfsigned + annotations: + argocd.argoproj.io/sync-wave: "-5" +spec: + selfSigned: {} +--- +# cert-manager reads ClusterIssuer CA secrets from its own namespace +# (--cluster-resource-namespace, which defaults to the release namespace), so +# this Certificate must land in cert-manager. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: bluefin-ca + namespace: cert-manager + annotations: + argocd.argoproj.io/sync-wave: "-5" +spec: + isCA: true + commonName: Bluefin Server CA + secretName: bluefin-ca + duration: 87600h # 10 years + renewBefore: 8760h # 1 year + privateKey: + algorithm: ECDSA + size: 256 + rotationPolicy: Always + issuerRef: + name: bluefin-selfsigned + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: bluefin-ca + annotations: + argocd.argoproj.io/sync-wave: "-4" +spec: + ca: + secretName: bluefin-ca diff --git a/files/cluster/manifests/pki/kustomization.yaml b/files/cluster/manifests/pki/kustomization.yaml new file mode 100644 index 00000000..25a48402 --- /dev/null +++ b/files/cluster/manifests/pki/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - issuers.yaml diff --git a/files/cluster/manifests/postgres/kustomization.yaml b/files/cluster/manifests/postgres/kustomization.yaml new file mode 100644 index 00000000..99bd600a --- /dev/null +++ b/files/cluster/manifests/postgres/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - postgres.yaml diff --git a/files/cluster/manifests/postgres/postgres.yaml b/files/cluster/manifests/postgres/postgres.yaml new file mode 100644 index 00000000..b76aa181 --- /dev/null +++ b/files/cluster/manifests/postgres/postgres.yaml @@ -0,0 +1,97 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-postgresql + namespace: kubeflex-system + labels: + app.kubernetes.io/name: postgresql + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + type: ClusterIP + ports: + - name: tcp-postgresql + port: 5432 + targetPort: postgresql + selector: + app.kubernetes.io/name: postgresql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres-postgresql + namespace: kubeflex-system + labels: + app.kubernetes.io/name: postgresql + annotations: + argocd.argoproj.io/sync-wave: "1" +spec: + serviceName: postgres-postgresql + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgresql + template: + metadata: + labels: + app.kubernetes.io/name: postgresql + spec: + securityContext: + fsGroup: 70 + containers: + - name: postgresql + image: docker.io/library/postgres@sha256:3c5c8892d184f738f4fe282d14ddaa613a38f00f4189d2d94725ebe6f2909ddb + env: + - name: POSTGRES_USER + value: "postgres" + # The superuser password lives in the kubeflex-postgres Secret, + # generated in-cluster by the Job in ../kubeflex/postgres-secret.yaml + # at sync wave 0, so it is never committed to git and never appears + # in this pod spec. See projectbluefin/server#98. + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: kubeflex-postgres + key: password + - name: POSTGRES_DB + value: "postgres" + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgresql + containerPort: 5432 + readinessProbe: + exec: + command: + - pg_isready + - -U + - postgres + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + securityContext: + runAsNonRoot: true + runAsUser: 70 + runAsGroup: 70 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + volumes: + # Deliberately ephemeral. The appliance ships no CSI driver and no + # default StorageClass, so a volumeClaimTemplate would leave this + # StatefulSet Pending forever and the root Application would never + # report Healthy. Postgres re-initialises from POSTGRES_PASSWORD on + # restart, and that password survives in the kubeflex-postgres Secret, + # so the database comes back consistent — just empty. Durable storage + # is gated on a storage provisioner landing in this tree. + - name: data + emptyDir: {} diff --git a/files/cluster/seed/00-cilium/cilium-values.yaml b/files/cluster/seed/00-cilium/cilium-values.yaml new file mode 100644 index 00000000..3826225e --- /dev/null +++ b/files/cluster/seed/00-cilium/cilium-values.yaml @@ -0,0 +1,42 @@ +# Helm values used to render cilium.yaml. Reproduce the rendered manifest with: +# +# curl -LO https://helm.cilium.io/cilium-1.20.2.tgz +# echo "b2afd87b7f75f875f92a14559f14f59b7babbb479d968e3fd625a20bf30ec20e cilium-1.20.2.tgz" | sha256sum -c +# helm template cilium ./cilium-1.20.2.tgz \ +# --namespace kube-system --include-crds --values cilium-values.yaml > cilium.yaml +# +# Helm is a build-time tool only. Nothing in this repository ships or runs Helm +# on the appliance; the seed is plain YAML applied with `kubectl apply -k`. + +# kube-proxy replacement. kubeadm runs with --skip-phases=addon/kube-proxy, so +# Cilium is the only thing implementing Service load balancing. This also turns +# on Cilium's hostPort support, which the kiosk proxy relies on. +kubeProxyReplacement: "true" + +# Every Cilium workload below is hostNetwork, and Bluefin Server is a +# single-node appliance whose kube-apiserver listens on the same host. Pointing +# at the loopback endpoint removes the chicken-and-egg of reaching the +# kubernetes Service before Service load balancing exists. +# +# The chart's `k8sServiceHost: auto` is not usable here: it resolves the +# cluster-info ConfigMap through Helm's `lookup`, which returns nothing during +# `helm template` and fails the render. +k8sServiceHost: "localhost" +k8sServicePort: "6443" + +# Single control-plane node; the chart default of 2 would leave one operator +# replica Pending forever. +operator: + replicas: 1 + +# Must agree with networking.podSubnet in /usr/share/bluefin/kubeadm.yaml. +ipam: + operator: + clusterPoolIPv4PodCIDRList: + - "10.244.0.0/16" + +# Hubble's TLS material is generated by Helm at render time, which would bake +# private keys into the shipped image. No certificate or key material ships in +# any Bluefin Server image, so Hubble stays off. +hubble: + enabled: false diff --git a/files/cluster/seed/00-cilium/cilium.yaml b/files/cluster/seed/00-cilium/cilium.yaml new file mode 100644 index 00000000..319d11a4 --- /dev/null +++ b/files/cluster/seed/00-cilium/cilium.yaml @@ -0,0 +1,1782 @@ +# Cilium 1.20.2, rendered from the upstream Helm chart at build time. +# +# chart: https://helm.cilium.io/cilium-1.20.2.tgz +# sha256: b2afd87b7f75f875f92a14559f14f59b7babbb479d968e3fd625a20bf30ec20e +# values: cilium-values.yaml (in this directory) +# +# Regenerate with the command in the cilium-values.yaml header. Do not hand-edit +# this file: customisation belongs in cilium-values.yaml or in kustomization.yaml +# so that the next chart bump is a re-render rather than a merge. +# +# Cilium ships in the seed rather than the Argo-owned tree because Argo's own +# pods need pod networking: a node with no CNI stays NotReady and carries a +# NoSchedule taint. +# +# All three images already carry sha256 digests; the chart sets useDigest: true. +--- +# Source: cilium/templates/cilium-secrets-namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 + annotations: +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-envoy/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-envoy" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd", "kvstore" or + # "doublewrite-readkvstore" / "doublewrite-readcrd". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful + # for seamless migrations from the kvstore mode to the crd mode. Consult the + # documentation for more information on how to perform the migration. + identity-allocation-mode: crd + + identity-heartbeat-timeout: "30m0s" + identity-gc-interval: "15m0s" + cilium-endpoint-gc-interval: "5m0s" + nodes-gc-interval: "5m0s" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + debug-verbose: "" + metrics-sampling-interval: "5m" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes + enable-policy: "default" + policy-cidr-match-mode: "" + # If you want metrics enabled in cilium-operator, set the port for + # which the Cilium Operator will have their metrics exposed. + # NOTE that this will open the port on the nodes where Cilium operator pod + # is scheduled. + operator-prometheus-serve-addr: ":9963" + enable-metrics: "true" + enable-policy-secrets-sync: "true" + policy-secrets-only-from-secrets-namespace: "true" + policy-secrets-namespace: "cilium-secrets" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "false" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "false" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: "5s" + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-policy-stats-map-max specifies the maximum number of entries in global + # policy stats map + bpf-policy-stats-map-max: "65536" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + bpf-lb-external-clusterip: "false" + bpf-lb-source-range-all-types: "false" +# When Gateway API is enabled this is forced to "true" so per-backend weights +# on TCPRoute/UDPRoute take effect (Maglev honors the LB algorithm annotation only +# when enabled). Paired with `bpf-lb-sock-hostns-only`, which is forced "true" +# for the same reason. + bpf-lb-algorithm-annotation: "false" + bpf-lb-mode-annotation: "false" + + bpf-distributed-lru: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: "default" + # Unique ID of the cluster. Must be unique across all connected clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "0" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + + routing-mode: "tunnel" + tunnel-protocol: "vxlan" + tunnel-source-port-range: "0-0" + service-no-backend-response: "reject" + policy-deny-response: "none" + + + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + enable-ipv4-masquerade: "true" + enable-ipv4-big-tcp: "false" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "true" + enable-tcx: "true" + datapath-mode: "veth" + enable-masquerade-to-route-source: "false" + + enable-xt-socket-fallback: "true" + install-no-conntrack-iptables-rules: "false" + iptables-random-fully: "false" + + auto-direct-node-routes: "false" + direct-routing-skip-unreachable: "false" + + + + enable-host-firewall: "false" + # List of devices used to attach bpf_host.o (implements BPF NodePort, + # host-firewall and BPF masquerading) + devices: "" + + kube-proxy-replacement: "true" + kube-proxy-replacement-healthz-bind-address: "" + enable-no-service-endpoints-routable: "true" + bpf-lb-sock: "false" + nodeport-addresses: "" + enable-health-check-nodeport: "true" + enable-dynamic-source-lookup-nodeport: "false" + enable-health-check-loadbalancer-ip: "false" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + bpf-lb-acceleration: "disabled" + enable-service-topology: "false" + enable-l2-neigh-discovery: "false" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + enable-k8s-networkpolicy: "true" + enable-endpoint-lockdown-on-policy-overflow: "false" + # Tell the agent to generate and write a CNI configuration file + write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist + cni-exclusive: "true" + cni-log-file: "/var/run/cilium/cilium-cni.log" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + health-check-icmp-failure-threshold: "3" + enable-well-known-identities: "false" + enable-node-selector-labels: "false" + synchronize-k8s-nodes: "true" + operator-api-serve-addr: "127.0.0.1:9234" + + enable-hubble: "false" + ipam: "cluster-pool" + ipam-cilium-node-update-rate: "15s" + cluster-pool-ipv4-cidr: "10.244.0.0/16" + cluster-pool-ipv4-mask-size: "24" + + default-lb-service-ipam: "lbipam" + egress-gateway-reconciliation-trigger-interval: "1s" + enable-vtep: "false" + vtep-endpoint: "" + vtep-cidr: "" + vtep-mask: "" + vtep-mac: "" + + packetization-layer-pmtud-mode: "blackhole" + procfs: "/host/proc" + bpf-root: "/sys/fs/bpf" + cgroup-root: "/run/cilium/cgroupv2" + + identity-management-mode: "agent" + enable-sctp: "false" + remove-cilium-node-taints: "true" + set-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + unmanaged-pod-watcher-interval: "15s" + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: "true" + dnsproxy-socket-linger-timeout: "10" + tofqdns-dns-reject-response-code: "refused" + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: "0s" + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-proxy-response-max-delay: "100ms" + tofqdns-preallocate-identities: "true" + agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" + + mesh-auth-enabled: "false" + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + mesh-auth-gc-interval: "5m0s" + + proxy-xff-num-trusted-hops-ingress: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-connect-timeout: "2" + proxy-initial-fetch-timeout: "30" + proxy-max-active-downstream-connections: "50000" + proxy-max-requests-per-connection: "0" + proxy-max-connection-duration-seconds: "0" + proxy-idle-timeout-seconds: "60" + proxy-max-concurrent-retries: "128" + proxy-use-original-source-address: "true" + proxy-cluster-max-connections: "1024" + proxy-cluster-max-pending-requests: "1024" + proxy-cluster-max-requests: "1024" + http-retry-count: "3" + http-stream-idle-timeout: "300" + envoy-node-locality-enabled: "false" + + external-envoy-proxy: "true" + envoy-base-id: "0" + envoy-access-log-enabled: "true" + envoy-access-log-buffer-size: "4096" + envoy-keep-cap-netbindservice: "false" + envoy-xds-mode: "ads" + max-connected-clusters: "255" + clustermesh-cache-ttl: "0s" + clustermesh-enable-endpoint-sync: "false" + clustermesh-enable-mcs-api: "false" + clustermesh-mcs-api-install-crds: "true" + clustermesh-default-global-namespace: "true" + policy-default-local-cluster: "true" + + nat-map-stats-entries: "32" + nat-map-stats-interval: "30s" + enable-lb-ipam: "true" + enable-non-default-deny-policies: "true" + enable-source-ip-verification: "true" + enable-dynamic-config: "true" + enable-drift-checker: "true" + + enable-datapath-plugins: "false" + datapath-plugins-state-dir: "/var/run/cilium/plugins" + +# Extra config allows adding arbitrary properties to the cilium config. +# By putting it at the end of the ConfigMap, it's also possible to override existing properties. +--- +# Source: cilium/templates/cilium-envoy/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-envoy-config + namespace: kube-system +data: + # Keep the key name as bootstrap-config.json to avoid breaking changes + bootstrap-config.json: | + {"admin":{"address":{"pipe":{"mode":432,"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"adsConfig":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"cdsConfig":{"ads":{},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"ads":{},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxConnections":1024,"maxRequests":1024,"maxRetries":128}]},"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + - ciliumdatapathplugins + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + # To synchronize garbage collection of such resources + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - delete +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cilium-config + verbs: + # allow patching of the configmap to set annotations + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + # To remove node taints + - nodes + # To set NetworkUnavailable false on startup + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + # to check apiserver connectivity + - namespaces + - secrets + - serviceaccounts + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + verbs: + - get + - list + - watch +- apiGroups: + # Watch CNP / CCNPs for validation and external groups. + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups'. + # This was removed in v1.20, but we must keep the permissions until v1.21 for + # upgrade phasing. + - create + - update + - deletecollection + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + # Set validity status on CNP / CCNP. + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + # To perform garbage collection of such resources + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + # To synchronize garbage collection of such resources + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + # To perform CiliumNode garbage collector + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + - ciliumcidrgroups + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - cilium.io + resources: + - ciliumbgpclusterconfigs/status + - ciliumbgppeerconfigs/status + verbs: + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - update + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io + - ciliumgatewayclassconfigs.cilium.io + - ciliumdatapathplugins.cilium.io +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions/status + verbs: + - update + resourceNames: + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgppeerconfigs + - ciliumdatapathplugins + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + verbs: + - deletecollection +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-operator/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - update + - patch +--- +# Source: cilium/templates/cilium-operator/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-operator-ztunnel + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +rules: +# ZTunnel DaemonSet management permissions +# Note: These permissions must always be granted (not conditional on encryption.type) +# because the controller needs to clean up stale DaemonSets when ztunnel is disabled. +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-config-agent + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: + - kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-tlsinterception-secrets + namespace: "cilium-secrets" + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-tlsinterception-secrets +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-operator-ztunnel + namespace: kube-system + labels: + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-operator-ztunnel +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-envoy/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-envoy + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9964" + labels: + k8s-app: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 + io.cilium/app: proxy +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium-envoy + ports: + - name: envoy-metrics + port: 9964 + protocol: TCP + targetPort: 9964 +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-agent + helm.sh/chart: cilium-1.20.2 +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: cilium-agent + labels: + k8s-app: cilium + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 + spec: + securityContext: + appArmorProfile: + type: Unconfined + seccompProfile: + type: Unconfined + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 300 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + - name: "require-k8s-connectivity" + value: "false" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory + divisor: '1' + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + lifecycle: + postStart: + exec: + command: + - "bash" + - "-c" + - | + set -o errexit + set -o pipefail + set -o nounset + + # When running in AWS ENI mode, it's likely that 'aws-node' has + # had a chance to install SNAT iptables rules. These can result + # in dropped traffic, so we should attempt to remove them. + # We do it using a 'postStart' hook since this may need to run + # for nodes which might have already been init'ed but may still + # have dangling rules. This is safe because there are no + # dependencies on anything that is part of the startup script + # itself, and can be safely run multiple times per node (e.g. in + # case of a restart). + if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; + then + echo 'Deleting iptables rules created by the AWS CNI VPC plugin' + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + fi + echo 'Done!' + + preStop: + exec: + command: + - /cni-uninstall.sh + ports: + - name: health + containerPort: 9879 + hostPort: 9879 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + - SYSLOG + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + # Unprivileged containers need to mount /proc/sys/net from the host + # to have write access + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + # Unprivileged containers need to mount /proc/sys/kernel from the host + # to have write access + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - name: bpf-maps + mountPath: /sys/fs/bpf + # Unprivileged containers can't set mount propagation to bidirectional + # in this case we will mount the bpf fs from an init container that + # is privileged and set the mount propagation from host to container + # in Cilium. + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + - name: cilium-netns + mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + - name: tmp + mountPath: /tmp + + initContainers: + - name: config + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + command: + - cilium-dbg + - build-config + - "--k8s-api-server-urls=" + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + volumeMounts: + - name: tmp + mountPath: /tmp + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + capabilities: + add: + - NET_ADMIN + drop: + - ALL + # Required to mount cgroup2 filesystem on the underlying Kubernetes node. + # We use nsenter command with host's cgroup and mount namespaces enabled. + - name: mount-cgroup + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + command: + - bash + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + - name: apply-sysctl-overwrites + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + env: + - name: BIN_PATH + value: /opt/cni/bin + command: + - bash + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + # Mount the bpf fs if it is not mounted. We will perform this task + # from a privileged container because the mount propagation bidirectional + # only works from privileged containers. + - name: mount-bpf-fs + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + args: + - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' + command: + - /bin/bash + - -c + - -- + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + terminationMessagePolicy: FallbackToLogsOnError + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium # wait-for-kube-proxy + # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent + - name: install-cni-binaries + image: "quay.io/cilium/cilium:v1.20.2@sha256:2939231d0d3e3ebddcd80fffa168b7ddcc78fdf0dc864d1c8c126ff523c54f01" + imagePullPolicy: IfNotPresent + command: + - "/install-plugin.sh" + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 100m + memory: 10Mi + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: cni-path + mountPath: /host/opt/cni/bin # .Values.cni.install + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + + volumes: + # For sharing configuration between the "config" initContainer and the agent + - name: tmp + emptyDir: {} + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To exec into pod network namespaces + - name: cilium-netns + hostPath: + path: /var/run/netns + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To mount cgroup2 filesystem on the host or apply sysctlfix + - name: hostproc + hostPath: + path: /proc + type: Directory + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Sharing socket with Cilium Envoy on the same node by using a host path + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + projected: + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + sources: + - secret: + name: cilium-clustermesh + optional: true + # note: items are not explicitly listed here, since the entries of this secret + # depend on the peers configured, and that would cause a restart of all agents + # at every addition/removal. Leaving the field empty makes each secret entry + # to be automatically projected into the volume as a file whose name is the key. + - secret: + name: clustermesh-apiserver-remote-cert + optional: true + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + # note: we configure the volume for the kvstoremesh-specific certificate + # regardless of whether KVStoreMesh is enabled or not, so that it can be + # automatically mounted in case KVStoreMesh gets subsequently enabled, + # without requiring an agent restart. + - secret: + name: clustermesh-apiserver-local-cert + optional: true + items: + - key: tls.key + path: local-etcd-client.key + - key: tls.crt + path: local-etcd-client.crt + - key: ca.crt + path: local-etcd-client-ca.crt + - name: host-proc-sys-net + hostPath: + path: /proc/sys/net + type: Directory + - name: host-proc-sys-kernel + hostPath: + path: /proc/sys/kernel + type: Directory +--- +# Source: cilium/templates/cilium-envoy/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium-envoy + namespace: kube-system + labels: + k8s-app: cilium-envoy + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy + name: cilium-envoy + helm.sh/chart: cilium-1.20.2 +spec: + selector: + matchLabels: + k8s-app: cilium-envoy + + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: cilium-envoy + name: cilium-envoy + app.kubernetes.io/name: cilium-envoy + app.kubernetes.io/part-of: cilium + helm.sh/chart: cilium-1.20.2 + spec: + securityContext: + appArmorProfile: + type: Unconfined + containers: + - name: cilium-envoy + image: "quay.io/cilium/cilium-envoy:v1.37.6-1789133542-cbec91f666af0bf742da986d43832932dbb26b82@sha256:af7382699576b9e65e9184efa52eeca0b58aea70ad6e511bf260c91d9f740463" + imagePullPolicy: IfNotPresent + command: + - /usr/bin/cilium-envoy-starter + args: + - '--' + - '-c /var/run/cilium/envoy/bootstrap-config.json' + - '--base-id 0' + - '--log-level info' + + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + initialDelaySeconds: 5 + timeoutSeconds: 5 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + initialDelaySeconds: 0 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9878 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + initialDelaySeconds: 0 + timeoutSeconds: 5 + + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + + ports: + - name: envoy-metrics + containerPort: 9964 + hostPort: 9964 + protocol: TCP + securityContext: + seLinuxOptions: + level: s0 + type: spc_t + capabilities: + add: + - NET_ADMIN + - SYS_ADMIN + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - name: envoy-sockets + mountPath: /var/run/cilium/envoy/sockets + readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true + - name: envoy-config + mountPath: /var/run/cilium/envoy/ + readOnly: true + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccountName: "cilium-envoy" + automountServiceAccountToken: true + terminationGracePeriodSeconds: 1 + hostNetwork: true + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cilium.io/no-schedule + operator: NotIn + values: + - "true" + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium-envoy + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: Exists + volumes: + - name: envoy-sockets + hostPath: + path: "/var/run/cilium/envoy/sockets" + type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "/var/run/cilium/envoy/artifacts" + type: DirectoryOrCreate + - name: envoy-config + configMap: + name: "cilium-envoy-config" + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + items: + - key: bootstrap-config.json + path: bootstrap-config.json + # To keep state between restarts / upgrades + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + helm.sh/chart: cilium-1.20.2 +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 1 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case + # of one replica and no user configured Recreate strategy. + # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the + # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9963" + prometheus.io/scrape: "true" + labels: + io.cilium/app: operator + name: cilium-operator + app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-operator + helm.sh/chart: cilium-1.20.2 + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: cilium-operator + image: "quay.io/cilium/operator-generic:v1.20.2@sha256:64d8798350e8569b8e7622563fed6e44dce2625f311e4651b774816516c744fc" + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "localhost" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + ports: + - name: health + containerPort: 9234 + hostPort: 9234 + - name: prometheus + containerPort: 9963 + hostPort: 9963 + protocol: TCP + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: health + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 5 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccountName: "cilium-operator" + automountServiceAccountToken: true + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + nodeSelector: + kubernetes.io/os: linux + + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.cilium.io/agent-not-ready + operator: Exists + + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/files/cluster/seed/00-cilium/kustomization.yaml b/files/cluster/seed/00-cilium/kustomization.yaml new file mode 100644 index 00000000..3495062b --- /dev/null +++ b/files/cluster/seed/00-cilium/kustomization.yaml @@ -0,0 +1,12 @@ +# Seed phase 00 — CNI. +# +# Applied by bluefin-cluster-bootstrap.service before anything else: +# `kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/00-cilium`. +# +# The seed is append-only and is never pruned; Argo owns pruning for everything +# downstream. See the "Closed decision: pruning" section of the cutover design. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - cilium.yaml diff --git a/files/cluster/seed/10-argocd-core/kustomization.yaml b/files/cluster/seed/10-argocd-core/kustomization.yaml new file mode 100644 index 00000000..07acb458 --- /dev/null +++ b/files/cluster/seed/10-argocd-core/kustomization.yaml @@ -0,0 +1,33 @@ +# Seed phase 10 — Argo CD Core. +# +# core-install.yaml is the upstream manifest, fetched and sha256-pinned by +# elements/bluefin-server/os-cluster-manifests.bst: +# +# https://raw.githubusercontent.com/argoproj/argo-cd/v3.5.3/manifests/core-install.yaml +# sha256: 1a87025d8eb2eae621653fd312fb9ca51df1b4b3b6992a030e3a9ef38e45c448 +# +# It is deliberately not vendored into this repository. A hand-maintained subset +# drifts from upstream and silently drops CRDs and RBAC; pinning the real file +# keeps the next Argo bump a two-line change. +# +# core-install.yaml carries no namespace on any object and ships no Namespace, +# so both are supplied here. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: argocd + +resources: + - namespace.yaml + - core-install.yaml + +patches: + # One root Application needs no template generator. The CRD stays in place; + # only the controller Deployment goes. + - path: remove-applicationset-controller.yaml + +images: + - name: quay.io/argoproj/argocd + digest: sha256:dd3f47d5a5e4da563a7a398506e892481b358a7cec50abdf320c71aa55904bfa + - name: public.ecr.aws/docker/library/redis + digest: sha256:08ad0b1d280850169a790dba1393ff7a90aef951fc19632cf4d3ce4f78e679ba diff --git a/files/cluster/seed/10-argocd-core/namespace.yaml b/files/cluster/seed/10-argocd-core/namespace.yaml new file mode 100644 index 00000000..15de758f --- /dev/null +++ b/files/cluster/seed/10-argocd-core/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: argocd + labels: + app.kubernetes.io/part-of: argocd diff --git a/files/cluster/seed/10-argocd-core/remove-applicationset-controller.yaml b/files/cluster/seed/10-argocd-core/remove-applicationset-controller.yaml new file mode 100644 index 00000000..3ff140b4 --- /dev/null +++ b/files/cluster/seed/10-argocd-core/remove-applicationset-controller.yaml @@ -0,0 +1,18 @@ +# Delete the ApplicationSet controller from upstream core-install.yaml. +# +# ApplicationSet generates Applications from templates. This appliance declares +# exactly one root Application by hand, so the controller has nothing to do and +# is a third idle workload plus a cluster-wide write surface. +# +# applicationsets.argoproj.io stays installed: an orphaned CRD costs nothing and +# removing it from the upstream manifest would fight every Argo bump. +# +# No namespace is named here on purpose: kustomize matches patches against the +# resource as written, and upstream core-install.yaml carries no namespace on +# any object. The `namespace: argocd` transformer in kustomization.yaml runs +# afterwards. +$patch: delete +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-applicationset-controller diff --git a/files/cluster/seed/15-gitd/deployment.yaml b/files/cluster/seed/15-gitd/deployment.yaml new file mode 100644 index 00000000..f643162f --- /dev/null +++ b/files/cluster/seed/15-gitd/deployment.yaml @@ -0,0 +1,116 @@ +# Serves /var/lib/bluefin/cluster.git over the cluster network. +# +# Argo's Application.source speaks git, Helm or OCI. There is no filesystem +# source, so Argo cannot read /usr/share/bluefin/cluster/ directly. This +# Deployment is the bridge: bluefin-cluster-repo.service initialises the bare +# repository from the image's manifest tree on first boot, and git-daemon +# publishes it to argocd-repo-server. +# +# It lives in the seed and cannot be an Argo manifest: the root Application +# needs the repository served before Argo can read anything. Same class of +# circular dependency as Cilium. +# +# The image is the Argo CD image that 10-argocd-core already pulls onto this +# node. It carries git (Debian's git package owns /usr/lib/git-core/git-daemon), +# so the bridge adds no second image to pull — which matters for a node that is +# expected to come up without a registry reachable. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bluefin-git-daemon + namespace: bluefin-system + labels: + app.kubernetes.io/name: bluefin-git-daemon + app.kubernetes.io/part-of: bluefin-server +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: bluefin-git-daemon + template: + metadata: + labels: + app.kubernetes.io/name: bluefin-git-daemon + app.kubernetes.io/part-of: bluefin-server + spec: + # The bare repository is a host path, so the pod is pinned to the node + # that owns it. Single-node appliance; this is a statement of fact, not a + # constraint being imposed. + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: git-daemon + image: quay.io/argoproj/argocd@sha256:dd3f47d5a5e4da563a7a398506e892481b358a7cec50abdf320c71aa55904bfa + command: + - git + - daemon + - --verbose + - --informative-errors + - --reuseaddr + - --export-all + - --listen=0.0.0.0 + - --port=9418 + - --base-path=/srv/git + - /srv/git + env: + # The bare repo is root-owned on the host and git-daemon runs as + # uid 999, which trips git's dubious-ownership guard. The mount is + # read-only, so there is nothing for an owner mismatch to protect. + - name: GIT_CONFIG_COUNT + value: "1" + - name: GIT_CONFIG_KEY_0 + value: safe.directory + - name: GIT_CONFIG_VALUE_0 + value: "*" + - name: HOME + value: /tmp + ports: + - name: git + containerPort: 9418 + readinessProbe: + tcpSocket: + port: git + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + tcpSocket: + port: git + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumeMounts: + - name: cluster-repo + mountPath: /srv/git/cluster.git + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: cluster-repo + hostPath: + path: /var/lib/bluefin/cluster.git + type: Directory + - name: tmp + emptyDir: {} diff --git a/files/cluster/seed/15-gitd/kustomization.yaml b/files/cluster/seed/15-gitd/kustomization.yaml new file mode 100644 index 00000000..fba9a756 --- /dev/null +++ b/files/cluster/seed/15-gitd/kustomization.yaml @@ -0,0 +1,12 @@ +# Seed phase 15 — manifest source bridge. +# +# Applied after Argo CD Core so the git URL exists before the root Application +# in phase 20 is created. See deployment.yaml for why this cannot be an +# Argo-managed manifest. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - namespace.yaml + - deployment.yaml + - service.yaml diff --git a/files/cluster/seed/15-gitd/namespace.yaml b/files/cluster/seed/15-gitd/namespace.yaml new file mode 100644 index 00000000..5353822a --- /dev/null +++ b/files/cluster/seed/15-gitd/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: bluefin-system + labels: + app.kubernetes.io/part-of: bluefin-server diff --git a/files/cluster/seed/15-gitd/service.yaml b/files/cluster/seed/15-gitd/service.yaml new file mode 100644 index 00000000..ad8b3420 --- /dev/null +++ b/files/cluster/seed/15-gitd/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: bluefin-git-daemon + namespace: bluefin-system + labels: + app.kubernetes.io/name: bluefin-git-daemon + app.kubernetes.io/part-of: bluefin-server +spec: + type: ClusterIP + ports: + - name: git + port: 9418 + targetPort: git + selector: + app.kubernetes.io/name: bluefin-git-daemon diff --git a/files/cluster/seed/20-root-app/application.yaml b/files/cluster/seed/20-root-app/application.yaml new file mode 100644 index 00000000..3ed5a039 --- /dev/null +++ b/files/cluster/seed/20-root-app/application.yaml @@ -0,0 +1,45 @@ +# The one Application. Everything downstream of here is Argo's. +# +# repoURL points at the in-cluster git-daemon from seed phase 15, which serves +# the bare repository that bluefin-cluster-repo.service initialises from +# /usr/share/bluefin/cluster/. No external git host is contacted, so the +# manifest tree stays available with the network unplugged. +# +# `.` is a kustomization that pulls in every component directory in order, so +# the whole appliance is one sync unit with sync waves for ordering rather than +# a fan-out of Applications that would each need their own health gate. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: bluefin-cluster + namespace: argocd + labels: + app.kubernetes.io/part-of: bluefin-server +spec: + project: bluefin + source: + repoURL: git://bluefin-git-daemon.bluefin-system.svc.cluster.local/cluster.git + targetRevision: main + path: . + destination: + server: https://kubernetes.default.svc + syncPolicy: + automated: + # Upstream Kubernetes ships no addon manager, so nothing else removes a + # resource that leaves this tree. Argo is the only pruner in the system: + # the seed itself is append-only and never pruned. + prune: true + # ...and the only thing correcting drift. + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + # cert-manager's CRDs exceed the annotation size limit that + # client-side apply's last-applied-configuration would need. + - Replace=false + retry: + limit: 10 + backoff: + duration: 10s + factor: 2 + maxDuration: 5m diff --git a/files/cluster/seed/20-root-app/appproject.yaml b/files/cluster/seed/20-root-app/appproject.yaml new file mode 100644 index 00000000..e01717bf --- /dev/null +++ b/files/cluster/seed/20-root-app/appproject.yaml @@ -0,0 +1,42 @@ +# Scopes what the root Application is allowed to touch. +# +# The tree it syncs installs cluster-scoped CRDs and webhooks and spans several +# namespaces, so the destination and cluster-resource allowlists are wide. They +# are still narrower than "*": a project that cannot create a new AppProject or +# a new Application cannot be used to escalate out of itself. +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: bluefin + namespace: argocd + labels: + app.kubernetes.io/part-of: bluefin-server +spec: + description: Bluefin Server appliance workload + sourceRepos: + - git://bluefin-git-daemon.bluefin-system.svc.cluster.local/cluster.git + destinations: + - server: https://kubernetes.default.svc + namespace: "*" + clusterResourceWhitelist: + - group: "" + kind: Namespace + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: cert-manager.io + kind: ClusterIssuer + - group: tenancy.kflex.kubestellar.org + kind: ControlPlane + namespaceResourceBlacklist: + - group: argoproj.io + kind: AppProject + - group: argoproj.io + kind: Application diff --git a/files/cluster/seed/20-root-app/kustomization.yaml b/files/cluster/seed/20-root-app/kustomization.yaml new file mode 100644 index 00000000..e35449e7 --- /dev/null +++ b/files/cluster/seed/20-root-app/kustomization.yaml @@ -0,0 +1,12 @@ +# Seed phase 20 — hand-off to Argo. +# +# Last seed phase. After this the seed is done: everything else in the +# appliance is reconciled by Argo CD Core from the git bridge. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: argocd + +resources: + - appproject.yaml + - application.yaml diff --git a/files/installer/repart.d/30-var.conf b/files/installer/repart.d/30-var.conf index 7f2ec345..586c2485 100644 --- a/files/installer/repart.d/30-var.conf +++ b/files/installer/repart.d/30-var.conf @@ -4,10 +4,40 @@ # creating /var entirely. # GrowFileSystem=yes: XFS grows to fill all remaining disk space after # the ESP and root-a partitions are placed. +# CopyFiles seeds both sysexts straight into systemd-sysext's own scan +# directory (/var/lib/extensions), so the first boot merges them without a +# network fetch. +# +# kubernetes.raw — kubeadm, kubelet, kubectl and the CNI plugins. +# systemd-sysupdate later replaces this file through the kubernetes.raw +# CurrentSymlink in files/os/sysupdate.kubernetes.d/70-kubernetes.transfer. +# +# containerd-flatcar.raw — Flatcar's containerd sysext (containerd, runc, ctr, +# containerd-shim-runc-v2) and the containerd.service that +# files/os/systemd/system/kubeadm-init.service hard-requires. FSDK 26.08 +# ships no containerd component, so this is the only provider. The image +# carries its own multi-user.target.wants symlink, so the merge enables the +# service; no preset of ours is involved. Pinned by +# elements/flatcar/containerd-sysext.bst. +# +# The filename keeps its upstream stem on purpose. systemd-sysext merges +# .raw only if the image carries extension-release.; this image +# ships extension-release.containerd-flatcar, so seeding it as +# containerd.raw would make sysext look for extension-release.containerd and +# refuse the extension at boot. +# +# The image is repacked before it is seeded: elements/flatcar/containerd-sysext.bst +# rewrites the extension-release to ID=_any and drops VERSION_ID, keeping the +# Flatcar release only in SYSEXT_VERSION_ID. So the merge is *not* gated on our +# os-release, and tests/unit/test_seeded_sysext_names.py asserts that decoupling. +# Bumping flatcar-version in include/flatcar.yml therefore only requires +# re-pinning the source ref of that element; it will not desynchronise the merge +# on hosts already carrying the old image. [Partition] Type=var Label=var Format=xfs GrowFileSystem=yes -CopyFiles=/k0s.raw:/lib/k0s/k0s.raw +CopyFiles=/kubernetes.raw:/lib/extensions/kubernetes.raw +CopyFiles=/containerd-flatcar.raw:/lib/extensions/containerd-flatcar.raw SizeMinBytes=4G diff --git a/files/k0s/kiosk/nginx.conf b/files/k0s/kiosk/nginx.conf deleted file mode 100644 index e34bf24b..00000000 --- a/files/k0s/kiosk/nginx.conf +++ /dev/null @@ -1,48 +0,0 @@ -pid /tmp/nginx.pid; - -events { - worker_connections 1024; -} - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - server { - listen 8080 ssl; - server_name _; - - ssl_certificate /etc/kubestellar-kiosk/cert.pem; - ssl_certificate_key /etc/kubestellar-kiosk/key.pem; - ssl_protocols TLSv1.2 TLSv1.3; - ssl_ciphers HIGH:!aNULL:!MD5; - - # Auto-redirect plain HTTP to HTTPS on the same port - error_page 497 https://$http_host$request_uri; - - location = /kiosk-gate.js { - alias /etc/kubestellar-kiosk/kiosk-gate.js; - add_header Cache-Control "no-store"; - } - - location = /kiosk-gate.css { - alias /etc/kubestellar-kiosk/kiosk-gate.css; - add_header Cache-Control "no-store"; - } - - location / { - proxy_pass http://kubestellar-console.kubestellar-console.svc.cluster.local:8080; - proxy_redirect ~^https?://[^/]+/(.*) /$1; - proxy_http_version 1.1; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header Accept-Encoding ""; - proxy_set_header Connection ""; - sub_filter_once on; - sub_filter '' ''; - sub_filter '' ''; - } - } -} diff --git a/files/k0s/kubeflex/generate-postgres-secret.sh b/files/k0s/kubeflex/generate-postgres-secret.sh deleted file mode 100755 index 7e7ea1b8..00000000 --- a/files/k0s/kubeflex/generate-postgres-secret.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# First-boot generator for the KubeStellar postgres superuser password. -# -# The postgres StatefulSet (20-postgres.yaml) reads its superuser password from -# the kubeflex-postgres Secret via secretKeyRef. This script creates that Secret -# once, with a random password, so no usable credential is committed to git or -# exposed in the pod spec (see projectbluefin/server#98). -# -# Idempotent: the password is generated only if the Secret already exists, so -# the initialized database stays accessible across re-boots. Run once per boot -# by k0s-first-boot.service, before k0s applies the manifests (its 15- name -# sorts before 20-postgres.yaml, so the Secret exists first). -set -euo pipefail - -manifest_dir="${KUBEFLEX_MANIFEST_DIR:-/var/lib/k0s/manifests/kubestellar}" -secret_file="$manifest_dir/15-kubeflex-postgres-secret.yaml" - -if [ -e "$secret_file" ]; then - exit 0 -fi - -mkdir -p "$manifest_dir" -password="$(head -c 32 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | cut -c1-30)" - -cat > "$secret_file" < k0s system extension raw image not found in /var/lib/extensions/" - echo "==> Running systemd-sysupdate to fetch and install the k0s sysext OTA..." - # Ignore errors if systemd-sysupdate can't connect (e.g. offline dev test) - systemd-sysupdate update || true + + # 0. Reject unsupported roles before touching anything on the host. This is + # a single-node control plane, so "controller" is the only real role; + # "server" is accepted as a backwards-compatible alias for it, and + # tests/unit/os-justfile_test.bats pins both spellings. + if [ "{{ROLE}}" != "controller" ] && [ "{{ROLE}}" != "server" ]; then + echo "ERROR: Invalid role '{{ROLE}}'. Supported: controller (alias: server)" >&2 + exit 1 fi - # 2. Setup k0s configuration directory - install -d -m 0755 /etc/k0s + # 1. Fetch the Kubernetes systemd-sysext if it is not installed yet. + # 70-kubernetes.transfer lands it in /var/lib/extensions/ behind the + # stable kubernetes.raw symlink. + if [ ! -e /var/lib/extensions/kubernetes.raw ]; then + echo "==> Kubernetes system extension not found in /var/lib/extensions/" + echo "==> Running systemd-sysupdate to fetch and install the Kubernetes sysext OTA..." + # Ignore errors if systemd-sysupdate can't connect (e.g. offline dev test) + systemd-sysupdate --component=kubernetes update || true + fi - # 3. Trigger systemd-sysext to merge extensions into /usr immediately - echo "==> Merging k0s system extension..." + # 2. Merge the extension into /usr now rather than waiting for a reboot. + echo "==> Merging the Kubernetes system extension..." systemctl enable --now systemd-sysext.service || true systemd-sysext merge || true - # 4. Trigger systemd-tmpfiles to seed declarative manifests - systemd-tmpfiles --create /usr/lib/tmpfiles.d/k0s-manifests.conf || true + # 3. A merge makes unit files appear under /usr; systemd does not notice + # them on its own. + systemctl daemon-reload - # 5. Enable and start the selected systemd service - if [ "{{ROLE}}" = "controller" ] || [ "{{ROLE}}" = "server" ]; then - echo "==> Enabling and starting k0s controller service..." - systemctl enable --now k0scontroller.service - else - echo "ERROR: Invalid role '{{ROLE}}'. Supported: controller" >&2 - exit 1 - fi + # 4. Bring the cluster up. bluefin-cluster-bootstrap.service carries + # Requires=/After= on kubeadm-init.service and bluefin-cluster-repo.service, + # so starting it runs kubeadm init and then the seed phases in order. + echo "==> Starting cluster bootstrap..." + systemctl enable --now bluefin-cluster-bootstrap.service - echo "==> k0s Kubernetes has been successfully configured and started!" + echo "==> Kubernetes has been successfully configured and started!" + echo "==> Inspect with: kubectl --kubeconfig /etc/kubernetes/admin.conf get nodes" diff --git a/files/os/libexec/bluefin-cluster-repo b/files/os/libexec/bluefin-cluster-repo new file mode 100755 index 00000000..2a134d47 --- /dev/null +++ b/files/os/libexec/bluefin-cluster-repo @@ -0,0 +1,75 @@ +#!/bin/bash +# Publish /usr/share/bluefin/cluster as a bare git repository that the +# in-cluster git-daemon serves to Argo CD. +# +# Argo CD's Application.source accepts git, Helm or OCI and has no filesystem +# type, so the image-delivered manifest tree cannot be read from /usr directly. +# This script bridges the two: it mirrors the read-only tree into a bare +# repository on /var, which the git-daemon Deployment exports over the cluster +# network. No external git host is ever contacted. +# +# It runs on every boot and is idempotent. Re-running after a system update +# publishes the manifests that update shipped; re-running with no change +# publishes nothing and creates no empty commit. +set -euo pipefail + +# git-daemon serves the repository from a non-root uid through a read-only +# hostPath mount, so every directory and object it walks must stay readable. +umask 022 + +SEED=/usr/share/bluefin/cluster +REPO=/var/lib/bluefin/cluster.git +BRANCH=main + +if [ ! -d "${SEED}" ]; then + echo "ERROR: cluster manifest tree is missing at ${SEED}" >&2 + exit 1 +fi + +# Keep the result independent of any stray host or user git configuration. +export GIT_CONFIG_SYSTEM=/dev/null +export GIT_CONFIG_GLOBAL=/dev/null +export GIT_AUTHOR_NAME="Bluefin Server" +export GIT_AUTHOR_EMAIL="root@localhost" +export GIT_COMMITTER_NAME="${GIT_AUTHOR_NAME}" +export GIT_COMMITTER_EMAIL="${GIT_AUTHOR_EMAIL}" + +mkdir -p "${REPO}" +if [ ! -e "${REPO}/HEAD" ]; then + git init --bare --initial-branch="${BRANCH}" "${REPO}" +fi + +# git-daemon refuses to serve a repository that is not explicitly exported. +touch "${REPO}/git-daemon-export-ok" + +# Build the tree through a scratch index so a failure leaves no partial state +# in the repository the daemon is serving. core.bare is overridden per +# invocation because `git add` refuses to run against a bare repository. +INDEX="${REPO}/bluefin-seed.index" +rm -f "${INDEX}" +GIT_INDEX_FILE="${INDEX}" git --git-dir="${REPO}" -c core.bare=false \ + --work-tree="${SEED}" add --all +TREE="$(GIT_INDEX_FILE="${INDEX}" git --git-dir="${REPO}" write-tree)" +rm -f "${INDEX}" + +PUBLISHED="" +if git --git-dir="${REPO}" rev-parse --verify --quiet "refs/heads/${BRANCH}^{tree}" >/dev/null; then + PUBLISHED="$(git --git-dir="${REPO}" rev-parse --verify "refs/heads/${BRANCH}^{tree}")" +fi + +if [ "${TREE}" = "${PUBLISHED}" ]; then + echo "cluster repository already matches ${SEED}" +else + if [ -n "${PUBLISHED}" ]; then + COMMIT="$(git --git-dir="${REPO}" commit-tree "${TREE}" \ + -p "refs/heads/${BRANCH}" -m "Bluefin cluster manifests")" + else + COMMIT="$(git --git-dir="${REPO}" commit-tree "${TREE}" \ + -m "Bluefin cluster manifests")" + fi + git --git-dir="${REPO}" update-ref "refs/heads/${BRANCH}" "${COMMIT}" + echo "published ${SEED} as ${BRANCH} ${COMMIT}" +fi + +git --git-dir="${REPO}" symbolic-ref HEAD "refs/heads/${BRANCH}" +git --git-dir="${REPO}" update-server-info diff --git a/files/os/systemd/system-preset/zz-enable-cluster-bootstrap.preset b/files/os/systemd/system-preset/zz-enable-cluster-bootstrap.preset new file mode 100644 index 00000000..dc9246b0 --- /dev/null +++ b/files/os/systemd/system-preset/zz-enable-cluster-bootstrap.preset @@ -0,0 +1,4 @@ +enable kubelet.service +enable kubeadm-init.service +enable bluefin-cluster-repo.service +enable bluefin-cluster-bootstrap.service diff --git a/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset b/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset deleted file mode 100644 index 60ae94e5..00000000 --- a/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset +++ /dev/null @@ -1 +0,0 @@ -enable k0s-first-boot.service diff --git a/files/os/systemd/system/bluefin-cluster-bootstrap.service b/files/os/systemd/system/bluefin-cluster-bootstrap.service new file mode 100644 index 00000000..6374af8c --- /dev/null +++ b/files/os/systemd/system/bluefin-cluster-bootstrap.service @@ -0,0 +1,96 @@ +[Unit] +Description=Seed the Bluefin cluster: CNI, Argo CD Core, manifest bridge, root Application +Documentation=file:///usr/share/bluefin/seed +# Deliberately no ConditionPathExists. A failed condition marks a unit +# *skipped* rather than failed, which would silently no-op the seed on first +# boot and leave a node with no CNI and no reconciler. Every phase below is a +# server-side apply and is idempotent on its own, so re-running is free. +# Wants=, not Requires=, on kubeadm-init.service. +# +# A job that dies with result 'dependency' never runs, so its own +# Restart=on-failure never engages and systemd never re-queues it. Under +# Requires= that made this unit permanently dead whenever kubeadm-init failed +# even once — and kubeadm-init has Restart=on-failure precisely because a first +# attempt can lose a race with the CRI socket, a slow etcd, an image pull or +# clock skew. Observed exactly that: kubeadm failed at 12:18:13, succeeded on +# retry at 12:18:59, and this unit stayed inactive forever with a control plane +# up and no CNI, no Argo and no console. +# +# With Wants= plus After=, a premature start is harmless: every phase is a +# server-side apply, the unit has Restart=on-failure/RestartSec=30s and +# TimeoutStartSec=1800, so it retries against the API server until the control +# plane answers. The chain converges instead of depending on kubeadm-init +# succeeding on the very first try. +Wants=kubeadm-init.service +After=kubeadm-init.service +# bluefin-cluster-repo.service stays a hard requirement: it is a base-OS unit +# that exists at transaction time, and without the mirrored manifests in +# /var/lib/bluefin/cluster.git there is nothing for the seed phases to apply. +Requires=bluefin-cluster-repo.service +After=bluefin-cluster-repo.service +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +Restart=on-failure +RestartSec=30s +# TimeoutStartSec must cover every bounded step below, or systemd kills the unit +# mid-chain and the kill reads as a generic timeout instead of naming the phase +# that stalled. The budget is partitioned rather than shared: +# +# ExecStartPre API server probe 600 +# phase 1 kubectl wait node Ready 300 +# phase 2 kubectl wait deployment Available 600 +# phase 2 kubectl rollout status statefulset 600 +# phase 3 kubectl wait deployment Available 600 +# ---- +# 2700 +# +# 3000 leaves headroom for the applies themselves without letting a genuinely +# stuck seed sit for an hour before Restart=on-failure gets a turn. +TimeoutStartSec=3000 +Environment=KUBECONFIG=/etc/kubernetes/admin.conf +# The seed is append-only and is never pruned: kubectl's allowlist --prune is +# alpha and incompatible with --server-side, and --applyset cannot span the +# namespaces this seed touches. Argo CD owns pruning for everything downstream. +# Wait for the API server before applying anything, and say so while waiting. +# +# This is the cost of Wants= over Requires= above. Under Requires=, a broken +# kubeadm-init produced exactly one legible line — "Job ... failed with result +# 'dependency'" — which is how the stranding bug was found in the first place. +# Under Wants= this unit instead retries every 30s until TimeoutStartSec, and in +# CI the 900s readiness gate fires first, so the operator would see a timeout +# with no stated cause and many rounds of kubectl connection errors to read +# through. +# +# Probing here collapses that into one self-describing line per attempt, and it +# sharpens the signal: once this passes, the API server is answering, so any +# failure in the phases below is a genuine apply failure rather than a control +# plane that has not arrived yet. `get --raw /readyz` is the cheapest call that +# proves the apiserver is serving, and needs no RBAC beyond the admin kubeconfig. +# +# The loop is bounded at 60 attempts (600s), not open-ended. An unbounded `until` +# would share this unit's TimeoutStartSec with the apply phases and could consume +# the entire budget before phase 1 ran — the systemd kill would then be +# attributed to the seed rather than to the control plane, which is exactly the +# ambiguity this probe exists to remove. Exiting non-zero instead hands the +# failure to Restart=on-failure with a line that names the cause. +ExecStartPre=/usr/bin/bash -c 'for i in $(seq 60); do /usr/bin/kubectl --request-timeout=5s get --raw /readyz >/dev/null 2>&1 && exit 0; echo "waiting for the Kubernetes API server; kubeadm-init has not finished (attempt ${i}/60)"; sleep 10; done; echo "API server never became ready after 600s; not attempting the seed" >&2; exit 1' +# Phase 1: CNI. Nothing else can schedule until the node leaves NotReady. +ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/00-cilium +ExecStart=/usr/bin/kubectl wait --for=condition=Ready node --all --timeout=300s +# Phase 2: the reconciler itself. The application controller is a StatefulSet, +# so `wait deployment --all` would not cover it. +ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/10-argocd-core +ExecStart=/usr/bin/kubectl wait --for=condition=Available deployment --all --namespace argocd --timeout=600s +ExecStart=/usr/bin/kubectl rollout status statefulset/argocd-application-controller --namespace argocd --timeout=600s +# Phase 3: the git bridge that lets Argo read /var/lib/bluefin/cluster.git. +ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/15-gitd +ExecStart=/usr/bin/kubectl wait --for=condition=Available deployment/bluefin-git-daemon --namespace bluefin-system --timeout=600s +# Phase 4: hand the cluster over. Argo owns everything from here. +ExecStart=/usr/bin/kubectl apply --server-side --force-conflicts -k /usr/share/bluefin/seed/20-root-app + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/bluefin-cluster-repo.service b/files/os/systemd/system/bluefin-cluster-repo.service new file mode 100644 index 00000000..1fbd487b --- /dev/null +++ b/files/os/systemd/system/bluefin-cluster-repo.service @@ -0,0 +1,32 @@ +[Unit] +Description=Publish the Bluefin cluster manifests as a local git repository +Documentation=file:///usr/share/bluefin/cluster +# Argo CD has no filesystem source type, so the image-delivered manifest tree +# has to be reachable over git before the root Application can sync. +RequiresMountsFor=/var/lib/bluefin +Before=bluefin-cluster-bootstrap.service + +[Service] +Type=oneshot +RemainAfterExit=yes +# Restart=on-failure is load-bearing, not boilerplate. +# +# bluefin-cluster-bootstrap.service declares Requires= on this unit, and a job +# that dies with result 'dependency' never runs at all, so the dependent's own +# Restart=on-failure never engages and systemd never re-queues it. Without a +# restart here, one transient failure of the publish — a git invocation that +# loses a race with /var/lib/bluefin being set up, a full filesystem that +# clears — would strand the entire seed permanently behind a node that +# otherwise looks healthy. That is the same failure mode kubeadm-init.service +# forced onto the seed and which the Wants= edge there exists to prevent. +# +# Restart=on-failure is valid for Type=oneshot (only always/on-success are +# refused), and the publish is idempotent: it initialises the bare repository +# when absent and pushes the current manifest tree otherwise. +Restart=on-failure +RestartSec=30s +StateDirectory=bluefin +ExecStart=/usr/libexec/bluefin-cluster-repo + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/k0s-first-boot-fetch.service b/files/os/systemd/system/k0s-first-boot-fetch.service deleted file mode 100644 index a8cee552..00000000 --- a/files/os/systemd/system/k0s-first-boot-fetch.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Fetch the k0s sysext when installation did not seed it -ConditionPathExists=!/var/lib/k0s/k0s.raw -Wants=network-online.target -After=network-online.target -Before=k0s-first-boot.service - -[Service] -Type=oneshot -Restart=on-failure -RestartSec=30s -ExecStart=/usr/bin/systemd-sysupdate --component=k0s update diff --git a/files/os/systemd/system/k0s-first-boot.service b/files/os/systemd/system/k0s-first-boot.service deleted file mode 100644 index 6e61f470..00000000 --- a/files/os/systemd/system/k0s-first-boot.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=Activate k0s sysext on boot -Wants=network-online.target -Wants=k0s-first-boot-fetch.service -After=network-online.target -After=k0s-first-boot-fetch.service -RequiresMountsFor=/var/lib/k0s - -[Service] -Type=oneshot -StateDirectory=k0s -Restart=on-failure -RestartSec=30s -ExecStartPre=/usr/bin/test -e /var/lib/k0s/k0s.raw -ExecStart=/usr/bin/install -D -m 0644 /var/lib/k0s/k0s.raw /run/extensions/k0s.raw -ExecStart=/usr/bin/systemd-sysext merge -ExecStart=/usr/bin/systemd-tmpfiles --create /usr/lib/tmpfiles.d/k0s-manifests.conf -ExecStart=/usr/share/k0s/kubeflex/generate-postgres-secret.sh -ExecStart=/usr/bin/systemctl daemon-reload -ExecStart=/usr/bin/systemctl enable --now k0scontroller.service -[Install] -WantedBy=multi-user.target diff --git a/files/os/systemd/system/kubeadm-init.service b/files/os/systemd/system/kubeadm-init.service new file mode 100644 index 00000000..954d25b7 --- /dev/null +++ b/files/os/systemd/system/kubeadm-init.service @@ -0,0 +1,132 @@ +[Unit] +Description=Initialise the Kubernetes control plane with kubeadm +Documentation=https://kubernetes.io/docs/reference/setup-tools/kubeadm/kubeadm-init/ +# The idempotency guard belongs here and nowhere else. A unit whose condition +# fails is marked *skipped*, and systemd treats a skipped unit as satisfied for +# everything ordered after it, so bluefin-cluster-bootstrap.service still runs. +# +# The guard is a stamp this unit writes itself, not /etc/kubernetes/admin.conf. +# admin.conf is not a completion marker: kubeadm writes it in the `kubeconfig` +# phase, which runs *before* etcd, control-plane, wait-control-plane, +# mark-control-plane and bootstrap-token. If `kubeadm init` dies after the +# kubeconfig phase, admin.conf is already on disk, so with the old condition the +# Restart=on-failure retry below was skipped instead of run, and the node kept a +# half-initialised control plane forever. The visible symptom was downstream and +# unhelpful: mark-control-plane never applied the control-plane role label, so +# the nodeSelector in files/cluster/seed/15-gitd/deployment.yaml never matched +# and bluefin-cluster-bootstrap phase 3 timed out even though the API server was +# answering. +# +# ExecStartPost writes the stamp only after kubeadm exits 0, so the condition +# now means "a previous init ran to completion" rather than "some init got as +# far as writing a kubeconfig". +ConditionPathExists=!/var/lib/bluefin/kubeadm-init.stamp +# Both /usr/bin/kubeadm and containerd.service arrive with the sysext merge, but +# from two different extensions, each seeded to /var/lib/extensions by +# files/installer/repart.d/30-var.conf: +# +# kubernetes.raw elements/oci/kubernetes-sysext.bst — kubeadm, kubelet, +# kubectl, CNI plugins. +# containerd-flatcar.raw elements/flatcar/containerd-sysext.bst — Flatcar's containerd +# sysext: containerd, runc, ctr, containerd-shim-runc-v2, and +# containerd.service itself. FSDK 26.08 ships no containerd +# component, so this import is the only provider. +# +# containerd-flatcar.raw carries its own multi-user.target.wants symlink, so the merge +# enables the service; no preset of ours is involved. +Requires=systemd-sysext.service +After=systemd-sysext.service +# Wants=, not Requires=, and the distinction is load-bearing. +# +# systemd resolves dependencies when it computes the boot transaction. At that +# moment containerd.service does not exist: it lives inside containerd-flatcar.raw +# and only appears once systemd-sysext merges /usr. Observed ordering on a real +# first boot: +# +# 12:06:10.283 Applying preset policy +# 12:06:11.751 Merged extensions into '/usr' <- containerd.service appears +# 12:06:13.141 Reached target Multi-User System +# +# With Requires=, the unresolvable dependency made systemd drop this unit's job +# from the transaction outright. It was never re-enqueued after the merge, so +# its conditions were never even evaluated (ConditionTimestamp empty), kubeadm +# never ran, and bluefin-cluster-bootstrap.service never ran either because it +# declares Requires=kubeadm-init.service. The machine booted to multi-user with +# no control plane and nothing in `systemctl list-units --failed` to show for it. +# +# kubelet.service survived the same boot precisely because it only declares +# After=containerd.service: a pure ordering edge on a unit that does not yet +# exist is ignored rather than fatal. +# +# Wants= is ignored the same way when unresolvable. Be clear about what After= +# then buys: nothing, at boot. containerd.service is absent from the boot +# transaction on *every* boot, not just the first, because the sysext merges +# mid-boot each time and the drop-in on systemd-sysext.service starts containerd +# imperatively rather than as a queued job. An After= against a unit that is not +# in the transaction orders nothing. +# +# It is kept because it costs nothing and is correct whenever containerd does +# enter a transaction — a later `systemctl restart containerd`, for instance. +# The ordering this unit actually relies on at boot is the ExecStartPre socket +# wait below. +Wants=containerd.service +After=containerd.service +Wants=network-online.target +After=network-online.target +RequiresMountsFor=/var/lib/kubelet +RequiresMountsFor=/var/lib/bluefin + +[Service] +Type=oneshot +# RemainAfterExit is what makes the ordering edge useful: the unit stays active +# after kubeadm exits, so dependents see a control plane that is still up. +RemainAfterExit=yes +Restart=on-failure +RestartSec=30s +TimeoutStartSec=900 +# Wait for the CRI socket rather than assuming After=containerd.service ordered +# us. It cannot: containerd.service arrives with the sysext merge, so it is not +# in the boot transaction when systemd computes ordering, and After= against a +# unit that is not in the transaction orders nothing. Observed without this: +# +# 12:18:13 [ERROR CRI]: dial unix /run/containerd/containerd.sock: +# connect: no such file or directory +# 12:18:43 Scheduled restart job, restart counter is at 1 +# 12:18:59 Finished Initialise the Kubernetes control plane +# +# kubeadm recovered on its own 30s later, but the damage was already done: +# bluefin-cluster-bootstrap.service declares Requires=kubeadm-init.service, and +# that first failure killed its job permanently with result 'dependency'. It is +# never re-queued when kubeadm later succeeds, so the node came up with a +# control plane and no CNI, no Argo, and no console — and nothing in +# `systemctl list-units --failed` to explain it. +# +# Waiting here makes the first attempt succeed, which keeps the Wants=/Requires= +# edges in the dependents honest. +# +# Bounded at 300 attempts (300s) rather than open-ended. An unbounded `until` +# shares TimeoutStartSec=900 with kubeadm itself, so a containerd that never +# arrives would burn the whole window in silence and the eventual systemd kill +# would look like "kubeadm was slow" rather than "there was no container +# runtime". Exiting non-zero hands it to Restart=on-failure with a line that +# names the cause, and still leaves kubeadm the larger share of the budget. +ExecStartPre=/usr/bin/bash -c 'for i in $(seq 300); do [ -S /run/containerd/containerd.sock ] && exit 0; sleep 1; done; echo "no CRI socket at /run/containerd/containerd.sock after 300s; containerd.service did not start" >&2; exit 1' +# Clear a half-initialised node before retrying. Reaching this point with +# /etc/kubernetes/admin.conf already present means a previous attempt wrote a +# kubeconfig and then failed before the stamp was written — kubeadm refuses to +# run against that state ("ERROR Port-6443 ... /etc/kubernetes/manifests is not +# empty"), so every retry would fail the same way forever. `kubeadm reset -f` +# tears down the manifests, etcd data and kubeconfigs so the next ExecStart is a +# clean first init. It is a no-op path on a genuinely first boot, where +# admin.conf does not exist. +ExecStartPre=/usr/bin/bash -c '[ -e /etc/kubernetes/admin.conf ] || exit 0; echo "found /etc/kubernetes/admin.conf with no completion stamp: a previous kubeadm init did not finish; resetting before retry" >&2; /usr/bin/kubeadm reset -f' +# Cilium provides the kube-proxy replacement; installing kube-proxy as well +# would leave two datapaths fighting over the same Services. +ExecStart=/usr/bin/kubeadm init --config /usr/share/bluefin/kubeadm.yaml --skip-phases=addon/kube-proxy +# The completion stamp the ConditionPathExists above reads. ExecStartPost only +# runs when every ExecStart succeeded, which is what makes the stamp mean +# "kubeadm init ran to completion" and not merely "kubeadm init was attempted". +ExecStartPost=/usr/bin/bash -c 'mkdir -p /var/lib/bluefin && : > /var/lib/bluefin/kubeadm-init.stamp' + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/kubelet.service b/files/os/systemd/system/kubelet.service new file mode 100644 index 00000000..9a0c847d --- /dev/null +++ b/files/os/systemd/system/kubelet.service @@ -0,0 +1,40 @@ +[Unit] +Description=kubelet: the Kubernetes node agent +Documentation=https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ +# /usr/bin/kubelet ships in the Kubernetes sysext, not in the base DDI, so the +# merge must complete before this unit can start. +Requires=systemd-sysext.service +After=systemd-sysext.service +Wants=network-online.target +After=network-online.target +After=containerd.service +# kubeadm writes the kubelet configuration, so the kubelet restarts until +# `kubeadm init` has run. Never give up on it. +StartLimitIntervalSec=0 + +[Service] +# The Kubernetes sysext ships the upstream CNI plugins at /usr/local/bin/cni, +# matching the flatcar/sysext-bakery layout. containerd and Cilium both look in +# /opt/cni/bin. Without this bridge the `loopback` plugin is missing, every pod +# sandbox fails to get a network, and the node never leaves NotReady — silently. +# +# /opt/cni/bin is on /var and survives reboots, so a plain no-clobber copy would +# pin the node to whatever plugins the sysext shipped on the day it was first +# installed: a sysext OTA update would advertise a new version in +# /usr/local/share/kubernetes-cni-version while the node kept executing the old +# binaries forever. The copy is therefore keyed on that stamp. +# +# Refreshing cannot clobber Cilium either way: /usr/local/bin/cni contains only +# the upstream containernetworking plugins, and the cilium agent installs +# cilium-cni, a name that is not in that set, so it is never a copy target. The +# no-clobber copy is kept for the unchanged case — it costs nothing and restores +# any plugin removed from /opt/cni/bin out of band. +ExecStartPre=/usr/bin/install -d -m 0755 /opt/cni +ExecStartPre=/usr/bin/install -d -m 0755 /opt/cni/bin +ExecStartPre=/bin/sh -c 'want=$(cat /usr/local/share/kubernetes-cni-version 2>/dev/null || echo unknown); have=$(cat /opt/cni/.bluefin-cni-version 2>/dev/null || echo none); if [ "$want" = "$have" ]; then cp -an /usr/local/bin/cni/. /opt/cni/bin/; else cp -a /usr/local/bin/cni/. /opt/cni/bin/ && printf "%s\\n" "$want" > /opt/cni/.bluefin-cni-version; fi' +ExecStart=/usr/bin/kubelet +Restart=always +RestartSec=10s + +[Install] +WantedBy=multi-user.target diff --git a/files/os/systemd/system/kubelet.service.d/10-kubeadm.conf b/files/os/systemd/system/kubelet.service.d/10-kubeadm.conf new file mode 100644 index 00000000..56d23d0d --- /dev/null +++ b/files/os/systemd/system/kubelet.service.d/10-kubeadm.conf @@ -0,0 +1,10 @@ +# Upstream kubeadm kubelet drop-in. kubeadm rewrites +# /var/lib/kubelet/kubeadm-flags.env on every `kubeadm init`, so node-specific +# flags are derived rather than baked into the image. +[Service] +Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" +Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" +EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env +EnvironmentFile=-/etc/default/kubelet +ExecStart= +ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS diff --git a/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf b/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf new file mode 100644 index 00000000..96c61f37 --- /dev/null +++ b/files/os/systemd/systemd-sysext.service.d/10-daemon-reload.conf @@ -0,0 +1,32 @@ +# A sysext merge makes unit files appear under /usr; it does not by itself make +# systemd notice them, and noticing them is still not the same as running them. +# +# The reload is necessary: without it, ordering edges on containerd.service and +# kubelet.service resolve against units systemd has never read. +# +# The explicit start is necessary too, and this is the subtle half. Flatcar's +# containerd sysext ships its own enablement symlink at +# /usr/lib/systemd/system/multi-user.target.wants/containerd.service. That +# symlink only becomes visible when the merge completes, which is *after* +# systemd has computed the boot transaction — and systemd does not retroactively +# pull newly-appeared .wants entries into a transaction already in flight. So +# whether containerd starts depends purely on whether the merge happened to land +# before multi-user.target was queued. +# +# That is a race, and it was observed resolving both ways on the same image: +# +# boot A merge 12:06:11.75, multi-user 12:06:13.14 -> containerd active +# boot B merge 12:21:10.24, multi-user not reached -> containerd inactive, +# kubeadm-init blocked +# forever on the CRI +# socket +# +# Starting it here removes the race: the merge and the start are the same +# transaction, in order. `start` is idempotent — if the .wants symlink did win +# the race and containerd is already running, this is a no-op. +# +# --no-block so a slow containerd start cannot stall systemd-sysext.service and +# with it every unit ordered after the merge. +[Service] +ExecStartPost=/usr/bin/systemctl daemon-reload +ExecStartPost=/usr/bin/systemctl start --no-block containerd.service diff --git a/files/os/sysupdate.k0s.d/70-k0s.transfer b/files/os/sysupdate.k0s.d/70-k0s.transfer deleted file mode 100644 index fa3cab9a..00000000 --- a/files/os/sysupdate.k0s.d/70-k0s.transfer +++ /dev/null @@ -1,13 +0,0 @@ -[Transfer] - -[Source] -Type=url-file -Path=https://github.com/projectbluefin/server/releases/latest/download/ -MatchPattern=k0s-@v.raw.zst - -[Target] -Type=regular-file -Path=/var/lib/k0s -MatchPattern=k0s-@v.raw -CurrentSymlink=k0s.raw -Mode=0644 diff --git a/files/os/sysupdate.kubernetes.d/70-kubernetes.transfer b/files/os/sysupdate.kubernetes.d/70-kubernetes.transfer new file mode 100644 index 00000000..fff4150c --- /dev/null +++ b/files/os/sysupdate.kubernetes.d/70-kubernetes.transfer @@ -0,0 +1,24 @@ +# systemd-sysupdate transfer for the Kubernetes sysext. +# +# The image lands directly in /var/lib/extensions, systemd-sysext's own scan +# directory, so systemd-sysext.service merges it at boot with no copy step. +# +# Upgrade track: Kubernetes does not support unattended cross-minor in-place +# upgrades, so this transfer must never carry the node across a minor. The +# asset name is fixed by .github/scripts/check-kubernetes-version.py and +# systemd-sysupdate has no version-range filter, so the minor track is held by +# what the release feed publishes: bumping include/kubernetes.yml across a +# minor is a deliberate release action, not something this file can drift into. +[Transfer] + +[Source] +Type=url-file +Path=https://github.com/projectbluefin/server/releases/latest/download/ +MatchPattern=kubernetes-@v.raw.zst + +[Target] +Type=regular-file +Path=/var/lib/extensions +MatchPattern=kubernetes-@v.raw +CurrentSymlink=kubernetes.raw +Mode=0644 diff --git a/include/aliases.yml b/include/aliases.yml index 6ff26ed3..aef91c40 100644 --- a/include/aliases.yml +++ b/include/aliases.yml @@ -3,7 +3,9 @@ aliases: pypi: https://files.pythonhosted.org/packages/ github: https://github.com/ + githubraw: https://raw.githubusercontent.com/ gitlab: https://gitlab.com/ gnome: https://gitlab.gnome.org/GNOME/ crates: https://static.crates.io/ + k8s: https://dl.k8s.io/ flatcar: https://flatcar.cdn.cncf.io/ diff --git a/include/arch.yml b/include/arch.yml index a8a79d60..871dfaec 100644 --- a/include/arch.yml +++ b/include/arch.yml @@ -13,7 +13,7 @@ # architectures mapped below, so an unmapped architecture cannot be selected. # # Included by: -# elements/oci/k0s-sysext.bst -> %{systemd-arch} +# elements/oci/kubernetes-sysext.bst -> %{systemd-arch} # elements/bluefin-server/os-release-flatcar.bst -> %{flatcar-board} variables: (?): diff --git a/include/k0s.yml b/include/k0s.yml deleted file mode 100644 index 5dd0fd28..00000000 --- a/include/k0s.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Single source of truth for the k0s version axis. -# -# k0s is an independently-pinned third-party payload, not an OS asset. Its -# version therefore does NOT live on the OS `release-version` axis declared in -# project.conf — that axis names the FSDK point release and is bumped by -# unrelated OS work. -# -# Only the two atoms below are edited when k0s is bumped. Every other spelling -# of the k0s version in this repository is derived from them: -# -# %{k0s-k8s-version} upstream Kubernetes version 1.36.4 -# %{k0s-patch} upstream k0s patch suffix number 0 -# %{k0s-upstream-tag} GitHub release tag, URL-escaped v1.36.4%2Bk0s.0 -# %{k0s-version} asset / extension-release version 1.36.4-k0s.0 -variables: - k0s-k8s-version: "1.36.4" - k0s-patch: "0" - - k0s-upstream-tag: "v%{k0s-k8s-version}%2Bk0s.%{k0s-patch}" - k0s-version: "%{k0s-k8s-version}-k0s.%{k0s-patch}" diff --git a/include/kubernetes.yml b/include/kubernetes.yml new file mode 100644 index 00000000..11b8be89 --- /dev/null +++ b/include/kubernetes.yml @@ -0,0 +1,29 @@ +# Single source of truth for the Kubernetes version axis. +# +# Kubernetes is an independently-pinned third-party payload, not an OS asset. +# Its version therefore does NOT live on the OS `release-version` axis declared +# in project.conf — that axis names the FSDK point release and is bumped by +# unrelated OS work. +# +# Only the two atoms below are edited when Kubernetes is bumped. Every other +# spelling of the Kubernetes version in this repository is derived from them: +# +# %{k8s-version} upstream Kubernetes version 1.36.4 +# %{cni-version} containernetworking/plugins tag 1.9.1 +# %{k8s-upstream-tag} dl.k8s.io release directory v1.36.4 +# %{cni-upstream-tag} CNI plugins GitHub release tag v1.9.1 +# +# The sysext asset is named `kubernetes-%{k8s-version}.raw`, which is the +# version oracle systemd-sysupdate parses back out through the `@v` wildcard +# in files/os/sysupdate.kubernetes.d/70-kubernetes.transfer. +# +# Included by: +# elements/kubernetes/kubernetes-bin.bst -> %{k8s-upstream-tag} +# elements/kubernetes/cni-plugins.bst -> %{cni-upstream-tag} +# elements/oci/kubernetes-sysext.bst -> %{k8s-version}, %{cni-version} +variables: + k8s-version: "1.36.4" + cni-version: "1.9.1" + + k8s-upstream-tag: "v%{k8s-version}" + cni-upstream-tag: "v%{cni-version}" diff --git a/scripts/lima-e2e-kubestellar-test.sh b/scripts/lima-e2e-kubestellar-test.sh index 79ecff63..cc604288 100755 --- a/scripts/lima-e2e-kubestellar-test.sh +++ b/scripts/lima-e2e-kubestellar-test.sh @@ -41,9 +41,9 @@ limactl start --tty=false "${INSTANCE_NAME}" & LIMA_START_PID=$! echo "==> Starting local kc-agent client connection..." -./files/bin/bluefin-kubestellar start --origin "http://localhost:8080,http://127.0.0.1:8080" +./files/bin/bluefin-kubestellar start --origin "https://localhost:8080,https://127.0.0.1:8080" echo "==> Running automated browser verification test..." -python3 tests/e2e/test_kubestellar_browser_login.py --console-url "http://127.0.0.1:8080" --agent-url "http://127.0.0.1:8585" --timeout 15 +python3 tests/e2e/test_kubestellar_browser_login.py --console-url "https://127.0.0.1:8080" --agent-url "http://127.0.0.1:8585" --timeout 15 echo "==> Lima VM end-to-end verification passed!" diff --git a/tests/e2e/test_kubestellar_browser_login.py b/tests/e2e/test_kubestellar_browser_login.py index bf975b32..e7fbee20 100755 --- a/tests/e2e/test_kubestellar_browser_login.py +++ b/tests/e2e/test_kubestellar_browser_login.py @@ -2,7 +2,7 @@ """Automated end-to-end browser test for KubeStellar Console. Tests: -1. Console availability at http://127.0.0.1:8080/ +1. Console availability at https://127.0.0.1:8080/ 2. Automated login flow (Cluster Access / GitHub OAuth / Dev Mode / Token Login) 3. Dashboard navigation and DOM rendering 4. Kiosk gate overlay and kc-agent interaction on port 8585 @@ -27,8 +27,8 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="KubeStellar Console automated browser verification") parser.add_argument( "--console-url", - default="http://127.0.0.1:8080", - help="Base URL of KubeStellar console (default: http://127.0.0.1:8080)", + default="https://127.0.0.1:8080", + help="Base URL of KubeStellar console (default: https://127.0.0.1:8080)", ) parser.add_argument( "--agent-url", diff --git a/tests/unit/bluefin-kubestellar_test.bats b/tests/unit/bluefin-kubestellar_test.bats index b7cdb299..4d65af18 100755 --- a/tests/unit/bluefin-kubestellar_test.bats +++ b/tests/unit/bluefin-kubestellar_test.bats @@ -165,7 +165,7 @@ run_isolated() { stub_kc_agent_foreground run_isolated --foreground [ "$status" -eq 0 ] - [[ "$output" =~ "Allowed origins: http://localhost:8080,http://127.0.0.1:8080" ]] + [[ "$output" =~ "Allowed origins: https://localhost:8080,https://127.0.0.1:8080" ]] } @test "foreground start execs kc-agent with -allowed-origins" { diff --git a/tests/unit/flash-installer_test.bats b/tests/unit/flash-installer_test.bats index 04857563..ed567d67 100644 --- a/tests/unit/flash-installer_test.bats +++ b/tests/unit/flash-installer_test.bats @@ -4,9 +4,11 @@ # # `flash-installer` is the only recipe in the repository that destroys data: it # pipes an exported installer image straight onto a block device with `dd`. -# Its four guards (device argument required, device must be a block device, an -# exported image must exist, and the operator must confirm) are the only thing -# standing between a typo and a wiped disk, and none of them were exercised. +# Its seven guards — device argument required, target must not be the disk +# backing the running system, target must have nothing mounted off it, target +# must be a block device, exactly one exported image must exist, and the +# operator must confirm — are the only thing standing between a typo and a +# wiped disk. # # The recipe is never run against a real device. Each test runs `just` inside a # private sandbox directory holding a copy of the Justfile, and `sudo`, `dd`, @@ -19,6 +21,11 @@ # device is a regular file instead. `test_block_device_guard_shape` asserts # that the substitution matched exactly once, so if the guard is ever reworded # or dropped the suite fails loudly rather than silently testing nothing. +# +# `lsblk` and `findmnt` are scriptable stubs: the guards added for the +# running-system and mounted-partition checks read them, so each test drives +# them through STUB_* environment variables rather than depending on whatever +# the host's real block topology happens to be. setup() { if ! command -v just >/dev/null 2>&1; then @@ -46,8 +53,21 @@ setup() { make_stub sudo 0 make_stub dd 0 - make_stub zstd 0 - make_stub lsblk 0 + make_stub sfdisk 0 + make_stub partprobe 0 + make_zstd_stub + make_topology_stubs + + # Default topology: target is an unrelated, unmounted disk. Individual + # tests override these to drive a specific guard. + export STUB_ROOT_SRC="composefs:0000" + export STUB_SYSROOT_SRC="/dev/nvme0n1p2" + export STUB_PKNAME="nvme0n1" + export STUB_SRC_KNAME="" + export STUB_TARGET_KNAME="sdz" + export STUB_MOUNTPOINTS="" + # zstd -t succeeds unless a test says otherwise. + export STUB_ZSTD_TEST_EXIT=0 } # make_stub @@ -63,6 +83,65 @@ EOF chmod +x "${STUB_DIR}/$1" } +# Scriptable `lsblk` and `findmnt`. +# +# The running-system and mounted-partition guards query real block topology, so +# a fixed stub would either always fire or never fire depending on the host. +# These answer from STUB_* variables, letting each test state the topology it +# is testing. Both still log, so assert_nothing_written stays meaningful. +make_topology_stubs() { + cat > "${STUB_DIR}/lsblk" <> "${LOG}" +case " \$* " in + *" KNAME "*) + # The recipe asks for KNAME twice: once for the target device, and + # once for the / or /sysroot source when PKNAME came back empty. + # Answer by argument so a whole-disk root is distinguishable from + # the flash target. + for arg in "\$@"; do last="\$arg"; done + case "\${last}" in + /dev/*) printf '%s\n' "\${STUB_SRC_KNAME:-}" ;; + *) printf '%s\n' "\${STUB_TARGET_KNAME:-}" ;; + esac + ;; + *" PKNAME "*) printf '%s\n' "\${STUB_PKNAME:-}" ;; + *MOUNTPOINTS*) printf '%s\n' "\${STUB_MOUNTPOINTS:-}" ;; +esac +exit 0 +EOF + chmod +x "${STUB_DIR}/lsblk" + + cat > "${STUB_DIR}/findmnt" <> "${LOG}" +case " \$* " in + *" /sysroot "*) printf '%s\n' "\${STUB_SYSROOT_SRC:-}" ;; + *" / "*) printf '%s\n' "\${STUB_ROOT_SRC:-}" ;; +esac +exit 0 +EOF + chmod +x "${STUB_DIR}/findmnt" +} + +# Scriptable `zstd`. +# +# The recipe now runs `zstd -t` as an integrity check before the confirmation +# prompt, and `zstd -dc` on the write path. Only the former needs to be able to +# fail, so it answers from STUB_ZSTD_TEST_EXIT while decompression stays a +# logging no-op. +make_zstd_stub() { + cat > "${STUB_DIR}/zstd" <> "${LOG}" +case " \$* " in + *" -t "*) exit "\${STUB_ZSTD_TEST_EXIT:-0}" ;; +esac +exit 0 +EOF + chmod +x "${STUB_DIR}/zstd" +} + # seed_image # # Places an exported installer artefact where the recipe looks for it. @@ -81,10 +160,14 @@ run_flash() { } # Nothing may reach the disk in any of the refusal paths. +# +# `zstd -t` is deliberately not refuted: the integrity check runs before the +# confirmation prompt and only reads the archive. `zstd -dc` is the decompress +# that feeds dd, and that must never happen on a refusal path. assert_nothing_written() { refute_log "sudo " refute_log "dd " - refute_log "zstd " + refute_log "zstd -dc" } assert_log() { @@ -130,6 +213,124 @@ refute_log() { assert_log "lsblk " } +# --- guard: the target must not back the running system ------------------- + +@test "flash-installer refuses the disk the running system is installed on" { + export STUB_TARGET_KNAME="nvme0n1" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"disk backing the running system"* ]] + assert_nothing_written +} + +@test "flash-installer resolves the system disk through /sysroot when / is composefs" { + # The regression this guard was rewritten for. On composefs/ostree hosts — + # Bluefin itself — `findmnt -no SOURCE /` returns a composefs digest, not a + # device. A guard that consults only / silently passes the root disk + # through. The real device is mounted at /sysroot. + export STUB_ROOT_SRC="composefs:99f5c50825f9ab07" + export STUB_SYSROOT_SRC="/dev/nvme0n1p2" + export STUB_TARGET_KNAME="nvme0n1" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"/sysroot is on /dev/nvme0n1p2"* ]] + assert_nothing_written +} + +@test "flash-installer resolves the system disk through / on a non-ostree host" { + # The other half of `for mp in / /sysroot`. Both cases above leave + # STUB_ROOT_SRC as a composefs digest, so without this the / iteration + # never runs against a /dev/* source — including on the ubuntu-24.04 + # runner that executes this suite in CI, where / is a plain block device + # and /sysroot does not exist. + export STUB_ROOT_SRC="/dev/sda1" + export STUB_SYSROOT_SRC="" + export STUB_PKNAME="sda" + export STUB_TARGET_KNAME="sda" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"/ is on /dev/sda1"* ]] + assert_nothing_written +} + +@test "flash-installer refuses a whole-disk root that PKNAME cannot resolve" { + # A filesystem directly on a whole-disk device has no parent, so + # `lsblk -no PKNAME` prints nothing. Without the KNAME fallback the + # comparison is skipped and the guard is inert on exactly the host it + # most needs to protect. + export STUB_ROOT_SRC="/dev/sda" + export STUB_SYSROOT_SRC="" + export STUB_PKNAME="" + export STUB_SRC_KNAME="sda" + export STUB_TARGET_KNAME="sda" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"disk backing the running system"* ]] + [[ "$output" == *"/ is on /dev/sda"* ]] + assert_nothing_written +} + +@test "flash-installer allows an unrelated disk when root is on a whole-disk device" { + export STUB_ROOT_SRC="/dev/sda" + export STUB_SYSROOT_SRC="" + export STUB_PKNAME="" + export STUB_SRC_KNAME="sda" + export STUB_TARGET_KNAME="sdb" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -eq 0 ] + assert_log "of=${FAKE_DEV}" +} + +@test "flash-installer allows an unrelated disk while the system disk is known" { + export STUB_TARGET_KNAME="sdb" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -eq 0 ] + assert_log "of=${FAKE_DEV}" +} + +# --- guard: the target must have nothing mounted off it ------------------- + +@test "flash-installer refuses a device with a mounted partition" { + export STUB_MOUNTPOINTS="/run/media/jorge/bluefin-root" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"has mounted partitions"* ]] + assert_nothing_written +} + +@test "flash-installer tells the operator how to unmount rather than just refusing" { + export STUB_MOUNTPOINTS="/run/media/jorge/bluefin-root" + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [[ "$output" == *"Unmount them first"* ]] +} + +# --- guard: exactly one exported image ------------------------------------ + +@test "flash-installer refuses to guess between two exported installers" { + seed_image "bluefin-server-installer-1.0.raw.zst" + seed_image "bluefin-server-installer-2.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"refusing to guess"* ]] + assert_nothing_written +} + +@test "flash-installer names both candidates so the operator can remove one" { + seed_image "bluefin-server-installer-1.0.raw.zst" + seed_image "bluefin-server-installer-2.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [[ "$output" == *"bluefin-server-installer-1.0.raw.zst"* ]] + [[ "$output" == *"bluefin-server-installer-2.0.raw.zst"* ]] +} + # --- guard 2: the target must be a block device --------------------------- @test "flash-installer rejects a path that is not a block device" { @@ -212,6 +413,33 @@ refute_log() { assert_log "sudo " } +# --- guard: the image must pass its integrity check ----------------------- + +@test "flash-installer refuses an image that fails its integrity check" { + export STUB_ZSTD_TEST_EXIT=1 + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -ne 0 ] + [[ "$output" == *"failed its integrity check"* ]] + refute_log "sudo " + refute_log "dd " +} + +@test "flash-installer checks integrity before warning about destruction" { + # Ordering matters: a corrupt download should cost nothing, not prompt the + # operator to destroy a disk and only then fail. + export STUB_ZSTD_TEST_EXIT=1 + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [[ "$output" != *"COMPLETELY DESTROYED"* ]] +} + +@test "flash-installer verifies the image it is about to write, not some other one" { + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + assert_log "zstd -t dist/bluefin-server-installer-1.0.raw.zst" +} + # --- the write itself ----------------------------------------------------- @test "flash-installer decompresses the discovered image onto the given device" { @@ -231,8 +459,54 @@ refute_log() { assert_log "conv=fsync" } +@test "flash-installer relocates the GPT backup header to the end of the device" { + # dd writes the image verbatim, so the backup header lands at the end of the + # IMAGE. On any medium larger than the image the table is half valid: the + # primary header points at a backup LBA that is not the device's last LBA. + # parted reports "Not all of the space available ... appears to be used", + # tools disagree about partition sizes between reads, and firmware that + # validates the backup header can refuse the medium. + # + # No other test can catch this: test-installer-artifact and the CI + # installer-test attach a drive sized exactly to the image, so device size + # always equals image size and the condition cannot arise. + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + [ "$status" -eq 0 ] + assert_log "sfdisk --relocate gpt-bak-std ${FAKE_DEV}" +} + +@test "flash-installer repairs the table after writing it, not before" { + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + # dd runs inside `sudo bash -c "... | dd of=..."`, so it never appears as a + # top-level `dd ` line in the call log — match the sudo invocation carrying it. + write_line=$(grep -n "dd of=${FAKE_DEV}" "$LOG" | head -1 | cut -d: -f1) + relocate_line=$(grep -n "relocate gpt-bak-std" "$LOG" | head -1 | cut -d: -f1) + [ -n "$write_line" ] + [ -n "$relocate_line" ] + [ "$relocate_line" -gt "$write_line" ] +} + +@test "flash-installer verifies the repaired table before declaring success" { + seed_image "bluefin-server-installer-1.0.raw.zst" + run_flash "$FAKE_DEV" "y" + assert_log "sfdisk --verify ${FAKE_DEV}" +} + @test "flash-installer reports success only after the write is attempted" { seed_image "bluefin-server-installer-1.0.raw.zst" run_flash "$FAKE_DEV" "y" [[ "$output" == *"Successfully flashed"* ]] } + +@test "the write pipeline runs under bash with pipefail so a dying zstd fails the flash" { + # Shape, not behaviour: `sudo` is a stub, so the nested shell never runs + # here. Under POSIX `sh -c` the pipeline's status is dd's alone, so zstd + # dying mid-stream leaves dd exiting 0 after writing a partial image and + # the success line printing over it. The outer `set -o pipefail` does not + # reach a nested shell. + grep -qF "sudo bash -c \"set -o pipefail;" "$JUSTFILE" + run grep -cF 'sudo sh -c "zstd -dc' "$JUSTFILE" + [ "$output" -eq 0 ] +} diff --git a/tests/unit/os-justfile_test.bats b/tests/unit/os-justfile_test.bats old mode 100644 new mode 100755 index a40d906b..767451d1 --- a/tests/unit/os-justfile_test.bats +++ b/tests/unit/os-justfile_test.bats @@ -2,11 +2,11 @@ # # Unit tests for the `k8s` recipe in files/os/justfile. # -# The recipe interacts with /etc/k0s and /var/lib/extensions, so every +# The recipe interacts with /var/lib/extensions and enables host units, so every # invocation runs inside an unprivileged user + mount namespace with tmpfs # masks over /etc and /var/lib. Nothing on the host is touched. systemctl, -# systemd-sysext, systemd-sysupdate, and systemd-tmpfiles are replaced by -# logging stubs on PATH, so no unit is ever enabled and no OTA is ever fetched. +# systemd-sysext and systemd-sysupdate are replaced by logging stubs on PATH, +# so no unit is ever enabled and no OTA is ever fetched. # # Results that must outlive the namespace (the call log) are copied back # into BATS_TEST_TMPDIR, which is outside both tmpfs masks. @@ -36,11 +36,10 @@ setup() { JUSTFILE_PARSES=0 fi - # Defaults: every helper succeeds, and the k0s sysext is not yet present. + # Defaults: every helper succeeds, and the sysext is not yet installed. make_stub systemctl 0 make_stub systemd-sysext 0 make_stub systemd-sysupdate 0 - make_stub systemd-tmpfiles 0 SYSEXT_PRESENT=0 } @@ -69,7 +68,7 @@ set -u mount -t tmpfs tmpfs /etc mount -t tmpfs tmpfs /var/lib mkdir -p /var/lib/extensions -[ "$SYSEXT_PRESENT" = "1" ] && : > /var/lib/extensions/k0s.raw +[ "$SYSEXT_PRESENT" = "1" ] && : > /var/lib/extensions/kubernetes.raw just --justfile "$JUSTFILE" --working-directory "$WORKDIR" k8s "$@" rc=$? @@ -83,11 +82,6 @@ calls() { # --- the recipe file itself -------------------------------------------------- -# This is the parse gate. It fails on the current tree: the `cat <> extension-release.k0s -""" - -GOOD_EXTENSION_RELEASE = "NAME=k0s\nID=_any\n" - -GOOD_TRANSFER = """[Source] -Type=url-file -MatchPattern=k0s-@v.raw.zst - -[Target] -MatchPattern=k0s-@v.raw -""" - -EXPECTED_VERSION = "1.36.4-k0s.0" - - -def _load_module(): - spec = importlib.util.spec_from_file_location("check_k0s_version", SCRIPT_PATH) - module = importlib.util.module_from_spec(spec) - sys.modules["check_k0s_version"] = module - spec.loader.exec_module(module) - return module - - -@pytest.fixture -def checker(tmp_path): - """Fresh module instance with its path constants rooted at tmp_path. - - ``ROOT`` and the five consumer paths are module-level globals derived from - the script's own location, so each test gets a re-imported copy repointed at - an isolated tree pre-populated with a passing configuration. - """ - module = _load_module() - module.ROOT = tmp_path - module.K0S_INCLUDE = tmp_path / "include" / "k0s.yml" - module.K0S_BIN = tmp_path / "elements" / "k0s" / "k0s-bin.bst" - module.K0S_SYSEXT = tmp_path / "elements" / "oci" / "k0s-sysext.bst" - module.EXTENSION_RELEASE = ( - tmp_path / "files" / "k0s" / "sysext" / "extension-release.k0s" - ) - module.TRANSFER = tmp_path / "files" / "os" / "sysupdate.d" / "70-k0s.transfer" - for path, text in ( - (module.K0S_INCLUDE, GOOD_INCLUDE), - (module.K0S_BIN, GOOD_BIN), - (module.K0S_SYSEXT, GOOD_SYSEXT), - (module.EXTENSION_RELEASE, GOOD_EXTENSION_RELEASE), - (module.TRANSFER, GOOD_TRANSFER), - ): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text, encoding="utf-8") - yield module - sys.modules.pop("check_k0s_version", None) - - -def _run(checker): - with pytest.raises(SystemExit) as excinfo: - checker.main() - return str(excinfo.value) - - -# --- read() --------------------------------------------------------------- - - -def test_read_returns_file_contents(checker, tmp_path): - target = tmp_path / "include" / "k0s.yml" - assert checker.read(target) == GOOD_INCLUDE - - -def test_read_exits_on_missing_file(checker, tmp_path): - missing = tmp_path / "include" / "gone.yml" - with pytest.raises(SystemExit) as excinfo: - checker.read(missing) - assert "expected file not found" in str(excinfo.value) - assert "include/gone.yml" in str(excinfo.value) - - -# --- scalar() ------------------------------------------------------------- - - -def test_scalar_reads_double_quoted_value(checker): - assert checker.scalar(GOOD_INCLUDE, "k0s-patch", "where") == "0" - - -def test_scalar_reads_single_quoted_value(checker): - text = "variables:\n k0s-patch: '3'\n" - assert checker.scalar(text, "k0s-patch", "where") == "3" - - -def test_scalar_ignores_unquoted_value(checker): - with pytest.raises(SystemExit) as excinfo: - checker.scalar("variables:\n k0s-patch: 0\n", "k0s-patch", "include/k0s.yml") - assert "does not declare a 'k0s-patch:' variable" in str(excinfo.value) - - -def test_scalar_exits_when_name_absent(checker): - with pytest.raises(SystemExit) as excinfo: - checker.scalar(GOOD_INCLUDE, "k0s-nope", "include/k0s.yml") - assert "include/k0s.yml does not declare a 'k0s-nope:' variable" in str( - excinfo.value - ) - - -# --- expand() ------------------------------------------------------------- - - -def test_expand_returns_literal_unchanged(checker): - assert checker.expand("1.36.4", {}, "where") == "1.36.4" - - -def test_expand_substitutes_nested_references(checker): - variables = {"a": "%{b}", "b": "leaf"} - assert checker.expand("%{a}", variables, "where") == "leaf" - - -def test_expand_exits_on_undefined_reference(checker): - with pytest.raises(SystemExit) as excinfo: - checker.expand("%{missing}", {}, "include/k0s.yml x") - assert "references undefined variable '%{missing}'" in str(excinfo.value) - - -def test_expand_exits_on_circular_reference(checker): - with pytest.raises(SystemExit) as excinfo: - checker.expand("%{a}", {"a": "%{b}", "b": "%{a}"}, "include/k0s.yml x") - assert "circular variable reference" in str(excinfo.value) - - -# --- fail() --------------------------------------------------------------- - - -def test_fail_reports_problem_and_fix(checker): - with pytest.raises(SystemExit) as excinfo: - checker.fail("something drifted", "do the thing") - message = str(excinfo.value) - assert "ERROR: something drifted" in message - assert "Fix: do the thing" in message - - -# --- module-level regexes ------------------------------------------------- - - -@pytest.mark.parametrize( - "candidate", - ["1.36.4-k0s.0", "v1.36.4", "1.36.4+k0s.0", "release~1^2"], -) -def test_safe_version_re_accepts_filename_safe_versions(checker, candidate): - assert checker.SAFE_VERSION_RE.match(candidate) - - -@pytest.mark.parametrize("candidate", ["1.36.4 k0s", "1.36.4/0", "", "a:b"]) -def test_safe_version_re_rejects_unsafe_versions(checker, candidate): - assert not checker.SAFE_VERSION_RE.match(candidate) - - -@pytest.mark.parametrize( - "candidate", - ["v1.36.4+k0s.0", "1.36.4%2Bk0s.0", "1.36.4-k0s.12"], -) -def test_literal_version_re_matches_every_spelling(checker, candidate): - assert checker.LITERAL_VERSION_RE.search(candidate) - - -def test_literal_version_re_ignores_templated_reference(checker): - assert not checker.LITERAL_VERSION_RE.search("%{k0s-upstream-tag}") - - -# --- main(): the passing path -------------------------------------------- - - -def test_main_accepts_a_fully_derived_configuration(checker, capsys): - checker.main() - out = capsys.readouterr().out - assert "OK: k0s v1.36.4%2Bk0s.0 pinned in include/k0s.yml" in out - assert f"asset k0s-{EXPECTED_VERSION}.raw.zst" in out - assert f"VERSION_ID={EXPECTED_VERSION}" in out - - -def test_main_tolerates_literal_version_inside_comments(checker, capsys): - checker.K0S_BIN.write_text( - "# pinned from the v1.36.4+k0s.0 release notes\n" + GOOD_BIN, - encoding="utf-8", - ) - checker.main() - assert "OK: k0s" in capsys.readouterr().out - - -# --- main(): version safety ---------------------------------------------- - - -def test_main_rejects_version_that_is_not_filename_safe(checker): - checker.K0S_INCLUDE.write_text( - GOOD_INCLUDE.replace('k0s-patch: "0"', 'k0s-patch: "0 rc1"'), - encoding="utf-8", - ) - assert "is not filename-safe" in _run(checker) - - -# --- main(): upstream download URL --------------------------------------- - - -def test_main_rejects_missing_source_url(checker): - checker.K0S_BIN.write_text("kind: manual\nsources: []\n", encoding="utf-8") - assert "declares no source 'url:'" in _run(checker) - - -def test_main_rejects_hardcoded_upstream_tag_in_url(checker): - checker.K0S_BIN.write_text( - "sources:\n" - " - kind: remote\n" - " url: github:k0sproject/k0s/releases/download/v1.36.4%2Bk0s.0/k0s-amd64\n", - encoding="utf-8", - ) - assert "hardcodes the upstream k0s release tag" in _run(checker) - - -# --- main(): sysext asset filename --------------------------------------- - - -def test_main_rejects_missing_fname(checker): - checker.K0S_SYSEXT.write_text( - 'echo "VERSION_ID=%{k0s-version}"\n', encoding="utf-8" - ) - assert 'declares no FNAME="..." asset name' in _run(checker) - - -def test_main_rejects_fname_on_the_os_release_axis(checker): - checker.K0S_SYSEXT.write_text( - GOOD_SYSEXT.replace("%{k0s-version}.raw", "%{release-version}.raw", 1), - encoding="utf-8", - ) - assert "names the k0s sysext on the OS release" in _run(checker) - - -def test_main_rejects_fname_not_derived_from_k0s_version(checker): - checker.K0S_SYSEXT.write_text( - GOOD_SYSEXT.replace('FNAME="k0s-%{k0s-version}.raw"', 'FNAME="k0s.raw"'), - encoding="utf-8", - ) - assert "does not derive the sysext asset name" in _run(checker) - - -def test_main_rejects_missing_generated_version_id(checker): - checker.K0S_SYSEXT.write_text( - 'FNAME="k0s-%{k0s-version}.raw"\n', encoding="utf-8" - ) - assert "does not generate VERSION_ID= from" in _run(checker) - - -# --- main(): extension-release.k0s --------------------------------------- - - -def test_main_rejects_hardcoded_version_id_in_extension_release(checker): - checker.EXTENSION_RELEASE.write_text( - "NAME=k0s\nID=_any\nVERSION_ID=1.36.4-k0s.0\n", encoding="utf-8" - ) - message = _run(checker) - assert "hardcodes a version" in message - assert "VERSION_ID=1.36.4-k0s.0" in message - - -def test_main_rejects_indented_hardcoded_version_id(checker): - checker.EXTENSION_RELEASE.write_text( - "NAME=k0s\n VERSION_ID=9.9.9\n", encoding="utf-8" - ) - assert "hardcodes a version" in _run(checker) - - -# --- main(): literal version scan ---------------------------------------- - - -@pytest.mark.parametrize( - "attr", - ["K0S_BIN", "K0S_SYSEXT", "EXTENSION_RELEASE"], -) -def test_main_rejects_literal_version_in_any_consumer(checker, attr): - path = getattr(checker, attr) - path.write_text( - path.read_text(encoding="utf-8") + "\nnote: shipped v1.36.4+k0s.0\n", - encoding="utf-8", - ) - message = _run(checker) - assert "restates a literal k0s version" in message - assert "Literal: v1.36.4+k0s.0" in message - - -# --- main(): sysupdate MatchPattern -------------------------------------- - - -def test_main_rejects_missing_match_pattern(checker): - checker.TRANSFER.write_text("[Source]\nType=url-file\n", encoding="utf-8") - assert "declares no [Source]" in _run(checker) - - -def test_main_rejects_match_pattern_without_zst_suffix(checker): - checker.TRANSFER.write_text( - "[Source]\nMatchPattern=k0s-@v.raw\n", encoding="utf-8" - ) - assert "declares no [Source]" in _run(checker) - - -def test_main_rejects_prefix_drift_between_fname_and_match_pattern(checker): - checker.TRANSFER.write_text( - "[Source]\nMatchPattern=kzeros-@v.raw.zst\n", encoding="utf-8" - ) - message = _run(checker) - assert "is not the pinned k0s version" in message - assert "'@v' captures : None" in message - - -def test_main_rejects_suffix_drift_between_fname_and_match_pattern(checker): - checker.K0S_SYSEXT.write_text( - GOOD_SYSEXT.replace( - 'FNAME="k0s-%{k0s-version}.raw"', 'FNAME="k0s-%{k0s-version}.erofs"' - ), - encoding="utf-8", - ) - assert "is not the pinned k0s version" in _run(checker) - - -def test_main_rejects_match_pattern_that_captures_extra_text(checker): - checker.TRANSFER.write_text( - "[Source]\nMatchPattern=k@v.raw.zst\n", encoding="utf-8" - ) - message = _run(checker) - assert "is not the pinned k0s version" in message - assert f"'@v' captures : '0s-{EXPECTED_VERSION}'" in message diff --git a/tests/unit/test_check_kubernetes_version.py b/tests/unit/test_check_kubernetes_version.py new file mode 100644 index 00000000..0843d8de --- /dev/null +++ b/tests/unit/test_check_kubernetes_version.py @@ -0,0 +1,481 @@ +"""Unit coverage for .github/scripts/check-kubernetes-version.py. + +The script is the only gate holding the Kubernetes version axis together. It is +run by ``just validate`` and by the ``check-kubernetes-version`` pre-commit +hook, so every fail-closed branch needs direct coverage: without it a branch +can regress to fail-open and nothing notices until the sysext silently stops +updating on deployed hosts. + +Covered here: ``read()``, ``scalar()``, ``expand()``, ``fail()``, +``source_urls()`` and each rejection path of ``main()`` — non-filename-safe +version, missing/hardcoded node-binary URLs, missing/hardcoded CNI URL, +missing/OS-axis/underived ``FNAME``, missing generated ``VERSION_ID``, missing +``/usr/local/share`` version stamps, a hardcoded ``VERSION_ID=`` in +extension-release.kubernetes, literal versions restated in any consumer, a +missing sysupdate ``MatchPattern``, and an asset name whose ``@v`` capture +disagrees with ``%{k8s-version}``. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_PATH = REPO_ROOT / ".github" / "scripts" / "check-kubernetes-version.py" + +GOOD_INCLUDE = """variables: + k8s-version: "1.36.4" + cni-version: "1.9.1" + + k8s-upstream-tag: "v%{k8s-version}" + cni-upstream-tag: "v%{cni-version}" +""" + +GOOD_BIN = """kind: manual +sources: + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubectl + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubeadm + - kind: remote + url: k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubelet +""" + +GOOD_CNI = """kind: manual +sources: + - kind: tar + url: github:containernetworking/plugins/releases/download/%{cni-upstream-tag}/cni-plugins-linux-amd64-%{cni-upstream-tag}.tgz +""" + +GOOD_SYSEXT = """kind: manual +config: + install-commands: + - | + FNAME="kubernetes-%{k8s-version}.raw" + echo "%{k8s-version}" > sysext/usr/local/share/kubernetes-version + echo "%{cni-version}" > sysext/usr/local/share/kubernetes-cni-version + echo "VERSION_ID=%{k8s-version}" >> extension-release.kubernetes +""" + +GOOD_EXTENSION_RELEASE = "NAME=kubernetes\nID=_any\n" + +GOOD_TRANSFER = """[Source] +Type=url-file +MatchPattern=kubernetes-@v.raw.zst + +[Target] +MatchPattern=kubernetes-@v.raw +""" + +EXPECTED_VERSION = "1.36.4" + + +def _load_module(): + spec = importlib.util.spec_from_file_location( + "check_kubernetes_version", SCRIPT_PATH + ) + module = importlib.util.module_from_spec(spec) + sys.modules["check_kubernetes_version"] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def checker(tmp_path): + """Fresh module instance with its path constants rooted at tmp_path. + + ``ROOT`` and the six consumer paths are module-level globals derived from + the script's own location, so each test gets a re-imported copy repointed at + an isolated tree pre-populated with a passing configuration. + """ + module = _load_module() + module.ROOT = tmp_path + module.K8S_INCLUDE = tmp_path / "include" / "kubernetes.yml" + module.K8S_BIN = tmp_path / "elements" / "kubernetes" / "kubernetes-bin.bst" + module.CNI_BIN = tmp_path / "elements" / "kubernetes" / "cni-plugins.bst" + module.K8S_SYSEXT = tmp_path / "elements" / "oci" / "kubernetes-sysext.bst" + module.EXTENSION_RELEASE = ( + tmp_path / "files" / "kubernetes" / "sysext" / "extension-release.kubernetes" + ) + module.TRANSFER = ( + tmp_path / "files" / "os" / "sysupdate.kubernetes.d" / "70-kubernetes.transfer" + ) + for path, text in ( + (module.K8S_INCLUDE, GOOD_INCLUDE), + (module.K8S_BIN, GOOD_BIN), + (module.CNI_BIN, GOOD_CNI), + (module.K8S_SYSEXT, GOOD_SYSEXT), + (module.EXTENSION_RELEASE, GOOD_EXTENSION_RELEASE), + (module.TRANSFER, GOOD_TRANSFER), + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + yield module + sys.modules.pop("check_kubernetes_version", None) + + +def _run(checker): + with pytest.raises(SystemExit) as excinfo: + checker.main() + return str(excinfo.value) + + +# --- read() --------------------------------------------------------------- + + +def test_read_returns_file_contents(checker, tmp_path): + target = tmp_path / "include" / "kubernetes.yml" + assert checker.read(target) == GOOD_INCLUDE + + +def test_read_exits_on_missing_file(checker, tmp_path): + missing = tmp_path / "include" / "gone.yml" + with pytest.raises(SystemExit) as excinfo: + checker.read(missing) + assert "expected file not found" in str(excinfo.value) + assert "include/gone.yml" in str(excinfo.value) + + +# --- scalar() ------------------------------------------------------------- + + +def test_scalar_reads_double_quoted_value(checker): + assert checker.scalar(GOOD_INCLUDE, "cni-version", "where") == "1.9.1" + + +def test_scalar_reads_single_quoted_value(checker): + text = "variables:\n cni-version: '1.9.2'\n" + assert checker.scalar(text, "cni-version", "where") == "1.9.2" + + +def test_scalar_ignores_unquoted_value(checker): + with pytest.raises(SystemExit) as excinfo: + checker.scalar( + "variables:\n cni-version: 1.9.1\n", + "cni-version", + "include/kubernetes.yml", + ) + assert "does not declare a 'cni-version:' variable" in str(excinfo.value) + + +def test_scalar_exits_when_name_absent(checker): + with pytest.raises(SystemExit) as excinfo: + checker.scalar(GOOD_INCLUDE, "k8s-nope", "include/kubernetes.yml") + assert "include/kubernetes.yml does not declare a 'k8s-nope:' variable" in str( + excinfo.value + ) + + +# --- expand() ------------------------------------------------------------- + + +def test_expand_returns_literal_unchanged(checker): + assert checker.expand("1.36.4", {}, "where") == "1.36.4" + + +def test_expand_substitutes_nested_references(checker): + variables = {"a": "%{b}", "b": "leaf"} + assert checker.expand("%{a}", variables, "where") == "leaf" + + +def test_expand_exits_on_undefined_reference(checker): + with pytest.raises(SystemExit) as excinfo: + checker.expand("%{missing}", {}, "include/kubernetes.yml x") + assert "references undefined variable '%{missing}'" in str(excinfo.value) + + +def test_expand_exits_on_circular_reference(checker): + with pytest.raises(SystemExit) as excinfo: + checker.expand("%{a}", {"a": "%{b}", "b": "%{a}"}, "include/kubernetes.yml x") + assert "circular variable reference" in str(excinfo.value) + + +# --- fail() --------------------------------------------------------------- + + +def test_fail_reports_problem_and_fix(checker): + with pytest.raises(SystemExit) as excinfo: + checker.fail("something drifted", "do the thing") + message = str(excinfo.value) + assert "ERROR: something drifted" in message + assert "Fix: do the thing" in message + + +# --- source_urls() -------------------------------------------------------- + + +def test_source_urls_collects_every_declared_url(checker): + assert checker.source_urls(GOOD_BIN) == [ + "k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubectl", + "k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubeadm", + "k8s:%{k8s-upstream-tag}/bin/linux/amd64/kubelet", + ] + + +def test_source_urls_returns_empty_without_sources(checker): + assert checker.source_urls("kind: manual\nsources: []\n") == [] + + +# --- module-level regexes ------------------------------------------------- + + +@pytest.mark.parametrize( + "candidate", + ["1.36.4", "v1.36.4", "1.9.1", "release~1^2"], +) +def test_safe_version_re_accepts_filename_safe_versions(checker, candidate): + assert checker.SAFE_VERSION_RE.match(candidate) + + +@pytest.mark.parametrize("candidate", ["1.36.4 rc1", "1.36.4/0", "", "a:b"]) +def test_safe_version_re_rejects_unsafe_versions(checker, candidate): + assert not checker.SAFE_VERSION_RE.match(candidate) + + +@pytest.mark.parametrize( + "candidate", + [ + "url: k8s:v1.36.4/bin/linux/amd64/kubeadm", + "cni-plugins-linux-amd64-v1.9.1.tgz", + "FNAME=kubernetes-1.36.4.raw", + ], +) +def test_literal_version_re_matches_every_spelling(checker, candidate): + assert checker.LITERAL_VERSION_RE.search(candidate) + + +@pytest.mark.parametrize( + "candidate", + [ + "%{k8s-upstream-tag}", + 'FNAME="kubernetes-%{k8s-version}.raw"', + "ref: b98f74a0f8522f0a83867178729c1aa70f2158f90c45a2ca8fa791db1c76b303", + "install -D -m 0755 cni-src/host-local", + ], +) +def test_literal_version_re_ignores_derived_and_incidental_text(checker, candidate): + assert not checker.LITERAL_VERSION_RE.search(candidate) + + +# --- main(): the passing path -------------------------------------------- + + +def test_main_accepts_a_fully_derived_configuration(checker, capsys): + checker.main() + out = capsys.readouterr().out + assert "OK: Kubernetes v1.36.4 and CNI plugins v1.9.1" in out + assert f"asset kubernetes-{EXPECTED_VERSION}.raw.zst" in out + assert f"VERSION_ID={EXPECTED_VERSION}" in out + + +def test_main_tolerates_literal_version_inside_comments(checker, capsys): + checker.K8S_BIN.write_text( + "# pinned from the v1.36.4 release notes\n" + GOOD_BIN, + encoding="utf-8", + ) + checker.main() + assert "OK: Kubernetes" in capsys.readouterr().out + + +# --- main(): version safety ---------------------------------------------- + + +def test_main_rejects_version_that_is_not_filename_safe(checker): + checker.K8S_INCLUDE.write_text( + GOOD_INCLUDE.replace('k8s-version: "1.36.4"', 'k8s-version: "1.36.4 rc1"'), + encoding="utf-8", + ) + assert "is not filename-safe" in _run(checker) + + +# --- main(): node binary download URLs ----------------------------------- + + +@pytest.mark.parametrize("binary", ["kubectl", "kubeadm", "kubelet"]) +def test_main_rejects_a_missing_node_binary_url(checker, binary): + checker.K8S_BIN.write_text( + "\n".join( + line for line in GOOD_BIN.splitlines() if not line.endswith(f"/{binary}") + ) + + "\n", + encoding="utf-8", + ) + message = _run(checker) + assert "declares no source 'url:'" in message + assert f"'/{binary}'" in message + + +def test_main_rejects_hardcoded_upstream_tag_in_node_binary_url(checker): + checker.K8S_BIN.write_text( + GOOD_BIN.replace("%{k8s-upstream-tag}", "v1.36.4", 1), encoding="utf-8" + ) + assert "hardcodes the upstream\n Kubernetes release tag" in _run(checker) + + +# --- main(): CNI plugins download URL ------------------------------------ + + +def test_main_rejects_missing_cni_url(checker): + checker.CNI_BIN.write_text("kind: manual\nsources: []\n", encoding="utf-8") + assert "cni-plugins.bst declares no source 'url:'" in _run(checker) + + +def test_main_rejects_hardcoded_cni_tag(checker): + checker.CNI_BIN.write_text( + GOOD_CNI.replace("%{cni-upstream-tag}", "v1.9.1"), encoding="utf-8" + ) + assert "hardcodes the upstream CNI" in _run(checker) + + +# --- main(): sysext asset filename --------------------------------------- + + +def test_main_rejects_missing_fname(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace('FNAME="kubernetes-%{k8s-version}.raw"', ""), + encoding="utf-8", + ) + assert 'declares no FNAME="..." asset name' in _run(checker) + + +def test_main_rejects_fname_on_the_os_release_axis(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace("%{k8s-version}.raw", "%{release-version}.raw", 1), + encoding="utf-8", + ) + assert "names the Kubernetes sysext on the\n OS release axis" in _run(checker) + + +def test_main_rejects_fname_not_derived_from_k8s_version(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace( + 'FNAME="kubernetes-%{k8s-version}.raw"', 'FNAME="kubernetes.raw"' + ), + encoding="utf-8", + ) + assert "does not derive the sysext asset\n name" in _run(checker) + + +def test_main_rejects_missing_generated_version_id(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace( + 'echo "VERSION_ID=%{k8s-version}" >> extension-release.kubernetes', "" + ), + encoding="utf-8", + ) + assert "does not generate VERSION_ID= from" in _run(checker) + + +# --- main(): /usr/local/share version stamps ----------------------------- + + +@pytest.mark.parametrize( + "stamp", ["kubernetes-version", "kubernetes-cni-version"] +) +def test_main_rejects_a_missing_version_stamp(checker, stamp): + checker.K8S_SYSEXT.write_text( + "\n".join( + line + for line in GOOD_SYSEXT.splitlines() + if not line.strip().endswith(f"sysext/usr/local/share/{stamp}") + ) + + "\n", + encoding="utf-8", + ) + message = _run(checker) + assert "does not generate" in message + assert f"/usr/local/share/{stamp}" in message + + +def test_main_rejects_cni_stamp_written_from_the_kubernetes_atom(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace( + 'echo "%{cni-version}" > sysext/usr/local/share/kubernetes-cni-version', + 'echo "%{k8s-version}" > sysext/usr/local/share/kubernetes-cni-version', + ), + encoding="utf-8", + ) + assert "/usr/local/share/kubernetes-cni-version" in _run(checker) + + +# --- main(): extension-release.kubernetes -------------------------------- + + +def test_main_rejects_hardcoded_version_id_in_extension_release(checker): + checker.EXTENSION_RELEASE.write_text( + "NAME=kubernetes\nID=_any\nVERSION_ID=1.36.4\n", encoding="utf-8" + ) + message = _run(checker) + assert "hardcodes a\n version" in message + assert "VERSION_ID=1.36.4" in message + + +def test_main_rejects_indented_hardcoded_version_id(checker): + checker.EXTENSION_RELEASE.write_text( + "NAME=kubernetes\n VERSION_ID=9.9.9\n", encoding="utf-8" + ) + assert "hardcodes a\n version" in _run(checker) + + +# --- main(): literal version scan ---------------------------------------- + + +@pytest.mark.parametrize( + "attr", + ["K8S_BIN", "CNI_BIN", "K8S_SYSEXT", "EXTENSION_RELEASE"], +) +def test_main_rejects_literal_version_in_any_consumer(checker, attr): + path = getattr(checker, attr) + path.write_text( + path.read_text(encoding="utf-8") + "\nnote: shipped v1.36.4\n", + encoding="utf-8", + ) + message = _run(checker) + assert "restates a literal Kubernetes or CNI" in message + assert "Literal: v1.36.4" in message + + +# --- main(): sysupdate MatchPattern -------------------------------------- + + +def test_main_rejects_missing_match_pattern(checker): + checker.TRANSFER.write_text("[Source]\nType=url-file\n", encoding="utf-8") + assert "declares no\n [Source]" in _run(checker) + + +def test_main_rejects_match_pattern_without_zst_suffix(checker): + checker.TRANSFER.write_text( + "[Source]\nMatchPattern=kubernetes-@v.raw\n", encoding="utf-8" + ) + assert "declares no\n [Source]" in _run(checker) + + +def test_main_rejects_prefix_drift_between_fname_and_match_pattern(checker): + checker.TRANSFER.write_text( + "[Source]\nMatchPattern=k8s-@v.raw.zst\n", encoding="utf-8" + ) + message = _run(checker) + assert "is not the pinned Kubernetes version" in message + assert "'@v' captures : None" in message + + +def test_main_rejects_suffix_drift_between_fname_and_match_pattern(checker): + checker.K8S_SYSEXT.write_text( + GOOD_SYSEXT.replace( + 'FNAME="kubernetes-%{k8s-version}.raw"', + 'FNAME="kubernetes-%{k8s-version}.erofs"', + ), + encoding="utf-8", + ) + assert "is not the pinned Kubernetes version" in _run(checker) + + +def test_main_rejects_match_pattern_that_captures_extra_text(checker): + checker.TRANSFER.write_text( + "[Source]\nMatchPattern=kuber@v.raw.zst\n", encoding="utf-8" + ) + message = _run(checker) + assert "is not the pinned Kubernetes version" in message + assert f"'@v' captures : 'netes-{EXPECTED_VERSION}'" in message diff --git a/tests/unit/test_cluster_bootstrap.py b/tests/unit/test_cluster_bootstrap.py new file mode 100644 index 00000000..128e8f62 --- /dev/null +++ b/tests/unit/test_cluster_bootstrap.py @@ -0,0 +1,543 @@ +"""Contracts for Kubernetes control-plane bring-up and cluster seeding. + +``systemd-analyze verify`` is not available in this repo's test environment, so +the unit invariants that a verifier would catch are asserted here instead: +ordering edges, the placement of the idempotency guard, and the fact that every +unit the preset enables can actually be enabled. + +The guard placement is the subtle one. systemd marks a unit whose ``Condition*`` +fails as *skipped*, not failed, and treats it as satisfied for everything +ordered after it. That is exactly what ``kubeadm-init.service`` wants on a +second boot, and exactly what ``bluefin-cluster-bootstrap.service`` must never +have: a condition there would silently no-op the seed. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +SYSTEM = ROOT / "files" / "os" / "systemd" / "system" +PRESET_DIR = ROOT / "files" / "os" / "systemd" / "system-preset" + +KUBEADM_INIT = SYSTEM / "kubeadm-init.service" +CLUSTER_REPO = SYSTEM / "bluefin-cluster-repo.service" +CLUSTER_BOOTSTRAP = SYSTEM / "bluefin-cluster-bootstrap.service" +KUBELET = SYSTEM / "kubelet.service" +KUBELET_DROPIN = SYSTEM / "kubelet.service.d" / "10-kubeadm.conf" +SYSEXT_DROPIN = ( + ROOT + / "files" + / "os" + / "systemd" + / "systemd-sysext.service.d" + / "10-daemon-reload.conf" +) +PRESET = PRESET_DIR / "zz-enable-cluster-bootstrap.preset" +REPO_HELPER = ROOT / "files" / "os" / "libexec" / "bluefin-cluster-repo" +KUBEADM_CONFIG = ROOT / "files" / "kubernetes" / "kubeadm.yaml" + +NETWORK = SYSTEM.parent / "network" / "20-wired.network" +NETWORK_PRESET = PRESET_DIR / "zz-enable-networkd.preset" + +ELEMENT = ROOT / "elements" / "bluefin-server" / "os-cluster-bootstrap.bst" +NETWORK_ELEMENT = ROOT / "elements" / "bluefin-server" / "os-networkd.bst" +SYSUPDATE_ELEMENT = ( + ROOT / "elements" / "bluefin-server" / "os-kubernetes-sysupdate.bst" +) +STACK = ROOT / "elements" / "bluefin-server" / "os-stack.bst" + +SEED_ROOT = "/usr/share/bluefin/seed" +SEED_PHASES = ("00-cilium", "10-argocd-core", "15-gitd", "20-root-app") + +UNITS = (KUBEADM_INIT, CLUSTER_REPO, CLUSTER_BOOTSTRAP, KUBELET) + + +def unit(path: Path) -> dict[str, dict[str, list[str]]]: + """Parse a unit file into ``{section: {key: [values...]}}``. + + ``configparser`` collapses repeated keys, and repeated keys are the whole + point of a unit file: ``ExecStart=`` order *is* the seed order. + """ + sections: dict[str, dict[str, list[str]]] = {} + current: dict[str, list[str]] | None = None + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith(("#", ";")): + continue + if line.startswith("[") and line.endswith("]"): + current = sections.setdefault(line[1:-1], {}) + continue + assert current is not None, f"{path.name}: {line!r} precedes any section" + key, _, value = line.partition("=") + current.setdefault(key.strip(), []).append(value.strip()) + return sections + + +def exec_starts(path: Path) -> list[str]: + return unit(path)["Service"].get("ExecStart", []) + + +@pytest.mark.parametrize("path", UNITS, ids=lambda p: p.name) +def test_every_unit_parses_and_can_be_enabled(path: Path) -> None: + sections = unit(path) + assert "Unit" in sections and "Service" in sections + assert sections["Install"]["WantedBy"] == ["multi-user.target"], ( + f"{path.name} has no [Install] WantedBy, so the preset cannot enable it" + ) + + +def test_preset_enables_exactly_the_units_that_are_shipped() -> None: + enabled = [ + line.removeprefix("enable ") + for line in PRESET.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + assert enabled == [ + "kubelet.service", + "kubeadm-init.service", + "bluefin-cluster-repo.service", + "bluefin-cluster-bootstrap.service", + ] + for name in enabled: + assert (SYSTEM / name).is_file(), f"preset enables missing unit {name}" + + +def test_the_idempotency_guard_lives_on_kubeadm_init_alone() -> None: + """A skipped unit satisfies its dependents; a skipped seed is a dead node.""" + init = unit(KUBEADM_INIT) + assert init["Unit"]["ConditionPathExists"] == [ + "!/var/lib/bluefin/kubeadm-init.stamp" + ] + + for path in (CLUSTER_BOOTSTRAP,): + guards = [ + key + for key in unit(path)["Unit"] + if key.startswith(("Condition", "Assert")) + ] + assert guards == [], ( + f"{path.name} carries {guards}; a failed condition marks the unit " + "skipped rather than failed, silently no-opping the first boot seed" + ) + + +def test_the_guard_is_a_completion_stamp_not_a_mid_run_artefact() -> None: + """admin.conf lands in kubeadm's `kubeconfig` phase, long before the control + plane is up and the control-plane role label is applied. Guarding on it made + Restart=on-failure skip the retry of a half-initialised node, which showed up + downstream as the 15-gitd nodeSelector never matching.""" + service = unit(KUBEADM_INIT)["Service"] + + guard = unit(KUBEADM_INIT)["Unit"]["ConditionPathExists"][0].removeprefix("!") + post = service.get("ExecStartPost", []) + assert any(guard in line for line in post), ( + f"nothing writes {guard}, so the condition can never become true and " + "kubeadm init would re-run on every boot" + ) + + pre = service.get("ExecStartPre", []) + assert any("kubeadm reset -f" in line for line in pre), ( + "a retry that finds /etc/kubernetes/admin.conf from a failed attempt " + "must reset first; kubeadm refuses to init over a populated " + "/etc/kubernetes/manifests, so every retry would fail identically" + ) + assert any( + "kubeadm reset -f" in line and "/etc/kubernetes/admin.conf" in line + for line in pre + ), "the reset must be conditional on the half-initialised state, not unconditional" + + +def test_kubeadm_init_waits_for_the_sysext_merge_and_the_runtime() -> None: + init = unit(KUBEADM_INIT)["Unit"] + requires = set(init.get("Requires", [])) + wants = set(init.get("Wants", [])) + after = set(init.get("After", [])) + + # systemd-sysext.service is part of systemd and exists when the boot + # transaction is computed, so requiring it is safe and correct: without the + # merge there is no /usr/bin/kubeadm to run. + assert "systemd-sysext.service" in requires + assert "systemd-sysext.service" in after + + # containerd.service is different. It arrives *inside* the merge, so it does + # not exist yet when systemd resolves dependencies. This previously read + # `Requires=containerd.service`, and systemd dropped kubeadm-init's job from + # the boot transaction outright — conditions never evaluated, kubeadm never + # ran, nothing in `systemctl list-units --failed`. Observed on a real boot: + # preset 12:06:10, merge 12:06:11, multi-user 12:06:13. + assert "containerd.service" not in requires, ( + "containerd.service does not exist when the boot transaction is " + "computed; a hard requirement silently drops this unit's job" + ) + assert "containerd.service" in wants + assert "containerd.service" in after + + assert "network-online.target" in after + + +def test_kubeadm_init_stays_active_so_dependents_can_order_on_it() -> None: + service = unit(KUBEADM_INIT)["Service"] + assert service["Type"] == ["oneshot"] + assert service["RemainAfterExit"] == ["yes"], ( + "without RemainAfterExit the unit deactivates immediately and ordering " + "on it tells a dependent nothing about the control plane" + ) + + +def test_kubeadm_init_reads_the_staged_config_and_skips_kube_proxy() -> None: + commands = exec_starts(KUBEADM_INIT) + assert commands == [ + "/usr/bin/kubeadm init --config /usr/share/bluefin/kubeadm.yaml " + "--skip-phases=addon/kube-proxy" + ], "Cilium supplies the kube-proxy replacement; installing both fights" + + +def test_sysext_merge_reloads_systemd_and_starts_the_merged_runtime() -> None: + """Making a merged unit visible is not the same as running it. + + containerd.service and its own multi-user.target.wants symlink both live + inside the sysext, so neither exists when systemd computes the boot + transaction. The reload makes them visible afterwards, but a reload does not + enqueue jobs for a target that is already active or already in flight — so + whether containerd starts comes down to whether the merge happened to land + before multi-user.target was queued. + + That race was observed resolving both ways on the same image:: + + boot A merge 12:06:11.75, multi-user 12:06:13.14 -> containerd active + boot B merge 12:21:10.24, multi-user not reached -> containerd inactive + + On boot B kubeadm-init blocked forever waiting for a CRI socket that never + appeared. The explicit start removes the race; it is idempotent when the + symlink did win. + """ + post = unit(SYSEXT_DROPIN)["Service"]["ExecStartPost"] + + assert "/usr/bin/systemctl daemon-reload" in post, ( + "without a reload, ordering edges resolve against units systemd has " + "never read" + ) + assert "/usr/bin/systemctl start --no-block containerd.service" in post, ( + "containerd.service must be started explicitly: its enablement symlink " + "arrives with the merge, too late for systemd to act on" + ) + assert post.index("/usr/bin/systemctl daemon-reload") < post.index( + "/usr/bin/systemctl start --no-block containerd.service" + ), "the unit must be read before it can be started" + + +def test_kubelet_is_configured_by_kubeadm_rather_than_by_the_image() -> None: + service = unit(KUBELET)["Service"] + assert service["Restart"] == ["always"] + assert unit(KUBELET)["Unit"]["StartLimitIntervalSec"] == ["0"], ( + "the kubelet restarts until kubeadm has written its config; a start " + "limit would leave the node permanently down" + ) + + dropin = unit(KUBELET_DROPIN)["Service"] + assert dropin["ExecStart"][0] == "", ( + "the drop-in must clear the inherited ExecStart before adding its own" + ) + assert "$KUBELET_KUBEADM_ARGS" in dropin["ExecStart"][1] + assert "-/var/lib/kubelet/kubeadm-flags.env" in dropin["EnvironmentFile"] + + +def test_cluster_repo_is_ordered_before_the_seed_and_is_re_runnable() -> None: + repo = unit(CLUSTER_REPO) + assert repo["Unit"]["Before"] == ["bluefin-cluster-bootstrap.service"] + assert repo["Service"]["Type"] == ["oneshot"] + assert repo["Service"]["StateDirectory"] == ["bluefin"] + assert exec_starts(CLUSTER_REPO) == ["/usr/libexec/bluefin-cluster-repo"] + + +def test_cluster_repo_helper_publishes_a_servable_bare_repository() -> None: + helper = REPO_HELPER.read_text(encoding="utf-8") + assert REPO_HELPER.stat().st_mode & 0o111, "the helper must be executable" + assert "REPO=/var/lib/bluefin/cluster.git" in helper + assert "SEED=/usr/share/bluefin/cluster" in helper + assert "BRANCH=main" in helper + assert "git init --bare" in helper + assert 'touch "${REPO}/git-daemon-export-ok"' in helper, ( + "git-daemon refuses to serve a repository that is not exported" + ) + assert "umask 022" in helper, ( + "git-daemon reads the hostPath mount as a non-root uid" + ) + + +def test_seed_runs_every_phase_in_order_under_the_admin_kubeconfig() -> None: + bootstrap = unit(CLUSTER_BOOTSTRAP) + assert bootstrap["Service"]["Environment"] == [ + "KUBECONFIG=/etc/kubernetes/admin.conf" + ] + applied = [ + command.rsplit(" ", 1)[-1] + for command in exec_starts(CLUSTER_BOOTSTRAP) + if " apply " in command + ] + assert applied == [f"{SEED_ROOT}/{phase}" for phase in SEED_PHASES] + + +def test_seed_applies_server_side_and_never_prunes() -> None: + for command in exec_starts(CLUSTER_BOOTSTRAP): + assert "prune" not in command, ( + f"{command}: kubectl rejects --prune alongside --server-side, and " + "the seed is append-only by design; Argo CD owns pruning downstream" + ) + if " apply " not in command: + continue + assert "--server-side --force-conflicts" in command, command + assert "-k " in command, f"{command} must apply a kustomization" + + +def test_seed_waits_for_the_api_server_before_it_applies_anything() -> None: + """The probe that keeps Wants= diagnosable and the first apply meaningful. + + Relaxing kubeadm-init to ``Wants=`` stopped one transient failure stranding + this unit, but it also removed the single legible + ``Job ... failed with result 'dependency'`` line that made the stranding + visible. Without a probe, a control plane that never arrives shows up as + thirty rounds of kubectl connection errors and then a gate timeout with no + stated cause. + + The probe also sharpens what a failure means: once it passes, the API server + is answering, so anything failing below is a genuine apply failure rather + than a control plane that has not arrived yet. + """ + pre = unit(CLUSTER_BOOTSTRAP)["Service"].get("ExecStartPre", []) + + assert pre, ( + "no ExecStartPre: the seed would start applying against an API server " + "that may not exist yet, and the retries would be unreadable" + ) + probe = " ".join(pre) + assert "/readyz" in probe, ( + "probe the apiserver's /readyz endpoint; it is the cheapest call that " + "proves it is serving and needs no RBAC beyond the admin kubeconfig" + ) + assert "exit 1" in probe, ( + "the probe must fail loudly rather than returning quietly, or a control " + "plane that never arrives is indistinguishable from one that did" + ) + assert probe_bound(unit(CLUSTER_BOOTSTRAP)["Service"]) > 0, ( + "the probe must declare a finite bound. `exit 1` alone is not enough: " + "`until cond; do ...; done; exit 1` has an unreachable failure path and " + "still loops forever, sharing TimeoutStartSec with the apply phases and " + "consuming the whole budget before phase 1 runs — so systemd's kill gets " + "attributed to the seed rather than to the control plane, which is the " + "exact ambiguity this probe exists to remove" + ) + assert "echo" in probe, ( + "the probe must say why it is waiting, or a stalled control plane is " + "indistinguishable from a slow one in the journal" + ) + + +def probe_bound(service: dict[str, list[str]]) -> int: + """Seconds the ExecStartPre probe is allowed to spend, from its own bound. + + The probe bounds itself by attempt count rather than a ``--timeout`` flag, + so this reads ``seq N`` and multiplies by the loop's sleep. Returns 0 when + no bound is declared, which the callers treat as a failure rather than as + "costs nothing" — an unbounded probe costs the entire budget. + """ + total = 0 + for step in service.get("ExecStartPre", []): + attempts = re.findall(r"seq (\d+)", step) + sleeps = re.findall(r"sleep (\d+)", step) + if attempts and sleeps: + total += int(attempts[0]) * int(sleeps[0]) + return total + + +def test_start_timeout_covers_every_bounded_step() -> None: + """A budget smaller than its parts kills the unit mid-chain. + + systemd enforces TimeoutStartSec across ExecStartPre plus every ExecStart. + If the bounded waits sum past it, the unit is killed partway through and the + kill reads as a generic timeout instead of naming the phase that stalled — + the same loss of attribution the probe above exists to prevent. + + Previously the four kubectl waits alone summed to 2100s against a 1800s + budget. + """ + service = unit(CLUSTER_BOOTSTRAP)["Service"] + budget = int(service["TimeoutStartSec"][0]) + + steps = service.get("ExecStartPre", []) + service.get("ExecStart", []) + declared = sum( + int(match) for step in steps for match in re.findall(r"--timeout=(\d+)s", step) + ) + bound = probe_bound(service) + + assert budget > declared + bound, ( + f"TimeoutStartSec={budget}s does not cover the bounded steps " + f"({declared}s of kubectl --timeout plus {bound}s of probe), so " + f"systemd can kill the seed mid-chain and the failure loses its cause" + ) + + +def test_seed_waits_for_the_node_to_leave_notready_before_argo() -> None: + commands = exec_starts(CLUSTER_BOOTSTRAP) + cilium = next(i for i, c in enumerate(commands) if c.endswith("/00-cilium")) + argocd = next( + i for i, c in enumerate(commands) if c.endswith("/10-argocd-core") + ) + waits = [ + c for c in commands[cilium + 1 : argocd] if "wait" in c and " node " in c + ] + assert waits and "--for=condition=Ready" in waits[0], ( + "without CNI the node stays NotReady and carries a NoSchedule taint, " + "so every Argo pod would be Pending forever" + ) + + +def test_seed_waits_for_the_git_bridge_before_handing_over_to_argo() -> None: + commands = exec_starts(CLUSTER_BOOTSTRAP) + gitd = next(i for i, c in enumerate(commands) if c.endswith("/15-gitd")) + root_app = next( + i for i, c in enumerate(commands) if c.endswith("/20-root-app") + ) + waits = commands[gitd + 1 : root_app] + assert any( + "deployment/bluefin-git-daemon" in c and "--namespace bluefin-system" in c + for c in waits + ), ( + "the root Application cannot sync until the daemon serving " + "/var/lib/bluefin/cluster.git is up" + ) + + +def test_seed_is_ordered_after_both_prerequisite_units() -> None: + """Ordering is required for both; hard dependency only for the repo. + + ``bluefin-cluster-repo.service`` is a base-OS unit and the seed has nothing + to apply without the mirrored manifests, so it stays ``Requires=``. + + ``kubeadm-init.service`` must not be, and the distinction is what keeps this + chain self-healing. A job that dies with ``result 'dependency'`` never runs, + so its own ``Restart=on-failure`` never engages and systemd never re-queues + it. Under ``Requires=``, a single failed kubeadm attempt killed this unit + permanently — observed with kubeadm failing at 12:18:13, succeeding on retry + at 12:18:59, and the seed staying inactive forever behind a working control + plane. + """ + bootstrap = unit(CLUSTER_BOOTSTRAP)["Unit"] + + for name in ("kubeadm-init.service", "bluefin-cluster-repo.service"): + assert name in bootstrap["After"], f"{name} must be ordered before the seed" + + assert "bluefin-cluster-repo.service" in bootstrap["Requires"] + assert "kubeadm-init.service" not in bootstrap.get("Requires", []), ( + "a hard requirement on kubeadm-init makes one transient kubeadm failure " + "strand the seed permanently; use Wants= so Restart=on-failure converges" + ) + assert "kubeadm-init.service" in bootstrap["Wants"] + + +def test_kubeadm_config_bootstraps_every_certificate_it_uses() -> None: + documents = [ + document + for document in yaml.safe_load_all( + KUBEADM_CONFIG.read_text(encoding="utf-8") + ) + if document + ] + kinds = {document["kind"]: document for document in documents} + assert set(kinds) == { + "InitConfiguration", + "ClusterConfiguration", + "KubeletConfiguration", + } + + kubelet = kinds["KubeletConfiguration"] + assert kubelet["serverTLSBootstrap"] is True, ( + "kubelet would otherwise generate a self-signed serving certificate" + ) + assert kubelet["rotateCertificates"] is True + assert kubelet["cgroupDriver"] == "systemd" + + +def test_kubeadm_config_does_not_restate_the_kubernetes_version() -> None: + cluster = next( + document + for document in yaml.safe_load_all( + KUBEADM_CONFIG.read_text(encoding="utf-8") + ) + if document and document["kind"] == "ClusterConfiguration" + ) + assert "kubernetesVersion" not in cluster, ( + "kubeadm derives the version from its own binary; restating it here " + "forks the single source of truth in include/kubernetes.yml" + ) + + +def test_kubeadm_config_schedules_workload_on_the_only_node() -> None: + init = next( + document + for document in yaml.safe_load_all( + KUBEADM_CONFIG.read_text(encoding="utf-8") + ) + if document and document["kind"] == "InitConfiguration" + ) + assert init["nodeRegistration"]["taints"] == [], ( + "the default control-plane NoSchedule taint leaves every workload " + "Pending on a single-node appliance" + ) + assert ( + init["nodeRegistration"]["criSocket"] + == "unix:///run/containerd/containerd.sock" + ) + + +def test_cluster_bootstrap_element_stages_units_config_and_helper() -> None: + element = ELEMENT.read_text(encoding="utf-8") + assert "path: files/os/systemd/system" in element + assert "path: files/os/systemd/systemd-sysext.service.d" in element + assert "path: files/kubernetes/kubeadm.yaml" in element + assert "path: files/os/libexec" in element + assert '"${OUT}/usr/share/bluefin/kubeadm.yaml"' in element + assert '"${OUT}/usr/libexec/bluefin-cluster-repo"' in element + assert "bluefin-server/os-cluster-bootstrap.bst" in STACK.read_text( + encoding="utf-8" + ) + + +def test_stack_pulls_in_git_for_the_manifest_bridge() -> None: + stack = STACK.read_text(encoding="utf-8") + assert "freedesktop-sdk.bst:components/git.bst" in stack, ( + "bluefin-cluster-repo.service shells out to git; without it the root " + "Application has nothing to sync from" + ) + assert "bluefin-server/os-cluster-manifests.bst" in stack + + +def test_kubernetes_sysupdate_transfer_is_packaged_as_a_component() -> None: + element = SYSUPDATE_ELEMENT.read_text(encoding="utf-8") + assert "path: files/os/sysupdate.kubernetes.d" in element + assert "target: /usr/lib/sysupdate.kubernetes.d" in element + assert "bluefin-server/os-kubernetes-sysupdate.bst" in STACK.read_text( + encoding="utf-8" + ) + + +def test_installed_os_configures_wired_dhcp_with_networkd() -> None: + assert NETWORK.read_text(encoding="utf-8") == ( + "[Match]\n" + "Name=e*\n" + "\n" + "[Network]\n" + "DHCP=ipv4\n" + ) + assert NETWORK_PRESET.read_text(encoding="utf-8") == ( + "enable systemd-networkd.service\n" + ) + network_element = NETWORK_ELEMENT.read_text(encoding="utf-8") + assert "path: files/os/systemd/network" in network_element + assert "target: /usr/lib/systemd/network" in network_element + assert "bluefin-server/os-networkd.bst" in STACK.read_text(encoding="utf-8") diff --git a/tests/unit/test_cluster_manifests.py b/tests/unit/test_cluster_manifests.py new file mode 100644 index 00000000..2eafb3c5 --- /dev/null +++ b/tests/unit/test_cluster_manifests.py @@ -0,0 +1,400 @@ +"""Contracts for the cluster bootstrap seed and the Argo CD-owned manifest tree. + +The appliance has no addon manager. Ordering, drift correction and pruning all +come from four seed phases plus one root Argo Application, so the properties +worth defending here are: every overlay still builds, every image is immutable, +and nothing in the tree is a credential someone could load. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +CLUSTER = ROOT / "files" / "cluster" +SEED = CLUSTER / "seed" +MANIFESTS = CLUSTER / "manifests" +ELEMENT = ROOT / "elements" / "bluefin-server" / "os-cluster-manifests.bst" + +SEED_PHASES = ("00-cilium", "10-argocd-core", "15-gitd", "20-root-app") + +# Upstream manifests are fetched and sha256-pinned by the BuildStream element +# rather than vendored, so they are absent from a source checkout. Each entry +# maps the missing file to a stand-in carrying the resources the overlay +# actually acts on — the ApplicationSet controller it deletes, and every image +# name its `images:` block claims to pin. Re-derive these when the pinned +# upstream version moves: a stand-in that no longer matches upstream is how a +# silently-unpinned image would slip through. +STAND_INS: dict[str, str] = { + "seed/10-argocd-core/core-install.yaml": """ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: applicationsets.argoproj.io +spec: + group: argoproj.io + names: + kind: ApplicationSet + plural: applicationsets + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-applicationset-controller +spec: + selector: + matchLabels: + app.kubernetes.io/name: argocd-applicationset-controller + template: + metadata: + labels: + app.kubernetes.io/name: argocd-applicationset-controller + spec: + containers: + - name: argocd-applicationset-controller + image: quay.io/argoproj/argocd:v3.5.3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-repo-server +spec: + selector: + matchLabels: + app.kubernetes.io/name: argocd-repo-server + template: + metadata: + labels: + app.kubernetes.io/name: argocd-repo-server + spec: + containers: + - name: argocd-repo-server + image: quay.io/argoproj/argocd:v3.5.3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argocd-redis +spec: + selector: + matchLabels: + app.kubernetes.io/name: argocd-redis + template: + metadata: + labels: + app.kubernetes.io/name: argocd-redis + spec: + containers: + - name: redis + image: public.ecr.aws/docker/library/redis:8.2.3-alpine +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: argocd-application-controller +spec: + serviceName: argocd-application-controller + selector: + matchLabels: + app.kubernetes.io/name: argocd-application-controller + template: + metadata: + labels: + app.kubernetes.io/name: argocd-application-controller + spec: + containers: + - name: argocd-application-controller + image: quay.io/argoproj/argocd:v3.5.3 +""", + "manifests/cert-manager/cert-manager.yaml": """ +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager + namespace: cert-manager +spec: + selector: + matchLabels: + app.kubernetes.io/name: cert-manager + template: + metadata: + labels: + app.kubernetes.io/name: cert-manager + spec: + containers: + - name: cert-manager-controller + image: "quay.io/jetstack/cert-manager-controller:v1.21.2" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-webhook + namespace: cert-manager +spec: + selector: + matchLabels: + app.kubernetes.io/name: webhook + template: + metadata: + labels: + app.kubernetes.io/name: webhook + spec: + containers: + - name: cert-manager-webhook + image: "quay.io/jetstack/cert-manager-webhook:v1.21.2" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-cainjector + namespace: cert-manager +spec: + selector: + matchLabels: + app.kubernetes.io/name: cainjector + template: + metadata: + labels: + app.kubernetes.io/name: cainjector + spec: + containers: + - name: cert-manager-cainjector + image: "quay.io/jetstack/cert-manager-cainjector:v1.21.2" +""", +} + +KUBECTL = shutil.which("kubectl") +requires_kubectl = pytest.mark.skipif( + KUBECTL is None, reason="kubectl is needed to build the kustomize overlays" +) + + +def _overlays() -> list[Path]: + """Every directory in files/cluster/ carrying a kustomization.yaml.""" + return sorted(p.parent for p in CLUSTER.rglob("kustomization.yaml")) + + +@pytest.fixture(scope="module") +def tree(tmp_path_factory: pytest.TempPathFactory) -> Path: + """files/cluster/ with the fetch-pinned upstream manifests stood in for.""" + staged = tmp_path_factory.mktemp("cluster") + shutil.copytree(CLUSTER, staged, dirs_exist_ok=True) + for relative, content in STAND_INS.items(): + target = staged / relative + assert not target.exists(), ( + f"{relative} is vendored into the repository; it is meant to be " + "fetched and sha256-pinned by os-cluster-manifests.bst" + ) + target.write_text(content, encoding="utf-8") + return staged + + +def _build(directory: Path) -> list[dict]: + result = subprocess.run( + [KUBECTL, "kustomize", str(directory)], + capture_output=True, + text=True, + check=False, + # kustomize reaches the network for remote bases and helmCharts. The + # tree is meant to be buildable on a machine with no network at all. + env={"PATH": "/usr/bin:/bin", "HOME": str(directory), "HTTP_PROXY": "http://127.0.0.1:1", "HTTPS_PROXY": "http://127.0.0.1:1"}, + ) + assert result.returncode == 0, f"kustomize build failed for {directory}:\n{result.stderr}" + return [doc for doc in yaml.safe_load_all(result.stdout) if doc] + + +def _images(docs: list[dict]) -> list[str]: + found: list[str] = [] + + def walk(node: object) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if key == "image" and isinstance(value, str): + found.append(value) + else: + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(docs) + return found + + +def test_seed_has_exactly_the_four_documented_phases() -> None: + # bluefin-cluster-bootstrap.service applies these by name, in this order. + # A fifth directory, or a renamed one, silently changes what first boot does. + assert sorted(p.name for p in SEED.iterdir() if p.is_dir()) == list(SEED_PHASES) + for phase in SEED_PHASES: + assert (SEED / phase / "kustomization.yaml").is_file(), ( + f"{phase} is applied with `kubectl apply -k` and needs a kustomization.yaml" + ) + + +@requires_kubectl +@pytest.mark.parametrize("overlay", _overlays(), ids=lambda p: str(p.relative_to(CLUSTER))) +def test_every_overlay_builds_offline(overlay: Path, tree: Path) -> None: + _build(tree / overlay.relative_to(CLUSTER)) + + +@requires_kubectl +@pytest.mark.parametrize("overlay", [*SEED_PHASES, "."], ids=lambda name: f"seed-{name}") +def test_every_seed_image_is_digest_pinned(overlay: str, tree: Path) -> None: + directory = tree / "manifests" if overlay == "." else tree / "seed" / overlay + for image in _images(_build(directory)): + assert "@sha256:" in image, f"{image} is a mutable reference in {directory.name}" + + +@requires_kubectl +def test_applicationset_controller_is_patched_out_but_its_crd_stays(tree: Path) -> None: + docs = _build(tree / "seed" / "10-argocd-core") + workloads = {(d["kind"], d["metadata"]["name"]) for d in docs} + assert ("Deployment", "argocd-applicationset-controller") not in workloads + assert ("CustomResourceDefinition", "applicationsets.argoproj.io") in workloads + # Deleting the wrong thing is the failure mode that matters: Core's other + # three workloads have to survive the patch. + assert ("Deployment", "argocd-repo-server") in workloads + assert ("Deployment", "argocd-redis") in workloads + assert ("StatefulSet", "argocd-application-controller") in workloads + + +@requires_kubectl +def test_root_application_owns_pruning_and_drift(tree: Path) -> None: + docs = _build(tree / "seed" / "20-root-app") + app = next(d for d in docs if d["kind"] == "Application") + project = next(d for d in docs if d["kind"] == "AppProject") + + automated = app["spec"]["syncPolicy"]["automated"] + assert automated["prune"] is True + assert automated["selfHeal"] is True + + # The manifest source must be the in-cluster bridge. An external repoURL + # would make first boot depend on a git host being reachable. + repo = app["spec"]["source"]["repoURL"] + assert repo.startswith("git://bluefin-git-daemon.bluefin-system.svc.cluster.local/") + assert repo in project["spec"]["sourceRepos"] + assert app["spec"]["project"] == project["metadata"]["name"] + + # A project that can mint Applications or AppProjects can escape its own + # scope, so those two kinds stay denied. + denied = {(e["group"], e["kind"]) for e in project["spec"]["namespaceResourceBlacklist"]} + assert ("argoproj.io", "Application") in denied + assert ("argoproj.io", "AppProject") in denied + + +def test_git_daemon_serves_the_bare_repo_read_only() -> None: + docs = list(yaml.safe_load_all((SEED / "15-gitd" / "deployment.yaml").read_text())) + pod = next(d for d in docs if d["kind"] == "Deployment")["spec"]["template"]["spec"] + volume = next(v for v in pod["volumes"] if v["name"] == "cluster-repo") + mount = next( + m for m in pod["containers"][0]["volumeMounts"] if m["name"] == "cluster-repo" + ) + + assert volume["hostPath"]["path"] == "/var/lib/bluefin/cluster.git" + assert mount["readOnly"] is True, "Argo must never be able to write to its own source" + assert pod["securityContext"]["runAsNonRoot"] is True + + +def test_no_loadable_secret_material_in_the_tree() -> None: + # No certificate or private key ships in any image, and no Secret in the + # tree carries a value. Credentials are generated in-cluster on first sync. + pem_markers = ("BEGIN PRIVATE KEY", "BEGIN RSA PRIVATE KEY", "BEGIN EC PRIVATE KEY", "BEGIN CERTIFICATE") + offenders: list[str] = [] + + for path in sorted(CLUSTER.rglob("*")): + if not path.is_file(): + continue + text = path.read_text(encoding="utf-8") + relative = path.relative_to(ROOT) + for marker in pem_markers: + if marker in text: + offenders.append(f"{relative}: contains {marker}") + if path.suffix not in (".yaml", ".yml"): + continue + for doc in yaml.safe_load_all(text): + if not isinstance(doc, dict) or doc.get("kind") != "Secret": + continue + name = doc.get("metadata", {}).get("name") + if doc.get("data") or doc.get("stringData"): + offenders.append(f"{relative}: Secret/{name} carries a value") + + assert offenders == [] + + +def test_console_has_no_developer_bypass() -> None: + # DEV_MODE made the appliance's only dashboard unauthenticated on every + # installed machine, and the shipped jwt-secret made its sessions forgeable + # by anyone who read this repository. + console = (MANIFESTS / "console" / "console.yaml").read_text(encoding="utf-8") + docs = list(yaml.safe_load_all(console)) + deployment = next(d for d in docs if d["kind"] == "Deployment") + env = {e["name"]: e for e in deployment["spec"]["template"]["spec"]["containers"][0]["env"]} + + assert "DEV_MODE" not in env + assert "ALLOW_DEV_MODE_IN_CLUSTER" not in env + assert "DEV_USER_LOGIN" not in env + + jwt = env["JWT_SECRET"]["valueFrom"]["secretKeyRef"] + assert "value" not in env["JWT_SECRET"], "the signing key must never be inline" + assert jwt["optional"] is False, "a console that starts without a signing key is a console with no sessions" + assert jwt["name"] == "kubestellar-console-auth" + + +def test_generated_credentials_are_never_overwritten() -> None: + # Re-running either generator must not rotate a live credential: the + # PostgreSQL password would lock the initialised database out, and the JWT + # key would invalidate every open console session on every re-sync. + for manifest, secret in ( + (MANIFESTS / "kubeflex" / "postgres-secret.yaml", "kubeflex-postgres"), + (MANIFESTS / "console" / "auth-secret.yaml", "kubestellar-console-auth"), + ): + job = next( + d for d in yaml.safe_load_all(manifest.read_text()) if d["kind"] == "Job" + ) + script = job["spec"]["template"]["spec"]["containers"][0]["command"][-1] + assert f"get secret {secret}" in script + assert "exit 0" in script + assert "/dev/urandom" in script + + +def test_element_pins_every_manifest_the_overlays_expect() -> None: + # The overlays name these files in their `resources:`; the element is what + # puts them there. A rename on either side is a build that succeeds and an + # appliance that never syncs. + element = yaml.safe_load(ELEMENT.read_text(encoding="utf-8")) + refs = {s["filename"]: s["ref"] for s in element["sources"] if s["kind"] == "remote"} + + assert refs == { + "core-install.yaml": "1a87025d8eb2eae621653fd312fb9ca51df1b4b3b6992a030e3a9ef38e45c448", + "cert-manager.yaml": "e03b668ec8675214af6b0a671699d088f2601fa3878e0dbe1b41d3feafd1879f", + } + + install = "\n".join(element["config"]["install-commands"]) + for relative in STAND_INS: + destination = relative.replace("seed/", "${SEED}/").replace( + "manifests/", "${CLUSTER}/" + ) + assert destination in install, f"element never stages {relative}" + + # core-install.yaml, not install.yaml: Core drops argocd-server and with it + # a second authenticated web surface on the appliance. + urls = [s["url"] for s in element["sources"] if s["kind"] == "remote"] + assert any(u.endswith("manifests/core-install.yaml") for u in urls) diff --git a/tests/unit/test_console_issue.py b/tests/unit/test_console_issue.py index af83ead1..70513b4a 100644 --- a/tests/unit/test_console_issue.py +++ b/tests/unit/test_console_issue.py @@ -15,7 +15,7 @@ def test_issue_file_exists_and_contains_kubestellar_url() -> None: assert ISSUE_FILE.is_file(), f"{ISSUE_FILE} must exist" content = ISSUE_FILE.read_text(encoding="utf-8") assert "Bluefin Server" in content - assert "KubeStellar Console: http://\\4:8080/" in content + assert "KubeStellar Console: https://\\4:8080/" in content def test_os_issue_element_target_usr_lib_issue_d() -> None: diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index d423c685..e9b0a4a1 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -62,25 +62,27 @@ def test_installer_wrapper_reads_kernel_command_line_without_cat() -> None: assert 'CMDLINE="$(cat /proc/cmdline' not in installer_element -def test_installer_stages_uncompressed_k0s_before_packing_cpio() -> None: +def test_installer_stages_uncompressed_sysext_before_packing_cpio() -> None: + """The sysext has to be in /layer before the initrd is packed, or the + offline installer ships no Kubernetes at all.""" installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") data = yaml.safe_load(installer_element) - k0s_dependency = next( + sysext_dependency = next( ( dependency for dependency in data["build-depends"] if isinstance(dependency, dict) - and dependency.get("filename") == "oci/k0s-sysext.bst" + and dependency.get("filename") == "oci/kubernetes-sysext.bst" ), None, ) - assert k0s_dependency == { - "filename": "oci/k0s-sysext.bst", - "config": {"location": "/k0s"}, + assert sysext_dependency == { + "filename": "oci/kubernetes-sysext.bst", + "config": {"location": "/kubernetes"}, } - seed_command = "cp /k0s/k0s-*.raw /layer/k0s.raw" + seed_command = "cp /kubernetes/kubernetes-*.raw /layer/kubernetes.raw" cpio_command = "| cpio --null --create --format=newc" assert seed_command in installer_element assert installer_element.index(seed_command) < installer_element.index( @@ -92,10 +94,19 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None: ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") assert "freedesktop-sdk.bst:components/kmod.bst" in ddi_element - assert 'depmod -b /layer "${KVER}"' in ddi_element + assert 'depmod -b /layer/usr "${KVER}"' in ddi_element assert "cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/" in ddi_element assert "tls-ca-bundle.pem" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service" in ddi_element + assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service" in ddi_element + justfile = JUSTFILE.read_text(encoding="utf-8") + assert "systemd.mask=systemd-homed-firstboot.service" in justfile + assert "hostfwd=tcp:127.0.0.1:2222-:22" in justfile + assert "systemd.wants=sshd.service" in justfile + assert "ssh.authorized_keys.root=" in justfile + assert "find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst'" in justfile + assert 'if [ "$ROOT_CODE" = "200" ]; then' in justfile + assert '[ "$ROOT_CODE" = "503" ]' not in justfile assert "ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service" in ddi_element assert "printf '127.0.0.1 localhost" in ddi_element assert "> /layer/etc/hosts" in ddi_element diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py deleted file mode 100644 index a185a611..00000000 --- a/tests/unit/test_k0s_first_boot.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Contracts for retry-safe k0s sysext first-boot activation.""" - -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[2] -SERVICE = ( - ROOT / "files" / "os" / "systemd" / "system" / "k0s-first-boot.service" -) -FETCH_SERVICE = ( - ROOT - / "files" - / "os" - / "systemd" - / "system" - / "k0s-first-boot-fetch.service" -) -PRESET = ( - ROOT - / "files" - / "os" - / "systemd" - / "system-preset" - / "zz-enable-k0s-first-boot.preset" -) -NETWORK = ROOT / "files" / "os" / "systemd" / "network" / "20-wired.network" -NETWORK_PRESET = ( - ROOT - / "files" - / "os" - / "systemd" - / "system-preset" - / "zz-enable-networkd.preset" -) -ELEMENT = ROOT / "elements" / "bluefin-server" / "os-k0s-first-boot.bst" -NETWORK_ELEMENT = ROOT / "elements" / "bluefin-server" / "os-networkd.bst" -K0S_UPDATE_ELEMENT = ( - ROOT / "elements" / "bluefin-server" / "os-k0s-sysupdate.bst" -) -STACK = ROOT / "elements" / "bluefin-server" / "os-stack.bst" - - -def test_seeded_install_skips_the_network_fetch() -> None: - assert FETCH_SERVICE.is_file(), "the conditional k0s fetch unit is missing" - fetch_service = FETCH_SERVICE.read_text(encoding="utf-8") - first_boot = SERVICE.read_text(encoding="utf-8") - - assert "ConditionPathExists=!/var/lib/k0s/k0s.raw" in fetch_service - assert ( - "ExecStart=/usr/bin/systemd-sysupdate --component=k0s update" - in fetch_service - ) - assert "systemd-sysupdate" not in first_boot - - -def test_missing_seed_fetches_before_activation_without_blocking_activation_retry() -> None: - assert FETCH_SERVICE.is_file(), "the conditional k0s fetch unit is missing" - fetch_service = FETCH_SERVICE.read_text(encoding="utf-8") - first_boot = SERVICE.read_text(encoding="utf-8") - - assert "Type=oneshot" in fetch_service - assert "Restart=on-failure" in fetch_service - assert "ConditionPathExists=!/var/lib/k0s/k0s.raw" in fetch_service - assert "Before=k0s-first-boot.service" in fetch_service - assert "Wants=k0s-first-boot-fetch.service" in first_boot - assert "Requires=k0s-first-boot-fetch.service" not in first_boot - assert "After=k0s-first-boot-fetch.service" in first_boot - assert ( - "ExecStartPre=/usr/bin/test -e /var/lib/k0s/k0s.raw" - in first_boot - ) - assert ( - "ExecStart=/usr/bin/install -D -m 0644 " - "/var/lib/k0s/k0s.raw /run/extensions/k0s.raw" - ) in first_boot - assert "ExecStart=/usr/bin/systemd-sysext merge" in first_boot - - -def test_k0s_first_boot_retries_until_controller_starts() -> None: - service = SERVICE.read_text(encoding="utf-8") - - assert "ConditionPathExists=" not in service - assert "Wants=network-online.target" in service - assert "After=network-online.target" in service - assert "Before=multi-user.target" not in service - assert "Type=oneshot" in service - assert "StateDirectory=k0s" in service - assert "Restart=on-failure" in service - assert ( - "ExecStart=/usr/bin/systemd-sysupdate --component=k0s update" - not in service - ) - assert "ExecStart=/usr/bin/systemd-sysupdate update" not in service - assert "ExecStart=/usr/bin/systemctl enable --now systemd-sysext.service" not in service - assert "ExecStart=/usr/bin/systemd-sysext merge" in service - assert ( - "ExecStart=/usr/bin/systemd-tmpfiles --create " - "/usr/lib/tmpfiles.d/k0s-manifests.conf" - ) in service - assert "ExecStart=/usr/bin/systemctl daemon-reload" in service - assert ( - "ExecStart=/usr/bin/systemctl enable --now k0scontroller.service" - in service - ) - assert ( - "ExecStartPost=/usr/bin/touch /var/lib/k0s/.first-boot-complete" - not in service - ) - assert "ConditionFirstBoot" not in service - - -def test_k0s_first_boot_is_packaged_and_enabled() -> None: - assert ( - PRESET.read_text(encoding="utf-8") - == "enable k0s-first-boot.service\n" - ) - assert "path: files/os/systemd/system" in ELEMENT.read_text(encoding="utf-8") - assert "target: /usr/lib/systemd/system" in ELEMENT.read_text(encoding="utf-8") - assert ( - "bluefin-server/os-k0s-first-boot.bst" - in STACK.read_text(encoding="utf-8") - ) - - -def test_installed_os_configures_wired_dhcp_with_networkd() -> None: - assert NETWORK.read_text(encoding="utf-8") == ( - "[Match]\n" - "Name=e*\n" - "\n" - "[Network]\n" - "DHCP=ipv4\n" - ) - assert NETWORK_PRESET.read_text(encoding="utf-8") == ( - "enable systemd-networkd.service\n" - ) - network_element = NETWORK_ELEMENT.read_text(encoding="utf-8") - assert "path: files/os/systemd/network" in network_element - assert "target: /usr/lib/systemd/network" in network_element - assert "bluefin-server/os-networkd.bst" in STACK.read_text(encoding="utf-8") - - -def test_k0s_sysupdate_transfer_is_packaged_as_a_component() -> None: - element = K0S_UPDATE_ELEMENT.read_text(encoding="utf-8") - assert "path: files/os/sysupdate.k0s.d" in element - assert "target: /usr/lib/sysupdate.k0s.d" in element - assert ( - "bluefin-server/os-k0s-sysupdate.bst" - in STACK.read_text(encoding="utf-8") - ) diff --git a/tests/unit/test_k0s_manifests.py b/tests/unit/test_k0s_manifests.py deleted file mode 100644 index 149ca421..00000000 --- a/tests/unit/test_k0s_manifests.py +++ /dev/null @@ -1,92 +0,0 @@ -from pathlib import Path - -import os -import subprocess - -import yaml - -ROOT = Path(__file__).resolve().parents[2] - - -def test_k0s_service_unit(): - unit = ROOT / "files" / "k0s" / "sysext" / "k0scontroller.service" - assert unit.is_file(), "k0scontroller.service missing" - text = unit.read_text() - assert "--disable-components=helm,autopilot" in text - assert "--enable-worker" in text - assert "--single" in text - - -def test_k0s_manifests_conf(): - conf = ROOT / "files" / "k0s" / "sysext" / "k0s-manifests.conf" - assert conf.is_file(), "k0s-manifests.conf missing" - text = conf.read_text() - assert "d /var/lib/k0s/manifests 0755 root root - -" in text - assert "C+ /var/lib/k0s/manifests/argocd - - - - /usr/share/k0s/manifests/argocd" in text - assert "C+ /var/lib/k0s/manifests/kubestellar - - - - /usr/share/k0s/manifests/kubestellar" in text - - -def test_k0s_manifest_files(): - argo_yaml = ROOT / "files" / "k0s" / "manifests" / "argocd" / "install.yaml" - assert argo_yaml.is_file(), "argocd install.yaml missing" - assert "namespace: argocd" in argo_yaml.read_text() - - ks_dir = ROOT / "files" / "k0s" / "manifests" / "kubestellar" - assert (ks_dir / "00-kubeflex-crds.yaml").is_file() - assert (ks_dir / "10-kubeflex-operator.yaml").is_file() - assert (ks_dir / "20-postgres.yaml").is_file() - assert (ks_dir / "30-kubestellar-core.yaml").is_file() - assert (ks_dir / "40-kubestellar-console.yaml").is_file() - assert (ks_dir / "41-kubestellar-kiosk-proxy.yaml").is_file() - - -def test_postgres_password_not_hardcoded(): - # #98: the postgres superuser password must not be committed to git in - # plaintext, and must not be the publicly documented kubeflex default. - manifest = ROOT / "files" / "k0s" / "manifests" / "kubestellar" / "20-postgres.yaml" - text = manifest.read_text() - assert "kubeflex" not in text.lower().replace("kubeflex-system", "").replace("kubeflex-postgres", "") - assert 'value: "kubeflex"' not in text - assert "POSTGRESQL_PASSWORD" in text - - -def test_postgres_password_from_secret(): - # The StatefulSet reads the password from a Secret, not a plaintext env. - docs = list(yaml.safe_load_all((ROOT / "files" / "k0s" / "manifests" / "kubestellar" / "20-postgres.yaml").read_text())) - statefulset = next(d for d in docs if d and d.get("kind") == "StatefulSet") - container = statefulset["spec"]["template"]["spec"]["containers"][0] - env = {e["name"]: e for e in container["env"]} - assert "value" not in env["POSTGRESQL_PASSWORD"] - ref = env["POSTGRESQL_PASSWORD"]["valueFrom"]["secretKeyRef"] - assert ref == {"name": "kubeflex-postgres", "key": "password"} - - -def test_k0s_first_boot_generates_postgres_secret_before_k0s(): - # The Secret must be staged before k0s applies the manifests, and the - # generated 15- file must sort before 20-postgres.yaml. - unit = ROOT / "files" / "os" / "systemd" / "system" / "k0s-first-boot.service" - text = unit.read_text() - lines = [l for l in text.splitlines() if l.startswith("ExecStart")] - gen = next((i for i, l in enumerate(lines) if "generate-postgres-secret.sh" in l), None) - k0s = next((i for i, l in enumerate(lines) if "k0scontroller.service" in l), None) - assert gen is not None, "first-boot service never runs the postgres secret generator" - assert k0s is not None, "first-boot service never starts k0scontroller" - assert gen < k0s, "postgres secret generator must run before k0s applies manifests" - - -def test_generate_postgres_secret_is_idempotent(tmp_path): - # Running the generator twice must not change an already-created password, - # so the initialized database stays accessible across re-boots. - script = ROOT / "files" / "k0s" / "kubeflex" / "generate-postgres-secret.sh" - env = dict(os.environ, KUBEFLEX_MANIFEST_DIR=str(tmp_path)) - run = lambda: subprocess.run(["/bin/bash", str(script)], env=env, check=True, capture_output=True, text=True) - run() - secret_file = tmp_path / "15-kubeflex-postgres-secret.yaml" - assert secret_file.is_file() - secret = yaml.safe_load(secret_file.read_text()) - assert secret["kind"] == "Secret" - assert secret["metadata"]["name"] == "kubeflex-postgres" - assert secret["stringData"]["password"] - first = secret_file.read_text() - run() - assert secret_file.read_text() == first, "password changed on re-run; DB would lose access" diff --git a/tests/unit/test_k0s_version.py b/tests/unit/test_k0s_version.py deleted file mode 100644 index ddf83633..00000000 --- a/tests/unit/test_k0s_version.py +++ /dev/null @@ -1,52 +0,0 @@ -import importlib.util -import re -from pathlib import Path -import yaml - -ROOT = Path(__file__).resolve().parents[2] - - -def test_k0s_version_ssot(): - k0s_yml = ROOT / "include" / "k0s.yml" - assert k0s_yml.is_file(), "include/k0s.yml missing" - data = yaml.safe_load(k0s_yml.read_text()) - vars_ = data.get("variables", {}) - assert vars_.get("k0s-k8s-version") == "1.36.4" - assert vars_.get("k0s-patch") == "0" - assert vars_.get("k0s-version") == "%{k0s-k8s-version}-k0s.%{k0s-patch}" - - -def test_k0s_bin_element(): - bin_bst = ROOT / "elements" / "k0s" / "k0s-bin.bst" - assert bin_bst.is_file(), "elements/k0s/k0s-bin.bst missing" - content = bin_bst.read_text() - assert "base/base-stack.bst" in content - assert "github:k0sproject/k0s/releases/download/" in content - assert "ca1e9e68107335846e8296777fce2ccd654284e6265b4b5d32c34ead872af98f" in content - - -def test_k0s_version_checker_reads_the_component_transfer(): - checker = ROOT / ".github" / "scripts" / "check-k0s-version.py" - spec = importlib.util.spec_from_file_location("check_k0s_version", checker) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - assert module.TRANSFER == ( - ROOT / "files" / "os" / "sysupdate.k0s.d" / "70-k0s.transfer" - ) - - -def test_k0s_version_pre_commit_hook_selects_the_relocated_transfer(): - config = yaml.safe_load((ROOT / ".pre-commit-config.yaml").read_text()) - hook = next( - hook - for repo in config["repos"] - if repo.get("repo") == "local" - for hook in repo.get("hooks", []) - if hook.get("id") == "check-k0s-version" - ) - selector = re.compile(hook["files"]) - assert ( - selector.fullmatch("files/os/sysupdate.k0s.d/70-k0s.transfer") is not None - ) - assert selector.fullmatch("files/os/sysupdate.d/70-k0s.transfer") is None diff --git a/tests/unit/test_kubeadm_config_documents.py b/tests/unit/test_kubeadm_config_documents.py new file mode 100644 index 00000000..ef7c1755 --- /dev/null +++ b/tests/unit/test_kubeadm_config_documents.py @@ -0,0 +1,104 @@ +"""`files/kubernetes/kubeadm.yaml` must parse the way kubeadm parses it. + +kubeadm reads `--config` through `k8s.io/apimachinery/pkg/util/yaml`, whose +YAMLReader splits the stream on lines that are exactly ``---`` and decodes each +chunk independently. Every chunk must carry both ``apiVersion`` and ``kind``; +an empty one is rejected with:: + + error: invalid configuration for GroupVersionKind /, Kind=: kind and + apiVersion is mandatory information that must be specified + +That is not a hypothetical. The file opened with a fourteen-line comment header +followed by a ``---`` separator, which made the header a document of its own. +`kubeadm init` failed on every boot, `kubeadm-init.service` restart-looped, +`bluefin-cluster-bootstrap.service` never ran because it declares +``Requires=kubeadm-init.service``, and the installer smoke gate timed out after +900 s waiting for a console that could never come up. Nothing else caught it: +the element builds, `just validate` resolves, and the unit suite passed. + +**This module deliberately does not use ``yaml.safe_load_all``.** PyYAML is +more forgiving than the Go splitter: on the broken file it returned three clean +documents and no empty one, so a PyYAML-based gate would have passed while +kubeadm kept failing. Emulating the splitter is the whole point. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +KUBEADM_CONFIG = ROOT / "files" / "kubernetes" / "kubeadm.yaml" + +# k8s.io/apimachinery/pkg/util/yaml YAMLReader: a separator is a line that is +# exactly `---`, optionally followed by whitespace. +SEPARATOR = re.compile(r"(?m)^---\s*$") + + +def chunks() -> list[str]: + return SEPARATOR.split(KUBEADM_CONFIG.read_text(encoding="utf-8")) + + +def test_config_exists() -> None: + assert KUBEADM_CONFIG.is_file(), f"{KUBEADM_CONFIG.relative_to(ROOT)} is missing" + + +def test_every_document_kubeadm_sees_has_apiversion_and_kind() -> None: + parts = chunks() + + assert parts, "the splitter produced no chunks at all" + + for index, chunk in enumerate(parts): + document = yaml.safe_load(chunk) if chunk.strip() else None + + assert document is not None, ( + f"chunk {index} of {KUBEADM_CONFIG.relative_to(ROOT)} carries no " + f"YAML nodes — only blank lines or comments. kubeadm decodes each " + f"`---`-separated chunk independently and rejects an empty one with " + f"'invalid configuration for GroupVersionKind /, Kind=: kind and " + f"apiVersion is mandatory'. A comment block above the first `---` " + f"becomes exactly this. Chunk content:\n{chunk[:200]}" + ) + assert document.get("apiVersion"), ( + f"chunk {index} has no apiVersion; kubeadm requires one per document" + ) + assert document.get("kind"), ( + f"chunk {index} has no kind; kubeadm requires one per document" + ) + + +def test_the_file_does_not_open_with_a_separator() -> None: + """The specific shape that produced the empty leading document. + + Stated directly as well as through the splitter, because this is the form + the mistake actually takes: a header comment, then `---`, then the first + real document. + """ + meaningful = [ + line + for line in KUBEADM_CONFIG.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + + assert meaningful, f"{KUBEADM_CONFIG.relative_to(ROOT)} has no content" + assert meaningful[0].strip() != "---", ( + "the first non-comment line is a `---` separator, so everything above " + "it becomes an empty document and kubeadm init fails. Drop the leading " + "separator; the header comment belongs to the first real document." + ) + + +@pytest.mark.parametrize("required", ["InitConfiguration", "ClusterConfiguration"]) +def test_the_kinds_kubeadm_init_needs_are_present(required: str) -> None: + kinds = { + (yaml.safe_load(c) or {}).get("kind") + for c in chunks() + if c.strip() + } + assert required in kinds, ( + f"{required} is missing; `kubeadm init --config` needs it to build the " + f"control plane. Found: {sorted(k for k in kinds if k)}" + ) diff --git a/tests/unit/test_kubernetes_version.py b/tests/unit/test_kubernetes_version.py new file mode 100644 index 00000000..3939075b --- /dev/null +++ b/tests/unit/test_kubernetes_version.py @@ -0,0 +1,146 @@ +"""Version-axis invariants for the Kubernetes sysext. + +These assert the properties that hold the axis together — that every consumer +derives its version from ``include/kubernetes.yml``, that the derived spellings +agree with each other, and that the pre-commit gate actually fires on every +file the checker reads — rather than restating the current contents of any one +file. +""" + +import importlib.util +import re +import subprocess +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / ".github" / "scripts" / "check-kubernetes-version.py" + +K8S_VERSION = "1.36.4" +CNI_VERSION = "1.9.1" + + +def _checker(): + spec = importlib.util.spec_from_file_location("check_kubernetes_version", CHECKER) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _atoms(): + data = yaml.safe_load((ROOT / "include" / "kubernetes.yml").read_text()) + return data["variables"] + + +def test_include_declares_the_two_atoms_the_axis_is_built_from(): + variables = _atoms() + assert variables["k8s-version"] == K8S_VERSION + assert variables["cni-version"] == CNI_VERSION + + +def test_derived_tags_expand_to_the_pinned_upstream_identifiers(): + checker = _checker() + variables = _atoms() + atoms = {k: variables[k] for k in ("k8s-version", "cni-version")} + assert ( + checker.expand(variables["k8s-upstream-tag"], atoms, "test") + == f"v{K8S_VERSION}" + ) + assert ( + checker.expand(variables["cni-upstream-tag"], atoms, "test") + == f"v{CNI_VERSION}" + ) + + +def test_version_axis_is_closed_over_its_own_atoms(): + """The FSDK point release must not be able to drag Kubernetes with it. + + Every derived spelling has to resolve using only k8s-version and + cni-version. If one reached for %{release-version} — or any other + project.conf variable — bumping the OS would move the Kubernetes sysext. + """ + checker = _checker() + variables = _atoms() + atoms = {k: variables[k] for k in ("k8s-version", "cni-version")} + for name, value in variables.items(): + if name in atoms: + continue + checker.expand(value, atoms, f"include/kubernetes.yml {name}") + + +def test_every_node_binary_is_fetched_from_the_same_derived_release(): + """kubeadm, kubelet and kubectl skewing apart is a supported-skew violation.""" + checker = _checker() + urls = checker.source_urls(checker.read(checker.K8S_BIN)) + fetched = {url.rsplit("/", 1)[1] for url in urls} + assert fetched == set(checker.NODE_BINARIES) + assert all("%{k8s-upstream-tag}" in url for url in urls) + + +def test_no_consumer_restates_a_literal_version(): + checker = _checker() + for path in ( + checker.K8S_BIN, + checker.CNI_BIN, + checker.K8S_SYSEXT, + checker.EXTENSION_RELEASE, + ): + for line in checker.read(path).splitlines(): + if line.lstrip().startswith("#"): + continue + assert not checker.LITERAL_VERSION_RE.search(line), ( + f"{path.relative_to(ROOT)}: {line.strip()}" + ) + + +def test_extension_release_defers_version_id_to_build_time(): + """A literal VERSION_ID would silently win over the generated one.""" + checker = _checker() + text = checker.read(checker.EXTENSION_RELEASE) + assert "NAME=kubernetes" in text + assert not any( + line.strip().startswith("VERSION_ID=") for line in text.splitlines() + ) + + +def test_checker_reads_the_component_scoped_transfer(): + checker = _checker() + assert checker.TRANSFER == ( + ROOT / "files" / "os" / "sysupdate.kubernetes.d" / "70-kubernetes.transfer" + ) + + +def test_pre_commit_hook_fires_on_every_file_the_checker_reads(): + checker = _checker() + config = yaml.safe_load((ROOT / ".pre-commit-config.yaml").read_text()) + hook = next( + hook + for repo in config["repos"] + if repo.get("repo") == "local" + for hook in repo.get("hooks", []) + if hook.get("id") == "check-kubernetes-version" + ) + selector = re.compile(hook["files"]) + consumers = [ + checker.K8S_INCLUDE, + checker.K8S_BIN, + checker.CNI_BIN, + checker.K8S_SYSEXT, + checker.EXTENSION_RELEASE, + checker.TRANSFER, + Path(CHECKER), + ] + for path in consumers: + relative = path.relative_to(ROOT).as_posix() + assert selector.fullmatch(relative) is not None, relative + assert selector.fullmatch("files/os/sysupdate.d/70-kubernetes.transfer") is None + + +def test_repository_satisfies_the_version_invariant(): + result = subprocess.run( + [sys.executable, str(CHECKER)], capture_output=True, text=True + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/unit/test_kubestellar_kiosk.py b/tests/unit/test_kubestellar_kiosk.py index ec608c72..3a1ff2ec 100644 --- a/tests/unit/test_kubestellar_kiosk.py +++ b/tests/unit/test_kubestellar_kiosk.py @@ -1,109 +1,142 @@ -"""Contracts for the KubeStellar kiosk proxy and authenticated agent gate.""" +"""Contracts for the KubeStellar kiosk proxy and its authenticated agent gate. +The kiosk is the appliance's only dashboard and its only public port. What is +defended here is that it is reachable, that it terminates TLS with material it +generated itself, that the gate it injects actually reaches the browser, and +that it reads nothing off the node. +""" + +from __future__ import annotations + +import re from pathlib import Path -ROOT = Path(__file__).resolve().parents[2] -KIOSK = ROOT / "files" / "k0s" / "kiosk" -KIOSK_CONF = KIOSK / "nginx.conf" -KIOSK_JS = KIOSK / "kiosk-gate.js" -KIOSK_CSS = KIOSK / "kiosk-gate.css" -TMPFILES = ROOT / "files" / "k0s" / "sysext" / "k0s-manifests.conf" -SYSEXT = ROOT / "elements" / "oci" / "k0s-sysext.bst" -CONSOLE_MANIFEST = ( - ROOT - / "files" - / "k0s" - / "manifests" - / "kubestellar" - / "40-kubestellar-console.yaml" -) -PROXY_MANIFEST = ( - ROOT - / "files" - / "k0s" - / "manifests" - / "kubestellar" - / "41-kubestellar-kiosk-proxy.yaml" -) - - -def test_kiosk_assets_are_packaged_and_seeded() -> None: - sysext = SYSEXT.read_text(encoding="utf-8") - tmpfiles = TMPFILES.read_text(encoding="utf-8") - - assert KIOSK_CONF.is_file() - assert KIOSK_JS.is_file() - assert KIOSK_CSS.is_file() - assert "freedesktop-sdk.bst:components/openssl.bst" in sysext - assert "keyout sysext/usr/share/k0s/kiosk/key.pem" in sysext - assert "path: files/k0s/kiosk" in sysext - assert "directory: kiosk-src" in sysext - assert "cp -a kiosk-src/. sysext/usr/share/k0s/kiosk/" in sysext - assert "C+ /var/lib/k0s/kiosk - - - - /usr/share/k0s/kiosk" in tmpfiles - - -def test_proxy_injects_only_csp_safe_same_origin_assets() -> None: - nginx = KIOSK_CONF.read_text(encoding="utf-8") +import yaml +ROOT = Path(__file__).resolve().parents[2] +KIOSK = ROOT / "files" / "cluster" / "manifests" / "kiosk" +ASSETS = KIOSK / "assets" +NGINX_CONF = ASSETS / "nginx.conf" +GATE_JS = ASSETS / "kiosk-gate.js" +GATE_CSS = ASSETS / "kiosk-gate.css" + + +def _doc(path: Path, kind: str) -> dict: + return next(d for d in yaml.safe_load_all(path.read_text(encoding="utf-8")) if d["kind"] == kind) + + +def test_assets_are_delivered_from_the_cluster_not_the_node() -> None: + # These three files used to arrive through a hostPath mount seeded by + # tmpfiles.d, which meant the workload's configuration lived outside the + # cluster and could not be reconciled or rolled. + kustomization = yaml.safe_load((KIOSK / "kustomization.yaml").read_text(encoding="utf-8")) + generator = kustomization["configMapGenerator"][0] + assert sorted(generator["files"]) == [ + "assets/kiosk-gate.css", + "assets/kiosk-gate.js", + "assets/nginx.conf", + ] + + pod = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"] + assert all("hostPath" not in volume for volume in pod["volumes"]) + assets = next(v for v in pod["volumes"] if v["name"] == "assets") + assert assets["configMap"]["name"] == generator["name"] + + +def test_nginx_writes_only_to_mounted_volumes() -> None: + # The container root filesystem is read-only. nginx opens its pid file and + # its proxy scratch paths at startup, so any path that is not on a writable + # mount is a pod that crash-loops before it ever serves a request. + container = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"]["containers"][0] + assert container["securityContext"]["readOnlyRootFilesystem"] is True + + writable = { + mount["mountPath"] + for mount in container["volumeMounts"] + if not mount.get("readOnly", False) + } + assert writable, "a read-only root filesystem needs somewhere for nginx to write" + + conf = NGINX_CONF.read_text(encoding="utf-8") + paths = re.findall(r"^\s*(?:pid|\w+_temp_path)\s+(\S+?);", conf, flags=re.MULTILINE) + assert paths, "nginx.conf declares no pid or temp paths to check" + for path in paths: + assert any(path.startswith(f"{root}/") for root in writable), ( + f"{path} is not on a writable mount ({sorted(writable)})" + ) + + +def test_proxy_injects_the_gate_same_origin_into_live_html() -> None: + conf = NGINX_CONF.read_text(encoding="utf-8") + + # The gate is injected by rewriting the console's HTML. nginx cannot rewrite + # a compressed response body, so upstream compression has to be refused or + # sub_filter silently does nothing and the gate never appears. + assert 'proxy_set_header Accept-Encoding "";' in conf + assert '' in conf + assert '' in conf + + # Both assets are served by this proxy on its own origin, so no CSP the + # console sets can block them and no third-party host is contacted. + for asset in ("kiosk-gate.js", "kiosk-gate.css"): + assert f"location = /{asset} {{" in conf + assert f"alias /etc/kubestellar-kiosk/assets/{asset};" in conf + + # CoreDNS on the kubeadm service CIDR; resolved at request time so the + # console Service moving does not require restarting the proxy. + assert "resolver 10.96.0.10" in conf assert ( - "proxy_pass " - "http://kubestellar-console.kubestellar-console.svc.cluster.local:8080;" - ) in nginx - assert "proxy_redirect" in nginx - assert 'proxy_set_header Accept-Encoding "";' in nginx - assert "sub_filter_types" not in nginx - assert ( - 'sub_filter \'\' ' - '\'\';' - ) in nginx - assert ( - 'sub_filter \'\' ' - '\'\';' - ) in nginx - assert "Content-Security-Policy" not in nginx + "http://kubestellar-console.kubestellar-console.svc.cluster.local:8080" in conf + ) -def test_gate_waits_for_session_and_blocks_until_agent_health() -> None: - script = KIOSK_JS.read_text(encoding="utf-8") - css = KIOSK_CSS.read_text(encoding="utf-8") +def test_gate_blocks_the_console_until_the_agent_is_reachable() -> None: + script = GATE_JS.read_text(encoding="utf-8") + css = GATE_CSS.read_text(encoding="utf-8") assert "kc-has-session" in script assert "http://127.0.0.1:8585/health" in script - assert "brew tap kubestellar/tap && brew install kc-agent" in script - assert ( - 'KAGENTI_CONTROLLER_URL="none" kc-agent -allowed-origins' - " ${window.location.origin}" - ) in script assert "aria-modal" in script + # The overlay has to actually trap interaction, or it is decoration. assert "addEventListener('keydown'" in script assert "pointer-events: auto" in css assert "z-index: 2147483647" in css -def test_console_provides_local_and_oauth_login_options() -> None: - console = CONSOLE_MANIFEST.read_text(encoding="utf-8") - - assert "name: DEV_MODE" in console - assert 'value: "true"' in console - assert "name: ALLOW_DEV_MODE_IN_CLUSTER" in console - assert "hostPort:" not in console - assert "name: GITHUB_CLIENT_ID" in console - assert "name: GITHUB_CLIENT_SECRET" in console - assert "name: kubestellar-console-github-oauth" in console - assert "key: client-id" in console - assert "key: client-secret" in console - assert "optional: true" in console - - -def test_proxy_is_the_only_public_console_endpoint() -> None: - proxy = PROXY_MANIFEST.read_text(encoding="utf-8") - - assert "name: kubestellar-kiosk-proxy" in proxy - assert "hostPort: 8080" in proxy - assert "mountPath: /etc/kubestellar-kiosk" in proxy - assert "hostPath:\n path: /var/lib/k0s/kiosk" in proxy - assert "readOnly: true" in proxy - assert ( - "nginx@sha256:62223d644fa234c3a1cc785ee14242ec47a77364226f1c811d2f669f96dc2ac8" - in proxy - ) +def test_tls_is_issued_in_cluster_and_short_lived() -> None: + # Replaces the 10-year self-signed key that was generated at image build + # time and was therefore identical on every installed appliance. + certificate = _doc(KIOSK / "certificate.yaml", "Certificate")["spec"] + assert certificate["issuerRef"]["kind"] == "ClusterIssuer" + assert certificate["issuerRef"]["name"] == "bluefin-ca" + assert certificate["privateKey"]["rotationPolicy"] == "Always" + + hours = int(certificate["duration"].removesuffix("h")) + assert hours <= 2400, "the kiosk certificate must stay under 100 days" + assert int(certificate["renewBefore"].removesuffix("h")) < hours + + container = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"]["containers"][0] + tls = next(m for m in container["volumeMounts"] if m["name"] == "tls") + conf = NGINX_CONF.read_text(encoding="utf-8") + assert f"ssl_certificate {tls['mountPath']}/tls.crt;" in conf + assert f"ssl_certificate_key {tls['mountPath']}/tls.key;" in conf + assert "listen 8080 ssl;" in conf + # Anyone arriving over plain HTTP on the TLS port gets redirected rather + # than an nginx error page. + assert "error_page 497 https://$http_host$request_uri;" in conf + + +def test_proxy_is_published_on_every_interface() -> None: + # Binding hostIP 127.0.0.1 made the console unreachable from the LAN and + # from anything but a single QEMU hostfwd. The endpoint is HTTPS-only and + # the console behind it authenticates. + container = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"]["containers"][0] + port = next(p for p in container["ports"] if p.get("hostPort")) + assert port["hostPort"] == 8080 + assert port["containerPort"] == 8080 + assert "hostIP" not in port + + +def test_proxy_image_is_immutable() -> None: + container = _doc(KIOSK / "deployment.yaml", "Deployment")["spec"]["template"]["spec"]["containers"][0] + assert "@sha256:" in container["image"] diff --git a/tests/unit/test_manual_element_strip_contract.py b/tests/unit/test_manual_element_strip_contract.py new file mode 100644 index 00000000..574312a7 --- /dev/null +++ b/tests/unit/test_manual_element_strip_contract.py @@ -0,0 +1,103 @@ +"""A staging-only `kind: manual` element must disable the binary stripper. + +FSDK's `manual` element runs default `strip-commands` after install. Those +commands shell out to `freedesktop-sdk-stripper`, a tool that arrives in the +sandbox with the FSDK build-system stacks — not with `base/base-stack.bst`. + +An element that only copies files into `%{install-root}` therefore has nothing +to strip *and* no stripper to run it, so the default turns a no-op into +`sh: freedesktop-sdk-stripper: command not found` and the build dies with +exit 127. The repository's answer is `variables: strip-binaries: ""`. + +`just validate` cannot catch this. It runs `bst show --deps all`, which +resolves the graph without executing a single build command, so a missing +`strip-binaries` override shows up only after a full `bst build` — eight +minutes into CI, and only for whoever pushes next. + +That is not hypothetical: `os-cluster-bootstrap.bst` and +`os-cluster-manifests.bst` both shipped without the override and both failed +at exit 127 in the `Build DDI artifacts` run for the Kubernetes cutover, after +`just validate` had passed clean. + +The discriminator is `config.build-commands`. An element that declares build +commands ran a compiler, so it has real ELF output and wants the stripper — +`uutils-coreutils.bst` is the one such element here, and it correctly leaves +the default alone. An element with no build commands only stages, and must +opt out. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ELEMENTS_DIR = ROOT / "elements" + + +def _manual_elements(): + for path in sorted(ELEMENTS_DIR.glob("**/*.bst")): + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if data.get("kind") == "manual": + yield path, data + + +STAGING_ONLY = [ + (path, data) + for path, data in _manual_elements() + if not (data.get("config") or {}).get("build-commands") +] + +COMPILING = [ + (path, data) + for path, data in _manual_elements() + if (data.get("config") or {}).get("build-commands") +] + + +@pytest.mark.parametrize( + "path,data", STAGING_ONLY, ids=[p.relative_to(ROOT).as_posix() for p, _ in STAGING_ONLY] +) +def test_staging_only_manual_elements_disable_the_stripper(path: Path, data: dict) -> None: + strip = (data.get("variables") or {}).get("strip-binaries", None) + + assert strip == "", ( + f"{path.relative_to(ROOT)} is a kind: manual element with no " + f"config.build-commands, so it only stages files and no stripper is in " + f"its sandbox. It must set:\n\n" + f" variables:\n" + f" strip-binaries: \"\"\n\n" + f"Without it the build fails with " + f"'freedesktop-sdk-stripper: command not found' (exit 127), which " + f"`just validate` cannot detect because it never runs a build command. " + f"Got strip-binaries={strip!r}." + ) + + +def test_compiling_manual_elements_keep_the_default_stripper() -> None: + # Guards the discriminator itself: if this list empties, the rule above has + # silently become unconditional and stops distinguishing anything. + assert COMPILING, ( + "no kind: manual element declares config.build-commands; the " + "staging-only rule above is no longer discriminating and its " + "build-commands check needs re-deriving" + ) + + for path, data in COMPILING: + strip = (data.get("variables") or {}).get("strip-binaries", None) + assert strip != "", ( + f"{path.relative_to(ROOT)} compiles (config.build-commands is set) " + f"but disables the stripper, so its binaries ship unstripped. " + f"Remove the strip-binaries override or move the build commands out." + ) + + +def test_at_least_one_manual_element_is_discovered() -> None: + # Guards the parser: a glob or kind-key regression would make the + # parametrized test above vacuously pass with zero cases. + assert STAGING_ONLY, ( + "no staging-only kind: manual elements found under elements/ — the " + "discovery in this module has regressed" + ) diff --git a/tests/unit/test_repart_layout.py b/tests/unit/test_repart_layout.py index e21e5b46..cb65f49e 100644 --- a/tests/unit/test_repart_layout.py +++ b/tests/unit/test_repart_layout.py @@ -38,7 +38,21 @@ def parse_size(value: str) -> int: def load_config(path: Path) -> configparser.ConfigParser: - parser = configparser.ConfigParser(strict=True) + """Parse a systemd-style config file. + + ``strict=False`` is required, not laxity: systemd documents several + directives as repeatable, and ``CopyFiles=`` in ``30-var.conf`` is one of + them — the installer seeds both the Kubernetes and the containerd sysext + onto ``/var``. ``configparser`` defaults to ``strict=True``, which raises + ``DuplicateOptionError`` on valid systemd input. + + With ``strict=False`` the last occurrence wins for single-value lookups, + which is fine for the size and label assertions in this module. Anything + asserting over a repeatable directive must read every occurrence instead — + see ``tests/unit/test_seeded_sysext_names.py``, which scans the raw text for + that reason. + """ + parser = configparser.ConfigParser(strict=False) # systemd drop-ins are case-sensitive; configparser lowercases keys by default. parser.optionxform = str parser.read_string(path.read_text(encoding="utf-8")) @@ -218,9 +232,33 @@ def test_var_is_a_growing_xfs_tail(): ) -def test_var_seeds_the_offline_k0s_sysext(): - var = next(s for s in partitions().values() if s["Type"] == "var") - assert var["CopyFiles"] == "/k0s.raw:/lib/k0s/k0s.raw" +def test_var_seeds_every_sysext_into_the_systemd_sysext_scan_directory(): + """An offline install must boot into a working cluster with no fetch. + + The destination is relative to the /var partition root, so + ``/lib/extensions/.raw`` is ``/var/lib/extensions/.raw`` on the + running system — one of systemd-sysext's own search paths, which is why no + unit has to copy the image anywhere before the merge. + + ``CopyFiles=`` is repeatable and there is more than one seed, so this reads + every occurrence out of the raw text rather than through configparser, which + keeps only the last. The naming contract between a seeded image and its + extension-release lives in tests/unit/test_seeded_sysext_names.py. + """ + text = (REPART_DIR / "30-var.conf").read_text(encoding="utf-8") + seeds = dict( + re.findall(r"^CopyFiles=([^:]+):(.+)$", text, re.MULTILINE) + ) + + assert seeds.get("/kubernetes.raw") == "/lib/extensions/kubernetes.raw", ( + "the Kubernetes sysext must be seeded for an offline first boot" + ) + assert seeds.get("/containerd-flatcar.raw") == ( + "/lib/extensions/containerd-flatcar.raw" + ), ( + "the containerd sysext must be seeded: kubeadm-init.service hard-requires " + "containerd.service and FSDK ships no containerd component" + ) def test_root_partition_label_is_matched_by_the_sysupdate_root_transfer(): diff --git a/tests/unit/test_seeded_sysext_names.py b/tests/unit/test_seeded_sysext_names.py new file mode 100644 index 00000000..262ce6e2 --- /dev/null +++ b/tests/unit/test_seeded_sysext_names.py @@ -0,0 +1,234 @@ +"""An installer-seeded sysext must keep the stem its extension-release names. + +systemd-sysext resolves an extension's metadata by filename stem. An image +merged as ``.raw`` is required to carry +``/usr/lib/extension-release.d/extension-release.``; if it does not, the +extension is refused. The refusal is silent from the build's point of view — +the element builds, ``just validate`` resolves, the installer writes a bootable +disk, and the failure only appears as a missing service on a running machine. + +``tests/unit/test_sysupdate_transfers.py`` already enforces this for the +Kubernetes sysext on the *sysupdate* path, where the name is fixed by the +transfer's ``CurrentSymlink``. This module covers the other path: the images +``files/installer/repart.d/30-var.conf`` seeds onto ``/var/lib/extensions`` at +install time, whose names are fixed by ``CopyFiles=`` and by the element that +stages them. + +The concrete hazard is the Flatcar containerd import. Upstream names it +``containerd-flatcar.raw`` and ships ``extension-release.containerd-flatcar``. +Shortening it to ``containerd.raw`` anywhere in the chain — the element's +install target, the installer's ``cp`` into the initrd, or the ``CopyFiles=`` +here — makes systemd-sysext look for ``extension-release.containerd``, find +nothing, and drop containerd.service. ``kubeadm-init.service`` then has an +unresolvable hard requirement and the control plane never starts. + +The chain is asserted end to end so a rename in any one link fails here. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +VAR_REPART = ROOT / "files" / "installer" / "repart.d" / "30-var.conf" +INSTALLER = ROOT / "elements" / "oci" / "bluefin-server-installer.bst" +CONTAINERD_ELEMENT = ROOT / "elements" / "flatcar" / "containerd-sysext.bst" +KUBEADM_INIT = ROOT / "files" / "os" / "systemd" / "system" / "kubeadm-init.service" + +# Units that only a seeded sysext can provide, mapped to the image that ships +# them. Derived from the image contents, not from configuration: Flatcar's +# containerd-flatcar.raw carries usr/lib/systemd/system/containerd.service. +SYSEXT_PROVIDED_UNITS = { + "containerd.service": "containerd-flatcar.raw", +} + +COPY_FILES_RE = re.compile(r"^CopyFiles=([^:]+):(.+)$", re.MULTILINE) + + +def seeded_extensions() -> dict[str, str]: + """``{staged source path: destination path}`` for every seeded sysext.""" + return { + src: dst + for src, dst in COPY_FILES_RE.findall(VAR_REPART.read_text(encoding="utf-8")) + if "/lib/extensions/" in dst + } + + +def test_seeded_sysexts_are_decoupled_from_the_host_os_release_version() -> None: + """A seeded sysext must survive an OS update, so its ID must be ``_any``. + + ``/var`` persists across an A/B root update, so an image sitting in + ``/var/lib/extensions`` outlives the os-release it was installed against. + systemd-sysext refuses an extension whose ``ID``/``VERSION_ID`` disagree with + the host, so an extension pinned to a specific version stops merging the + first time the OS is updated — taking containerd.service, and therefore + kubeadm-init.service and the whole cluster, down on an otherwise routine + reboot. + + ``files/kubernetes/sysext/extension-release.kubernetes`` already uses + ``ID=_any`` for this reason. Flatcar's containerd image ships + ``ID=flatcar VERSION_ID=4593.2.5`` instead, which is correct for Flatcar and + wrong here, so ``elements/flatcar/containerd-sysext.bst`` repacks it. This + asserts the repack is still happening: dropping it to "import verbatim" is a + tempting simplification that reintroduces the failure. + """ + element = CONTAINERD_ELEMENT.read_text(encoding="utf-8") + + assert "unsquashfs" in element and "mksquashfs" in element, ( + f"{CONTAINERD_ELEMENT.relative_to(ROOT)} no longer repacks the upstream " + "image. Imported verbatim it carries ID=flatcar with a fixed VERSION_ID, " + "so the merge breaks on the first OS update that bumps flatcar-version." + ) + assert "ID=_any" in element, ( + f"{CONTAINERD_ELEMENT.relative_to(ROOT)} must rewrite the " + "extension-release with ID=_any so the sysext is independent of the " + "host os-release version, matching " + "files/kubernetes/sysext/extension-release.kubernetes" + ) + + +def test_every_sysext_provided_unit_a_host_unit_depends_on_is_actually_seeded() -> None: + """Depending on a sysext-provided unit obliges the installer to seed it. + + Deleting the ``CopyFiles=`` line for containerd would restore exactly the + failure this whole arrangement exists to fix: kubeadm-init.service depends + on ``containerd.service``, nothing provides it, and + bluefin-cluster-bootstrap.service never runs because it declares + ``Requires=kubeadm-init.service``. The machine boots to multi-user.target + with no control plane. + + Both ``Requires=`` and ``Wants=`` count. The unit deliberately uses ``Wants=`` + — see ``test_sysext_provided_units_are_not_hard_required`` — so matching only + ``Requires=`` here would make this gate silently vacuous. + """ + unit = without_comments(KUBEADM_INIT.read_text(encoding="utf-8")) + seeded_names = {Path(dst).name for dst in seeded_extensions().values()} + + for dep_unit, image in SYSEXT_PROVIDED_UNITS.items(): + if not any(f"{kw}={dep_unit}" in unit for kw in ("Requires", "Wants")): + continue + assert image in seeded_names, ( + f"{KUBEADM_INIT.relative_to(ROOT)} depends on {dep_unit}, which " + f"only {image} provides, but {VAR_REPART.relative_to(ROOT)} does " + f"not seed it into /lib/extensions. Seeded images: " + f"{sorted(seeded_names) or 'none'}. The control plane will not start." + ) + + +def test_sysext_provided_units_are_not_hard_required() -> None: + """A unit that only exists after the merge cannot be a ``Requires=``. + + systemd resolves dependencies when it computes the boot transaction, which + happens before systemd-sysext merges ``/usr``. Observed on a real first boot + of this image:: + + 12:06:10.283 Applying preset policy + 12:06:11.751 Merged extensions into '/usr' <- containerd.service appears + 12:06:13.141 Reached target Multi-User System + + With ``Requires=containerd.service`` the unresolvable dependency made systemd + drop kubeadm-init.service's job from the transaction entirely. It was never + re-enqueued after the merge, so its conditions were never evaluated, kubeadm + never ran, and nothing appeared in ``systemctl list-units --failed``. + kubelet.service survived the same boot because it only declares + ``After=containerd.service``; a pure ordering edge on a missing unit is + ignored rather than fatal. + + ``Wants=`` is ignored the same way, and ``After=`` still orders correctly once + the merge makes the unit real. + """ + unit = without_comments(KUBEADM_INIT.read_text(encoding="utf-8")) + + for dep_unit in SYSEXT_PROVIDED_UNITS: + assert f"Requires={dep_unit}" not in unit, ( + f"{KUBEADM_INIT.relative_to(ROOT)} hard-requires {dep_unit}, which " + f"does not exist when systemd computes the boot transaction — it " + f"arrives with the sysext merge. systemd will drop this unit's job " + f"and never re-enqueue it, so kubeadm never runs and the failure is " + f"invisible to `systemctl list-units --failed`. Use Wants= plus " + f"After=." + ) + assert f"After={dep_unit}" in unit, ( + f"{KUBEADM_INIT.relative_to(ROOT)} must still order itself " + f"After={dep_unit}, or kubeadm can run before the CRI socket exists." + ) + + +def test_seeded_images_land_in_the_sysext_scan_directory() -> None: + seeded = seeded_extensions() + + assert seeded, ( + f"{VAR_REPART.relative_to(ROOT)} seeds no sysext into /lib/extensions; " + "the installer's offline-merge path has regressed" + ) + + for src, dst in seeded.items(): + assert dst.startswith("/lib/extensions/"), ( + f"{dst} is not under /lib/extensions, so systemd-sysext will not " + f"scan it (/var is mounted at /var, making this /var/lib/extensions)" + ) + assert dst.endswith(".raw"), f"{dst} is not a .raw sysext image" + assert Path(src).name == Path(dst).name, ( + f"{VAR_REPART.relative_to(ROOT)} renames {src} to {dst} in flight. " + "The seeded name is what systemd-sysext resolves " + "extension-release. against; renaming here breaks the merge." + ) + + +def test_installer_stages_every_seeded_image_under_its_seeded_name() -> None: + installer = INSTALLER.read_text(encoding="utf-8") + + for src in seeded_extensions(): + # repart reads CopyFiles= sources from the installer's own root, which + # is the staged /layer tree. + staged = f"/layer{src}" + assert staged in installer, ( + f"{VAR_REPART.relative_to(ROOT)} seeds {src}, but " + f"{INSTALLER.relative_to(ROOT)} never stages {staged} into the " + "initrd, so repart has nothing to copy at install time" + ) + + +def without_comments(text: str) -> str: + """Effective directives only, for ``#``-commented formats. + + The prose in these files deliberately names the wrong spelling in order to + warn against it, so scanning raw text would flag the warning as the defect. + """ + return "\n".join( + line for line in text.splitlines() if not line.lstrip().startswith("#") + ) + + +def test_containerd_keeps_its_upstream_extension_release_stem() -> None: + """The specific rename that would silently drop containerd.service. + + Flatcar's image carries extension-release.containerd-flatcar, so every link + in the chain must say ``containerd-flatcar.raw``, never ``containerd.raw``. + """ + element = CONTAINERD_ELEMENT.read_text(encoding="utf-8") + + assert '"%{install-root}/containerd-flatcar.raw"' in element, ( + f"{CONTAINERD_ELEMENT.relative_to(ROOT)} must install the image as " + "containerd-flatcar.raw to match extension-release.containerd-flatcar" + ) + + # Only the two `#`-commented files are scanned for the forbidden spelling. + # The element's `description: |` is a YAML block scalar, not a comment, and + # it names containerd.raw on purpose to explain the hazard. The element is + # pinned by the positive assertion above instead, which is the stronger + # check: it fails if the install target is anything but containerd-flatcar.raw. + for path in (INSTALLER, VAR_REPART): + # `containerd.raw` as a whole token — `containerd-flatcar.raw` must not + # trip this. + match = re.search( + r"(? None: - justfile = JUSTFILE.read_text(encoding="utf-8") - start = justfile.index("export-sysext:") - end = justfile.index("\n# Write the raw GPT installer image", start) - - assert "k3s-" not in justfile[start:end] diff --git a/tests/unit/test_sysupdate_transfers.py b/tests/unit/test_sysupdate_transfers.py index 5cf22c61..493828d9 100644 --- a/tests/unit/test_sysupdate_transfers.py +++ b/tests/unit/test_sysupdate_transfers.py @@ -2,9 +2,9 @@ ``tests/unit/test_repart_layout.py`` already pins ``50-root.transfer`` against the installer repart config. The other two OTA transfer definitions — -``60-uki.transfer`` (the boot UKI) and ``70-k0s.transfer`` (the k0s sysext) — -have no coverage at all, and neither does the source-side artifact naming that -systemd-sysupdate matches against. +``60-uki.transfer`` (the boot UKI) and ``70-kubernetes.transfer`` (the +Kubernetes sysext) — have no coverage at all, and neither does the source-side +artifact naming that systemd-sysupdate matches against. These tests assert the drift-prone contracts: @@ -12,9 +12,10 @@ * every source pulls from this repo's own release feed over https * every source ``MatchPattern`` corresponds to an artifact name that some element under ``elements/`` actually emits (``bluefin-server-ddi-.raw.zst``, - ``bluefin-server-.efi``, ``k0s-.raw.zst``) -* the k0s sysext transfer lands in ``/var/lib/extensions`` under a name that - ``files/k0s/sysext/extension-release.k0s`` (``NAME=k0s``) can merge + ``bluefin-server-.efi``, ``kubernetes-.raw.zst``) +* the Kubernetes sysext transfer lands in ``/var/lib/extensions`` under a name + that ``files/kubernetes/sysext/extension-release.kubernetes`` + (``NAME=kubernetes``) can merge """ import configparser @@ -25,17 +26,22 @@ REPO_ROOT = Path(__file__).resolve().parents[2] SYSUPDATE_DIR = REPO_ROOT / "files" / "os" / "sysupdate.d" -K0S_SYSUPDATE_DIR = REPO_ROOT / "files" / "os" / "sysupdate.k0s.d" -K0S_TRANSFER = K0S_SYSUPDATE_DIR / "70-k0s.transfer" +KUBERNETES_SYSUPDATE_DIR = REPO_ROOT / "files" / "os" / "sysupdate.kubernetes.d" +KUBERNETES_TRANSFER = KUBERNETES_SYSUPDATE_DIR / "70-kubernetes.transfer" ELEMENTS_DIR = REPO_ROOT / "elements" -EXTENSION_RELEASE = REPO_ROOT / "files" / "k0s" / "sysext" / "extension-release.k0s" +EXTENSION_RELEASE = ( + REPO_ROOT / "files" / "kubernetes" / "sysext" / "extension-release.kubernetes" +) RELEASE_FEED = "https://github.com/projectbluefin/server/releases/latest/download/" def transfer_paths() -> list[Path]: return sorted( - (*SYSUPDATE_DIR.glob("*.transfer"), *K0S_SYSUPDATE_DIR.glob("*.transfer")) + ( + *SYSUPDATE_DIR.glob("*.transfer"), + *KUBERNETES_SYSUPDATE_DIR.glob("*.transfer"), + ) ) @@ -63,9 +69,11 @@ def split_match_pattern(pattern: str) -> tuple[str, str]: def test_sysupdate_directories_are_populated(): generic = sorted(p.name for p in SYSUPDATE_DIR.glob("*.transfer")) - k0s = sorted(p.name for p in K0S_SYSUPDATE_DIR.glob("*.transfer")) + kubernetes = sorted( + p.name for p in KUBERNETES_SYSUPDATE_DIR.glob("*.transfer") + ) assert generic == ["50-root.transfer", "60-uki.transfer"] - assert k0s == ["70-k0s.transfer"] + assert kubernetes == ["70-kubernetes.transfer"] @pytest.mark.parametrize("path", transfer_paths(), ids=lambda p: p.name) @@ -126,7 +134,7 @@ def test_every_source_artifact_is_produced_by_an_element(path: Path): load_transfer(path)["Source"]["MatchPattern"] ) # Elements name artifacts with a BuildStream variable in the version slot, - # e.g. FNAME="k0s-%{k0s-version}.raw" plus a .zst compression step. + # e.g. FNAME="kubernetes-%{k8s-version}.raw" plus a .zst compression step. produced = re.compile( re.escape(prefix) + r"%\{[a-z0-9-]+\}" + re.escape(suffix.removesuffix(".zst")) ) @@ -158,27 +166,33 @@ def test_uki_source_and_target_names_agree(): ) -def test_k0s_sysext_transfer_lands_in_the_system_extension_directory(): - target = load_transfer(K0S_TRANSFER)["Target"] +def test_kubernetes_sysext_transfer_lands_where_systemd_sysext_scans() -> None: + """The image must be merged by systemd-sysext.service, not by a copy step. + + ``/var/lib/extensions`` is one of systemd-sysext's own search paths, so an + image landing there is merged at boot with no unit shuffling it into + ``/run/extensions`` first. + """ + target = load_transfer(KUBERNETES_TRANSFER)["Target"] assert target.get("Type") == "regular-file", ( - "the k0s sysext is delivered as a decompressed regular file" + "the Kubernetes sysext is delivered as a decompressed regular file" ) - assert target.get("Path") == "/var/lib/k0s", ( - f"k0s sysext target path is {target.get('Path')!r}; the persistent " - "staging path must remain outside systemd-sysext's early scan" + assert target.get("Path") == "/var/lib/extensions", ( + f"sysext target path is {target.get('Path')!r}; anywhere else and " + "systemd-sysext.service never sees the image" ) assert target.get("Mode") == "0644", ( - f"k0s sysext mode is {target.get('Mode')!r}; the image must be readable " + f"sysext mode is {target.get('Mode')!r}; the image must be readable " "by systemd-sysext at merge time" ) -def test_k0s_sysext_transfer_maintains_a_stable_current_symlink(): - target = load_transfer(K0S_TRANSFER)["Target"] +def test_kubernetes_sysext_transfer_maintains_a_stable_current_symlink(): + target = load_transfer(KUBERNETES_TRANSFER)["Target"] symlink = target.get("CurrentSymlink") - assert symlink == "k0s.raw", ( - f"CurrentSymlink is {symlink!r}; the boot activation unit requires a " - "stable filename, so a version bump otherwise stops merging k0s" + assert symlink == "kubernetes.raw", ( + f"CurrentSymlink is {symlink!r}; the merge name must be stable, so a " + "version bump otherwise stops merging the sysext" ) prefix, suffix = split_match_pattern(target["MatchPattern"]) assert symlink == f"{prefix.rstrip('-')}{suffix}", ( @@ -187,23 +201,28 @@ def test_k0s_sysext_transfer_maintains_a_stable_current_symlink(): ) -def test_k0s_sysext_transfer_decompresses_the_release_asset(): - parser = load_transfer(K0S_TRANSFER) +def test_kubernetes_sysext_transfer_decompresses_the_release_asset(): + parser = load_transfer(KUBERNETES_TRANSFER) source = parser["Source"]["MatchPattern"] target = parser["Target"]["MatchPattern"] - assert source.endswith(".raw.zst"), f"k0s release asset {source!r} is not zstd" + assert source == "kubernetes-@v.raw.zst", ( + f"release asset pattern is {source!r}; " + ".github/scripts/check-kubernetes-version.py cross-checks this exact " + "spelling against the name elements/oci/kubernetes-sysext.bst emits" + ) assert target == source.removesuffix(".zst"), ( - f"k0s sysext installs as {target!r} but downloads {source!r}; a still " + f"sysext installs as {target!r} but downloads {source!r}; a still " "compressed image cannot be mounted by systemd-sysext" ) -def test_k0s_sysext_image_name_matches_its_extension_release_name(): +def test_kubernetes_sysext_image_name_matches_its_extension_release_name(): """systemd-sysext requires ``extension-release.`` to agree. - The image installed by ``70-k0s.transfer`` is merged through the stable - ``k0s.raw`` symlink, so ``NAME=`` in ``files/k0s/sysext/extension-release.k0s`` - must be ``k0s`` or the merge is rejected at boot. + The image installed by ``70-kubernetes.transfer`` is merged through the + stable ``kubernetes.raw`` symlink, so ``NAME=`` in + ``files/kubernetes/sysext/extension-release.kubernetes`` must be + ``kubernetes`` or the merge is rejected at boot. """ assert EXTENSION_RELEASE.is_file(), f"{EXTENSION_RELEASE} missing" fields = dict( @@ -211,9 +230,7 @@ def test_k0s_sysext_image_name_matches_its_extension_release_name(): for line in EXTENSION_RELEASE.read_text().splitlines() if "=" in line and not line.startswith("#") ) - symlink = load_transfer(K0S_TRANSFER)["Target"][ - "CurrentSymlink" - ] + symlink = load_transfer(KUBERNETES_TRANSFER)["Target"]["CurrentSymlink"] image_name = symlink.removesuffix(".raw") assert EXTENSION_RELEASE.name == f"extension-release.{image_name}", ( f"{EXTENSION_RELEASE.name} does not match the installed image name " @@ -239,11 +256,3 @@ def test_every_source_artifact_is_staged_in_release_workflow(path: Path): assert re.search(rf"cp\s+.*{re.escape(prefix)}\*.*dist/release/", build_yml), ( f"{path.name} source asset prefix {prefix!r} is not staged to dist/release/ in .github/workflows/build.yml" ) - - -def test_k0s_release_staging_does_not_use_legacy_k3s_name() -> None: - build_yml = (REPO_ROOT / ".github" / "workflows" / "build.yml").read_text() - assert re.search(r"cp\s+.*k0s-\*\.raw\.zst.*dist/release/", build_yml) - assert not re.search( - r"cp\s+.*k3s-\*\.raw\.zst.*dist/release/", build_yml - ) diff --git a/tests/unit/test_unit_dependency_provenance.py b/tests/unit/test_unit_dependency_provenance.py new file mode 100644 index 00000000..ce3e9821 --- /dev/null +++ b/tests/unit/test_unit_dependency_provenance.py @@ -0,0 +1,128 @@ +"""Every hard dependency of a shipped unit must have a named provider. + +`just validate` resolves the BuildStream graph, but a systemd `Requires=` is not +a bst node — the graph cannot see it. A unit that requires a service nothing +ships still builds, still validates, and then fails at boot with a job that +cannot be resolved. That is a silent, boot-only failure mode. + +This module closes it. Every unit named by `Requires=` or `BindsTo=` in a unit +this repository stages must be either: + + * staged by this repository (a peer file in files/os/systemd/system), or + * listed in EXTERNAL_PROVIDERS with the component that supplies it. + +Adding a hard dependency on something nothing provides therefore fails here +rather than on a running machine. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +UNIT_DIR = ROOT / "files" / "os" / "systemd" / "system" + +# Units this repository does not stage, mapped to what does supply them. +# +# containerd.service is the load-bearing entry. FSDK 26.08 ships no containerd +# component at all, so nothing can be composed from it — see +# docs/superpowers/specs/2026-09-19-flatcar-runtime-parity.md, which records +# `no elements/components/containerd.bst` in the pinned junction. +# +# Flatcar publishes containerd as a standalone systemd-sysext, the same delivery +# mechanism this repository already uses for Kubernetes. +# elements/flatcar/containerd-sysext.bst imports that image, the installer stages +# it into the initrd, and files/installer/repart.d/30-var.conf seeds it to +# /var/lib/extensions/containerd-flatcar.raw so the first boot merges it offline. The +# image carries its own multi-user.target.wants symlink, so the merge is what +# enables the service. +EXTERNAL_PROVIDERS = { + "containerd.service": ( + "Flatcar containerd sysext via elements/flatcar/containerd-sysext.bst, " + "seeded to /var/lib/extensions by files/installer/repart.d/30-var.conf" + ), + "systemd-sysext.service": "systemd, present in the base image", + "network-online.target": "systemd", + "systemd-networkd.service": "systemd", + "var.mount": "generated from the installer's /var partition", +} + +HARD_DEPENDENCY_KEYS = ("Requires=", "BindsTo=") + + +def staged_units() -> set[str]: + return {path.name for path in UNIT_DIR.iterdir() if path.is_file()} + + +def hard_dependencies() -> list[tuple[str, str]]: + """Return (unit_file, required_unit) for every hard dependency declared.""" + found: list[tuple[str, str]] = [] + for unit in sorted(UNIT_DIR.glob("*.service")): + for raw in unit.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + for key in HARD_DEPENDENCY_KEYS: + if line.startswith(key): + # systemd accepts a space-separated list and repeated keys. + for name in line[len(key):].split(): + found.append((unit.name, name)) + return found + + +def test_units_declare_at_least_one_hard_dependency() -> None: + """Guard the parser itself: a silent zero-match would make this suite vacuous.""" + assert hard_dependencies(), "parsed no Requires=/BindsTo= at all; parser is broken" + + +@pytest.mark.parametrize( + ("unit", "required"), + hard_dependencies(), + ids=lambda value: value.replace(".service", ""), +) +def test_hard_dependency_has_a_provider(unit: str, required: str) -> None: + local = staged_units() + assert required in local or required in EXTERNAL_PROVIDERS, ( + f"{unit} declares a hard dependency on {required!r}, which this repository " + f"does not stage and EXTERNAL_PROVIDERS does not account for. Either ship " + f"the unit, or record who supplies it — a Requires= on a unit nothing " + f"provides fails only at boot, where nothing else catches it." + ) + + +def test_kubelet_bridges_the_cni_plugin_directory() -> None: + """Plugins ship at the bakery path; containerd and Cilium read /opt/cni/bin. + + Without the bridge the loopback plugin is absent, every pod sandbox fails to + get a network, and the node never leaves NotReady with no obvious cause. + + The bridge must also refresh: /opt/cni/bin lives on /var and outlives a + sysext update, so a copy that can only ever add files would leave the node + running the plugins it first booted with while the version stamp advertises + newer ones. + """ + kubelet = (UNIT_DIR / "kubelet.service").read_text(encoding="utf-8") + + assert "/opt/cni/bin" in kubelet, "kubelet does not populate /opt/cni/bin" + assert "/usr/local/bin/cni" in kubelet, ( + "kubelet does not read the sysext's CNI plugin directory" + ) + + bridge = next( + line for line in kubelet.splitlines() if "cp -a" in line and "cni" in line + ) + assert "-an" in bridge or "--no-clobber" in bridge, ( + "the copy must not clobber when the sysext is unchanged; cilium-cni is " + "not a name the sysext ships, but an unconditional overwrite of a " + "steady-state /opt/cni/bin buys nothing and widens the blast radius" + ) + assert "/usr/local/share/kubernetes-cni-version" in bridge, ( + "the bridge must key off the sysext's CNI version stamp, or a sysext " + "update never reaches /opt/cni/bin and the node runs stale plugins " + "while advertising the new version" + ) + + body = kubelet.split("[Service]", 1)[1] + assert body.index("/opt/cni/bin") < body.index("ExecStart=/usr/bin/kubelet"), ( + "the bridge must run before kubelet starts" + ) diff --git a/workflows/server-release-watch-and-fix.md b/workflows/server-release-watch-and-fix.md index fa352027..ffa05089 100644 --- a/workflows/server-release-watch-and-fix.md +++ b/workflows/server-release-watch-and-fix.md @@ -28,11 +28,11 @@ Specifies the automated loop that monitors GitHub Actions release runs on `proje #### Stage B: Release Staging & Contract Audit While the remote build is compiling in CI, audit the local tree against the release contract: -1. **Element Graph**: `just validate` (checks release version and k0s version alignment). +1. **Element Graph**: `just validate` (checks release version and Kubernetes version alignment). 2. **Unit Suite**: `pytest tests/unit` and `bats tests/unit`. 3. **Docs & Skills**: `python3 .github/scripts/docs-checks.py`. 4. **Sysupdate Asset Parity**: Verify every `.transfer` target pattern in - `files/os/sysupdate.d/*.transfer` and `files/os/sysupdate.k0s.d/*.transfer` + `files/os/sysupdate.d/*.transfer` and `files/os/sysupdate.kubernetes.d/*.transfer` is emitted by an element under `elements/` AND staged into `dist/release/` in `.github/workflows/build.yml`. @@ -66,7 +66,7 @@ Once the GitHub Actions `release` job finishes: - `bluefin-server-installer-*.raw.zst` - `bluefin-server-*.efi` - `bluefin-server-ddi-*.raw.zst` - - `k0s-*.raw.zst` + - `kubernetes-*.raw.zst` - `SHA256SUMS` - `SHA256SUMS.gpg` 3. Verify GPG signature against `SHA256SUMS`.