diff --git a/docs/skills/kubeadm-sysext.md b/docs/skills/kubeadm-sysext.md index fea013a6..483cbf1e 100644 --- a/docs/skills/kubeadm-sysext.md +++ b/docs/skills/kubeadm-sysext.md @@ -125,6 +125,24 @@ off. `/etc/iscsi/iscsid.conf` comes from the factory `/etc` (`30-bluefin-iscsi.conf` restores it and creates `/var/lib/iscsi`), which is what democratic-csi's `chroot /host ... iscsiadm` node plugin needs. +## NFS + +The base image ships an NFS client built from source +(`bluefin-server/nfs-utils.bst`, `bluefin-server/rpcbind.bst`; FSDK 26.08 +has neither): `mount.nfs`/`mount.nfs4`/`umount.nfs` in `/usr/bin` (reached +through `/sbin -> usr/sbin -> bin`), `rpc.statd`, `sm-notify`, `nfsidmap`, +`nfsstat`, `showmount`. Without `mount.nfs`, util-linux `mount -t nfs` +falls through to the new mount API and the kernel refuses (`fsconfig() +failed: NFS: mount program didn't pass remote address`), which is what +kubelet reported for every NFS PV. Client only: no server daemons, no +GSS/Kerberos (`sec=krb5*` mounts are unsupported). `80-bluefin-nfs.preset` +enables `nfs-client.target` and `rpcbind.socket`; `mount.nfs` starts +`rpc-statd.service` on demand for NFSv3 locking (NFSv4 needs neither). +`/var/lib/nfs/statd` comes from tmpfiles.d, owned by `rpcuser`; on a +diskless node it is lost at reboot, so NFSv3 servers are not notified when a +rebooted diskless client held locks. NFSv4 id mapping uses the kernel +`request-key` upcall to `nfsidmap` (`/etc/request-key.d/id_resolver.conf`). + ## Known gaps - aarch64: the arm64 release binaries are pinned, but no aarch64 image has diff --git a/elements/bluefin-server/nfs-utils.bst b/elements/bluefin-server/nfs-utils.bst new file mode 100644 index 00000000..5d40f77a --- /dev/null +++ b/elements/bluefin-server/nfs-utils.bst @@ -0,0 +1,173 @@ +kind: manual +description: | + NFS client from nfs-utils: mount.nfs / mount.nfs4 / umount.nfs (the + helpers util-linux mount(8) execs for -t nfs), rpc.statd and sm-notify + (NFSv3 locking), nfsidmap (NFSv4 id mapping through request-key), + nfsstat, showmount, rpcdebug, nfsconf and nfsrahead, with upstream's + nfs-client.target, rpc-statd*.service and udev rules. + + Without mount.nfs, util-linux mount(8) hands "-t nfs" straight to the new + mount API and the kernel refuses it: "fsconfig() failed: NFS: mount + program didn't pass remote address", which fails every kubelet NFS volume. + FSDK 26.08 has libtirpc and keyutils but no nfs-utils element, so it is + built here from the upstream tarball. + + Client only: the server (rpc.nfsd, rpc.mountd, exportfs, fsidd, + rpc.idmapd, nfs-server.service and friends, rpc_pipefs, the generators) + is dropped after install. Not built: GSS/Kerberos (no krb5 in the image), + nfsdcld/nfsdcltrack/nfsdctl (server-side), junctions (libxml2), LDAP, + tcp-wrappers. The python tools (mountstats, nfsiostat, rpcctl, + nfsdclnts) are dropped as the image has no python. + + Runtime state: /usr is read-only and /var starts empty (tmpfs on diskless + nodes), so the statd state directory comes from tmpfiles.d, owned by the + rpcuser account (sysusers.d) that rpc.statd and sm-notify drop to. + rpc.statd registers with rpcbind (bluefin-server/rpcbind.bst). + +build-depends: +- base/base-stack.bst +- freedesktop-sdk.bst:public-stacks/buildsystem-autotools.bst +# configure hard-requires libevent and sqlite3 even though only the dropped +# fsidd/idmapd/nfsdcld link them. +- freedesktop-sdk.bst:components/libevent.bst +- freedesktop-sdk.bst:components/sqlite.bst + +depends: +- freedesktop-sdk.bst:public-stacks/runtime-minimal.bst +- freedesktop-sdk.bst:components/libtirpc.bst +- freedesktop-sdk.bst:components/keyutils.bst +- freedesktop-sdk.bst:components/libcap.bst +# libmount for mount.nfs and nfsrahead, libuuid/libblkid headers for configure. +- freedesktop-sdk.bst:components/util-linux-libs.bst +# start-statd runs systemctl and flock; the kernel calls request-key for +# nfsidmap (keyutils, in depends above). +- freedesktop-sdk.bst:components/systemd.bst +- freedesktop-sdk.bst:components/util-linux.bst +- bluefin-server/rpcbind.bst + +variables: + nfs-utils-version: 2.8.7 + statd-statedir: "%{localstatedir}/lib/nfs/statd" + +config: + configure-commands: + - | + ./configure \ + --prefix="%{prefix}" \ + --bindir="%{bindir}" \ + --sbindir="%{sbindir}" \ + --libdir="%{libdir}" \ + --libexecdir="%{libexecdir}" \ + --includedir="%{includedir}" \ + --datadir="%{datadir}" \ + --mandir="%{mandir}" \ + --sysconfdir="%{sysconfdir}" \ + --localstatedir="%{localstatedir}" \ + --enable-shared \ + --disable-static \ + --disable-sbin-override \ + --enable-libmount-mount \ + --enable-mountconfig \ + --enable-ipv6 \ + --with-rpcgen=internal \ + --with-systemd="%{indep-libdir}/systemd/system" \ + --with-statdpath="%{statd-statedir}" \ + --with-statduser=rpcuser \ + --with-start-statd="%{sbindir}/start-statd" \ + --with-pluginpath="%{libdir}/libnfsidmap" \ + --disable-gss \ + --disable-svcgss \ + --disable-nfsv4server \ + --disable-nfsdcld \ + --disable-nfsdcltrack \ + --disable-nfsdctl \ + --disable-junction \ + --disable-ldap \ + --without-tcp-wrappers + + build-commands: + - make -j%{max-jobs} + + install-commands: + - | + set -eu + make -j1 install DESTDIR="%{install-root}" + find "%{install-root}" -name '*.la' -delete + + # Client only: drop the server daemons, their units and generators, the + # python tools, the internal rpcgen (a build tool), and the /var state the + # install hook seeds (etab/rmtab are server files; statd's directory is + # created at boot by tmpfiles.d below). + - | + set -eu + root="%{install-root}" + for p in rpc.mountd rpc.nfsd exportfs fsidd rpc.idmapd \ + mountstats nfsiostat rpcctl nfsdclnts rpcgen; do + rm -f "${root}%{sbindir}/${p}" + done + for u in nfs-server.service nfs-mountd.service nfs-idmapd.service \ + fsidd.service proc-fs-nfsd.mount \ + var-lib-nfs-rpc_pipefs.mount rpc_pipefs.target; do + rm -f "${root}%{indep-libdir}/systemd/system/${u}" + done + rm -rf "${root}%{indep-libdir}/systemd/system-generators" + rm -rf "${root}%{localstatedir}" + for p in mount.nfs mount.nfs4 umount.nfs umount.nfs4 rpc.statd sm-notify \ + start-statd nfsidmap nfsstat showmount rpcdebug nfsconf; do + [ -e "${root}%{sbindir}/${p}" ] || { echo "ERROR: ${p} missing" >&2; exit 1; } + done + for u in nfs-client.target nfs-utils.service rpc-statd.service rpc-statd-notify.service; do + [ -e "${root}%{indep-libdir}/systemd/system/${u}" ] || { echo "ERROR: ${u} missing" >&2; exit 1; } + done + + # rpc.statd and sm-notify drop privileges to the owner of the sm/ + # directory; a fixed uid (Fedora's) keeps installed nodes' /var/lib/nfs + # valid across image updates. systemd-sysusers runs at image build + # (oci/bluefin-server-usr.bst), so the account is in the factory /etc. + - | + set -eu + root="%{install-root}" + install -d "${root}%{indep-libdir}/sysusers.d" "${root}%{indep-libdir}/tmpfiles.d" + printf 'u rpcuser 29 "RPC Service User" %{localstatedir}/lib/nfs\n' \ + > "${root}%{indep-libdir}/sysusers.d/nfs-utils.conf" + cat > "${root}%{indep-libdir}/tmpfiles.d/nfs-utils.conf" < "${d}/10-bluefin-statedir.conf" <): + - '%{libdir}/libnfsidmap.so' + +sources: +- kind: tar + url: kernel_pub:linux/utils/nfs-utils/%{nfs-utils-version}/nfs-utils-%{nfs-utils-version}.tar.xz + ref: 59d0f1e17b18efaa60ea3ccf89a9cad3217f8d3b23c18d2fe34b25c8969d60ae diff --git a/elements/bluefin-server/os-base.bst b/elements/bluefin-server/os-base.bst index 6d68f8cd..0132c5d6 100644 --- a/elements/bluefin-server/os-base.bst +++ b/elements/bluefin-server/os-base.bst @@ -39,4 +39,9 @@ depends: # iSCSI initiator (iscsiadm, iscsid; socket-activated) for iSCSI CSI volumes. - freedesktop-sdk.bst:components/open-iscsi.bst +# NFS client (mount.nfs, rpc.statd; nfs-client.target) for NFS CSI/PV volumes. +# Built from source: FSDK 26.08 has no nfs-utils or rpcbind element. +- bluefin-server/nfs-utils.bst +- bluefin-server/rpcbind.bst + - bluefin-server/kernel-modules.bst diff --git a/elements/bluefin-server/rpcbind.bst b/elements/bluefin-server/rpcbind.bst new file mode 100644 index 00000000..a11c671a --- /dev/null +++ b/elements/bluefin-server/rpcbind.bst @@ -0,0 +1,61 @@ +kind: manual +description: | + rpcbind, the ONC RPC portmapper, for the NFS client. rpc.statd (NFSv3 + locking, bluefin-server/nfs-utils.bst) registers its listener with the + local rpcbind and exits when there is none, and NFS servers look statd up + through it (port 111) to send SM_NOTIFY after they reboot. FSDK 26.08 has + libtirpc but no rpcbind element, so it is built here from the upstream + tarball. + + Socket-activated: 80-bluefin-nfs.preset enables rpcbind.socket and keeps + rpcbind.service off, so the daemon starts on the first RPC registration. + It runs as the "rpc" user (sysusers.d) and keeps its warm-start state in + /run/rpcbind, which it creates itself. The service unit depends on + rpcbind.target from systemd. + +build-depends: +- base/base-stack.bst +- freedesktop-sdk.bst:public-stacks/buildsystem-autotools.bst + +depends: +- freedesktop-sdk.bst:public-stacks/runtime-minimal.bst +- freedesktop-sdk.bst:components/libtirpc.bst +- freedesktop-sdk.bst:components/systemd-libs.bst + +variables: + rpcbind-version: 1.3.1 + +config: + configure-commands: + - | + ./configure \ + --prefix="%{prefix}" \ + --bindir="%{bindir}" \ + --sbindir="%{sbindir}" \ + --sysconfdir="%{sysconfdir}" \ + --mandir="%{mandir}" \ + --enable-warmstarts \ + --with-statedir=/run/rpcbind \ + --with-rpcuser=rpc \ + --with-systemdsystemunitdir="%{indep-libdir}/systemd/system" + + build-commands: + - make -j%{max-jobs} + + install-commands: + - make -j1 install DESTDIR="%{install-root}" + + # Same fixed uid as Fedora; systemd-sysusers applies this at image build + # (oci/bluefin-server-usr.bst), so the account is in the factory /etc. + - | + install -d "%{install-root}%{indep-libdir}/sysusers.d" + printf 'u rpc 32 "Rpcbind Daemon" -\n' \ + > "%{install-root}%{indep-libdir}/sysusers.d/rpcbind.conf" + + - | + %{install-extra} + +sources: +- kind: tar + url: sourceforge_download:project/rpcbind/rpcbind/%{rpcbind-version}/rpcbind-%{rpcbind-version}.tar.bz2 + ref: 8e551b445f5172f8ef218845af342fb2a2d741b57ce8c777c781a01ea92f92a5 diff --git a/files/os/systemd/system-preset/80-bluefin-nfs.preset b/files/os/systemd/system-preset/80-bluefin-nfs.preset new file mode 100644 index 00000000..8622d3d0 --- /dev/null +++ b/files/os/systemd/system-preset/80-bluefin-nfs.preset @@ -0,0 +1,8 @@ +# NFS client. nfs-client.target orders NFS mounts after remote-fs-pre and +# runs sm-notify once at boot; mount.nfs starts rpc-statd.service itself +# (start-statd) the first time an NFSv3 mount needs locking, which pulls in +# rpcbind through rpcbind.socket. Nothing listens until then except the +# portmapper socket. FSDK has no catch-all "disable *". +enable nfs-client.target +enable rpcbind.socket +disable rpcbind.service diff --git a/include/aliases.yml b/include/aliases.yml index 21a62d59..58a1b795 100644 --- a/include/aliases.yml +++ b/include/aliases.yml @@ -7,3 +7,4 @@ aliases: gitlab: https://gitlab.com/ crates: https://static.crates.io/ sourceforge_download: https://downloads.sourceforge.net/ + kernel_pub: https://www.kernel.org/pub/ diff --git a/tests/unit/test_nfs_client.py b/tests/unit/test_nfs_client.py new file mode 100644 index 00000000..ceb73641 --- /dev/null +++ b/tests/unit/test_nfs_client.py @@ -0,0 +1,115 @@ +"""Contracts for the NFS client (nfs-utils + rpcbind) shipped in the base image. + +Without /sbin/mount.nfs, util-linux mount(8) passes "-t nfs" to the new +mount API and the kernel rejects it ("fsconfig() failed: NFS: mount program +didn't pass remote address"), so every kubelet NFS volume fails. The client +is built from source: FSDK 26.08 has libtirpc and keyutils but no nfs-utils +or rpcbind element. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ELEMENTS = ROOT / "elements" / "bluefin-server" +OS_BASE = ELEMENTS / "os-base.bst" +NFS_UTILS = ELEMENTS / "nfs-utils.bst" +RPCBIND = ELEMENTS / "rpcbind.bst" +PRESET = ROOT / "files" / "os" / "systemd" / "system-preset" / "80-bluefin-nfs.preset" +ALIASES = ROOT / "include" / "aliases.yml" + + +def element(path: Path) -> dict: + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def configure_flags(path: Path) -> set[str]: + script = element(path)["config"]["configure-commands"][0] + return {token.rstrip("\\").strip() for token in script.split() if token.startswith("--")} + + +def preset_lines() -> list[list[str]]: + return [line.split() for line in PRESET.read_text().splitlines() if line and not line.startswith("#")] + + +def test_nfs_client_is_in_the_base_stack_only() -> None: + depends = element(OS_BASE)["depends"] + assert "bluefin-server/nfs-utils.bst" in depends + assert "bluefin-server/rpcbind.bst" in depends + for sysext in (ROOT / "elements").rglob("*sysext*.bst"): + assert "nfs-utils" not in sysext.read_text(encoding="utf-8"), sysext + assert "bluefin-server/rpcbind.bst" in element(NFS_UTILS)["depends"] + + +def test_sources_are_upstream_tarballs_pinned_by_sha256() -> None: + aliases = yaml.safe_load(ALIASES.read_text(encoding="utf-8"))["aliases"] + assert aliases["kernel_pub"] == "https://www.kernel.org/pub/" + for path, prefix in ((NFS_UTILS, "kernel_pub:linux/utils/nfs-utils/"), (RPCBIND, "sourceforge_download:project/rpcbind/")): + (source,) = element(path)["sources"] + assert source["kind"] == "tar" + assert source["url"].startswith(prefix), source["url"] + assert re.fullmatch(r"[0-9a-f]{64}", source["ref"]), path + + +def test_manual_elements_build_in_the_fsdk_autotools_sandbox() -> None: + for path in (NFS_UTILS, RPCBIND): + data = element(path) + assert data["kind"] == "manual" + assert "freedesktop-sdk.bst:public-stacks/buildsystem-autotools.bst" in data["build-depends"] + assert "freedesktop-sdk.bst:components/libtirpc.bst" in data["depends"] + + +def test_nfs_utils_is_a_client_only_build() -> None: + flags = configure_flags(NFS_UTILS) + # No mount helper without these: /usr/sbin is a symlink to bin in FSDK, + # and the default "sbin override" would install into a literal /sbin. + assert "--disable-sbin-override" in flags + assert "--enable-libmount-mount" in flags + assert '--with-systemd="%{indep-libdir}/systemd/system"' in flags + assert "--with-rpcgen=internal" in flags + # Server and Kerberos pieces stay out (no krb5, libxml2, libnl in the image). + for flag in ("--disable-gss", "--disable-svcgss", "--disable-nfsv4server", "--disable-nfsdcld", + "--disable-nfsdcltrack", "--disable-nfsdctl", "--disable-junction", "--disable-ldap"): + assert flag in flags, flag + install = "\n".join(element(NFS_UTILS)["config"]["install-commands"]) + for server_bit in ("rpc.mountd", "rpc.nfsd", "exportfs", "nfs-server.service", "system-generators"): + assert server_bit in install, server_bit + for kept in ("mount.nfs", "rpc.statd", "sm-notify", "nfsidmap", "nfsstat", "nfs-client.target", "rpc-statd.service"): + assert kept in install, kept + + +def test_statd_state_and_user_come_from_tmpfiles_and_sysusers() -> None: + install = "\n".join(element(NFS_UTILS)["config"]["install-commands"]) + flags = configure_flags(NFS_UTILS) + assert '--with-statdpath="%{statd-statedir}"' in flags + assert "--with-statduser=rpcuser" in flags + assert 'u rpcuser 29 "RPC Service User"' in install + assert "d %{statd-statedir}/sm 0700 rpcuser rpcuser -" in install + assert "d %{statd-statedir}/sm.bak 0700 rpcuser rpcuser -" in install + assert "After=systemd-tmpfiles-setup.service" in install + # /usr is read-only and /var starts empty: nothing may ship under /var. + assert 'rm -rf "${root}%{localstatedir}"' in install + # NFSv4 id mapping upcall (request-key -> nfsidmap). + assert "request-key.d/id_resolver.conf" in install + + +def test_rpcbind_is_socket_activated_and_unprivileged() -> None: + flags = configure_flags(RPCBIND) + assert "--with-rpcuser=rpc" in flags + assert "--with-statedir=/run/rpcbind" in flags + assert '--with-systemdsystemunitdir="%{indep-libdir}/systemd/system"' in flags + assert 'u rpc 32 "Rpcbind Daemon" -' in "\n".join(element(RPCBIND)["config"]["install-commands"]) + lines = preset_lines() + assert ["enable", "rpcbind.socket"] in lines + assert ["disable", "rpcbind.service"] in lines + + +def test_preset_enables_the_client_target_and_nothing_else() -> None: + lines = preset_lines() + assert ["enable", "nfs-client.target"] in lines + enabled = {unit for verb, unit in lines if verb == "enable"} + assert enabled == {"nfs-client.target", "rpcbind.socket"}