diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 8a4b26b7..d5e17667 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -4,7 +4,7 @@ description: Build, export, flash, and release the Bluefin Server installer medi metadata: type: how-to status: stable - last_updated: "2026-09-08" + last_updated: "2026-09-26" context7-sources: - /systemd/systemd - /apache/buildstream @@ -161,6 +161,7 @@ The release process is driven by `.github/workflows/build.yml`: | "Put the DDI in the initrd cpio." | The DDI is 2 GiB+. The initrd cpio step must run before the DDI is placed in `/layer`. | | "Store the DDI in the ESP (FAT32)." | FAT32 has a 4 GiB per-file limit. Use a separate XFS partition. | | "Add an 8 GiB minimum size floor to the DDI." | The rootfs is immutable. It never grows in-place. Content + overhead is enough. | +| "`chmod 4755` in an element's install-commands makes the file setuid in the image." | No. BuildStream artifacts keep one executable bit per file, so every staged file is 0644/0755 (FSDK works around this with `initial-script` chmods at image assembly). `flatcar-usr.bst` records Flatcar's setuid/setgid files in `usr/lib/bluefin-server/setuid-modes` (after its own removals) and `bluefin-server-ddi.bst` re-applies them right before `mkfs.xfs`, then asserts the written image with `xfs_db -c 'path /usr/bin/sudo' -c 'print core.mode'`; special modes must be set in the `script` element that writes the image. (A tmpfiles `z` line at boot would technically work, since the UKI cmdline is `rw`, but it would mutate a signed A/B slot after the fact and the first `sudo` before tmpfiles ran would fail — the bits belong in the shipped image.) | ## Red flags diff --git a/elements/flatcar/flatcar-usr.bst b/elements/flatcar/flatcar-usr.bst index 0c6ca19d..4a57e0fa 100644 --- a/elements/flatcar/flatcar-usr.bst +++ b/elements/flatcar/flatcar-usr.bst @@ -6,6 +6,14 @@ description: | (systemd-sysupdate, systemd-creds, Kured), and removes /usr/lib/modules to leave kernel module management exclusively to flatcar-kernel.bst. + BuildStream artifacts record a single executable bit per file (REAPI + FileNode.is_executable), so the setuid/setgid bits Flatcar ships on sudo, + su, passwd, mount and friends are dropped when this install root is + captured. The element records them in usr/lib/bluefin-server/setuid-modes + while the archive modes are still intact (after its own removals, so the + list only names files that ship); oci/bluefin-server-ddi.bst restores + them before mkfs.xfs and verifies the written image with xfs_db. + (@): - include/arch.yml - include/flatcar.yml @@ -14,6 +22,7 @@ build-depends: - base/base-stack.bst - freedesktop-sdk.bst:components/tar.bst - freedesktop-sdk.bst:components/gzip.bst + - freedesktop-sdk.bst:components/findutils.bst variables: strip-binaries: "" @@ -97,3 +106,26 @@ config: rm "%{install-root}/usr/lib/systemd/system/local-fs.target.wants/oem.mount" rm "%{install-root}/usr/lib/systemd/system/dev-disk-by\\x2dlabel-OEM.device" rm -r "%{install-root}/usr/lib/systemd/system/systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d" + + # Record every setuid/setgid file (octal mode, path relative to the + # install root) before the artifact capture flattens the modes to 0755. + # This runs after the removals above so the list never names a file this + # element deletes (a stale entry would fail the DDI restore). Paths must + # not contain leading/trailing whitespace or newlines (`read` would + # mangle them); Flatcar ships none such. Fail loudly if the list is + # empty or misses sudo: that means the archive was extracted without its + # special bits and the image would ship an unusable sudo/su/passwd. + mkdir -p "%{install-root}/usr/lib/bluefin-server" + find "%{install-root}/usr" -type f -perm /6000 -printf '%m usr/%P\n' \ + | LC_ALL=C sort > "%{install-root}/usr/lib/bluefin-server/setuid-modes" + SETUID_SUDO_SEEN=0 + while read -r mode path; do + case "${mode}" in [2-7][0-7][0-7][0-7]) ;; *) echo "ERROR: unexpected mode '${mode}' for ${path} in setuid-modes" >&2; exit 1 ;; esac + if [ "${path}" = usr/bin/sudo ]; then + SETUID_SUDO_SEEN=1 + fi + done < "%{install-root}/usr/lib/bluefin-server/setuid-modes" + if [ "${SETUID_SUDO_SEEN}" -ne 1 ]; then + echo "ERROR: usr/bin/sudo is not setuid in the extracted Flatcar /usr; refusing to record an empty setuid-modes list" >&2 + exit 1 + fi diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index bee1ed53..2bdbbc36 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -40,6 +40,41 @@ config: rm -f "/layer/usr/lib/modules/${KVER}/vmlinux" "/layer/usr/lib/modules/${KVER}/System.map" depmod -b /layer/usr "${KVER}" + # Restore the setuid/setgid bits recorded by flatcar/flatcar-usr.bst. + # BuildStream artifacts keep only an executable bit per file, so the + # staged /layer has sudo, su, passwd, mount, ... at 0755; mkfs.xfs -p + # copies whatever mode it finds, so fix them here, before the image is + # written. A missing or empty list is a build error, not a silent 0755. + SETUID_MODES=/layer/usr/lib/bluefin-server/setuid-modes + if [ ! -s "${SETUID_MODES}" ]; then + echo "ERROR: ${SETUID_MODES} is missing or empty; flatcar-usr.bst must record the setuid/setgid files" >&2 + exit 1 + fi + # mkfs.xfs -p has to open every file it copies, and the build sandbox + # (buildbox-fuse) enforces the mode bits itself, so a mode without an + # owner-read bit -- Flatcar ships dbus-daemon-launch-helper as 4110, + # ---s--x--- -- fails the image build with "Permission denied". Keep + # u+r on such files: root reads them regardless on a real system. + while read -r mode path; do + if [ ! -f "/layer/${path}" ]; then + echo "ERROR: setuid-modes lists ${path} but it is not in /layer (stale manifest? flatcar-usr.bst records after its removals)" >&2 + exit 1 + fi + if (( (8#${mode} & 8#400) == 0 )); then + mode="$(printf '%04o' $(( 8#${mode} | 8#400 )))" + echo "NOTE: ${path} recorded without owner read; applying ${mode} so mkfs.xfs can copy it" + fi + chmod "${mode}" "/layer/${path}" + done < "${SETUID_MODES}" + SUDO_MODE="$(stat -c %a /layer/usr/bin/sudo)" + if [ "${SUDO_MODE}" != 4755 ]; then + echo "ERROR: /usr/bin/sudo is mode ${SUDO_MODE} after restoring setuid-modes, expected 4755" >&2 + exit 1 + fi + echo "Restored $(wc -l < "${SETUID_MODES}") setuid/setgid modes from ${SETUID_MODES}:" + cat "${SETUID_MODES}" + ls -l /layer/usr/bin/sudo + # Stage extracted CA certificate bundle into target DDI rootfs # Remote execution may hard-link the staged CA files to the build root; # replace the target tree before copying rather than copying onto itself. @@ -117,6 +152,18 @@ config: echo "Allocating XFS image of ${TARGET_BYTES} bytes" mkfs.xfs -f -L bluefin-root -m reflink=0,rmapbt=0 -n parent=0 -i exchange=0 -p /layer "${OUT}/${FNAME}" + # Prove the image, not just /layer, carries the setuid bit: BuildStream + # only shows element output on failure, so this check is what makes a + # green build evidence that mkfs.xfs -p copied the mode into the DDI. + SUDO_IMAGE_MODE="$(xfs_db -r -c 'path /usr/bin/sudo' -c 'print core.mode' "${OUT}/${FNAME}")" + case "${SUDO_IMAGE_MODE}" in + "core.mode = 0104755") echo "Verified in image: /usr/bin/sudo ${SUDO_IMAGE_MODE}" ;; + *) + echo "ERROR: /usr/bin/sudo in ${FNAME} is '${SUDO_IMAGE_MODE}', expected 'core.mode = 0104755'" >&2 + exit 1 + ;; + esac + zstd --rm -T0 -19 -q "${OUT}/${FNAME}" -o "${OUT}/${FNAME}.zst" ( cd "${OUT}" diff --git a/tests/unit/test_flatcar_usr.py b/tests/unit/test_flatcar_usr.py index b84e2f43..2aee5659 100644 --- a/tests/unit/test_flatcar_usr.py +++ b/tests/unit/test_flatcar_usr.py @@ -101,6 +101,9 @@ def test_flatcar_usr_element_structure() -> None: assert "base/base-stack.bst" in build_deps, "Must depend on base/base-stack.bst" assert "freedesktop-sdk.bst:components/tar.bst" in build_deps assert "freedesktop-sdk.bst:components/gzip.bst" in build_deps + assert "freedesktop-sdk.bst:components/findutils.bst" in build_deps, ( + "find -perm/-printf records the setuid/setgid modes" + ) # Prebuilt binaries must not be stripped variables = data.get("variables", {}) @@ -180,20 +183,10 @@ def test_flatcar_usr_removes_modules() -> None: ) -def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: - """Simulate execution of flatcar-usr.bst install-commands against a staged /usr. - - Verifies: - 1. Removed binaries are absent. - 2. Removed units are absent. - 3. bootengine.img is preserved. - 4. Module directory /usr/lib/modules is removed to avoid collision with flatcar-kernel.bst. - 5. Retained files (bash, sshd, systemd, crictl) remain untouched. - """ +def _populate_mock_usr(install_root: Path, *, sudo_mode: int = 0o4755) -> tuple[str, Path, Path]: + """Lay out the files flatcar-container.tar.gz delivers (as far as the script touches them).""" flatcar_yml_data = yaml.safe_load(FLATCAR_YML.read_text(encoding="utf-8")) kver = flatcar_yml_data["variables"]["flatcar-kver"] - - install_root = tmp_path / "install-root" usr = install_root / "usr" # Populate mock files that flatcar-container.tar.gz delivers @@ -202,9 +195,18 @@ def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: for b in [ "bash", "crictl", "sshd", "update_engine", "update_engine_client", "update_engine_stub", "locksmithctl", "ignition", "coreos-cloudinit", - "flatcar-update", "download_sysext" + "flatcar-update", "download_sysext", "sudo", "su", "umount", "unix_chkpwd" ]: (bin_dir / b).write_text(f"mock-{b}", encoding="utf-8") + # Flatcar ships these with special bits; the archive extraction keeps them. + # umount sorts after sudo so the manifest's last line is not sudo (the + # real list ends with polkit-agent-helper-1), exercising the loop's + # last-iteration path under set -e. + (bin_dir / "sudo").chmod(sudo_mode) + (bin_dir / "su").chmod(0o4755) + (bin_dir / "umount").chmod(0o4755) + (bin_dir / "unix_chkpwd").chmod(0o2755) + (bin_dir / "bash").chmod(0o755) locksmith_dir = usr / "lib" / "locksmith" locksmith_dir.mkdir(parents=True) @@ -260,6 +262,23 @@ def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: (mod_dir / "kernel" / "drivers" / "driver.ko").write_bytes(b"mock-driver") (mod_dir / "modules.dep").write_text("mock modules.dep", encoding="utf-8") + return kver, bin_dir, bootengine + + +def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: + """Simulate execution of flatcar-usr.bst install-commands against a staged /usr. + + Verifies: + 1. Removed binaries are absent. + 2. Removed units are absent. + 3. bootengine.img is preserved. + 4. Module directory /usr/lib/modules is removed to avoid collision with flatcar-kernel.bst. + 5. Retained files (bash, sshd, systemd, crictl) remain untouched. + """ + install_root = tmp_path / "install-root" + usr = install_root / "usr" + kver, bin_dir, bootengine = _populate_mock_usr(install_root) + # Extract script from flatcar-usr.bst and adapt variables install_commands = _load_element_data().get("config", {}).get("install-commands", []) script = "\n".join(install_commands) @@ -300,3 +319,42 @@ def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: # 5. Assert kept binaries remain for kept in ["bash", "crictl", "sshd"]: assert (bin_dir / kept).is_file(), f"Kept binary {kept} must remain in output" + + # 6. Every setuid/setgid file is recorded (octal mode, install-root-relative + # path) so bluefin-server-ddi.bst can restore the bits the artifact loses. + # The manifest is recorded after the rm block, so it never names a file + # this element removes, and its last line is not sudo. + manifest = usr / "lib" / "bluefin-server" / "setuid-modes" + assert manifest.read_text(encoding="utf-8") == ( + "2755 usr/bin/unix_chkpwd\n" + "4755 usr/bin/su\n" + "4755 usr/bin/sudo\n" + "4755 usr/bin/umount\n" + ) + for rel_path in REMOVED_BINARIES: + assert rel_path not in manifest.read_text(encoding="utf-8") + + +def test_flatcar_usr_refuses_an_extraction_that_lost_the_setuid_bits(tmp_path: Path) -> None: + """If sudo comes out of the archive without setuid the build must fail, not ship 0755.""" + install_root = tmp_path / "install-root" + kver, _, _ = _populate_mock_usr(install_root, sudo_mode=0o755) + + install_commands = _load_element_data().get("config", {}).get("install-commands", []) + script = "\n".join(install_commands) + script = script.replace("%{install-root}", str(install_root)) + script = script.replace("%{flatcar-kver}", kver) + script = re.sub(r"tar\s+.*flatcar-container\.tar\.gz\s+-C\s+.*", "# tar extract bypassed", script) + + result = subprocess.run(["bash", "-c", script], capture_output=True, text=True) + assert result.returncode == 1 + assert "usr/bin/sudo is not setuid" in result.stderr + + +def test_flatcar_usr_records_the_manifest_after_its_removals() -> None: + """A removed file must never end up in setuid-modes (the DDI restore would fail on it).""" + install_commands = _load_element_data().get("config", {}).get("install-commands", []) + script = "\n".join(install_commands) + last_rm = max(m.end() for m in re.finditer(r"^\s*rm .*$", script, re.MULTILINE)) + record = script.index("-printf '%m usr/%P\\n'") + assert last_rm < record diff --git a/tests/unit/test_setuid_modes.py b/tests/unit/test_setuid_modes.py new file mode 100644 index 00000000..c495d607 --- /dev/null +++ b/tests/unit/test_setuid_modes.py @@ -0,0 +1,135 @@ +"""Setuid/setgid bits survive the BuildStream artifact round trip into the DDI. + +BuildStream artifacts record one executable bit per file (REAPI +``FileNode.is_executable``), so every file flatcar-usr.bst extracts is staged +into the DDI build at 0644/0755. Flatcar ships ``sudo``, ``su``, ``passwd``, +``mount`` and friends setuid root; without a fix the installed image has +``/usr/bin/sudo`` at 0755 and sudo refuses to run. flatcar-usr.bst records the +modes in ``usr/lib/bluefin-server/setuid-modes`` and bluefin-server-ddi.bst +restores them before ``mkfs.xfs -p`` copies the tree. +""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +DDI_ELEMENT = REPO_ROOT / "elements" / "oci" / "bluefin-server-ddi.bst" +FLATCAR_USR_ELEMENT = REPO_ROOT / "elements" / "flatcar" / "flatcar-usr.bst" + + +def _ddi_script() -> str: + data = yaml.safe_load(DDI_ELEMENT.read_text(encoding="utf-8")) + return "\n".join(data["config"]["commands"]) + + +def _restore_block(script: str) -> str: + """From the manifest assignment to the `ls -l sudo` that logs the restored tree.""" + start = script.index("SETUID_MODES=/layer/usr/lib/bluefin-server/setuid-modes") + end = script.index("ls -l /layer/usr/bin/sudo\n") + len("ls -l /layer/usr/bin/sudo\n") + return script[start:end] + + +def _run_restore(tmp_path: Path, manifest: str | None, sudo_mode: int = 0o755) -> tuple[Path, subprocess.CompletedProcess]: + layer = tmp_path / "layer" + for rel, mode in ( + ("usr/bin/sudo", sudo_mode), + ("usr/bin/su", 0o755), + ("usr/bin/unix_chkpwd", 0o755), + ("usr/libexec/dbus-daemon-launch-helper", 0o644), + ("usr/bin/bash", 0o755), + ): + path = layer / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("mock", encoding="utf-8") + path.chmod(mode) + (layer / "usr" / "lib" / "bluefin-server").mkdir(parents=True) + if manifest is not None: + (layer / "usr" / "lib" / "bluefin-server" / "setuid-modes").write_text(manifest, encoding="utf-8") + block = _restore_block(_ddi_script()).replace("/layer", str(layer)) + result = subprocess.run(["bash", "-euo", "pipefail", "-c", block], capture_output=True, text=True) + return layer, result + + +def _mode(path: Path) -> int: + return stat.S_IMODE(os.stat(path).st_mode) + + +def test_flatcar_usr_records_and_ddi_restores_the_same_file() -> None: + flatcar_usr = FLATCAR_USR_ELEMENT.read_text(encoding="utf-8") + ddi = DDI_ELEMENT.read_text(encoding="utf-8") + + assert "find \"%{install-root}/usr\" -type f -perm /6000 -printf '%m usr/%P\\n'" in flatcar_usr + assert '"%{install-root}/usr/lib/bluefin-server/setuid-modes"' in flatcar_usr + assert "SETUID_MODES=/layer/usr/lib/bluefin-server/setuid-modes" in ddi + # Restoration happens after the tree is final and before the image is cut, + # and the written image is checked with xfs_db afterwards, so a green build + # proves the mode reached the DDI (BuildStream hides element output on + # success, so the ls -l/cat lines alone would prove nothing). + script = _ddi_script() + mkfs = script.index("mkfs.xfs -f -L bluefin-root") + assert script.index('depmod -b /layer/usr "${KVER}"') < script.index("SETUID_MODES=") < mkfs + xfs_db = script.index("xfs_db -r -c 'path /usr/bin/sudo' -c 'print core.mode'") + assert mkfs < xfs_db < script.index("zstd --rm") + assert '"core.mode = 0104755")' in script + # The manifest is recorded after flatcar-usr's removals, never before. + assert flatcar_usr.index('rm -r "%{install-root}/usr/lib/systemd/system/systemd-fsck@') < flatcar_usr.index("-printf '%m usr/%P") + + +def test_ddi_restores_every_recorded_mode(tmp_path: Path) -> None: + layer, result = _run_restore( + tmp_path, + "2755 usr/bin/unix_chkpwd\n" + "4110 usr/libexec/dbus-daemon-launch-helper\n" + "4755 usr/bin/su\n" + "4755 usr/bin/sudo\n", + ) + assert result.returncode == 0, result.stderr + assert _mode(layer / "usr/bin/sudo") == 0o4755 + assert _mode(layer / "usr/bin/su") == 0o4755 + assert _mode(layer / "usr/bin/unix_chkpwd") == 0o2755 + # 4110 (---s--x---) gains owner read: mkfs.xfs -p must be able to open the + # file under buildbox-fuse, which enforces the mode bits itself (the + # round-1 CI build failed with "Permission denied" on exactly this file). + assert _mode(layer / "usr/libexec/dbus-daemon-launch-helper") == 0o4510 + assert "NOTE: usr/libexec/dbus-daemon-launch-helper recorded without owner read; applying 4510" in result.stdout + assert _mode(layer / "usr/bin/bash") == 0o755 + + +def test_ddi_fails_when_the_manifest_is_missing(tmp_path: Path) -> None: + layer, result = _run_restore(tmp_path, None) + assert result.returncode == 1 + assert "setuid-modes is missing or empty" in result.stderr + assert _mode(layer / "usr/bin/sudo") == 0o755 + + +def test_ddi_fails_when_the_manifest_is_empty(tmp_path: Path) -> None: + _, result = _run_restore(tmp_path, "") + assert result.returncode == 1 + assert "setuid-modes is missing or empty" in result.stderr + + +def test_ddi_fails_when_the_manifest_lists_a_missing_file(tmp_path: Path) -> None: + layer, result = _run_restore(tmp_path, "4755 usr/bin/sudo\n4755 usr/bin/gone\n") + assert result.returncode == 1 + assert "setuid-modes lists usr/bin/gone but it is not in" in result.stderr + assert "stale manifest" in result.stderr + + +def test_ddi_logs_the_restored_modes(tmp_path: Path) -> None: + _, result = _run_restore(tmp_path, "4755 usr/bin/su\n4755 usr/bin/sudo\n") + assert result.returncode == 0, result.stderr + assert "Restored 2 setuid/setgid modes" in result.stdout + assert "4755 usr/bin/sudo" in result.stdout + assert "-rwsr-xr-x" in result.stdout + + +def test_ddi_fails_when_sudo_is_still_not_setuid(tmp_path: Path) -> None: + _, result = _run_restore(tmp_path, "4755 usr/bin/su\n") + assert result.returncode == 1 + assert "/usr/bin/sudo is mode 755" in result.stderr