From 970045ec57de20baae56f82dac58909761e94f1c Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 20 Oct 2026 20:29:33 -0700 Subject: [PATCH 01/12] nvx-rs: add guest operations for a layered root Add `nvx::guest::root`, the operations through which a guest init or agent, whether or not it is PID 1, bootstraps the guest for a layered root, assembles the root, mounts live shares in it, and tears it down. Each returns typed results and owned handles. `bootstrap` creates the agent's and the workload's sibling cgroups of a cgroup `Layout`, and moves every process of their parent into the agent's, kernel threads excepted, in passes until none is left, so that a process that a moved one starts meanwhile moves too. It enables the `pids` controller, and `memory` when a memory value needs it, writes the agent's `memory.low` and the workload's `memory.max` and `pids.max` that the caller gives, and sets `user.max_user_namespaces` to 0, so that user namespaces stay unavailable whatever the kernel supports. It mounts the cgroup v2 hierarchy first unless the cgroups' parent is a cgroup. `assemble` finds the distro and scratch block devices by the base addresses of their MMIO windows in `/proc/iomem`, polling until they appear, checks each one's expected size, and flushes its buffers. It checks the EROFS image's expected UUID in its superblock, reading nothing else, before it mounts the image read-only, then mounts the ext4 scratch `nosuid,nodev`, with separate upper and work directories, and the overlay with `metacopy=on,xino=on`. It returns a `Root` that owns the mounts, and unmounts them again when a later step fails. `Root::mount_share` and `Root::mount_aggregate` mount the VMM's virtio-fs live shares with the shell init's checks, unchanged: canonical absolute targets, reserved targets, overlaps, no symbolic link along the path in the root, and directory or file targets. `Root::teardown` unmounts the shares in reverse mount order, then the overlay, the image and the scratch. It attempts every unmount and reports each one that failed; dropping a root unmounts the same way and ignores failures. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off. The mounts, unmounts, buffer flushes and control-file writes go through a private seam, so that unit tests check the order and flags of each mount and unmount, the typed errors, and the failed unmounts that teardown reports, against procfs, sysfs and cgroup fixtures in a temporary directory. NVX's managed layered root still runs the shell init; this change only adds the operations. Part of #621. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- doc/project-structure.md | 2 +- nvx-rs/README.md | 23 +- nvx-rs/src/guest/mod.rs | 8 +- nvx-rs/src/guest/root.rs | 805 ++++++++++++++++++++++++++ nvx-rs/src/guest/root/block.rs | 225 ++++++++ nvx-rs/src/guest/root/bootstrap.rs | 364 ++++++++++++ nvx-rs/src/guest/root/kernel.rs | 374 ++++++++++++ nvx-rs/src/guest/root/share.rs | 569 +++++++++++++++++++ nvx-rs/src/guest/root/tests.rs | 884 +++++++++++++++++++++++++++++ nvx-rs/src/guest/sys/filesystem.rs | 17 + nvx-rs/src/guest/sys/ioctl.rs | 25 +- nvx-rs/src/guest/sys/mod.rs | 7 +- nvx-rs/src/lib.rs | 10 +- 13 files changed, 3291 insertions(+), 22 deletions(-) create mode 100644 nvx-rs/src/guest/root.rs create mode 100644 nvx-rs/src/guest/root/block.rs create mode 100644 nvx-rs/src/guest/root/bootstrap.rs create mode 100644 nvx-rs/src/guest/root/kernel.rs create mode 100644 nvx-rs/src/guest/root/share.rs create mode 100644 nvx-rs/src/guest/root/tests.rs diff --git a/doc/project-structure.md b/doc/project-structure.md index 1eda9ff09..901be564c 100644 --- a/doc/project-structure.md +++ b/doc/project-structure.md @@ -105,7 +105,7 @@ nvx/ | | |-- net/ CIDR parsing and subtraction, and hostname and guest-network forms | | |-- host/ Artifacts, launch, network and egress-policy lowering, sessions, VM instances, sandbox stores, the sandbox launch profile, and the `nvx-rs` CLI (feature `cli`) | | |-- testing/ Fake OpenVMM (feature `testing`) -| | |-- guest/ Programs that run in Linux guests, the guest's NVXS connection, the libraries through which any guest init or agent starts a program, collects its children's statuses, settles its executions' cgroups, drives each execution, and bind-mounts host mappings below the roots that it names, and the typed calls through which it runs the time ABI's steps and the reseed (feature `guest`) +| | |-- guest/ Programs that run in Linux guests, the guest's NVXS connection, the libraries through which any guest init or agent starts a program, collects its children's statuses, settles its executions' cgroups, drives each execution, bind-mounts host mappings below the roots that it names, and assembles and tears down a layered root, and the typed calls through which it runs the time ABI's steps and the reseed (feature `guest`) | | `-- bin/ nvx-rs CLI, nvx-fake-openvmm test double, and the guest binaries | |-- tests/ Fake-OpenVMM instance lifecycle, sandbox store, CLI, NVXS session, guest-binary, and hypervisor managed-lifecycle tests, the store's record fixtures, and the golden fixtures that Python recorded | |-- build.rs Stages the bundled artifacts (feature `bundled`) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 166a6ea22..891febdd9 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -41,8 +41,9 @@ nvx-rs/ │ ├── guest/ feature `guest`: the programs that run in Linux guests, the guest's NVXS │ │ connection, the libraries through which a guest init or agent starts a │ │ program, collects its children's statuses, settles its executions' -│ │ cgroups, and bind-mounts host mappings below the roots that it names, and -│ │ the typed calls through which it runs the time ABI's steps and the reseed +│ │ cgroups, bind-mounts host mappings below the roots that it names, and +│ │ assembles and tears down a layered root, and the typed calls through which +│ │ it runs the time ABI's steps and the reseed │ └── bin/ nvx-rs, nvx-fake-openvmm, and the guest binaries ├── tests/ fake-OpenVMM, CLI, NVXS session, process-supervision, store, and guest-binary │ tests, the store's record fixtures, and the golden fixtures that Python @@ -85,6 +86,7 @@ nvx-rs/ | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on` whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev`. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own @@ -584,14 +586,15 @@ Linux images ship all nine at their established `/sbin/nvx-*` paths. The module's libraries are not programs: `spawn` starts a program in a new process, `process` collects the status of a child that the caller names, `cgroup` places, kills, and settles the processes of an execution, `exec` -drives one program as an execution, and `mappings` bind-mounts host mappings -below the roots that its caller names. `nvx-managed-agent` launches, drives, -collects, and settles its workloads through them, and applies its host mapping -table through `mappings`, with its mount of the host's export and `/` as the -roots, and any other guest init or agent can use them too. Any guest init or -agent can also run the steps of `nvx-time` and `nvx-reseed` in its own process, -through the `nvx::guest::time` and `nvx::guest::reseed` calls, on which both -programs run. +drives one program as an execution, `mappings` bind-mounts host mappings below +the roots that its caller names, and `root` bootstraps the guest for a layered +root, assembles the root, mounts live shares in it, and tears it down. +`nvx-managed-agent` launches, drives, collects, and settles its workloads +through them, and applies its host mapping table through `mappings`, with its +mount of the host's export and `/` as the roots, and any other guest init or +agent can use them too. Any guest init or agent can also run the steps of +`nvx-time` and `nvx-reseed` in its own process, through the `nvx::guest::time` +and `nvx::guest::reseed` calls, on which both programs run. | Binary | Program | | --- | --- | diff --git a/nvx-rs/src/guest/mod.rs b/nvx-rs/src/guest/mod.rs index 3a43c32f4..8d5aad92d 100644 --- a/nvx-rs/src/guest/mod.rs +++ b/nvx-rs/src/guest/mod.rs @@ -3,9 +3,10 @@ //! [`spawn`], through which it starts a program; [`process`], through which it collects the //! status of each child that it names, and of no other; [`cgroup`], through which it places, //! accounts, kills, and settles the processes of its executions; [`mappings`], through which it -//! bind-mounts host mappings below the roots that it names; and [`exec`], which drives each -//! program as one execution, with its launch, its bounded output and input, its ending, and its -//! outcome. +//! bind-mounts host mappings below the roots that it names; [`exec`], which drives each program +//! as one execution, with its launch, its bounded output and input, its ending, and its outcome; +//! and [`root`], through which it bootstraps the guest for a layered root, assembles the root, +//! mounts live shares in it, and tears it down. //! //! Each other module holds one program, and the crate's binary of the same name runs its `main`. //! Two of them also offer their steps as typed calls, which any guest init or agent runs in its @@ -90,6 +91,7 @@ pub mod mmio_write; pub mod nvxs; pub mod process; pub mod reseed; +pub mod root; pub mod spawn; #[cfg(target_arch = "x86_64")] pub mod time; diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs new file mode 100644 index 000000000..75527cf4f --- /dev/null +++ b/nvx-rs/src/guest/root.rs @@ -0,0 +1,805 @@ +//! Operations that bootstrap a guest for a layered root, assemble the root, mount live shares in +//! it, and tear it down, for a guest init or agent, whether or not it is PID 1. +//! +//! - [`bootstrap`] prepares the guest once: it creates the agent's and the workload's sibling +//! cgroups of a [`Layout`](super::cgroup::Layout), moves every process of their parent into the +//! agent's, applies the agent's and the workload's resource values, and keeps user namespaces +//! unavailable. +//! - [`assemble`] finds the block devices of an [`Assembly`] by their MMIO windows, checks and +//! flushes them, and mounts the root: a read-only EROFS image, the `distro` layer, under an +//! overlay whose upper and work directories are on an ext4 scratch. It returns the [`Root`] +//! that owns the mounts. +//! - [`Root::mount_share`] and [`Root::mount_aggregate`] mount the VMM's live shares at targets in +//! the root, after the checks that [live shares](#live-shares) lists. +//! - [`Root::teardown`] unmounts, in order, the live shares in reverse mount order, the overlay, +//! the layer, and the scratch. It attempts each, whatever failed before, and reports every +//! unmount that failed. A root that is dropped tears down the same way, but ignores failures. +//! +//! A failure is an [`Error`] that names what failed. The operations need the guest's root user, +//! in the guest's initial mount and cgroup namespaces. They never collect a child's status, +//! change the caller's root directory or credentials, start an async runtime, or power the VM +//! off, so a caller that is not PID 1 keeps its own reaping, transport, and shutdown. +//! +//! # Runtime directory +//! +//! The root's mounts are below a runtime directory that the caller chooses, such as `/run/nvx`, +//! which holds: +//! +//! - `layers/distro`, the EROFS image; +//! - `scratch`, the scratch, with the overlay's `upper` and `work` directories; +//! - `rootfs`, the overlay, which is the root; +//! - and `shares`, an aggregate live share, whose root lists every child, out of the workload's +//! reach. +//! +//! # Live shares +//! +//! A live share is a virtio-fs device that the VMM exports from the host: one host directory, +//! mounted at a target in the root, or an aggregate of host directories and files, each of which, +//! a child, is bound at its own target. Before it mounts anything for a share, the root checks +//! that each target: +//! +//! - is absolute and canonical: not `/`, without an empty, `.`, or `..` component, and without a +//! trailing `/`; +//! - holds no `\`, `=`, `,`, or NUL; +//! - is not where a container runtime mounts or binds in the root: `/proc`, `/sys`, `/dev`, +//! `/.nvx-agent`, `/etc/machine-id`, `/run/nvx/time`, or below them, nor `/etc`, `/run`, or +//! `/run/nvx`, below which it mounts; +//! - is not another share's target, nor inside one, nor holds one; +//! - crosses no symbolic link in the root; +//! - and is a directory, or a regular file for a child that is a host file. +//! +//! It creates each missing component of a target in the root: a directory with mode 0755, or a +//! file child's file with mode 0644. +//! +//! ```no_run +//! use std::path::Path; +//! use std::time::Duration; +//! +//! use nvx::guest::cgroup::Layout; +//! use nvx::guest::root::{self, Access, Assembly, Device, Resources}; +//! +//! # fn main() -> Result<(), Box> { +//! let layout = Layout::new("/sys/fs/cgroup/agent", "/sys/fs/cgroup/container")?; +//! let resources = Resources { +//! agent_memory_low: Some(16 << 20), +//! ..Resources::default() +//! }; +//! let bootstrap = root::bootstrap(&layout, &resources)?; +//! let mut root = root::assemble(&Assembly { +//! runtime: "/run/nvx".into(), +//! distro: Device { mmio: 0xd000_0000, sectors: None }, +//! distro_uuid: None, +//! scratch: Device { mmio: 0xd000_1000, sectors: None }, +//! timeout: Duration::from_secs(30), +//! })?; +//! root.mount_share("workspace", Path::new("/workspace"), Access::ReadWrite)?; +//! // Run the workload in `bootstrap.workload()`, in the root at `root.path()`; then: +//! root.teardown()?; +//! # Ok(()) +//! # } +//! ``` + +mod block; +mod bootstrap; +mod kernel; +mod share; +#[cfg(test)] +mod tests; + +use std::fmt; +use std::fs; +use std::io; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use self::kernel::{Kernel, Paths, SystemKernel}; +use super::sys::{self, Errno, MountFlags}; + +pub use self::bootstrap::{Bootstrap, BootstrapStep, Resources, bootstrap}; +pub use self::share::{Access, ShareChild, ShareKind, ShareRefusal}; + +/// The EROFS image's mount point, below the runtime directory. +const DISTRO_DIRECTORY: &str = "layers/distro"; +/// The scratch's mount point, below the runtime directory. +const SCRATCH_DIRECTORY: &str = "scratch"; +/// The overlay's mount point, the root, below the runtime directory. +const ROOT_DIRECTORY: &str = "rootfs"; +/// The overlay's upper directory, below the scratch. +const UPPER_DIRECTORY: &str = "upper"; +/// The overlay's work directory, below the scratch. +const WORK_DIRECTORY: &str = "work"; +/// The overlay's options other than its directories: metadata-only copy-up, and inode numbers +/// that stay unique across the layers. +const OVERLAY_OPTIONS: &[u8] = b"metacopy=on,xino=on"; +/// The characters that a runtime directory may not hold: the separators and escape of the +/// overlay's options, and the C string's terminator. +const RUNTIME_RESERVED: &[u8] = b",:\\=\0"; + +/// The role of a block device in a layered root. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Role { + /// The read-only EROFS image, the overlay's lower layer. + Distro, + /// The writable ext4 scratch, which holds the overlay's upper and work directories. + Scratch, +} + +impl fmt::Display for Role { + /// Writes `distro` or `scratch`. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Distro => "distro", + Self::Scratch => "scratch", + }) + } +} + +/// A block device that the VMM attaches for a role, which the base address of its MMIO window +/// names, as the VMM declares it on the kernel command line. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Device { + /// The base address of the device's MMIO window. + pub mmio: u64, + /// The size that the device must have, in 512-byte sectors, if the caller expects one. + pub sectors: Option, +} + +/// A step on a found block device, which an [`Error::Device`] names. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DeviceStep { + /// Reading its size. + Size, + /// Flushing its buffers. + Flush, + /// Reading its EROFS superblock. + Superblock, +} + +impl fmt::Display for DeviceStep { + /// Writes the step as the verb phrase of a failure: `failed to {step} the device`. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Size => "read the size of", + Self::Flush => "flush the buffers of", + Self::Superblock => "read the superblock of", + }) + } +} + +/// What [`assemble`] mounts, and where. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Assembly { + /// The runtime directory: an absolute path without `,`, `:`, `\`, `=`, or NUL, which the + /// overlay's options reserve. + pub runtime: PathBuf, + /// The device of the EROFS image. + pub distro: Device, + /// The UUID that the EROFS image's superblock must hold, if the caller expects one. + pub distro_uuid: Option<[u8; 16]>, + /// The device of the ext4 scratch. + pub scratch: Device, + /// How long to wait for each device to appear. + pub timeout: Duration, +} + +/// Assembles the layered root of `assembly`, as the [module](self) describes, and returns the +/// root that owns its mounts. In order, it: +/// +/// 1. checks the runtime directory; +/// 2. creates its `layers/distro`, `scratch`, and `rootfs` directories, as `mkdir -p` does; +/// 3. finds the distro device, polling for it every 50 ms until `timeout` passes, checks its +/// size if the caller expects one, and flushes its buffers; +/// 4. checks the expected UUID, if any, in the device's EROFS superblock, and reads nothing else; +/// 5. mounts the device's EROFS image read-only at `layers/distro`; +/// 6. finds, checks, and flushes the scratch device the same way; +/// 7. mounts its ext4 file system at `scratch` with `nosuid` and `nodev`; +/// 8. creates the scratch's `upper` and `work` directories unless they exist; +/// 9. and mounts the overlay at `rootfs`, with the distro layer as its lower directory, and +/// `metacopy=on,xino=on`. +/// +/// A failure unmounts what the assembly mounted, as [`Root::teardown`] does, and reports both. +pub fn assemble(assembly: &Assembly) -> Result { + assemble_with(Box::new(SystemKernel), &Paths::default(), assembly) +} + +/// Implements [`assemble`] with `kernel`, and with the kernel's interfaces at `paths`. +fn assemble_with( + kernel: Box, + paths: &Paths, + assembly: &Assembly, +) -> Result { + let runtime = assembly.runtime.as_os_str().as_bytes(); + if !runtime.starts_with(b"/") || runtime.iter().any(|byte| RUNTIME_RESERVED.contains(byte)) { + return Err(AssemblyError { + error: Error::Runtime { + path: assembly.runtime.clone(), + }, + cleanup: None, + }); + } + let mut root = Root { + kernel, + runtime: assembly.runtime.clone(), + path: assembly.runtime.join(ROOT_DIRECTORY), + overlay: None, + distro: None, + scratch: None, + shares: Vec::new(), + claimed: Vec::new(), + }; + match root.mount_layers(paths, assembly) { + Ok(()) => Ok(root), + Err(error) => Err(AssemblyError { + error, + cleanup: root.unmount().err(), + }), + } +} + +/// The handle of an assembled layered root, which owns its mounts: those of the assembly, and +/// those of its live shares. +/// +/// [`teardown`](Self::teardown) unmounts them and reports each failure. Dropping the root +/// unmounts them the same way, and ignores failures. +pub struct Root { + /// The kernel that mounts and unmounts. + kernel: Box, + /// The runtime directory. + runtime: PathBuf, + /// The overlay's mount point, which is the root. + path: PathBuf, + /// The overlay's mount point, while it is mounted. + overlay: Option, + /// The EROFS image's mount point, while it is mounted. + distro: Option, + /// The scratch's mount point, while it is mounted. + scratch: Option, + /// The live shares' mounts and their mount points, in mount order. + shares: Vec<(Mount, PathBuf)>, + /// The targets of the live shares, which another share's target may not overlap. + claimed: Vec, +} + +impl Root { + /// Returns the root: the overlay's mount point, `rootfs` in the runtime directory. + pub fn path(&self) -> &Path { + &self.path + } + + /// Returns the runtime directory. + pub fn runtime(&self) -> &Path { + &self.runtime + } + + /// Unmounts the root's mounts, in order: the live shares in reverse mount order, the overlay, + /// the EROFS image, and the scratch. + /// + /// Each unmount is attempted, whatever failed before, without detaching a busy mount, and + /// every one that failed is reported, with the mount that it leaves in place. + pub fn teardown(mut self) -> Result<(), TeardownError> { + self.unmount() + } + + /// Implements [`assemble`]'s steps after the first. + fn mount_layers(&mut self, paths: &Paths, assembly: &Assembly) -> Result<(), Error> { + let distro = self.runtime.join(DISTRO_DIRECTORY); + let scratch = self.runtime.join(SCRATCH_DIRECTORY); + for directory in [&distro, &scratch, &self.path] { + fs::create_dir_all(directory).map_err(|error| Error::MountPoint { + path: directory.clone(), + error, + })?; + } + let device = block::prepare( + self.kernel.as_mut(), + paths, + Role::Distro, + assembly.distro, + assembly.timeout, + )?; + if let Some(expected) = assembly.distro_uuid { + match block::erofs_uuid(&device) { + Ok(Some(actual)) if actual == expected => {} + Ok(Some(actual)) => return Err(Error::Uuid { expected, actual }), + Ok(None) => return Err(Error::NotErofs { device }), + Err(error) => { + return Err(Error::Device { + role: Role::Distro, + step: DeviceStep::Superblock, + device, + error, + }); + } + } + } + let source = device.as_os_str().as_bytes(); + self.mount_at( + &Mount::Distro, + Some(source), + &distro, + Some(b"erofs"), + MountFlags::MS_RDONLY, + None, + )?; + self.distro = Some(distro.clone()); + let device = block::prepare( + self.kernel.as_mut(), + paths, + Role::Scratch, + assembly.scratch, + assembly.timeout, + )?; + self.mount_at( + &Mount::Scratch, + Some(device.as_os_str().as_bytes()), + &scratch, + Some(b"ext4"), + MountFlags::MS_NOSUID | MountFlags::MS_NODEV, + None, + )?; + self.scratch = Some(scratch.clone()); + let upper = scratch.join(UPPER_DIRECTORY); + let work = scratch.join(WORK_DIRECTORY); + for directory in [&upper, &work] { + create_directory(directory).map_err(|error| Error::MountPoint { + path: directory.clone(), + error, + })?; + } + let options = [ + b"lowerdir=".as_slice(), + distro.as_os_str().as_bytes(), + b",upperdir=", + upper.as_os_str().as_bytes(), + b",workdir=", + work.as_os_str().as_bytes(), + b",", + OVERLAY_OPTIONS, + ] + .concat(); + let root = self.path.clone(); + self.mount_at( + &Mount::Overlay, + Some(b"overlay"), + &root, + Some(b"overlay"), + MountFlags::empty(), + Some(&options), + )?; + self.overlay = Some(root); + Ok(()) + } + + /// Mounts `source` at `target` with `filesystem`, `flags`, and the file system's own options + /// `data`, and reports a failure as one to mount `mount`. + fn mount_at( + &mut self, + mount: &Mount, + source: Option<&[u8]>, + target: &Path, + filesystem: Option<&[u8]>, + flags: MountFlags, + data: Option<&[u8]>, + ) -> Result<(), Error> { + let mounted = (|| { + let source = source.map(sys::c_string).transpose()?; + let filesystem = filesystem.map(sys::c_string).transpose()?; + let data = data.map(sys::c_string).transpose()?; + let target = sys::path_c_string(target)?; + self.kernel.mount( + source.as_deref(), + &target, + filesystem.as_deref(), + flags, + data.as_deref(), + ) + })(); + mounted.map_err(|error| Error::Mount { + mount: mount.clone(), + error: error.into(), + }) + } + + /// Unmounts what the root has mounted, as [`teardown`](Self::teardown) describes, and + /// forgets it, failed unmounts included. + fn unmount(&mut self) -> Result<(), TeardownError> { + let assembly = [ + (Mount::Overlay, self.overlay.take()), + (Mount::Distro, self.distro.take()), + (Mount::Scratch, self.scratch.take()), + ]; + let mounts: Vec<(Mount, PathBuf)> = self + .shares + .drain(..) + .rev() + .chain( + assembly + .into_iter() + .filter_map(|(mount, path)| Some((mount, path?))), + ) + .collect(); + self.claimed.clear(); + let mut failures = Vec::new(); + for (mount, path) in mounts { + let unmounted = + sys::path_c_string(&path).and_then(|target| self.kernel.unmount(&target)); + if let Err(error) = unmounted { + failures.push(Unmount { + mount, + path, + error: error.into(), + }); + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(TeardownError { failures }) + } + } +} + +impl Drop for Root { + /// Unmounts what the root has mounted, as [`Root::teardown`] does, and ignores failures. + fn drop(&mut self) { + let _ = self.unmount(); + } +} + +impl fmt::Debug for Root { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Root") + .field("runtime", &self.runtime) + .field("path", &self.path) + .field("overlay", &self.overlay) + .field("distro", &self.distro) + .field("scratch", &self.scratch) + .field("shares", &self.shares) + .finish_non_exhaustive() + } +} + +/// Creates the directory `path` unless a directory, and not a symbolic link, is there already. +fn create_directory(path: &Path) -> io::Result<()> { + match fs::create_dir(path) { + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + if fs::symlink_metadata(path)?.is_dir() { + Ok(()) + } else { + Err(Errno::ENOTDIR.into()) + } + } + result => result, + } +} + +/// A mount of a layered root. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Mount { + /// The EROFS image, at `layers/distro` in the runtime directory. + Distro, + /// The scratch, at `scratch` in the runtime directory. + Scratch, + /// The overlay, which is the root, at `rootfs` in the runtime directory. + Overlay, + /// A live share, at its target in the root. + Share { + /// The share's virtio-fs tag. + tag: String, + /// The share's target in the root. + target: PathBuf, + }, + /// An aggregate live share, at `shares` in the runtime directory. + Aggregate { + /// The aggregate's virtio-fs tag. + tag: String, + }, + /// A child of an aggregate live share, bound at its target in the root. + Child { + /// The child's name in the aggregate. + name: String, + /// The child's target in the root. + target: PathBuf, + }, +} + +impl fmt::Display for Mount { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Distro => formatter.write_str("the distro layer"), + Self::Scratch => formatter.write_str("the scratch"), + Self::Overlay => formatter.write_str("the overlay"), + Self::Share { tag, target } => { + write!(formatter, "the live share {tag} at {}", target.display()) + } + Self::Aggregate { tag } => write!(formatter, "the aggregate live share {tag}"), + Self::Child { name, target } => { + write!( + formatter, + "the live share child {name} at {}", + target.display() + ) + } + } + } +} + +/// Why a root operation failed. Its text names what failed, and the system's error, if any. +#[derive(Debug)] +#[non_exhaustive] +pub enum Error { + /// A step of [`bootstrap`] failed. + Bootstrap { + /// The step that failed. + step: BootstrapStep, + /// The step's error. + error: io::Error, + }, + /// The runtime directory is not an absolute path, or holds a reserved character. + Runtime { + /// The runtime directory. + path: PathBuf, + }, + /// A mount point, or a directory below one, could not be created. + MountPoint { + /// The path that could not be created. + path: PathBuf, + /// The error. + error: io::Error, + }, + /// The kernel's MMIO windows, or a window's devices, could not be read. + Lookup { + /// The role of the device that was looked up. + role: Role, + /// The base address of its MMIO window. + mmio: u64, + /// The error. + error: io::Error, + }, + /// No block device appeared at an MMIO window in time. + Missing { + /// The role of the device that did not appear. + role: Role, + /// The base address of its MMIO window. + mmio: u64, + }, + /// A device's size is not the expected one. + Size { + /// The device's role. + role: Role, + /// The device's node. + device: PathBuf, + /// The expected size, in 512-byte sectors. + expected: u64, + /// The device's size, in 512-byte sectors. + actual: u64, + }, + /// A step on a device failed. + Device { + /// The device's role. + role: Role, + /// The step that failed. + step: DeviceStep, + /// The device's node. + device: PathBuf, + /// The error. + error: io::Error, + }, + /// The distro device holds no EROFS superblock, so it has no UUID to check. + NotErofs { + /// The device's node. + device: PathBuf, + }, + /// The EROFS image's UUID is not the expected one. + Uuid { + /// The expected UUID. + expected: [u8; 16], + /// The image's UUID. + actual: [u8; 16], + }, + /// A mount failed. + Mount { + /// The mount that failed. + mount: Mount, + /// The error. + error: io::Error, + }, + /// A bound child of an aggregate live share could not be restricted to its access, `nosuid`, + /// and `nodev`. It stays bound, and the root unmounts it when it is torn down. + Restrict { + /// The child's mount. + mount: Mount, + /// The error. + error: io::Error, + }, + /// A live share was refused before anything was mounted for it. + Share(ShareRefusal), +} + +impl fmt::Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Bootstrap { step, error } => write!(formatter, "failed to {step}: {error}"), + Self::Runtime { path } => write!( + formatter, + "the runtime directory is not an absolute path without `,`, `:`, `\\`, or `=`: {}", + path.display() + ), + Self::MountPoint { path, error } => { + write!(formatter, "failed to create {}: {error}", path.display()) + } + Self::Lookup { role, mmio, error } => write!( + formatter, + "failed to look up the {role} device at MMIO {mmio:#x}: {error}" + ), + Self::Missing { role, mmio } => write!( + formatter, + "the {role} device at MMIO {mmio:#x} did not appear" + ), + Self::Size { + role, + device, + expected, + actual, + } => write!( + formatter, + "the {role} device {} has {actual} sectors, expected {expected}", + device.display() + ), + Self::Device { + role, + step, + device, + error, + } => write!( + formatter, + "failed to {step} the {role} device {}: {error}", + device.display() + ), + Self::NotErofs { device } => write!( + formatter, + "the distro device {} holds no EROFS image", + device.display() + ), + Self::Uuid { expected, actual } => write!( + formatter, + "the distro layer's UUID is {}, expected {}", + Uuid(actual), + Uuid(expected) + ), + Self::Mount { mount, error } => write!(formatter, "failed to mount {mount}: {error}"), + Self::Restrict { mount, error } => { + write!(formatter, "failed to restrict {mount}: {error}") + } + Self::Share(refusal) => refusal.fmt(formatter), + } + } +} + +impl std::error::Error for Error {} + +/// Why [`assemble`] failed, and how unmounting what it had mounted failed, if it did. +#[derive(Debug)] +pub struct AssemblyError { + /// The assembly's failure. + error: Error, + /// The failures to unmount what the assembly had mounted. + cleanup: Option, +} + +impl AssemblyError { + /// Returns the assembly's failure. + pub fn error(&self) -> &Error { + &self.error + } + + /// Returns the failures to unmount what the assembly had mounted, if any: those mounts stay. + pub fn cleanup(&self) -> Option<&TeardownError> { + self.cleanup.as_ref() + } + + /// Returns the assembly's failure and the cleanup's. + pub fn into_parts(self) -> (Error, Option) { + (self.error, self.cleanup) + } +} + +impl fmt::Display for AssemblyError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.error.fmt(formatter)?; + match &self.cleanup { + Some(cleanup) => write!(formatter, "; then {cleanup}"), + None => Ok(()), + } + } +} + +impl std::error::Error for AssemblyError {} + +/// The unmounts of a teardown that failed, in the order in which they were attempted. +#[derive(Debug)] +pub struct TeardownError { + /// The failed unmounts, which is never empty. + failures: Vec, +} + +impl TeardownError { + /// Returns the failed unmounts, in the order in which they were attempted. + pub fn failures(&self) -> &[Unmount] { + &self.failures + } +} + +impl fmt::Display for TeardownError { + /// Writes each failure, separated by `; `. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for (index, failure) in self.failures.iter().enumerate() { + if index > 0 { + formatter.write_str("; ")?; + } + failure.fmt(formatter)?; + } + Ok(()) + } +} + +impl std::error::Error for TeardownError {} + +/// An unmount that failed, which left its mount in place. +#[derive(Debug)] +pub struct Unmount { + /// The mount. + mount: Mount, + /// Its mount point. + path: PathBuf, + /// The error. + error: io::Error, +} + +impl Unmount { + /// Returns the mount that stays in place. + pub fn mount(&self) -> &Mount { + &self.mount + } + + /// Returns its mount point. + pub fn path(&self) -> &Path { + &self.path + } + + /// Returns the error of its unmount. + pub fn error(&self) -> &io::Error { + &self.error + } +} + +impl fmt::Display for Unmount { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "failed to unmount {}: {}", + self.mount, self.error + ) + } +} + +/// A UUID, which displays in its canonical form: 32 lowercase hexadecimal digits, in groups of 8, +/// 4, 4, 4, and 12 that hyphens separate. +struct Uuid<'a>(&'a [u8; 16]); + +impl fmt::Display for Uuid<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + for (index, byte) in self.0.iter().enumerate() { + if matches!(index, 4 | 6 | 8 | 10) { + formatter.write_str("-")?; + } + write!(formatter, "{byte:02x}")?; + } + Ok(()) + } +} diff --git a/nvx-rs/src/guest/root/block.rs b/nvx-rs/src/guest/root/block.rs new file mode 100644 index 000000000..40b2e031e --- /dev/null +++ b/nvx-rs/src/guest/root/block.rs @@ -0,0 +1,225 @@ +//! The block devices that the VMM attaches in the root's roles, which the base addresses of their +//! MMIO windows name. +//! +//! The VMM declares each device on the kernel command line, so the kernel names its MMIO window +//! `virtio-mmio.N` in `/proc/iomem`, and creates its block device below the window's device in +//! `/sys/devices/virtio-mmio-cmdline`. + +use std::fs::{self, File}; +use std::io; +use std::os::unix::fs::FileExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use super::kernel::{Kernel, Paths}; +use super::{Device, DeviceStep, Error, Role}; +use crate::guest::parse::{parse_decimal, parse_hexadecimal}; +use crate::guest::sys; + +/// The interval between two polls for a block device that has not appeared yet. +pub(super) const POLL_INTERVAL: Duration = Duration::from_millis(50); +/// The prefix of the name of a virtio MMIO window in `/proc/iomem`. +const WINDOW_PREFIX: &[u8] = b"virtio-mmio."; +/// The directory, below sysfs, of the virtio MMIO devices that the kernel command line declares. +const WINDOW_DEVICES: &str = "devices/virtio-mmio-cmdline"; +/// The byte offset of the EROFS superblock on its device. +const EROFS_SUPERBLOCK: u64 = 1024; +/// The EROFS superblock's magic number, which it starts with, little-endian. +const EROFS_MAGIC: u32 = 0xE0F5_E1E2; +/// The range of the file system's UUID in the EROFS superblock. +const EROFS_UUID: std::ops::Range = 48..64; + +/// Finds the block device of `role` that the VMM attached at `device`'s MMIO window, polling +/// until it appears for as long as `timeout` allows, checks its size if the caller expects one, +/// and flushes its buffers, so that later reads come from the device. Returns the path of its +/// node. +pub(super) fn prepare( + kernel: &mut dyn Kernel, + paths: &Paths, + role: Role, + device: Device, + timeout: Duration, +) -> Result { + let (name, path) = resolve(kernel, paths, role, device.mmio, timeout)?; + if let Some(expected) = device.sectors { + let size = paths.sys.join("class/block").join(&name).join("size"); + let actual = fs::read(&size) + .and_then(|text| parse_count(&text)) + .map_err(|error| Error::Device { + role, + step: DeviceStep::Size, + device: path.clone(), + error, + })?; + if actual != expected { + return Err(Error::Size { + role, + device: path, + expected, + actual, + }); + } + } + let flushed = sys::path_c_string(&path).and_then(|device| kernel.flush(&device)); + flushed.map_err(|error| Error::Device { + role, + step: DeviceStep::Flush, + device: path.clone(), + error: error.into(), + })?; + Ok(path) +} + +/// Returns the name and the node of the block device that the VMM attached at `mmio`, polling +/// for it every [`POLL_INTERVAL`] while `timeout` allows another poll. +fn resolve( + kernel: &mut dyn Kernel, + paths: &Paths, + role: Role, + mmio: u64, + timeout: Duration, +) -> Result<(String, PathBuf), Error> { + let polls = timeout.as_millis() / POLL_INTERVAL.as_millis() + 1; + for poll in 0..polls { + if poll > 0 { + kernel.sleep(POLL_INTERVAL); + } + let found = find(paths, mmio).map_err(|error| Error::Lookup { role, mmio, error })?; + if let Some(name) = found { + let node = paths.dev.join(&name); + // The kernel creates the node right after the device's sysfs directory. + if fs::symlink_metadata(&node).is_ok() { + return Ok((name, node)); + } + } + } + Err(Error::Missing { role, mmio }) +} + +/// Returns the name of the block device below the virtio MMIO window at `mmio`, or `None` while +/// the kernel lists no such window, or has not probed it yet. +fn find(paths: &Paths, mmio: u64) -> io::Result> { + let windows = fs::read(paths.proc.join("iomem"))?; + for line in windows.split(|&byte| byte == b'\n') { + // `START-END : NAME`, indented below the window that holds it. + let mut fields = line + .split(|&byte| byte == b' ') + .filter(|field| !field.is_empty()); + let (Some(range), Some(b":"), Some(name)) = (fields.next(), fields.next(), fields.next()) + else { + continue; + }; + let start = range.split(|&byte| byte == b'-').next().unwrap_or(range); + if parse_hexadecimal(start) != Some(mmio) || !name.starts_with(WINDOW_PREFIX) { + continue; + } + let Ok(name) = std::str::from_utf8(name) else { + continue; + }; + if let Some(block) = block_device(&paths.sys.join(WINDOW_DEVICES).join(name))? { + return Ok(Some(block)); + } + } + Ok(None) +} + +/// Returns the name of the first block device, in byte order, of the first virtio device of the +/// window device `window`, or `None` while there is none. +fn block_device(window: &Path) -> io::Result> { + for device in sorted_entries(window)? { + if !device.starts_with("virtio") { + continue; + } + if let Some(block) = sorted_entries(&window.join(&device).join("block"))? + .into_iter() + .next() + { + return Ok(Some(block)); + } + } + Ok(None) +} + +/// Returns the names of the entries of `directory` that are UTF-8, in byte order, or none if it +/// does not exist yet. +fn sorted_entries(directory: &Path) -> io::Result> { + let entries = match fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut names = Vec::new(); + for entry in entries { + if let Ok(name) = entry?.file_name().into_string() { + names.push(name); + } + } + names.sort_unstable(); + Ok(names) +} + +/// Parses the canonical decimal count of a sysfs attribute and its newline, such as a block +/// device's size in 512-byte sectors. +fn parse_count(text: &[u8]) -> io::Result { + text.strip_suffix(b"\n") + .and_then(parse_decimal) + .ok_or_else(|| io::Error::from(sys::Errno::EPROTO)) +} + +/// Reads the UUID of the EROFS file system on the block device `device` from its superblock, and +/// nothing else, or returns `None` if the device holds no EROFS superblock. +pub(super) fn erofs_uuid(device: &Path) -> io::Result> { + let mut superblock = [0; EROFS_UUID.end]; + match File::open(device)?.read_exact_at(&mut superblock, EROFS_SUPERBLOCK) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error), + } + let magic = u32::from_le_bytes(superblock[..4].try_into().expect("four bytes")); + if magic != EROFS_MAGIC { + return Ok(None); + } + Ok(Some( + superblock[EROFS_UUID] + .try_into() + .expect("a UUID is 16 bytes"), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sizes_are_canonical_decimal_counts() { + assert_eq!(parse_count(b"2048\n").unwrap(), 2048); + assert_eq!(parse_count(b"0\n").unwrap(), 0); + for text in [&b"2048"[..], b"", b"\n", b" 1\n", b"0x10\n", b"1\n\n"] { + assert!(parse_count(text).is_err(), "{}", text.escape_ascii()); + } + } + + #[test] + fn the_superblock_names_an_erofs_uuid_or_none() { + let temporary = tempfile::tempdir().unwrap(); + let device = temporary.path().join("device"); + let uuid: [u8; 16] = std::array::from_fn(|index| index as u8 + 1); + let mut image = vec![0u8; 4096]; + image[1024..1028].copy_from_slice(&EROFS_MAGIC.to_le_bytes()); + image[1024 + 48..1024 + 64].copy_from_slice(&uuid); + fs::write(&device, &image).unwrap(); + assert_eq!(erofs_uuid(&device).unwrap(), Some(uuid)); + // Another magic number, or a device too short for a superblock, holds no EROFS. + image[1024] ^= 1; + fs::write(&device, &image).unwrap(); + assert_eq!(erofs_uuid(&device).unwrap(), None); + fs::write(&device, &image[..1060]).unwrap(); + assert_eq!(erofs_uuid(&device).unwrap(), None); + assert_eq!( + erofs_uuid(&temporary.path().join("missing")) + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); + } +} diff --git a/nvx-rs/src/guest/root/bootstrap.rs b/nvx-rs/src/guest/root/bootstrap.rs new file mode 100644 index 000000000..a4766607b --- /dev/null +++ b/nvx-rs/src/guest/root/bootstrap.rs @@ -0,0 +1,364 @@ +//! The guest's bootstrap: the agent's and the workload's sibling cgroups, and user namespaces. + +use std::fmt; +use std::fs; +use std::io; +use std::os::unix::fs::DirBuilderExt; +use std::path::Path; + +use super::Error; +use super::kernel::{Kernel, Paths, SystemKernel}; +use crate::guest::cgroup::{ExecutionCgroup, Layout}; +use crate::guest::parse::parse_decimal; +use crate::guest::sys::{self, Errno, MountFlags}; + +/// The flag of a kernel thread among a process's flags in `/proc/PID/stat`: Linux's +/// `PF_KTHREAD`. +const KERNEL_THREAD: u64 = 0x0020_0000; +/// The most passes that the placement makes over the parent cgroup's processes. A process that a +/// moved one starts meanwhile starts in the parent too, and the next pass moves it. +const PLACEMENT_PASSES: usize = 16; +/// The sysctl, below procfs, of the user namespaces that each user may create. +const USER_NAMESPACES: &str = "sys/user/max_user_namespaces"; + +/// The resource values that [`bootstrap`] applies, each only when the caller gives it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Resources { + /// The agent's `memory.low`, in bytes: memory of the agent's cgroup that the kernel reclaims + /// only when no unprotected memory is left. + pub agent_memory_low: Option, + /// The workload cgroup's `memory.max`, in bytes. + pub workload_memory_max: Option, + /// The workload cgroup's `pids.max`: the most processes in it and below it. + pub workload_pids_max: Option, +} + +impl Resources { + /// Returns whether a value needs the `memory` controller. + fn need_memory(&self) -> bool { + self.agent_memory_low.is_some() || self.workload_memory_max.is_some() + } +} + +/// A step of [`bootstrap`], which an [`Error::Bootstrap`] names. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum BootstrapStep { + /// Checking that the layout's cgroups are siblings. + Layout, + /// Setting `user.max_user_namespaces` to 0. + UserNamespaces, + /// Mounting the cgroup v2 hierarchy at the cgroups' parent. + Hierarchy, + /// Creating the agent's cgroup. + AgentCgroup, + /// Finding the parent cgroup's processes, and moving them into the agent's cgroup. + Placement, + /// Enabling the controllers of the resource values in the parent's `cgroup.subtree_control`. + Controllers, + /// Writing the agent's `memory.low`. + AgentMemoryLow, + /// Creating the workload cgroup. + WorkloadCgroup, + /// Writing the workload cgroup's `memory.max`. + WorkloadMemoryMax, + /// Writing the workload cgroup's `pids.max`. + WorkloadPidsMax, +} + +impl fmt::Display for BootstrapStep { + /// Writes the step as the verb phrase of a failure: `failed to {step}`. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Layout => "check that the agent's and the workload cgroups are siblings", + Self::UserNamespaces => "disable user namespaces", + Self::Hierarchy => "mount the cgroup hierarchy", + Self::AgentCgroup => "create the agent's cgroup", + Self::Placement => "place the agent's processes in its cgroup", + Self::Controllers => "enable the cgroup controllers", + Self::AgentMemoryLow => "set the agent's memory reservation", + Self::WorkloadCgroup => "create the workload cgroup", + Self::WorkloadMemoryMax => "set the workload's memory limit", + Self::WorkloadPidsMax => "set the workload's process limit", + }) + } +} + +/// The cgroups that [`bootstrap`] prepared, and the processes that it moved into the agent's. +#[derive(Clone, Debug)] +pub struct Bootstrap { + /// The agent's cgroup and the workload cgroup. + layout: Layout, + /// The handle of the workload cgroup itself. + workload: ExecutionCgroup, + /// The processes that the placement moved, in the order in which it moved them. + moved: Vec, +} + +impl Bootstrap { + /// Returns the agent's cgroup and the workload cgroup. + pub fn layout(&self) -> &Layout { + &self.layout + } + + /// Returns the handle of the workload cgroup itself, through which a caller that runs one + /// execution at a time places, kills, and settles it, and below which another prepares the + /// cgroup of each execution. + pub fn workload(&self) -> &ExecutionCgroup { + &self.workload + } + + /// Returns the IDs of the processes that the bootstrap moved into the agent's cgroup, in the + /// order in which it moved them. + pub fn moved(&self) -> &[u32] { + &self.moved + } +} + +/// Bootstraps the guest for a layered root: user namespaces, and the agent's and the workload's +/// cgroups of `layout`, which must be siblings. In order, it: +/// +/// 1. sets `user.max_user_namespaces` to 0, so that no process creates a user namespace, even on +/// a kernel that supports them; +/// 2. mounts the cgroup v2 hierarchy, `nosuid,nodev,noexec`, at the cgroups' parent, unless the +/// parent is a cgroup already, which its `cgroup.controllers` shows; +/// 3. creates the agent's cgroup unless it exists; +/// 4. moves every process of the parent cgroup into the agent's cgroup, except kernel threads, +/// which stay: the caller, and every daemon that its prelude started. It repeats until no +/// such process is left, for at most 16 passes, so that one that a moved process starts +/// meanwhile moves too, and fails with `EBUSY` if one is still left; +/// 5. enables the `pids` controller, and the `memory` controller when a memory value needs it, +/// in the parent's `cgroup.subtree_control`, in one write; +/// 6. writes the agent's `memory.low`; +/// 7. creates the workload cgroup unless it exists; +/// 8. and writes its `memory.max` and `pids.max`. +/// +/// Each value is written only when `resources` holds one. A failure names its step, and leaves +/// the earlier steps' changes in place. The bootstrap collects no status, and starts and stops +/// no process. +pub fn bootstrap(layout: &Layout, resources: &Resources) -> Result { + bootstrap_with(&mut SystemKernel, &Paths::default(), layout, resources) +} + +/// Implements [`bootstrap`] with `kernel`, and with procfs at `paths.proc`. +pub(super) fn bootstrap_with( + kernel: &mut dyn Kernel, + paths: &Paths, + layout: &Layout, + resources: &Resources, +) -> Result { + let failed = |step| move |error| Error::Bootstrap { step, error }; + let parent = match (layout.agent().parent(), layout.workload().parent()) { + (Some(agent), Some(workload)) if agent == workload => agent, + _ => return Err(failed(BootstrapStep::Layout)(Errno::EINVAL.into())), + }; + write(kernel, &paths.proc.join(USER_NAMESPACES), b"0") + .map_err(failed(BootstrapStep::UserNamespaces))?; + mount_hierarchy(kernel, parent).map_err(failed(BootstrapStep::Hierarchy))?; + create_cgroup(layout.agent()).map_err(failed(BootstrapStep::AgentCgroup))?; + let moved = + place(kernel, paths, parent, layout.agent()).map_err(failed(BootstrapStep::Placement))?; + let controllers: &[u8] = if resources.need_memory() { + b"+pids +memory" + } else { + b"+pids" + }; + write(kernel, &parent.join("cgroup.subtree_control"), controllers) + .map_err(failed(BootstrapStep::Controllers))?; + if let Some(low) = resources.agent_memory_low { + write( + kernel, + &layout.agent().join("memory.low"), + low.to_string().as_bytes(), + ) + .map_err(failed(BootstrapStep::AgentMemoryLow))?; + } + create_cgroup(layout.workload()).map_err(failed(BootstrapStep::WorkloadCgroup))?; + for (value, file, step) in [ + ( + resources.workload_memory_max, + "memory.max", + BootstrapStep::WorkloadMemoryMax, + ), + ( + resources.workload_pids_max, + "pids.max", + BootstrapStep::WorkloadPidsMax, + ), + ] { + if let Some(value) = value { + write( + kernel, + &layout.workload().join(file), + value.to_string().as_bytes(), + ) + .map_err(failed(step))?; + } + } + let workload = + ExecutionCgroup::open(layout, None).map_err(failed(BootstrapStep::WorkloadCgroup))?; + Ok(Bootstrap { + layout: layout.clone(), + workload, + moved, + }) +} + +/// Writes `value` to the control file `path` through `kernel`. +fn write(kernel: &mut dyn Kernel, path: &Path, value: &[u8]) -> io::Result<()> { + let path = sys::path_c_string(path)?; + Ok(kernel.write(&path, value)?) +} + +/// Mounts the cgroup v2 hierarchy at `parent`, which must exist, unless `parent` is a cgroup. +fn mount_hierarchy(kernel: &mut dyn Kernel, parent: &Path) -> io::Result<()> { + match fs::symlink_metadata(parent.join("cgroup.controllers")) { + Ok(_) => return Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + if !fs::metadata(parent)?.is_dir() { + return Err(Errno::ENOTDIR.into()); + } + let target = sys::path_c_string(parent)?; + Ok(kernel.mount( + Some(c"cgroup2"), + &target, + Some(c"cgroup2"), + MountFlags::MS_NOSUID | MountFlags::MS_NODEV | MountFlags::MS_NOEXEC, + None, + )?) +} + +/// Creates the cgroup `path` unless it exists. A file in its place is `ENOTDIR`. +fn create_cgroup(path: &Path) -> io::Result<()> { + match fs::DirBuilder::new().mode(0o755).create(path) { + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + if fs::symlink_metadata(path)?.is_dir() { + Ok(()) + } else { + Err(Errno::ENOTDIR.into()) + } + } + result => result, + } +} + +/// Moves every process of the cgroup `parent` into the cgroup `agent`, except kernel threads, in +/// passes until a pass finds none to move, and returns the IDs that it moved. +fn place( + kernel: &mut dyn Kernel, + paths: &Paths, + parent: &Path, + agent: &Path, +) -> io::Result> { + let members = parent.join("cgroup.procs"); + let destination = sys::path_c_string(&agent.join("cgroup.procs"))?; + let mut moved = Vec::new(); + for _ in 0..PLACEMENT_PASSES { + let mut found = false; + for process in processes(&fs::read(&members)?)? { + match fs::read(paths.proc.join(process.to_string()).join("stat")) { + Ok(stat) if is_kernel_thread(&stat).ok_or(Errno::EPROTO)? => continue, + Ok(_) => {} + // The process has ended. + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) if error.raw_os_error() == Some(Errno::ESRCH as i32) => continue, + Err(error) => return Err(error), + } + found = true; + match kernel.write(&destination, process.to_string().as_bytes()) { + Ok(()) => moved.push(process), + Err(Errno::ESRCH) => {} + Err(error) => return Err(error.into()), + } + } + if !found { + return Ok(moved); + } + } + Err(Errno::EBUSY.into()) +} + +/// Parses a cgroup's `cgroup.procs`: one decimal process ID per line. +fn processes(members: &[u8]) -> io::Result> { + let Some(members) = members.strip_suffix(b"\n") else { + return if members.is_empty() { + Ok(Vec::new()) + } else { + Err(Errno::EPROTO.into()) + }; + }; + members + .split(|&byte| byte == b'\n') + .map(|line| { + parse_decimal(line) + .and_then(|process| u32::try_from(process).ok()) + .filter(|&process| process != 0) + .ok_or_else(|| Errno::EPROTO.into()) + }) + .collect() +} + +/// Returns whether the `/proc/PID/stat` record `stat` is a kernel thread's, or `None` for a +/// malformed record. Its flags are the seventh field after the command, which is parenthesized +/// and may hold any character, a `)` and spaces included. +fn is_kernel_thread(stat: &[u8]) -> Option { + let end = stat.iter().rposition(|&byte| byte == b')')?; + let flags = stat[end + 1..] + .split(|&byte| byte == b' ') + .filter(|field| !field.is_empty()) + .nth(6)?; + Some(parse_decimal(flags)? & KERNEL_THREAD != 0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kernel_threads_are_known_by_their_flags() { + let user = b"512 (sh) S 1 512 512 34816 512 4194560 147 0 0 0 0 0 0 0 20 0 1 0 72\n"; + let kernel = b"2 (kthreadd) S 0 0 0 0 -1 2129984 0 0 0 0 0 0 0 0 20 0 1 0 1\n"; + assert_eq!(is_kernel_thread(user), Some(false)); + assert_eq!(is_kernel_thread(kernel), Some(true)); + // A command with a parenthesis and spaces of its own. + let tricky = b"77 (a) b (c)) S 1 77 77 0 -1 4194560 0 0 0 0 0 0 0 0 20 0 1 0 9\n"; + assert_eq!(is_kernel_thread(tricky), Some(false)); + for malformed in [ + &b""[..], + b"77 (sh S 1 77 77 0 -1 4194560", + b"77 (sh) S 1 77 77 0 -1", + b"77 (sh) S 1 77 77 0 -1 0x10 0", + ] { + assert_eq!( + is_kernel_thread(malformed), + None, + "{}", + malformed.escape_ascii() + ); + } + } + + #[test] + fn membership_lists_are_decimal_lines() { + assert_eq!(processes(b"").unwrap(), Vec::::new()); + assert_eq!(processes(b"1\n42\n4194304\n").unwrap(), [1, 42, 4_194_304]); + for malformed in [ + &b"1"[..], + b"\n", + b"1\n\n", + b"0\n", + b"x\n", + b"-1\n", + b"4294967296\n", + ] { + assert_eq!( + processes(malformed).unwrap_err().raw_os_error(), + Some(Errno::EPROTO as i32), + "{}", + malformed.escape_ascii() + ); + } + } +} diff --git a/nvx-rs/src/guest/root/kernel.rs b/nvx-rs/src/guest/root/kernel.rs new file mode 100644 index 000000000..3462d83ce --- /dev/null +++ b/nvx-rs/src/guest/root/kernel.rs @@ -0,0 +1,374 @@ +//! The kernel interfaces of the root operations, which tests replace. +//! +//! The operations read the kernel's text files and walk directories through the standard +//! library, at the paths of a [`Paths`], so that a test relocates them to a temporary directory. +//! What a test cannot do in a temporary directory, such as a mount, an unmount, or a flush of a +//! block device's buffers, and the writes to the kernel's control files, whose effects a test +//! simulates, go through a [`Kernel`]. + +use std::ffi::CStr; +use std::path::PathBuf; +use std::time::Duration; + +use crate::guest::sys::{self, Errno, Mode, MountFlags, OpenFlags}; + +/// The kernel's operations that the root operations perform, other than reads: mounts and +/// unmounts, flushes of a block device's buffers, writes of a control file, and the sleeps +/// between two polls for a block device. +/// +/// [`SystemKernel`] performs them. The tests' fake records them, simulates the effect of a +/// cgroup's membership file, and fails the ones that a test chooses. +pub(super) trait Kernel: Send + Sync { + /// Mounts `source` at `target`, as `mount(2)` does, with the file system's own options + /// `data`. + fn mount( + &mut self, + source: Option<&CStr>, + target: &CStr, + filesystem: Option<&CStr>, + flags: MountFlags, + data: Option<&CStr>, + ) -> Result<(), Errno>; + + /// Unmounts the file system mounted at `target`, without detaching a busy one. + fn unmount(&mut self, target: &CStr) -> Result<(), Errno>; + + /// Writes the dirty buffers of the block device `device`, then drops its cached ones. + fn flush(&mut self, device: &CStr) -> Result<(), Errno>; + + /// Writes `value` whole to the existing control file `path`, such as a cgroup's or a + /// sysctl's, and reports a failure to close it. + fn write(&mut self, path: &CStr, value: &[u8]) -> Result<(), Errno>; + + /// Sleeps for `duration`. + fn sleep(&mut self, duration: Duration); +} + +/// The kernel itself. +#[derive(Clone, Copy, Debug, Default)] +pub(super) struct SystemKernel; + +impl Kernel for SystemKernel { + fn mount( + &mut self, + source: Option<&CStr>, + target: &CStr, + filesystem: Option<&CStr>, + flags: MountFlags, + data: Option<&CStr>, + ) -> Result<(), Errno> { + sys::mount_with_options(source, target, filesystem, flags, data) + } + + fn unmount(&mut self, target: &CStr) -> Result<(), Errno> { + sys::unmount(target) + } + + fn flush(&mut self, device: &CStr) -> Result<(), Errno> { + let descriptor = sys::open( + device, + OpenFlags::O_RDONLY | OpenFlags::O_CLOEXEC, + Mode::empty(), + )?; + sys::flush_block_buffers(&descriptor)?; + descriptor.close() + } + + fn write(&mut self, path: &CStr, value: &[u8]) -> Result<(), Errno> { + let descriptor = sys::open( + path, + OpenFlags::O_WRONLY | OpenFlags::O_CLOEXEC, + Mode::empty(), + )?; + descriptor.write_all(value)?; + descriptor.close() + } + + fn sleep(&mut self, duration: Duration) { + std::thread::sleep(duration); + } +} + +/// The directories of the kernel's interfaces that the root operations read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Paths { + /// The mount point of procfs. + pub(super) proc: PathBuf, + /// The mount point of sysfs. + pub(super) sys: PathBuf, + /// The directory of the device nodes. + pub(super) dev: PathBuf, +} + +impl Default for Paths { + /// Returns the paths at which the guest mounts the kernel's interfaces. + fn default() -> Self { + Self { + proc: "/proc".into(), + sys: "/sys".into(), + dev: "/dev".into(), + } + } +} + +#[cfg(test)] +pub(super) use fake::{Call, FakeKernel, Operation}; + +/// The tests' kernel, which records each operation and performs none. +#[cfg(test)] +mod fake { + use std::ffi::CStr; + use std::path::{Path, PathBuf}; + use std::sync::{Arc, Mutex, PoisonError}; + use std::time::Duration; + + use super::Kernel; + use crate::guest::sys::{Errno, MountFlags}; + + /// An operation of the kernel, which a test fails. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(in crate::guest::root) enum Operation { + /// A mount, other than a remount, which its target names. + Mount, + /// A remount, which its target names. + Remount, + /// An unmount, which its target names. + Unmount, + /// A flush of a block device's buffers, which the device names. + Flush, + /// A write of a control file, which its path names. + Write, + } + + /// One operation that the fake kernel recorded, with its paths and options as text. + #[derive(Clone, Debug, PartialEq, Eq)] + pub(in crate::guest::root) enum Call { + /// A mount. + Mount { + /// The source, if any. + source: Option, + /// The target. + target: String, + /// The file system type, if any. + filesystem: Option, + /// The flags. + flags: MountFlags, + /// The file system's own options, if any. + data: Option, + }, + /// An unmount of a target. + Unmount(String), + /// A flush of a block device's buffers. + Flush(String), + /// A write of a value to a control file. + Write(String, String), + } + + /// The type of the action that runs after each sleep. + type SleepAction = Box; + + /// What the fake kernel recorded and what it fails. + #[derive(Default)] + struct State { + /// The operations, in order. + calls: Vec, + /// The operations that fail, with their error, each at the path that names it. + failures: Vec<(Operation, PathBuf, Errno)>, + /// The processes that a write of their ID to a membership file leaves where they are. + unmovable: Vec, + /// The sleeps so far. + sleeps: usize, + /// What happens after each sleep, given the number of sleeps so far. + on_sleep: Option, + } + + /// A kernel that records each operation, performs no mount, unmount, or flush, and fails the + /// operations that a test chooses. + /// + /// A write of a control file writes the regular file at its path, which it creates if + /// needed, except that a write of a process ID to a cgroup's `cgroup.procs` moves the ID + /// there from the `cgroup.procs` of the cgroup's parent, as the kernel moves the process. + /// Its clones share what they record, so a test keeps one while a root owns another. + #[derive(Clone, Default)] + pub(in crate::guest::root) struct FakeKernel(Arc>); + + impl FakeKernel { + /// Runs `action` on the shared state. + fn with(&self, action: impl FnOnce(&mut State) -> T) -> T { + action(&mut self.0.lock().unwrap_or_else(PoisonError::into_inner)) + } + + /// Makes the operations of kind `operation` at `path` fail with `error`. + pub(in crate::guest::root) fn fail( + &self, + operation: Operation, + path: impl Into, + error: Errno, + ) { + self.with(|state| state.failures.push((operation, path.into(), error))); + } + + /// Makes a write of the ID `process` to a membership file leave the process where it is, + /// as for a process that reappears as fast as it is moved. + pub(in crate::guest::root) fn pin(&self, process: u32) { + self.with(|state| state.unmovable.push(process)); + } + + /// Runs `action` after each sleep, with the number of sleeps so far. + pub(in crate::guest::root) fn on_sleep(&self, action: impl FnMut(usize) + Send + 'static) { + self.with(|state| state.on_sleep = Some(Box::new(action))); + } + + /// Returns the operations so far, in order. + pub(in crate::guest::root) fn calls(&self) -> Vec { + self.with(|state| state.calls.clone()) + } + + /// Returns the targets of the unmounts so far, in order. + pub(in crate::guest::root) fn unmounts(&self) -> Vec { + self.calls() + .into_iter() + .filter_map(|call| match call { + Call::Unmount(target) => Some(target), + _ => None, + }) + .collect() + } + + /// Returns the targets of the mounts so far, in order. + pub(in crate::guest::root) fn mounts(&self) -> Vec { + self.calls() + .into_iter() + .filter_map(|call| match call { + Call::Mount { target, .. } => Some(target), + _ => None, + }) + .collect() + } + + /// Returns the paths and values of the writes so far, in order. + pub(in crate::guest::root) fn writes(&self) -> Vec<(String, String)> { + self.calls() + .into_iter() + .filter_map(|call| match call { + Call::Write(path, value) => Some((path, value)), + _ => None, + }) + .collect() + } + + /// Returns the number of sleeps so far. + pub(in crate::guest::root) fn sleeps(&self) -> usize { + self.with(|state| state.sleeps) + } + + /// Records `call`, and returns the error of the failure of `operation` at `path`, if a + /// test chose one. + fn record(&self, call: Call, operation: Operation, path: &CStr) -> Result<(), Errno> { + let path = text(path); + self.with(|state| { + state.calls.push(call); + state + .failures + .iter() + .find(|(failing, failing_path, _)| { + *failing == operation && failing_path.as_os_str() == path.as_str() + }) + .map_or(Ok(()), |&(_, _, error)| Err(error)) + }) + } + } + + /// Returns `text` as a `String`, replacing what is not UTF-8. + fn text(text: &CStr) -> String { + text.to_string_lossy().into_owned() + } + + /// Moves the ID `process` into the membership file `path`, from that of the parent of + /// `path`'s cgroup. + fn move_process(path: &Path, process: u32) -> Result<(), Errno> { + let source = path + .parent() + .and_then(Path::parent) + .map(|parent| parent.join("cgroup.procs")) + .ok_or(Errno::EINVAL)?; + let listed = std::fs::read_to_string(&source).map_err(|_| Errno::ENOENT)?; + let line = process.to_string(); + let remaining: String = listed + .lines() + .filter(|listed| *listed != line) + .map(|listed| format!("{listed}\n")) + .collect(); + std::fs::write(&source, remaining).map_err(|_| Errno::EIO)?; + let mut members = std::fs::read_to_string(path).unwrap_or_default(); + members.push_str(&format!("{process}\n")); + std::fs::write(path, members).map_err(|_| Errno::EIO) + } + + impl Kernel for FakeKernel { + fn mount( + &mut self, + source: Option<&CStr>, + target: &CStr, + filesystem: Option<&CStr>, + flags: MountFlags, + data: Option<&CStr>, + ) -> Result<(), Errno> { + let call = Call::Mount { + source: source.map(text), + target: text(target), + filesystem: filesystem.map(text), + flags, + data: data.map(text), + }; + let operation = if flags.contains(MountFlags::MS_REMOUNT) { + Operation::Remount + } else { + Operation::Mount + }; + self.record(call, operation, target) + } + + fn unmount(&mut self, target: &CStr) -> Result<(), Errno> { + self.record(Call::Unmount(text(target)), Operation::Unmount, target) + } + + fn flush(&mut self, device: &CStr) -> Result<(), Errno> { + self.record(Call::Flush(text(device)), Operation::Flush, device) + } + + fn write(&mut self, path: &CStr, value: &[u8]) -> Result<(), Errno> { + let call = Call::Write(text(path), String::from_utf8_lossy(value).into_owned()); + self.record(call, Operation::Write, path)?; + let path = PathBuf::from(text(path)); + let process = std::str::from_utf8(value) + .ok() + .and_then(|value| value.trim_end().parse::().ok()); + match process { + Some(process) if path.ends_with("cgroup.procs") => { + if self.with(|state| state.unmovable.contains(&process)) { + Ok(()) + } else { + move_process(&path, process) + } + } + _ => std::fs::write(&path, value).map_err(|_| Errno::EIO), + } + } + + fn sleep(&mut self, _duration: Duration) { + let (sleeps, mut action) = self.with(|state| { + state.sleeps += 1; + (state.sleeps, state.on_sleep.take()) + }); + if let Some(action) = action.as_mut() { + action(sleeps); + } + self.with(|state| { + if state.on_sleep.is_none() { + state.on_sleep = action; + } + }); + } + } +} diff --git a/nvx-rs/src/guest/root/share.rs b/nvx-rs/src/guest/root/share.rs new file mode 100644 index 000000000..fd89f5bec --- /dev/null +++ b/nvx-rs/src/guest/root/share.rs @@ -0,0 +1,569 @@ +//! Live shares: virtio-fs devices that the VMM exports from the host, mounted at targets in a +//! layered root after the checks that the [module](super#live-shares) lists, which are the ones +//! that NVX's shell init applies. + +use std::ffi::OsStr; +use std::fmt; +use std::fs; +use std::io; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::path::{Path, PathBuf}; + +use super::{Error, Mount, Root}; +use crate::guest::sys::{self, MountFlags}; + +/// The targets that the runtime mounts or binds over in the root, each with everything below it. +const RESERVED_TREES: [&[u8]; 6] = [ + b"/proc", + b"/sys", + b"/dev", + b"/.nvx-agent", + b"/etc/machine-id", + b"/run/nvx/time", +]; +/// The directories that the runtime creates or mounts below, without the paths below them. +const RESERVED_DIRECTORIES: [&[u8]; 3] = [b"/etc", b"/run", b"/run/nvx"]; +/// The directory, below the runtime directory, at which an aggregate share is mounted. +const AGGREGATE_DIRECTORY: &str = "shares"; +/// The file system type of a live share. +const VIRTIOFS: &[u8] = b"virtiofs"; + +/// Whether the workload may write a live share. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Access { + /// The share is mounted read-only. + ReadOnly, + /// The share is mounted read-write. + ReadWrite, +} + +impl Access { + /// Returns the mount flags of a share with this access, without set-user-ID programs or + /// device nodes. + fn flags(self) -> MountFlags { + let flags = MountFlags::MS_NOSUID | MountFlags::MS_NODEV; + match self { + Self::ReadOnly => flags | MountFlags::MS_RDONLY, + Self::ReadWrite => flags, + } + } +} + +impl fmt::Display for Access { + /// Writes `ro` or `rw`, as a mount option names the access. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::ReadOnly => "ro", + Self::ReadWrite => "rw", + }) + } +} + +/// What a child of an aggregate share is: a host directory, or a host file. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum ShareKind { + /// A directory, bound at a directory of the root. + #[default] + Directory, + /// A regular file, bound at a regular file of the root. + File, +} + +/// A child of an aggregate share: an entry of the aggregate, bound at a target in the root. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ShareChild { + /// The name of the child's entry in the aggregate: letters, digits, `.`, `_`, and `-`, other + /// than `.` and `..`. + pub name: String, + /// The absolute path in the root at which the child is bound. + pub target: PathBuf, + /// Whether the workload may write the child. + pub access: Access, + /// What the child is. + pub kind: ShareKind, +} + +/// Why a live share was refused before anything was mounted for it. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ShareRefusal { + /// The virtio-fs tag is not a nonempty name of letters, digits, `_`, and `-`. + Tag(String), + /// An aggregate has no children. + NoChildren, + /// A child's name is not a nonempty name of letters, digits, `.`, `_`, and `-`, or is `.` or + /// `..`. + ChildName(String), + /// Two children of an aggregate have this name. + DuplicateChild(String), + /// The target is not an absolute path. + Relative(PathBuf), + /// The target is `/`, or has an empty, `.`, or `..` component, or a trailing `/`. + NotCanonical(PathBuf), + /// The target holds a reserved character: `\`, `=`, `,`, or NUL. + ReservedCharacter(PathBuf), + /// The runtime mounts at the target, or below it, or binds over it. + Reserved(PathBuf), + /// The target is the target of another share, or one inside it or that holds it. + Overlap { + /// The other share's target. + claimed: PathBuf, + /// The refused target. + target: PathBuf, + }, + /// A component of the target's path in the root is a symbolic link. + SymbolicLink(PathBuf), + /// The target, or a component of its path, exists in the root but is not a directory. + NotDirectory(PathBuf), + /// A file child's target exists in the root but is not a regular file. + NotFile(PathBuf), +} + +impl fmt::Display for ShareRefusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Tag(tag) => write!(formatter, "live share tag is invalid: {tag}"), + Self::NoChildren => formatter.write_str("the aggregate live share has no children"), + Self::ChildName(name) => write!(formatter, "live share child is invalid: {name}"), + Self::DuplicateChild(name) => { + write!(formatter, "live share child is not unique: {name}") + } + Self::Relative(target) => write!( + formatter, + "live share target must be an absolute path: {}", + target.display() + ), + Self::NotCanonical(target) => write!( + formatter, + "live share target is not canonical: {}", + target.display() + ), + Self::ReservedCharacter(target) => write!( + formatter, + "live share target contains a reserved character: {}", + target.display() + ), + Self::Reserved(target) => write!( + formatter, + "live share target is reserved: {}", + target.display() + ), + Self::Overlap { claimed, target } => write!( + formatter, + "live share targets overlap: {} and {}", + claimed.display(), + target.display() + ), + Self::SymbolicLink(target) => write!( + formatter, + "live share target crosses a symbolic link: {}", + target.display() + ), + Self::NotDirectory(target) => write!( + formatter, + "live share target is not a directory: {}", + target.display() + ), + Self::NotFile(target) => write!( + formatter, + "live share target is not a regular file: {}", + target.display() + ), + } + } +} + +impl Root { + /// Mounts the virtio-fs device `tag` at `target` in the root, a directory, with `access`, + /// `nosuid`, and `nodev`. + /// + /// The target is checked as the [module](super#live-shares) describes, against the targets + /// of the root's other shares, then created in the root if it is missing; nothing is mounted + /// for a refused one. The mount belongs to the root, which unmounts it first when it is torn + /// down. + pub fn mount_share(&mut self, tag: &str, target: &Path, access: Access) -> Result<(), Error> { + check_tag(tag)?; + check_target(target)?; + self.check_overlap(target, &[])?; + let point = mount_point(&self.path, target, ShareKind::Directory)?; + let mount = Mount::Share { + tag: tag.to_owned(), + target: target.to_owned(), + }; + self.mount_at( + &mount, + Some(tag.as_bytes()), + &point, + Some(VIRTIOFS), + access.flags(), + None, + )?; + self.shares.push((mount, point)); + self.claimed.push(target.to_owned()); + Ok(()) + } + + /// Mounts the aggregate virtio-fs device `tag`, with `access`, `nosuid`, and `nodev`, at + /// `shares` in the runtime directory, outside the root, where the workload cannot list its + /// children. Then binds each child, in order, at its target in the root, and restricts the + /// bind to the child's access, `nosuid`, and `nodev`. + /// + /// Every child is checked first: its name, which no other child has, and its target, as the + /// [module](super#live-shares) describes, against the other children's and the root's other + /// shares' targets. Each target is created in the root if it is missing. Nothing is mounted + /// unless every child passes. The mounts belong to the root, which unmounts them first when + /// it is torn down, in reverse mount order, those of an aggregate that failed part of the way + /// included. + pub fn mount_aggregate( + &mut self, + tag: &str, + access: Access, + children: &[ShareChild], + ) -> Result<(), Error> { + check_tag(tag)?; + if children.is_empty() { + return Err(Error::Share(ShareRefusal::NoChildren)); + } + let mut targets: Vec = Vec::with_capacity(children.len()); + let mut points = Vec::with_capacity(children.len()); + for (index, child) in children.iter().enumerate() { + check_name(&child.name)?; + if children[..index] + .iter() + .any(|earlier| earlier.name == child.name) + { + return Err(Error::Share(ShareRefusal::DuplicateChild( + child.name.clone(), + ))); + } + check_target(&child.target)?; + self.check_overlap(&child.target, &targets)?; + targets.push(child.target.clone()); + points.push(mount_point(&self.path, &child.target, child.kind)?); + } + let directory = self.runtime.join(AGGREGATE_DIRECTORY); + fs::create_dir_all(&directory).map_err(|error| Error::MountPoint { + path: directory.clone(), + error, + })?; + let aggregate = Mount::Aggregate { + tag: tag.to_owned(), + }; + self.mount_at( + &aggregate, + Some(tag.as_bytes()), + &directory, + Some(VIRTIOFS), + access.flags(), + None, + )?; + self.shares.push((aggregate, directory.clone())); + for (child, point) in children.iter().zip(points) { + let mount = Mount::Child { + name: child.name.clone(), + target: child.target.clone(), + }; + let source = directory.join(&child.name); + self.mount_at( + &mount, + Some(source.as_os_str().as_bytes()), + &point, + None, + MountFlags::MS_BIND, + None, + )?; + self.shares.push((mount.clone(), point.clone())); + self.claimed.push(child.target.clone()); + // A bind takes its source's flags; only a remount of the bind changes them. + let restricted = sys::path_c_string(&point).and_then(|point| { + self.kernel.mount( + None, + &point, + None, + MountFlags::MS_REMOUNT | MountFlags::MS_BIND | child.access.flags(), + None, + ) + }); + restricted.map_err(|error| Error::Restrict { + mount, + error: error.into(), + })?; + } + Ok(()) + } + + /// Refuses `target` if it overlaps the target of one of the root's shares, or one of + /// `pending`. + fn check_overlap(&self, target: &Path, pending: &[PathBuf]) -> Result<(), Error> { + let below = |inner: &Path, outer: &Path| { + let mut inner = inner.as_os_str().as_bytes().to_vec(); + let mut outer = outer.as_os_str().as_bytes().to_vec(); + inner.push(b'/'); + outer.push(b'/'); + inner.starts_with(&outer) + }; + match self + .claimed + .iter() + .chain(pending) + .find(|claimed| below(target, claimed) || below(claimed, target)) + { + Some(claimed) => Err(Error::Share(ShareRefusal::Overlap { + claimed: claimed.clone(), + target: target.to_owned(), + })), + None => Ok(()), + } + } +} + +/// Refuses a virtio-fs tag that is not a nonempty name of letters, digits, `_`, and `-`. +fn check_tag(tag: &str) -> Result<(), Error> { + if tag.is_empty() + || !tag + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')) + { + return Err(Error::Share(ShareRefusal::Tag(tag.to_owned()))); + } + Ok(()) +} + +/// Refuses an aggregate child's name that is not a nonempty name of letters, digits, `.`, `_`, +/// and `-`, or that is `.` or `..`. +fn check_name(name: &str) -> Result<(), Error> { + if matches!(name, "" | "." | "..") + || !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + return Err(Error::Share(ShareRefusal::ChildName(name.to_owned()))); + } + Ok(()) +} + +/// Refuses a target that is not canonical and absolute, that holds a reserved character, or that +/// the runtime reserves. +fn check_target(target: &Path) -> Result<(), Error> { + let bytes = target.as_os_str().as_bytes(); + // A trailing `/` makes `/` itself, and a target that ends in one, `.`, or `..`, a form below. + let terminated = [bytes, b"/"].concat(); + let refusal = if !bytes.starts_with(b"/") { + Some(ShareRefusal::Relative(target.to_owned())) + } else if [b"//".as_slice(), b"/./", b"/../"] + .iter() + .any(|form| terminated.windows(form.len()).any(|part| part == *form)) + { + Some(ShareRefusal::NotCanonical(target.to_owned())) + } else if bytes + .iter() + .any(|byte| matches!(byte, b'\\' | b'=' | b',' | b'\0')) + { + Some(ShareRefusal::ReservedCharacter(target.to_owned())) + } else if RESERVED_DIRECTORIES.contains(&bytes) + || RESERVED_TREES.iter().any(|tree| { + bytes == *tree || (bytes.starts_with(tree) && bytes.get(tree.len()) == Some(&b'/')) + }) + { + Some(ShareRefusal::Reserved(target.to_owned())) + } else { + None + }; + refusal.map_or(Ok(()), |refusal| Err(Error::Share(refusal))) +} + +/// Walks the components of `target` from `root`, creating each that is missing, and returns the +/// target's path below `root`. Each component is a directory, except the last of a file child, +/// which is a regular file, and none is a symbolic link. +fn mount_point(root: &Path, target: &Path, kind: ShareKind) -> Result { + let refused = |refusal: fn(PathBuf) -> ShareRefusal| Err(Error::Share(refusal(target.into()))); + let components: Vec<&[u8]> = target.as_os_str().as_bytes()[1..] + .split(|&byte| byte == b'/') + .collect(); + let mut path = root.to_path_buf(); + for (index, component) in components.iter().enumerate() { + path.push(OsStr::from_bytes(component)); + let file = kind == ShareKind::File && index + 1 == components.len(); + match fs::symlink_metadata(&path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return refused(ShareRefusal::SymbolicLink); + } + Ok(metadata) if file && !metadata.is_file() => return refused(ShareRefusal::NotFile), + Ok(metadata) if !file && !metadata.is_dir() => { + return refused(ShareRefusal::NotDirectory); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => { + create(&path, file).map_err(|error| Error::MountPoint { + path: path.clone(), + error, + })?; + } + Err(error) => return Err(Error::MountPoint { path, error }), + } + } + Ok(path) +} + +/// Creates the missing component `path` of a target: an empty regular file with mode 0644 if +/// `file` is set, and otherwise a directory with mode 0755, whatever the process's umask. +fn create(path: &Path, file: bool) -> io::Result<()> { + let mode = if file { + // An exclusive creation follows no symbolic link either. + fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o644) + .open(path)?; + 0o644 + } else { + fs::create_dir(path)?; + 0o755 + }; + fs::set_permissions(path, fs::Permissions::from_mode(mode)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Returns the refusal of `target`, if it is refused. + fn refusal(target: &str) -> Option { + match check_target(Path::new(target)) { + Ok(()) => None, + Err(Error::Share(refusal)) => Some(refusal), + Err(error) => panic!("{target}: {error}"), + } + } + + #[test] + fn targets_are_canonical_absolute_paths_outside_the_runtime_s_mounts() { + for target in [ + "/workspace", + "/opt/hostedtoolcache", + "/a/b/c", + "/a/...", + "/.hidden", + "/etc/app", + "/etc/machine-ids", + "/run/app", + "/run/nvx/state", + "/run/nvx/timer", + "/procs", + "/devices", + "/.nvx-agents", + ] { + assert_eq!(refusal(target), None, "{target}"); + } + for target in ["workspace", "", "./a"] { + assert!( + matches!(refusal(target), Some(ShareRefusal::Relative(_))), + "{target}" + ); + } + for target in [ + "/", "//a", "/a//b", "/a/", "/a/.", "/a/./b", "/a/..", "/../a", + ] { + assert!( + matches!(refusal(target), Some(ShareRefusal::NotCanonical(_))), + "{target}" + ); + } + for target in ["/a\\b", "/a=b", "/a,b"] { + assert!( + matches!(refusal(target), Some(ShareRefusal::ReservedCharacter(_))), + "{target}" + ); + } + for target in [ + "/proc", + "/proc/self", + "/sys", + "/sys/fs/cgroup", + "/dev", + "/dev/shm", + "/.nvx-agent", + "/.nvx-agent/tool", + "/etc", + "/etc/machine-id", + "/etc/machine-id/x", + "/run", + "/run/nvx", + "/run/nvx/time", + "/run/nvx/time/state", + ] { + assert!( + matches!(refusal(target), Some(ShareRefusal::Reserved(_))), + "{target}" + ); + } + } + + #[test] + fn tags_and_child_names_are_plain_names() { + for tag in ["microvm", "a_b-C9"] { + assert!(check_tag(tag).is_ok(), "{tag}"); + } + for tag in ["", "a.b", "a b", "a/b", "é"] { + assert!(check_tag(tag).is_err(), "{tag}"); + } + for name in ["0", "a.b_C-9", "..."] { + assert!(check_name(name).is_ok(), "{name}"); + } + for name in ["", ".", "..", "a/b", "a b", "a,b"] { + assert!(check_name(name).is_err(), "{name}"); + } + } + + #[test] + fn mount_points_are_created_with_fixed_modes_and_never_through_links() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path(); + let point = mount_point(root, Path::new("/opt/tools"), ShareKind::Directory).unwrap(); + assert_eq!(point, root.join("opt/tools")); + let point = mount_point(root, Path::new("/etc/app/config"), ShareKind::File).unwrap(); + for (path, mode, directory) in [ + (root.join("opt"), 0o755, true), + (root.join("opt/tools"), 0o755, true), + (root.join("etc/app"), 0o755, true), + (point.clone(), 0o644, false), + ] { + let metadata = fs::symlink_metadata(&path).unwrap(); + assert_eq!(metadata.is_dir(), directory, "{}", path.display()); + assert_eq!( + metadata.permissions().mode() & 0o7777, + mode, + "{}", + path.display() + ); + } + // Existing components of the right kinds are kept. + mount_point(root, Path::new("/etc/app/config"), ShareKind::File).unwrap(); + mount_point(root, Path::new("/opt"), ShareKind::Directory).unwrap(); + + std::os::unix::fs::symlink(root.join("opt"), root.join("link")).unwrap(); + let refused = |target: &str, kind| match mount_point(root, Path::new(target), kind) { + Err(Error::Share(refusal)) => refusal, + other => panic!("{target}: {other:?}"), + }; + assert_eq!( + refused("/link/new", ShareKind::Directory), + ShareRefusal::SymbolicLink("/link/new".into()) + ); + assert_eq!( + refused("/link", ShareKind::File), + ShareRefusal::SymbolicLink("/link".into()) + ); + assert_eq!( + refused("/etc/app/config/x", ShareKind::Directory), + ShareRefusal::NotDirectory("/etc/app/config/x".into()) + ); + assert_eq!( + refused("/opt/tools", ShareKind::File), + ShareRefusal::NotFile("/opt/tools".into()) + ); + // Nothing is created through the link. + assert!(!root.join("opt/new").exists()); + } +} diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs new file mode 100644 index 000000000..4ac001cee --- /dev/null +++ b/nvx-rs/src/guest/root/tests.rs @@ -0,0 +1,884 @@ +//! Tests of the root operations with the fake kernel, and with the kernel's interfaces, a cgroup +//! hierarchy, and the runtime directory in a temporary directory. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use super::kernel::{Call, FakeKernel, Operation, Paths}; +use super::*; +use crate::guest::cgroup::Layout; + +/// The base address of the distro device's MMIO window. +const DISTRO: u64 = 0xd000_0000; +/// The base address of the scratch device's MMIO window. +const SCRATCH: u64 = 0x1_d000_1000; +/// The UUID of the test's EROFS image. +const UUID: [u8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + +/// A guest in a temporary directory, and the fake kernel that the operations get clones of. +struct Guest { + /// The temporary directory. + directory: tempfile::TempDir, + /// The kernel's interfaces, below the temporary directory. + paths: Paths, + /// The fake kernel. + kernel: FakeKernel, +} + +impl Guest { + /// Returns a guest with the distro device at window 0 and the scratch at window 1, both + /// attached, and nothing else. + fn new() -> Self { + let guest = Self::bare(); + attach(guest.directory.path(), 0, "vda", 4096); + attach(guest.directory.path(), 1, "vdb", 8192); + guest + } + + /// Returns a guest whose `/proc/iomem` declares the distro device's window 0 and the + /// scratch's window 1, without their devices. + fn bare() -> Self { + let directory = tempfile::tempdir().unwrap(); + let paths = Paths { + proc: directory.path().join("proc"), + sys: directory.path().join("sys"), + dev: directory.path().join("dev"), + }; + fs::create_dir_all(&paths.dev).unwrap(); + write( + &paths.proc.join("iomem"), + format!( + "00000000-00000fff : Reserved\n {DISTRO:08x}-{:08x} : virtio-mmio.0\n\ + {SCRATCH:016x}-{:016x} : virtio-mmio.1\n", + DISTRO + 0x1ff, + SCRATCH + 0x1ff, + ), + ); + Self { + directory, + paths, + kernel: FakeKernel::default(), + } + } + + /// Returns `relative`'s path below the temporary directory. + fn path(&self, relative: &str) -> PathBuf { + self.directory.path().join(relative) + } + + /// Returns `relative`'s path below the temporary directory, as the fake kernel records it. + fn text(&self, relative: &str) -> String { + self.path(relative).display().to_string() + } + + /// Returns an assembly of the guest's devices, in its runtime directory, `run/nvx`. + fn assembly(&self) -> Assembly { + Assembly { + runtime: self.path("run/nvx"), + distro: Device { + mmio: DISTRO, + sectors: None, + }, + distro_uuid: None, + scratch: Device { + mmio: SCRATCH, + sectors: None, + }, + timeout: Duration::from_millis(100), + } + } + + /// Assembles `assembly` with a clone of the guest's kernel. + fn assemble(&self, assembly: &Assembly) -> Result { + assemble_with(Box::new(self.kernel.clone()), &self.paths, assembly) + } +} + +/// Writes `contents` to `path`, after creating its parent directories. +fn write(path: &Path, contents: impl AsRef<[u8]>) { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, contents).unwrap(); +} + +/// Attaches the block device `name`, of `sectors` sectors, below the MMIO window `index` of the +/// guest in `root`: its sysfs directory, its size, and its node. +fn attach(root: &Path, index: usize, name: &str, sectors: u64) { + probe(root, index, name, sectors); + write(&root.join("dev").join(name), ""); +} + +/// Creates the sysfs directory and size of the block device `name` below the MMIO window +/// `index` of the guest in `root`, without its node. +fn probe(root: &Path, index: usize, name: &str, sectors: u64) { + let window = format!("sys/devices/virtio-mmio-cmdline/virtio-mmio.{index}"); + fs::create_dir_all( + root.join(window) + .join(format!("virtio{index}/block/{name}")), + ) + .unwrap(); + write( + &root.join("sys/class/block").join(name).join("size"), + format!("{sectors}\n"), + ); +} + +/// Returns an EROFS image's first 4 KiB, whose superblock holds `uuid`. +fn erofs_image(uuid: [u8; 16]) -> Vec { + let mut image = vec![0; 4096]; + image[1024..1028].copy_from_slice(&0xE0F5_E1E2_u32.to_le_bytes()); + image[1024 + 48..1024 + 64].copy_from_slice(&uuid); + image +} + +/// Returns the error of a failed system call, as an operation reports it. +fn os_error(errno: Errno) -> io::Error { + io::Error::from_raw_os_error(errno as i32) +} + +/// Returns a recorded mount. +fn mount_call( + source: Option<&str>, + target: &str, + filesystem: Option<&str>, + flags: MountFlags, + data: Option<&str>, +) -> Call { + Call::Mount { + source: source.map(str::to_owned), + target: target.to_owned(), + filesystem: filesystem.map(str::to_owned), + flags, + data: data.map(str::to_owned), + } +} + +#[test] +fn assembly_mounts_the_image_and_the_scratch_under_the_overlay() { + let guest = Guest::new(); + let root = guest.assemble(&guest.assembly()).unwrap(); + let runtime = guest.text("run/nvx"); + let options = format!( + "lowerdir={runtime}/layers/distro,upperdir={runtime}/scratch/upper,\ + workdir={runtime}/scratch/work,metacopy=on,xino=on" + ); + assert_eq!( + guest.kernel.calls(), + [ + Call::Flush(guest.text("dev/vda")), + mount_call( + Some(&guest.text("dev/vda")), + &format!("{runtime}/layers/distro"), + Some("erofs"), + MountFlags::MS_RDONLY, + None, + ), + Call::Flush(guest.text("dev/vdb")), + mount_call( + Some(&guest.text("dev/vdb")), + &format!("{runtime}/scratch"), + Some("ext4"), + MountFlags::MS_NOSUID | MountFlags::MS_NODEV, + None, + ), + mount_call( + Some("overlay"), + &format!("{runtime}/rootfs"), + Some("overlay"), + MountFlags::empty(), + Some(&options), + ), + ] + ); + for directory in ["layers/distro", "scratch/upper", "scratch/work", "rootfs"] { + assert!( + guest.path("run/nvx").join(directory).is_dir(), + "{directory}" + ); + } + assert_eq!(root.path(), guest.path("run/nvx/rootfs")); + assert_eq!(root.runtime(), guest.path("run/nvx")); + root.teardown().unwrap(); +} + +#[test] +fn devices_are_found_when_they_appear_and_missing_ones_time_out() { + let guest = Guest::bare(); + attach(guest.directory.path(), 1, "vdb", 8192); + let root_directory = guest.directory.path().to_owned(); + // The kernel probes the device first, then creates its node. + guest.kernel.on_sleep(move |sleeps| match sleeps { + 1 => probe(&root_directory, 0, "vda", 4096), + 2 => write(&root_directory.join("dev/vda"), ""), + _ => {} + }); + let mut assembly = guest.assembly(); + assembly.timeout = Duration::from_secs(1); + guest.assemble(&assembly).unwrap().teardown().unwrap(); + assert_eq!(guest.kernel.sleeps(), 2); + + // A timeout of 100 ms allows two more polls, 50 ms apart. + let guest = Guest::bare(); + attach(guest.directory.path(), 0, "vda", 4096); + let error = guest.assemble(&guest.assembly()).unwrap_err(); + assert!( + matches!( + error.error(), + Error::Missing { + role: Role::Scratch, + mmio: SCRATCH, + } + ), + "{error:?}" + ); + assert!(error.cleanup().is_none()); + assert_eq!(guest.kernel.sleeps(), 2); + assert_eq!( + error.to_string(), + "the scratch device at MMIO 0x1d0001000 did not appear" + ); + // The distro layer, which was mounted, is unmounted again. + assert_eq!( + guest.kernel.unmounts(), + [guest.text("run/nvx/layers/distro")] + ); +} + +#[test] +fn a_device_of_another_size_is_refused_before_it_is_flushed() { + let guest = Guest::new(); + let mut assembly = guest.assembly(); + assembly.distro.sectors = Some(2048); + let error = guest.assemble(&assembly).unwrap_err().into_parts().0; + assert!( + matches!( + &error, + Error::Size { + role: Role::Distro, + device, + expected: 2048, + actual: 4096, + } if *device == guest.path("dev/vda") + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "the distro device {} has 4096 sectors, expected 2048", + guest.text("dev/vda") + ) + ); + assert!(guest.kernel.calls().is_empty()); + + assembly.distro.sectors = Some(4096); + assembly.scratch.sectors = Some(8192); + guest.assemble(&assembly).unwrap().teardown().unwrap(); +} + +#[test] +fn the_expected_uuid_is_checked_in_the_superblock_before_the_mount() { + let guest = Guest::new(); + fs::write(guest.path("dev/vda"), erofs_image(UUID)).unwrap(); + let mut assembly = guest.assembly(); + assembly.distro_uuid = Some(UUID); + guest.assemble(&assembly).unwrap().teardown().unwrap(); + + let guest = Guest::new(); + fs::write(guest.path("dev/vda"), erofs_image(UUID)).unwrap(); + let mut expected = UUID; + expected[15] = 0xff; + assembly = guest.assembly(); + assembly.distro_uuid = Some(expected); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Uuid { expected: e, actual: UUID } if *e == expected), + "{error:?}" + ); + assert_eq!( + error.to_string(), + "the distro layer's UUID is 01020304-0506-0708-090a-0b0c0d0e0f10, \ + expected 01020304-0506-0708-090a-0b0c0d0e0fff" + ); + assert!(guest.kernel.mounts().is_empty()); + assert!(guest.kernel.unmounts().is_empty()); + + // A device without the EROFS magic number has no UUID to check. + let guest = Guest::new(); + fs::write(guest.path("dev/vda"), vec![0; 4096]).unwrap(); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::NotErofs { device } if *device == guest.path("dev/vda")), + "{error:?}" + ); + assert!(guest.kernel.mounts().is_empty()); +} + +#[test] +fn a_failed_assembly_unmounts_what_it_mounted_and_reports_what_stays() { + let guest = Guest::new(); + guest.kernel.fail( + Operation::Mount, + guest.path("run/nvx/rootfs"), + Errno::EINVAL, + ); + let error = guest.assemble(&guest.assembly()).unwrap_err(); + assert!( + matches!( + error.error(), + Error::Mount { mount: Mount::Overlay, error } if error.raw_os_error() == Some(Errno::EINVAL as i32) + ), + "{error:?}" + ); + assert!(error.cleanup().is_none()); + assert_eq!( + guest.kernel.unmounts(), + [ + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch") + ] + ); + + // A failed unmount of the cleanup is reported too, and the next one still attempted. + let guest = Guest::new(); + guest.kernel.fail( + Operation::Mount, + guest.path("run/nvx/rootfs"), + Errno::EINVAL, + ); + guest.kernel.fail( + Operation::Unmount, + guest.path("run/nvx/layers/distro"), + Errno::EBUSY, + ); + let error = guest.assemble(&guest.assembly()).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "failed to mount the overlay: {}; then failed to unmount the distro layer: {}", + os_error(Errno::EINVAL), + os_error(Errno::EBUSY) + ) + ); + let (_, cleanup) = error.into_parts(); + let failures = cleanup.unwrap(); + let [failure] = failures.failures() else { + panic!("{failures:?}"); + }; + assert_eq!(failure.mount(), &Mount::Distro); + assert_eq!(failure.path(), guest.path("run/nvx/layers/distro")); + assert_eq!(failure.error().raw_os_error(), Some(Errno::EBUSY as i32)); + assert_eq!( + guest.kernel.unmounts(), + [ + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch") + ] + ); +} + +#[test] +fn a_runtime_directory_that_the_overlay_options_reserve_is_refused() { + let guest = Guest::new(); + for runtime in ["run/nvx", "/run/a,b", "/run/a:b", "/run/a\\b", "/run/a=b"] { + let mut assembly = guest.assembly(); + assembly.runtime = runtime.into(); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Runtime { path } if *path == Path::new(runtime)), + "{runtime}: {error:?}" + ); + } + assert!(guest.kernel.calls().is_empty()); +} + +/// Returns a child of an aggregate share. +fn child(name: &str, target: &str, access: Access, kind: ShareKind) -> ShareChild { + ShareChild { + name: name.to_owned(), + target: target.into(), + access, + kind, + } +} + +#[test] +fn shares_are_mounted_restricted_and_torn_down_first_in_reverse_order() { + let guest = Guest::new(); + let mut root = guest.assemble(&guest.assembly()).unwrap(); + let assembled = guest.kernel.calls().len(); + root.mount_share("one", Path::new("/srv/one"), Access::ReadOnly) + .unwrap(); + root.mount_aggregate( + "all", + Access::ReadWrite, + &[ + child("a", "/data/a", Access::ReadWrite, ShareKind::Directory), + child("b", "/etc/app/config", Access::ReadOnly, ShareKind::File), + ], + ) + .unwrap(); + let rootfs = guest.text("run/nvx/rootfs"); + let shares = guest.text("run/nvx/shares"); + let restricted = MountFlags::MS_NOSUID | MountFlags::MS_NODEV; + let remount = MountFlags::MS_REMOUNT | MountFlags::MS_BIND | restricted; + assert_eq!( + guest.kernel.calls()[assembled..], + [ + mount_call( + Some("one"), + &format!("{rootfs}/srv/one"), + Some("virtiofs"), + restricted | MountFlags::MS_RDONLY, + None, + ), + mount_call(Some("all"), &shares, Some("virtiofs"), restricted, None), + mount_call( + Some(&format!("{shares}/a")), + &format!("{rootfs}/data/a"), + None, + MountFlags::MS_BIND, + None, + ), + mount_call(None, &format!("{rootfs}/data/a"), None, remount, None), + mount_call( + Some(&format!("{shares}/b")), + &format!("{rootfs}/etc/app/config"), + None, + MountFlags::MS_BIND, + None, + ), + mount_call( + None, + &format!("{rootfs}/etc/app/config"), + None, + remount | MountFlags::MS_RDONLY, + None, + ), + ] + ); + assert!(guest.path("run/nvx/rootfs/srv/one").is_dir()); + assert!(guest.path("run/nvx/rootfs/etc/app/config").is_file()); + + root.teardown().unwrap(); + assert_eq!( + guest.kernel.unmounts(), + [ + format!("{rootfs}/etc/app/config"), + format!("{rootfs}/data/a"), + shares, + format!("{rootfs}/srv/one"), + rootfs, + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch"), + ] + ); +} + +#[test] +fn refused_shares_mount_nothing() { + let guest = Guest::new(); + let mut root = guest.assemble(&guest.assembly()).unwrap(); + root.mount_share("srv", Path::new("/srv"), Access::ReadWrite) + .unwrap(); + let mounts = guest.kernel.mounts().len(); + let refusal = |result: Result<(), Error>| match result { + Err(Error::Share(refusal)) => refusal, + other => panic!("{other:?}"), + }; + let directory = ShareKind::Directory; + let rw = Access::ReadWrite; + assert_eq!( + refusal(root.mount_share("a b", Path::new("/a"), rw)), + ShareRefusal::Tag("a b".into()) + ); + assert_eq!( + refusal(root.mount_share("t", Path::new("/proc/x"), rw)), + ShareRefusal::Reserved("/proc/x".into()) + ); + assert_eq!( + refusal(root.mount_share("t", Path::new("/"), rw)), + ShareRefusal::NotCanonical("/".into()) + ); + assert_eq!( + refusal(root.mount_share("t", Path::new("/srv/inner"), rw)), + ShareRefusal::Overlap { + claimed: "/srv".into(), + target: "/srv/inner".into() + } + ); + assert_eq!( + refusal(root.mount_aggregate("t", rw, &[])), + ShareRefusal::NoChildren + ); + assert_eq!( + refusal(root.mount_aggregate("t", rw, &[child("..", "/a", rw, directory)])), + ShareRefusal::ChildName("..".into()) + ); + assert_eq!( + refusal(root.mount_aggregate( + "t", + rw, + &[ + child("x", "/a", rw, directory), + child("x", "/b", rw, directory) + ] + )), + ShareRefusal::DuplicateChild("x".into()) + ); + assert_eq!( + refusal(root.mount_aggregate( + "t", + rw, + &[ + child("x", "/b/c", rw, directory), + child("y", "/b", rw, directory) + ] + )), + ShareRefusal::Overlap { + claimed: "/b/c".into(), + target: "/b".into() + } + ); + assert_eq!( + refusal(root.mount_aggregate("t", rw, &[child("x", "/", rw, directory)])), + ShareRefusal::NotCanonical("/".into()) + ); + assert_eq!(guest.kernel.mounts().len(), mounts); + + // A share that fails to mount claims no target. + guest.kernel.fail( + Operation::Mount, + guest.path("run/nvx/rootfs/opt"), + Errno::ENODEV, + ); + let error = root.mount_share("opt", Path::new("/opt"), rw).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "failed to mount the live share opt at /opt: {}", + os_error(Errno::ENODEV) + ) + ); + assert_eq!(root.claimed, [PathBuf::from("/srv")]); +} + +#[test] +fn a_child_that_cannot_be_restricted_stays_mounted_until_teardown() { + let guest = Guest::new(); + let mut root = guest.assemble(&guest.assembly()).unwrap(); + guest.kernel.fail( + Operation::Remount, + guest.path("run/nvx/rootfs/data/a"), + Errno::EPERM, + ); + let error = root + .mount_aggregate( + "all", + Access::ReadOnly, + &[ + child("a", "/data/a", Access::ReadOnly, ShareKind::Directory), + child("b", "/data/b", Access::ReadOnly, ShareKind::Directory), + ], + ) + .unwrap_err(); + assert!( + matches!( + &error, + Error::Restrict { mount: Mount::Child { name, target }, .. } + if name == "a" && *target == Path::new("/data/a") + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "failed to restrict the live share child a at /data/a: {}", + os_error(Errno::EPERM) + ) + ); + root.teardown().unwrap(); + assert_eq!( + guest.kernel.unmounts(), + [ + guest.text("run/nvx/rootfs/data/a"), + guest.text("run/nvx/shares"), + guest.text("run/nvx/rootfs"), + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch"), + ] + ); +} + +#[test] +fn a_failed_unmount_is_reported_and_the_later_ones_still_attempted() { + let guest = Guest::new(); + let mut root = guest.assemble(&guest.assembly()).unwrap(); + root.mount_share("srv", Path::new("/srv"), Access::ReadWrite) + .unwrap(); + guest.kernel.fail( + Operation::Unmount, + guest.path("run/nvx/rootfs/srv"), + Errno::EBUSY, + ); + guest.kernel.fail( + Operation::Unmount, + guest.path("run/nvx/rootfs"), + Errno::EBUSY, + ); + let error = root.teardown().unwrap_err(); + let failures: Vec<_> = error + .failures() + .iter() + .map(|failure| (failure.mount().clone(), failure.path().to_owned())) + .collect(); + assert_eq!( + failures, + [ + ( + Mount::Share { + tag: "srv".into(), + target: "/srv".into() + }, + guest.path("run/nvx/rootfs/srv") + ), + (Mount::Overlay, guest.path("run/nvx/rootfs")), + ] + ); + assert_eq!( + error.to_string(), + format!( + "failed to unmount the live share srv at /srv: {busy}; \ + failed to unmount the overlay: {busy}", + busy = os_error(Errno::EBUSY) + ) + ); + assert_eq!( + guest.kernel.unmounts(), + [ + guest.text("run/nvx/rootfs/srv"), + guest.text("run/nvx/rootfs"), + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch"), + ] + ); +} + +#[test] +fn a_dropped_root_is_torn_down_once() { + let guest = Guest::new(); + let root = guest.assemble(&guest.assembly()).unwrap(); + guest.kernel.fail( + Operation::Unmount, + guest.path("run/nvx/rootfs"), + Errno::EBUSY, + ); + drop(root); + let unmounts = [ + guest.text("run/nvx/rootfs"), + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch"), + ]; + assert_eq!(guest.kernel.unmounts(), unmounts); + + // A root that was torn down has nothing left to unmount when it is dropped. + let guest = Guest::new(); + guest + .assemble(&guest.assembly()) + .unwrap() + .teardown() + .unwrap(); + assert_eq!(guest.kernel.unmounts().len(), 3); +} + +/// A process of the bootstrap's parent cgroup. +enum Process { + /// A process, which the bootstrap moves. + User(u32), + /// A kernel thread, which stays. + Kernel(u32), + /// A process that ended after the cgroup listed it, which has no `/proc` entry. + Ended(u32), +} + +/// Prepares a cgroup hierarchy at `cgroup` in `guest`, whose root, already a cgroup if +/// `mounted`, holds `processes`, and returns the layout of its `agent` and `container` cgroups. +fn hierarchy(guest: &Guest, mounted: bool, processes: &[Process]) -> Layout { + let cgroup = guest.path("cgroup"); + write( + &guest.paths.proc.join("sys/user/max_user_namespaces"), + "15019\n", + ); + if mounted { + write(&cgroup.join("cgroup.controllers"), "cpu io memory pids\n"); + } + write(&cgroup.join("cgroup.subtree_control"), ""); + let mut members = String::new(); + for process in processes { + let (id, flags) = match *process { + Process::User(id) => (id, Some(0x0040_0100)), + Process::Kernel(id) => (id, Some(0x0020_8040)), + Process::Ended(id) => (id, None), + }; + members.push_str(&format!("{id}\n")); + if let Some(flags) = flags { + let stat = format!("{id} (p {id}) S 0 {id} {id} 0 -1 {flags} 0 0 0 0 0 0 0 0 20 0 1\n"); + write(&guest.paths.proc.join(id.to_string()).join("stat"), stat); + } + } + write(&cgroup.join("cgroup.procs"), members); + Layout::new(cgroup.join("agent"), cgroup.join("container")).unwrap() +} + +/// Bootstraps `layout` with `resources` and a clone of `guest`'s kernel. +fn bootstrap_in(guest: &Guest, layout: &Layout, resources: &Resources) -> Result { + bootstrap::bootstrap_with(&mut guest.kernel.clone(), &guest.paths, layout, resources) +} + +#[test] +fn bootstrap_moves_every_process_but_kernel_threads_and_applies_the_resources() { + let guest = Guest::bare(); + let processes = [ + Process::User(1), + Process::Kernel(2), + Process::User(45), + Process::Ended(99), + ]; + let layout = hierarchy(&guest, true, &processes); + let resources = Resources { + agent_memory_low: Some(16 << 20), + workload_memory_max: Some(512 << 20), + workload_pids_max: Some(65), + }; + let bootstrap = bootstrap_in(&guest, &layout, &resources).unwrap(); + assert_eq!(bootstrap.moved(), [1, 45]); + assert_eq!(bootstrap.layout(), &layout); + assert_eq!(bootstrap.workload().path(), layout.workload()); + let writes: Vec<(String, String)> = [ + ("proc/sys/user/max_user_namespaces", "0"), + ("cgroup/agent/cgroup.procs", "1"), + ("cgroup/agent/cgroup.procs", "45"), + ("cgroup/cgroup.subtree_control", "+pids +memory"), + ("cgroup/agent/memory.low", "16777216"), + ("cgroup/container/memory.max", "536870912"), + ("cgroup/container/pids.max", "65"), + ] + .into_iter() + .map(|(path, value)| (guest.text(path), value.to_owned())) + .collect(); + assert_eq!(guest.kernel.writes(), writes); + assert!(guest.kernel.mounts().is_empty()); + assert_eq!( + fs::read_to_string(guest.path("cgroup/cgroup.procs")).unwrap(), + "2\n99\n" + ); + assert_eq!( + fs::read_to_string(guest.path("cgroup/agent/cgroup.procs")).unwrap(), + "1\n45\n" + ); + assert!(guest.path("cgroup/container").is_dir()); +} + +#[test] +fn bootstrap_mounts_the_hierarchy_unless_the_parent_is_a_cgroup() { + let guest = Guest::bare(); + let layout = hierarchy(&guest, false, &[Process::User(1)]); + bootstrap_in(&guest, &layout, &Resources::default()).unwrap(); + assert_eq!( + guest.kernel.calls()[1], + mount_call( + Some("cgroup2"), + &guest.text("cgroup"), + Some("cgroup2"), + MountFlags::MS_NOSUID | MountFlags::MS_NODEV | MountFlags::MS_NOEXEC, + None, + ) + ); + // Without a memory value, only the `pids` controller is enabled, and nothing else written. + let writes = guest.kernel.writes(); + assert_eq!( + writes[2..], + [(guest.text("cgroup/cgroup.subtree_control"), "+pids".into())] + ); +} + +#[test] +fn bootstrap_names_the_failed_step_and_keeps_the_earlier_changes() { + let guest = Guest::bare(); + let layout = hierarchy(&guest, true, &[Process::User(1)]); + guest.kernel.fail( + Operation::Write, + guest.path("cgroup/container/memory.max"), + Errno::EINVAL, + ); + let resources = Resources { + agent_memory_low: Some(1 << 20), + workload_memory_max: Some(1), + workload_pids_max: Some(8), + }; + let error = bootstrap_in(&guest, &layout, &resources).unwrap_err(); + assert!( + matches!( + &error, + Error::Bootstrap { step: BootstrapStep::WorkloadMemoryMax, error } + if error.raw_os_error() == Some(Errno::EINVAL as i32) + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "failed to set the workload's memory limit: {}", + os_error(Errno::EINVAL) + ) + ); + let writes = guest.kernel.writes(); + assert_eq!( + writes.last().unwrap().0, + guest.text("cgroup/container/memory.max") + ); + assert_eq!( + fs::read_to_string(guest.path("cgroup/agent/memory.low")).unwrap(), + "1048576" + ); + + // Cgroups that are not siblings are refused before anything changes. + let guest = Guest::bare(); + hierarchy(&guest, true, &[Process::User(1)]); + let cgroup = guest.path("cgroup"); + let layout = Layout::new(cgroup.join("agent"), cgroup.join("workloads/container")).unwrap(); + let error = bootstrap_in(&guest, &layout, &Resources::default()).unwrap_err(); + assert!( + matches!( + &error, + Error::Bootstrap { step: BootstrapStep::Layout, error } + if error.raw_os_error() == Some(Errno::EINVAL as i32) + ), + "{error:?}" + ); + assert!(guest.kernel.calls().is_empty()); +} + +#[test] +fn bootstrap_fails_while_a_process_keeps_returning_to_the_parent() { + let guest = Guest::bare(); + let layout = hierarchy(&guest, true, &[Process::User(1), Process::User(7)]); + guest.kernel.pin(7); + let error = bootstrap_in(&guest, &layout, &Resources::default()).unwrap_err(); + assert!( + matches!( + &error, + Error::Bootstrap { step: BootstrapStep::Placement, error } + if error.raw_os_error() == Some(Errno::EBUSY as i32) + ), + "{error:?}" + ); + let attempts = guest + .kernel + .writes() + .into_iter() + .filter(|(path, value)| path.ends_with("agent/cgroup.procs") && value == "7") + .count(); + assert_eq!(attempts, 16); +} diff --git a/nvx-rs/src/guest/sys/filesystem.rs b/nvx-rs/src/guest/sys/filesystem.rs index 9ccfff8e0..fba63ac24 100644 --- a/nvx-rs/src/guest/sys/filesystem.rs +++ b/nvx-rs/src/guest/sys/filesystem.rs @@ -78,6 +78,23 @@ pub(crate) fn mount( nix::mount::mount(source, target, filesystem, flags, None::<&CStr>) } +/// Mounts `source` on `target`, as [`mount`] does, with the file system's own options `data`, +/// such as an overlay's layers. +pub(crate) fn mount_with_options( + source: Option<&CStr>, + target: &CStr, + filesystem: Option<&CStr>, + flags: MountFlags, + data: Option<&CStr>, +) -> Result<(), Errno> { + nix::mount::mount(source, target, filesystem, flags, data) +} + +/// Unmounts the file system mounted at `target`. A busy one stays mounted: `EBUSY`. +pub(crate) fn unmount(target: &CStr) -> Result<(), Errno> { + nix::mount::umount(target) +} + /// Changes the owner and group of `path`, following the final symlink like C `chown`. pub(crate) fn chown(path: &CStr, uid: u32, gid: u32) -> Result<(), Errno> { nix::unistd::chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) diff --git a/nvx-rs/src/guest/sys/ioctl.rs b/nvx-rs/src/guest/sys/ioctl.rs index f48274986..5eac19830 100644 --- a/nvx-rs/src/guest/sys/ioctl.rs +++ b/nvx-rs/src/guest/sys/ioctl.rs @@ -41,6 +41,14 @@ nix::ioctl_none!( 0x07 ); +nix::ioctl_none!( + /// Writes a block device's dirty buffers and drops its cached ones: Linux `BLKFLSBUF`, + /// `_IO(0x12, 97)`. + blkflsbuf, + 0x12, + 97 +); + /// Returns the number of bytes available for reading from `descriptor`. /// /// The local integer outlives `ioctl`, and the wrapper returns it only after the kernel reports @@ -73,9 +81,18 @@ pub(crate) fn reseed_random( Ok(()) } +/// Writes the dirty buffers of the block device that `descriptor` opened, then drops its cached +/// buffers, as `blockdev --flushbufs` does, so that later reads come from the device. It needs +/// `CAP_SYS_ADMIN`. +pub(crate) fn flush_block_buffers(descriptor: &FileDescriptor) -> Result<(), Errno> { + // SAFETY: `BLKFLSBUF` reads no argument. + unsafe { blkflsbuf(descriptor.raw()) }?; + Ok(()) +} + #[cfg(test)] mod tests { - //! ABI checks for the random-device ioctls. + //! ABI checks for the ioctls. use super::*; @@ -90,4 +107,10 @@ mod tests { assert_eq!(nix::request_code_none!(b'R', 0x07) as u32, ioc(0, 0x07, 0)); assert_eq!(size_of::(), 8 + SEED_BYTES); } + + #[test] + fn the_buffer_flush_ioctl_number_matches_linux() { + // `BLKFLSBUF` in linux/fs.h. + assert_eq!(nix::request_code_none!(0x12, 97) as u32, 0x1261); + } } diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index 6c9b17426..0b66bb6be 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -84,10 +84,11 @@ pub(crate) use fd::{ }; pub(crate) use filesystem::{ MountFlags, PATH_MAX, ScratchDirectory, TmpfsMount, change_directory, check_exists, chmod, - chown, current_directory, enter_private_mount_namespace, mount, path_c_string, path_exists, - read_file, read_line, read_number, write_file, write_number, + chown, current_directory, enter_private_mount_namespace, mount, mount_with_options, + path_c_string, path_exists, read_file, read_line, read_number, unmount, write_file, + write_number, }; -pub(crate) use ioctl::{console_pending, reseed_random}; +pub(crate) use ioctl::{console_pending, flush_block_buffers, reseed_random}; pub(crate) use mapping::MemoryMapping; #[cfg(target_arch = "x86_64")] pub(crate) use msr::{MsrAccess, MsrFile}; diff --git a/nvx-rs/src/lib.rs b/nvx-rs/src/lib.rs index 54c9d0fa4..704fb01f4 100644 --- a/nvx-rs/src/lib.rs +++ b/nvx-rs/src/lib.rs @@ -27,10 +27,12 @@ //! init or agent starts a program in a new process (`guest::spawn`), collects the status of //! each child that it names (`guest::process`), places, kills, and settles the processes of //! its executions in cgroups (`guest::cgroup`), bind-mounts host mappings below the roots that -//! it names (`guest::mappings`), and drives each program as one execution, with its launch, -//! bounded output, input, ending, and outcome (`guest::exec`), and the typed calls through -//! which it runs the time ABI's steps (`guest::time`) and reseeds after a restore -//! (`guest::reseed`) in its own process. The crate's `nvx-*` guest binaries run the programs. +//! it names (`guest::mappings`), drives each program as one execution, with its launch, bounded +//! output, input, ending, and outcome (`guest::exec`), and bootstraps the guest for a layered +//! root, assembles the root, mounts live shares in it, and tears it down (`guest::root`), and +//! the typed calls through which it runs the time ABI's steps (`guest::time`) and reseeds after +//! a restore (`guest::reseed`) in its own process. The crate's `nvx-*` guest binaries run the +//! programs. //! //! # Example //! From a40cf60f232147b4837f72501000dc37c8f3fdf2 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 20 Oct 2026 20:48:25 -0700 Subject: [PATCH 02/12] nvx-rs: mount the guest's merged root nosuid,nodev The overlay that `assemble` mounts is the workload's root, so mount it `nosuid,nodev`, as #621 requires of the merged root: those flags apply to each mount, and the scratch's do not reach the files of the overlay. NVX's workloads already run with `no_new_privs` and their own `/dev`, so nothing that they do changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/README.md | 2 +- nvx-rs/src/guest/root.rs | 11 ++++++----- nvx-rs/src/guest/root/tests.rs | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 891febdd9..8cc4cd84c 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -86,7 +86,7 @@ nvx-rs/ | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on` whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev`. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 75527cf4f..6064ce520 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -7,8 +7,8 @@ //! unavailable. //! - [`assemble`] finds the block devices of an [`Assembly`] by their MMIO windows, checks and //! flushes them, and mounts the root: a read-only EROFS image, the `distro` layer, under an -//! overlay whose upper and work directories are on an ext4 scratch. It returns the [`Root`] -//! that owns the mounts. +//! overlay whose upper and work directories are on an ext4 scratch, and which is mounted +//! `nosuid,nodev`, as the scratch is. It returns the [`Root`] that owns the mounts. //! - [`Root::mount_share`] and [`Root::mount_aggregate`] mount the VMM's live shares at targets in //! the root, after the checks that [live shares](#live-shares) lists. //! - [`Root::teardown`] unmounts, in order, the live shares in reverse mount order, the overlay, @@ -197,8 +197,8 @@ pub struct Assembly { /// 6. finds, checks, and flushes the scratch device the same way; /// 7. mounts its ext4 file system at `scratch` with `nosuid` and `nodev`; /// 8. creates the scratch's `upper` and `work` directories unless they exist; -/// 9. and mounts the overlay at `rootfs`, with the distro layer as its lower directory, and -/// `metacopy=on,xino=on`. +/// 9. and mounts the overlay at `rootfs`, `nosuid` and `nodev`, with the distro layer as its +/// lower directory, and `metacopy=on,xino=on`. /// /// A failure unmounts what the assembly mounted, as [`Root::teardown`] does, and reports both. pub fn assemble(assembly: &Assembly) -> Result { @@ -361,12 +361,13 @@ impl Root { ] .concat(); let root = self.path.clone(); + // Neither set-ID programs nor device nodes of the image or the upper take effect. self.mount_at( &Mount::Overlay, Some(b"overlay"), &root, Some(b"overlay"), - MountFlags::empty(), + MountFlags::MS_NOSUID | MountFlags::MS_NODEV, Some(&options), )?; self.overlay = Some(root); diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 4ac001cee..971a326c7 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -186,7 +186,7 @@ fn assembly_mounts_the_image_and_the_scratch_under_the_overlay() { Some("overlay"), &format!("{runtime}/rootfs"), Some("overlay"), - MountFlags::empty(), + MountFlags::MS_NOSUID | MountFlags::MS_NODEV, Some(&options), ), ] From fcd9ed70ef6651aad1fc5b94d315ed9ef1355ba4 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 20 Oct 2026 21:14:24 -0700 Subject: [PATCH 03/12] nvx-rs: let the guest's last placement pass empty the parent The bootstrap's placement returned EBUSY whenever its sixteenth pass found a process, even when that pass moved the last one. Check the parent cgroup once more, without moving anything, before failing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/src/guest/root/bootstrap.rs | 45 +++++++++++++++++++----------- nvx-rs/src/guest/root/kernel.rs | 23 ++++++++++----- nvx-rs/src/guest/root/tests.rs | 21 +++++++++++++- 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/nvx-rs/src/guest/root/bootstrap.rs b/nvx-rs/src/guest/root/bootstrap.rs index a4766607b..c2972e8d9 100644 --- a/nvx-rs/src/guest/root/bootstrap.rs +++ b/nvx-rs/src/guest/root/bootstrap.rs @@ -15,7 +15,7 @@ use crate::guest::sys::{self, Errno, MountFlags}; /// The flag of a kernel thread among a process's flags in `/proc/PID/stat`: Linux's /// `PF_KTHREAD`. const KERNEL_THREAD: u64 = 0x0020_0000; -/// The most passes that the placement makes over the parent cgroup's processes. A process that a +/// The most passes in which the placement moves the parent cgroup's processes. A process that a /// moved one starts meanwhile starts in the parent too, and the next pass moves it. const PLACEMENT_PASSES: usize = 16; /// The sysctl, below procfs, of the user namespaces that each user may create. @@ -245,7 +245,7 @@ fn create_cgroup(path: &Path) -> io::Result<()> { } /// Moves every process of the cgroup `parent` into the cgroup `agent`, except kernel threads, in -/// passes until a pass finds none to move, and returns the IDs that it moved. +/// passes until none is left, and returns the IDs that it moved. fn place( kernel: &mut dyn Kernel, paths: &Paths, @@ -256,28 +256,41 @@ fn place( let destination = sys::path_c_string(&agent.join("cgroup.procs"))?; let mut moved = Vec::new(); for _ in 0..PLACEMENT_PASSES { - let mut found = false; - for process in processes(&fs::read(&members)?)? { - match fs::read(paths.proc.join(process.to_string()).join("stat")) { - Ok(stat) if is_kernel_thread(&stat).ok_or(Errno::EPROTO)? => continue, - Ok(_) => {} - // The process has ended. - Err(error) if error.kind() == io::ErrorKind::NotFound => continue, - Err(error) if error.raw_os_error() == Some(Errno::ESRCH as i32) => continue, - Err(error) => return Err(error), - } - found = true; + let left = user_processes(paths, &members)?; + if left.is_empty() { + return Ok(moved); + } + for process in left { match kernel.write(&destination, process.to_string().as_bytes()) { Ok(()) => moved.push(process), Err(Errno::ESRCH) => {} Err(error) => return Err(error.into()), } } - if !found { - return Ok(moved); + } + // The last pass may have moved the last of them. + if user_processes(paths, &members)?.is_empty() { + Ok(moved) + } else { + Err(Errno::EBUSY.into()) + } +} + +/// Returns the processes that the membership file `members` lists, except kernel threads and +/// processes that have ended. +fn user_processes(paths: &Paths, members: &Path) -> io::Result> { + let mut left = Vec::new(); + for process in processes(&fs::read(members)?)? { + match fs::read(paths.proc.join(process.to_string()).join("stat")) { + Ok(stat) if is_kernel_thread(&stat).ok_or(Errno::EPROTO)? => {} + Ok(_) => left.push(process), + // The process has ended. + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) if error.raw_os_error() == Some(Errno::ESRCH as i32) => {} + Err(error) => return Err(error), } } - Err(Errno::EBUSY.into()) + Ok(left) } /// Parses a cgroup's `cgroup.procs`: one decimal process ID per line. diff --git a/nvx-rs/src/guest/root/kernel.rs b/nvx-rs/src/guest/root/kernel.rs index 3462d83ce..f65f6e259 100644 --- a/nvx-rs/src/guest/root/kernel.rs +++ b/nvx-rs/src/guest/root/kernel.rs @@ -174,8 +174,9 @@ mod fake { calls: Vec, /// The operations that fail, with their error, each at the path that names it. failures: Vec<(Operation, PathBuf, Errno)>, - /// The processes that a write of their ID to a membership file leaves where they are. - unmovable: Vec, + /// The processes that writes of their ID to a membership file leave where they are, each + /// with the number of such writes left. + unmovable: Vec<(u32, usize)>, /// The sleeps so far. sleeps: usize, /// What happens after each sleep, given the number of sleeps so far. @@ -208,10 +209,10 @@ mod fake { self.with(|state| state.failures.push((operation, path.into(), error))); } - /// Makes a write of the ID `process` to a membership file leave the process where it is, - /// as for a process that reappears as fast as it is moved. - pub(in crate::guest::root) fn pin(&self, process: u32) { - self.with(|state| state.unmovable.push(process)); + /// Makes the next `writes` writes of the ID `process` to a membership file leave the + /// process where it is, as for a process that reappears as fast as it is moved. + pub(in crate::guest::root) fn pin(&self, process: u32, writes: usize) { + self.with(|state| state.unmovable.push((process, writes))); } /// Runs `action` after each sleep, with the number of sleeps so far. @@ -346,7 +347,15 @@ mod fake { .and_then(|value| value.trim_end().parse::().ok()); match process { Some(process) if path.ends_with("cgroup.procs") => { - if self.with(|state| state.unmovable.contains(&process)) { + let pinned = self.with(|state| { + state + .unmovable + .iter_mut() + .find(|(pinned, writes)| *pinned == process && *writes > 0) + .map(|(_, writes)| *writes -= 1) + .is_some() + }); + if pinned { Ok(()) } else { move_process(&path, process) diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 971a326c7..10d5813d0 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -864,7 +864,7 @@ fn bootstrap_names_the_failed_step_and_keeps_the_earlier_changes() { fn bootstrap_fails_while_a_process_keeps_returning_to_the_parent() { let guest = Guest::bare(); let layout = hierarchy(&guest, true, &[Process::User(1), Process::User(7)]); - guest.kernel.pin(7); + guest.kernel.pin(7, usize::MAX); let error = bootstrap_in(&guest, &layout, &Resources::default()).unwrap_err(); assert!( matches!( @@ -882,3 +882,22 @@ fn bootstrap_fails_while_a_process_keeps_returning_to_the_parent() { .count(); assert_eq!(attempts, 16); } + +#[test] +fn bootstrap_succeeds_when_the_last_pass_empties_the_parent() { + let guest = Guest::bare(); + let layout = hierarchy(&guest, true, &[Process::User(1), Process::User(7)]); + // The sixteenth pass is the first that moves the process. + guest.kernel.pin(7, 15); + let bootstrap = bootstrap_in(&guest, &layout, &Resources::default()).unwrap(); + assert_eq!(bootstrap.moved()[0], 1); + assert_eq!(bootstrap.moved()[1..], [7; 16]); + assert_eq!( + fs::read_to_string(guest.path("cgroup/cgroup.procs")).unwrap(), + "" + ); + assert_eq!( + fs::read_to_string(guest.path("cgroup/agent/cgroup.procs")).unwrap(), + "1\n7\n" + ); +} From 78f5cfd138e8a4fc3a3faaf6ccfae958c9cc5e9b Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 20 Oct 2026 21:31:23 -0700 Subject: [PATCH 04/12] nvx-rs: let the guest bootstrap run on a kernel without sysctls A kernel built without sysctls lacks user.max_user_namespaces, so the bootstrap failed with ENOENT there, even when the kernel has no user namespaces to limit. Accept the missing sysctl when /proc/self/ns/user shows that the kernel has no user namespaces, and keep failing otherwise, since such a kernel's user namespaces cannot be limited. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/src/guest/root/bootstrap.rs | 26 +++++++++++++++++--- nvx-rs/src/guest/root/tests.rs | 38 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/nvx-rs/src/guest/root/bootstrap.rs b/nvx-rs/src/guest/root/bootstrap.rs index c2972e8d9..f3bb42089 100644 --- a/nvx-rs/src/guest/root/bootstrap.rs +++ b/nvx-rs/src/guest/root/bootstrap.rs @@ -20,6 +20,9 @@ const KERNEL_THREAD: u64 = 0x0020_0000; const PLACEMENT_PASSES: usize = 16; /// The sysctl, below procfs, of the user namespaces that each user may create. const USER_NAMESPACES: &str = "sys/user/max_user_namespaces"; +/// The calling process's user namespace, below procfs, which only a kernel with user namespaces +/// has. +const OWN_USER_NAMESPACE: &str = "self/ns/user"; /// The resource values that [`bootstrap`] applies, each only when the caller gives it. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] @@ -119,7 +122,8 @@ impl Bootstrap { /// cgroups of `layout`, which must be siblings. In order, it: /// /// 1. sets `user.max_user_namespaces` to 0, so that no process creates a user namespace, even on -/// a kernel that supports them; +/// a kernel that supports them. A kernel without sysctls lacks it, which the bootstrap accepts +/// only when the kernel has no user namespaces either, and otherwise fails with `ENOENT`; /// 2. mounts the cgroup v2 hierarchy, `nosuid,nodev,noexec`, at the cgroups' parent, unless the /// parent is a cgroup already, which its `cgroup.controllers` shows; /// 3. creates the agent's cgroup unless it exists; @@ -152,8 +156,7 @@ pub(super) fn bootstrap_with( (Some(agent), Some(workload)) if agent == workload => agent, _ => return Err(failed(BootstrapStep::Layout)(Errno::EINVAL.into())), }; - write(kernel, &paths.proc.join(USER_NAMESPACES), b"0") - .map_err(failed(BootstrapStep::UserNamespaces))?; + disable_user_namespaces(kernel, paths).map_err(failed(BootstrapStep::UserNamespaces))?; mount_hierarchy(kernel, parent).map_err(failed(BootstrapStep::Hierarchy))?; create_cgroup(layout.agent()).map_err(failed(BootstrapStep::AgentCgroup))?; let moved = @@ -210,6 +213,23 @@ fn write(kernel: &mut dyn Kernel, path: &Path, value: &[u8]) -> io::Result<()> { Ok(kernel.write(&path, value)?) } +/// Sets `user.max_user_namespaces` to 0, through `kernel`, with procfs at `paths.proc`. Without +/// the sysctl, the kernel needs no limit if it has no user namespaces, and otherwise cannot be +/// limited, which is the write's `ENOENT`. +fn disable_user_namespaces(kernel: &mut dyn Kernel, paths: &Paths) -> io::Result<()> { + let Err(error) = write(kernel, &paths.proc.join(USER_NAMESPACES), b"0") else { + return Ok(()); + }; + if error.kind() != io::ErrorKind::NotFound { + return Err(error); + } + match fs::symlink_metadata(paths.proc.join(OWN_USER_NAMESPACE)) { + Err(missing) if missing.kind() == io::ErrorKind::NotFound => Ok(()), + Ok(_) => Err(error), + Err(other) => Err(other), + } +} + /// Mounts the cgroup v2 hierarchy at `parent`, which must exist, unless `parent` is a cgroup. fn mount_hierarchy(kernel: &mut dyn Kernel, parent: &Path) -> io::Result<()> { match fs::symlink_metadata(parent.join("cgroup.controllers")) { diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 10d5813d0..0ece9ae89 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -860,6 +860,44 @@ fn bootstrap_names_the_failed_step_and_keeps_the_earlier_changes() { assert!(guest.kernel.calls().is_empty()); } +#[test] +fn bootstrap_accepts_a_missing_sysctl_only_without_user_namespaces() { + // A kernel without sysctls lacks the limit, which one without user namespaces needs no more. + let guest = Guest::bare(); + let layout = hierarchy(&guest, true, &[Process::User(1)]); + guest.kernel.fail( + Operation::Write, + guest.path("proc/sys/user/max_user_namespaces"), + Errno::ENOENT, + ); + bootstrap_in(&guest, &layout, &Resources::default()).unwrap(); + assert_eq!( + fs::read_to_string(guest.path("cgroup/agent/cgroup.procs")).unwrap(), + "1\n" + ); + + // One with user namespaces cannot be limited, so nothing else changes. + let guest = Guest::bare(); + let layout = hierarchy(&guest, true, &[Process::User(1)]); + guest.kernel.fail( + Operation::Write, + guest.path("proc/sys/user/max_user_namespaces"), + Errno::ENOENT, + ); + write(&guest.paths.proc.join("self/ns/user"), ""); + let error = bootstrap_in(&guest, &layout, &Resources::default()).unwrap_err(); + assert!( + matches!( + &error, + Error::Bootstrap { step: BootstrapStep::UserNamespaces, error } + if error.raw_os_error() == Some(Errno::ENOENT as i32) + ), + "{error:?}" + ); + assert_eq!(guest.kernel.calls().len(), 1); + assert!(!guest.path("cgroup/agent").exists()); +} + #[test] fn bootstrap_fails_while_a_process_keeps_returning_to_the_parent() { let guest = Guest::bare(); From 1fe364cf0d70843b1936fedbaa5687b58a54c333 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 20 Oct 2026 21:47:46 -0700 Subject: [PATCH 05/12] nvx-rs: refuse a guest root at an occupied runtime directory A root unmounts what it mounted by its paths, so a second assembly at the same runtime directory stacked its mounts on the first root's, and dropping the first handle then unmounted the second root's mounts. Refuse an assembly when a file system is mounted already at one of its mount points, before it mounts anything, with a new Error::Occupied. A mount that a failed unmount left in place keeps the directory occupied too. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/README.md | 2 +- nvx-rs/src/guest/root.rs | 36 +++++++++++++++++++-- nvx-rs/src/guest/root/kernel.rs | 50 +++++++++++++++++++++++------ nvx-rs/src/guest/root/tests.rs | 56 +++++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 13 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 8cc4cd84c..518780f18 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -86,7 +86,7 @@ nvx-rs/ | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory where no other root is mounted: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 6064ce520..564e8ff1c 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -8,7 +8,8 @@ //! - [`assemble`] finds the block devices of an [`Assembly`] by their MMIO windows, checks and //! flushes them, and mounts the root: a read-only EROFS image, the `distro` layer, under an //! overlay whose upper and work directories are on an ext4 scratch, and which is mounted -//! `nosuid,nodev`, as the scratch is. It returns the [`Root`] that owns the mounts. +//! `nosuid,nodev`, as the scratch is. It returns the [`Root`] that owns the mounts, and refuses +//! a runtime directory where another root's are. //! - [`Root::mount_share`] and [`Root::mount_aggregate`] mount the VMM's live shares at targets in //! the root, after the checks that [live shares](#live-shares) lists. //! - [`Root::teardown`] unmounts, in order, the live shares in reverse mount order, the overlay, @@ -189,7 +190,8 @@ pub struct Assembly { /// root that owns its mounts. In order, it: /// /// 1. checks the runtime directory; -/// 2. creates its `layers/distro`, `scratch`, and `rootfs` directories, as `mkdir -p` does; +/// 2. creates its `layers/distro`, `scratch`, and `rootfs` directories, as `mkdir -p` does, and +/// refuses them if a file system is mounted at one already, such as another root's; /// 3. finds the distro device, polling for it every 50 ms until `timeout` passes, checks its /// size if the caller expects one, and flushes its buffers; /// 4. checks the expected UUID, if any, in the device's EROFS superblock, and reads nothing else; @@ -293,6 +295,23 @@ impl Root { error, })?; } + // Another root's handle unmounts what it mounted by its path, so no root stacks on it. + for directory in [&distro, &scratch, &self.path] { + match self.kernel.is_mount_point(directory) { + Ok(false) => {} + Ok(true) => { + return Err(Error::Occupied { + path: directory.clone(), + }); + } + Err(error) => { + return Err(Error::MountPoint { + path: directory.clone(), + error, + }); + } + } + } let device = block::prepare( self.kernel.as_mut(), paths, @@ -546,13 +565,19 @@ pub enum Error { /// The runtime directory. path: PathBuf, }, - /// A mount point, or a directory below one, could not be created. + /// A mount point, or a directory below one, could not be created, or checked for a mount. MountPoint { /// The path that could not be created. path: PathBuf, /// The error. error: io::Error, }, + /// A file system is mounted already where the assembly mounts, such as one of another root + /// at the same runtime directory. + Occupied { + /// The mount point. + path: PathBuf, + }, /// The kernel's MMIO windows, or a window's devices, could not be read. Lookup { /// The role of the device that was looked up. @@ -634,6 +659,11 @@ impl fmt::Display for Error { Self::MountPoint { path, error } => { write!(formatter, "failed to create {}: {error}", path.display()) } + Self::Occupied { path } => write!( + formatter, + "a file system is mounted at {} already", + path.display() + ), Self::Lookup { role, mmio, error } => write!( formatter, "failed to look up the {role} device at MMIO {mmio:#x}: {error}" diff --git a/nvx-rs/src/guest/root/kernel.rs b/nvx-rs/src/guest/root/kernel.rs index f65f6e259..f4d459c54 100644 --- a/nvx-rs/src/guest/root/kernel.rs +++ b/nvx-rs/src/guest/root/kernel.rs @@ -3,21 +3,25 @@ //! The operations read the kernel's text files and walk directories through the standard //! library, at the paths of a [`Paths`], so that a test relocates them to a temporary directory. //! What a test cannot do in a temporary directory, such as a mount, an unmount, or a flush of a -//! block device's buffers, and the writes to the kernel's control files, whose effects a test -//! simulates, go through a [`Kernel`]. +//! block device's buffers, and the writes to the kernel's control files and the check for a +//! mount point, whose effects and answers a test simulates, go through a [`Kernel`]. use std::ffi::CStr; -use std::path::PathBuf; +use std::fs; +use std::io; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; use std::time::Duration; use crate::guest::sys::{self, Errno, Mode, MountFlags, OpenFlags}; /// The kernel's operations that the root operations perform, other than reads: mounts and /// unmounts, flushes of a block device's buffers, writes of a control file, and the sleeps -/// between two polls for a block device. +/// between two polls for a block device; and the check for a mount point, which depends on them. /// -/// [`SystemKernel`] performs them. The tests' fake records them, simulates the effect of a -/// cgroup's membership file, and fails the ones that a test chooses. +/// [`SystemKernel`] performs them. The tests' fake records the operations, simulates the effect +/// of a cgroup's membership file and its own mounts, and fails the operations that a test +/// chooses. pub(super) trait Kernel: Send + Sync { /// Mounts `source` at `target`, as `mount(2)` does, with the file system's own options /// `data`. @@ -40,6 +44,10 @@ pub(super) trait Kernel: Send + Sync { /// sysctl's, and reports a failure to close it. fn write(&mut self, path: &CStr, value: &[u8]) -> Result<(), Errno>; + /// Returns whether a file system other than its parent directory's is mounted at the + /// directory `path`. + fn is_mount_point(&mut self, path: &Path) -> io::Result; + /// Sleeps for `duration`. fn sleep(&mut self, duration: Duration); } @@ -84,6 +92,11 @@ impl Kernel for SystemKernel { descriptor.close() } + fn is_mount_point(&mut self, path: &Path) -> io::Result { + let parent = path.parent().ok_or(Errno::EINVAL)?; + Ok(fs::metadata(path)?.dev() != fs::metadata(parent)?.dev()) + } + fn sleep(&mut self, duration: Duration) { std::thread::sleep(duration); } @@ -177,6 +190,8 @@ mod fake { /// The processes that writes of their ID to a membership file leave where they are, each /// with the number of such writes left. unmovable: Vec<(u32, usize)>, + /// The targets of the mounts that no unmount has undone, in mount order. + mounted: Vec, /// The sleeps so far. sleeps: usize, /// What happens after each sleep, given the number of sleeps so far. @@ -189,7 +204,9 @@ mod fake { /// A write of a control file writes the regular file at its path, which it creates if /// needed, except that a write of a process ID to a cgroup's `cgroup.procs` moves the ID /// there from the `cgroup.procs` of the cgroup's parent, as the kernel moves the process. - /// Its clones share what they record, so a test keeps one while a root owns another. + /// The mount points that it reports are the targets of its mounts, other than remounts, that + /// no unmount has undone. Its clones share what they record, so a test keeps one while a + /// root owns another. #[derive(Clone, Default)] pub(in crate::guest::root) struct FakeKernel(Arc>); @@ -327,11 +344,22 @@ mod fake { } else { Operation::Mount }; - self.record(call, operation, target) + self.record(call, operation, target)?; + if operation == Operation::Mount { + self.with(|state| state.mounted.push(PathBuf::from(text(target)))); + } + Ok(()) } fn unmount(&mut self, target: &CStr) -> Result<(), Errno> { - self.record(Call::Unmount(text(target)), Operation::Unmount, target) + self.record(Call::Unmount(text(target)), Operation::Unmount, target)?; + let target = PathBuf::from(text(target)); + self.with(|state| { + if let Some(last) = state.mounted.iter().rposition(|point| *point == target) { + state.mounted.remove(last); + } + }); + Ok(()) } fn flush(&mut self, device: &CStr) -> Result<(), Errno> { @@ -365,6 +393,10 @@ mod fake { } } + fn is_mount_point(&mut self, path: &Path) -> std::io::Result { + Ok(self.with(|state| state.mounted.iter().any(|point| point == path))) + } + fn sleep(&mut self, _duration: Duration) { let (sleeps, mut action) = self.with(|state| { state.sleeps += 1; diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 0ece9ae89..8b63d61db 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -393,6 +393,62 @@ fn a_runtime_directory_that_the_overlay_options_reserve_is_refused() { assert!(guest.kernel.calls().is_empty()); } +#[test] +fn a_runtime_directory_where_another_root_is_mounted_is_refused() { + let guest = Guest::new(); + let first = guest.assemble(&guest.assembly()).unwrap(); + let calls = guest.kernel.calls(); + let error = guest.assemble(&guest.assembly()).unwrap_err(); + assert!( + matches!(error.error(), Error::Occupied { path } if *path == guest.path("run/nvx/layers/distro")), + "{error:?}" + ); + assert!(error.cleanup().is_none()); + assert_eq!( + error.to_string(), + format!( + "a file system is mounted at {} already", + guest.text("run/nvx/layers/distro") + ) + ); + // The refused assembly touched none of the mounts, which the first root still owns. + assert_eq!(guest.kernel.calls(), calls); + first.teardown().unwrap(); + assert_eq!( + guest.kernel.unmounts(), + [ + guest.text("run/nvx/rootfs"), + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch"), + ] + ); + guest + .assemble(&guest.assembly()) + .unwrap() + .teardown() + .unwrap(); + + // An overlay that a failed unmount left in place keeps the runtime directory occupied. + let guest = Guest::new(); + guest.kernel.fail( + Operation::Unmount, + guest.path("run/nvx/rootfs"), + Errno::EBUSY, + ); + guest + .assemble(&guest.assembly()) + .unwrap() + .teardown() + .unwrap_err(); + let mounts = guest.kernel.mounts().len(); + let error = guest.assemble(&guest.assembly()).unwrap_err(); + assert!( + matches!(error.error(), Error::Occupied { path } if *path == guest.path("run/nvx/rootfs")), + "{error:?}" + ); + assert_eq!(guest.kernel.mounts().len(), mounts); +} + /// Returns a child of an aggregate share. fn child(name: &str, target: &str, access: Access, kind: ShareKind) -> ShareChild { ShareChild { From 1d225f894c5e74bd2d143825a5327849a8e86277 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 20 Oct 2026 22:09:56 -0700 Subject: [PATCH 06/12] nvx-rs: hold a guest root's runtime directory while it lives Two assemblies for one runtime directory could both pass the check for mounted file systems before either mounted anything, one waiting for its device while the other finished. The later one then mounted over the earlier root, and the two handles, which unmount by path, shared mount points. An assembly now takes an exclusive, non-blocking flock of the runtime directory before it checks the mount points, and the root holds it until it is torn down or dropped, after its unmounts. A runtime directory that another root holds, in this process or another, fails the assembly with Error::Reserved before it checks or mounts anything. The mount-point check stays for file systems that no root holds, such as one that a failed teardown left. A test runs a second assembly while the first waits for its device. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/README.md | 2 +- nvx-rs/src/guest/root.rs | 63 +++++++++++++++++++++++----- nvx-rs/src/guest/root/tests.rs | 77 ++++++++++++++++++++++++++++++++-- nvx-rs/src/guest/sys/fd.rs | 11 +++++ nvx-rs/src/guest/sys/mod.rs | 2 +- 5 files changed, 138 insertions(+), 17 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 518780f18..3a237eb5b 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -86,7 +86,7 @@ nvx-rs/ | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory where no other root is mounted: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 564e8ff1c..95a21c434 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -8,8 +8,8 @@ //! - [`assemble`] finds the block devices of an [`Assembly`] by their MMIO windows, checks and //! flushes them, and mounts the root: a read-only EROFS image, the `distro` layer, under an //! overlay whose upper and work directories are on an ext4 scratch, and which is mounted -//! `nosuid,nodev`, as the scratch is. It returns the [`Root`] that owns the mounts, and refuses -//! a runtime directory where another root's are. +//! `nosuid,nodev`, as the scratch is. It returns the [`Root`] that owns the mounts and holds +//! the [runtime directory](#runtime-directory). //! - [`Root::mount_share`] and [`Root::mount_aggregate`] mount the VMM's live shares at targets in //! the root, after the checks that [live shares](#live-shares) lists. //! - [`Root::teardown`] unmounts, in order, the live shares in reverse mount order, the overlay, @@ -32,6 +32,12 @@ //! - and `shares`, an aggregate live share, whose root lists every child, out of the workload's //! reach. //! +//! A root unmounts its mounts by their paths, so it holds the exclusive lock of its runtime +//! directory, from the start of its assembly until it is torn down. An assembly refuses a runtime +//! directory that another root holds, in this process or another, and one where a file system is +//! mounted already, such as one that a failed teardown left. The lock is advisory: it keeps out +//! the roots of these operations, not other mounts. +//! //! # Live shares //! //! A live share is a virtio-fs device that the VMM exports from the host: one host directory, @@ -190,8 +196,10 @@ pub struct Assembly { /// root that owns its mounts. In order, it: /// /// 1. checks the runtime directory; -/// 2. creates its `layers/distro`, `scratch`, and `rootfs` directories, as `mkdir -p` does, and -/// refuses them if a file system is mounted at one already, such as another root's; +/// 2. creates its `layers/distro`, `scratch`, and `rootfs` directories, as `mkdir -p` does, takes +/// the runtime directory's exclusive lock, which the root then holds, and refuses the runtime +/// directory if another root holds it, or if a file system is mounted at one of the three +/// already; /// 3. finds the distro device, polling for it every 50 ms until `timeout` passes, checks its /// size if the caller expects one, and flushes its buffers; /// 4. checks the expected UUID, if any, in the device's EROFS superblock, and reads nothing else; @@ -231,6 +239,7 @@ fn assemble_with( scratch: None, shares: Vec::new(), claimed: Vec::new(), + reservation: None, }; match root.mount_layers(paths, assembly) { Ok(()) => Ok(root), @@ -245,7 +254,8 @@ fn assemble_with( /// those of its live shares. /// /// [`teardown`](Self::teardown) unmounts them and reports each failure. Dropping the root -/// unmounts them the same way, and ignores failures. +/// unmounts them the same way, and ignores failures. Until then, the root holds the lock of its +/// runtime directory, where no other root assembles. pub struct Root { /// The kernel that mounts and unmounts. kernel: Box, @@ -263,6 +273,9 @@ pub struct Root { shares: Vec<(Mount, PathBuf)>, /// The targets of the live shares, which another share's target may not overlap. claimed: Vec, + /// The exclusive lock of the runtime directory, from the start of the assembly; dropping the + /// root releases it after the unmounts. + reservation: Option, } impl Root { @@ -295,7 +308,7 @@ impl Root { error, })?; } - // Another root's handle unmounts what it mounted by its path, so no root stacks on it. + self.reservation = Some(reserve(&self.runtime)?); for directory in [&distro, &scratch, &self.path] { match self.kernel.is_mount_point(directory) { Ok(false) => {} @@ -483,6 +496,22 @@ impl fmt::Debug for Root { } } +/// Takes the exclusive lock of the runtime directory `runtime`, without waiting for another root +/// that holds it. +fn reserve(runtime: &Path) -> Result { + sys::path_c_string(runtime) + .and_then(|path| sys::try_lock_directory(&path)) + .map_err(|error| match error { + Errno::EWOULDBLOCK => Error::Reserved { + path: runtime.to_owned(), + }, + error => Error::MountPoint { + path: runtime.to_owned(), + error: error.into(), + }, + }) +} + /// Creates the directory `path` unless a directory, and not a symbolic link, is there already. fn create_directory(path: &Path) -> io::Result<()> { match fs::create_dir(path) { @@ -565,15 +594,22 @@ pub enum Error { /// The runtime directory. path: PathBuf, }, - /// A mount point, or a directory below one, could not be created, or checked for a mount. + /// A mount point or a directory below one could not be created or checked for a mount, or + /// the runtime directory could not be locked. MountPoint { - /// The path that could not be created. + /// The path that could not be prepared. path: PathBuf, /// The error. error: io::Error, }, - /// A file system is mounted already where the assembly mounts, such as one of another root - /// at the same runtime directory. + /// Another root holds the runtime directory: one that has not been torn down, or one whose + /// assembly is in progress, in this process or another. + Reserved { + /// The runtime directory. + path: PathBuf, + }, + /// A file system is mounted already where the assembly mounts, such as one that a failed + /// teardown left. Occupied { /// The mount point. path: PathBuf, @@ -657,8 +693,13 @@ impl fmt::Display for Error { path.display() ), Self::MountPoint { path, error } => { - write!(formatter, "failed to create {}: {error}", path.display()) + write!(formatter, "failed to prepare {}: {error}", path.display()) } + Self::Reserved { path } => write!( + formatter, + "another root holds the runtime directory {}", + path.display() + ), Self::Occupied { path } => write!( formatter, "a file system is mounted at {} already", diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 8b63d61db..db7b73a48 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -4,6 +4,7 @@ use std::fs; use std::io; use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use super::kernel::{Call, FakeKernel, Operation, Paths}; @@ -394,21 +395,21 @@ fn a_runtime_directory_that_the_overlay_options_reserve_is_refused() { } #[test] -fn a_runtime_directory_where_another_root_is_mounted_is_refused() { +fn a_runtime_directory_that_another_root_holds_is_refused() { let guest = Guest::new(); let first = guest.assemble(&guest.assembly()).unwrap(); let calls = guest.kernel.calls(); let error = guest.assemble(&guest.assembly()).unwrap_err(); assert!( - matches!(error.error(), Error::Occupied { path } if *path == guest.path("run/nvx/layers/distro")), + matches!(error.error(), Error::Reserved { path } if *path == guest.path("run/nvx")), "{error:?}" ); assert!(error.cleanup().is_none()); assert_eq!( error.to_string(), format!( - "a file system is mounted at {} already", - guest.text("run/nvx/layers/distro") + "another root holds the runtime directory {}", + guest.text("run/nvx") ) ); // The refused assembly touched none of the mounts, which the first root still owns. @@ -422,12 +423,80 @@ fn a_runtime_directory_where_another_root_is_mounted_is_refused() { guest.text("run/nvx/scratch"), ] ); + // A root that is torn down or dropped holds the runtime directory no longer. + drop(guest.assemble(&guest.assembly()).unwrap()); guest .assemble(&guest.assembly()) .unwrap() .teardown() .unwrap(); + // An assembly that starts while another one waits for its distro device is refused before + // it checks or mounts anything, and the first one goes on. + let guest = Guest::bare(); + attach(guest.directory.path(), 1, "vdb", 8192); + let refused = Arc::new(Mutex::new(None)); + let (kernel, paths, assembly) = (guest.kernel.clone(), guest.paths.clone(), guest.assembly()); + let (directory, outcome) = (guest.directory.path().to_owned(), Arc::clone(&refused)); + guest.kernel.on_sleep(move |sleeps| { + if sleeps == 1 { + attach(&directory, 0, "vda", 4096); + let calls = kernel.calls().len(); + let result = assemble_with(Box::new(kernel.clone()), &paths, &assembly); + *outcome.lock().unwrap() = Some((result.map(drop), kernel.calls().len() - calls)); + } + }); + let mut assembly = guest.assembly(); + assembly.timeout = Duration::from_secs(1); + let first = guest.assemble(&assembly).unwrap(); + let (result, calls) = refused.lock().unwrap().take().unwrap(); + let error = result.unwrap_err(); + assert!( + matches!(error.error(), Error::Reserved { path } if *path == guest.path("run/nvx")), + "{error:?}" + ); + assert!(error.cleanup().is_none()); + assert_eq!(calls, 0); + assert_eq!( + guest.kernel.mounts(), + [ + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch"), + guest.text("run/nvx/rootfs"), + ] + ); + first.teardown().unwrap(); +} + +#[test] +fn a_runtime_directory_where_a_file_system_is_mounted_is_refused() { + // A file system that no root mounted, at the distro layer's mount point. + let guest = Guest::new(); + let distro = guest.path("run/nvx/layers/distro"); + guest + .kernel + .clone() + .mount( + Some(c"tmpfs"), + &sys::path_c_string(&distro).unwrap(), + Some(c"tmpfs"), + MountFlags::empty(), + None, + ) + .unwrap(); + let calls = guest.kernel.calls(); + let error = guest.assemble(&guest.assembly()).unwrap_err(); + assert!( + matches!(error.error(), Error::Occupied { path } if *path == distro), + "{error:?}" + ); + assert!(error.cleanup().is_none()); + assert_eq!( + error.to_string(), + format!("a file system is mounted at {} already", distro.display()) + ); + assert_eq!(guest.kernel.calls(), calls); + // An overlay that a failed unmount left in place keeps the runtime directory occupied. let guest = Guest::new(); guest.kernel.fail( diff --git a/nvx-rs/src/guest/sys/fd.rs b/nvx-rs/src/guest/sys/fd.rs index 73b3ca0f9..48989186b 100644 --- a/nvx-rs/src/guest/sys/fd.rs +++ b/nvx-rs/src/guest/sys/fd.rs @@ -316,6 +316,17 @@ pub(crate) fn lock_file(path: &CStr, mode: Mode) -> Result { } } +/// Opens the directory `path` and takes its exclusive lock without waiting, or fails with +/// `EWOULDBLOCK` while another open file description holds a lock on it, in this process or +/// another. +pub(crate) fn try_lock_directory(path: &CStr) -> Result { + let flags = OpenFlags::O_RDONLY | OpenFlags::O_DIRECTORY | OpenFlags::O_CLOEXEC; + let descriptor = nix::fcntl::open(path, flags, Mode::empty())?; + Flock::lock(descriptor, FlockArg::LockExclusiveNonblock) + .map(|lock| FileLock { _lock: lock }) + .map_err(|(_, error)| error) +} + /// A file that this process creates, which dropping the owner closes and removes unless the /// process [keeps](Self::keep) or [renames](Self::rename) it first. #[derive(Debug)] diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index 0b66bb6be..ffd78dd11 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -80,7 +80,7 @@ pub(crate) use fd::close_inherited_descriptors; pub(crate) use fd::set_descriptor_limit; pub(crate) use fd::{ CreatedFile, FileDescriptor, FileLock, Mode, OpenFlags, create_directory, create_fifo, - create_memfd, create_pipe, lock_file, open, open_at, page_size, unlink, + create_memfd, create_pipe, lock_file, open, open_at, page_size, try_lock_directory, unlink, }; pub(crate) use filesystem::{ MountFlags, PATH_MAX, ScratchDirectory, TmpfsMount, change_directory, check_exists, chmod, From 78ed49c66149fd9969b08885b30f052904ba45e5 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 20 Oct 2026 22:36:37 -0700 Subject: [PATCH 07/12] nvx-rs: keep a guest root's aggregate share from stacking An aggregate live share is mounted at shares in the runtime directory, outside the overlay. A teardown that failed to unmount it alone, but unmounted the overlay, the image, and the scratch, left a runtime directory that the next assembly accepted, and that root's aggregate then covered the leaked mount, which outlived its teardown. An assembly now also refuses a runtime directory where a file system is mounted at shares, which may be missing. A second mount_aggregate call stacked another aggregate at the same path, so a failed unmount of the newer one was retried under the older one's name, and the older one was never attempted. A root now has one aggregate at most: once it has mounted one, even one whose children failed to bind, it refuses another with ShareRefusal::SecondAggregate. The tests' fake kernel, as the kernel does, now fails to check a missing directory for a mount. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/README.md | 2 +- nvx-rs/src/guest/root.rs | 19 +++++--- nvx-rs/src/guest/root/kernel.rs | 6 ++- nvx-rs/src/guest/root/share.rs | 20 +++++++-- nvx-rs/src/guest/root/tests.rs | 77 +++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 13 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 3a237eb5b..8c58a727f 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -86,7 +86,7 @@ nvx-rs/ | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root, with one aggregate per root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 95a21c434..b76c3d716 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -35,15 +35,15 @@ //! A root unmounts its mounts by their paths, so it holds the exclusive lock of its runtime //! directory, from the start of its assembly until it is torn down. An assembly refuses a runtime //! directory that another root holds, in this process or another, and one where a file system is -//! mounted already, such as one that a failed teardown left. The lock is advisory: it keeps out -//! the roots of these operations, not other mounts. +//! mounted already at one of these mount points, such as one that a failed teardown left. The +//! lock is advisory: it keeps out the roots of these operations, not other mounts. //! //! # Live shares //! //! A live share is a virtio-fs device that the VMM exports from the host: one host directory, //! mounted at a target in the root, or an aggregate of host directories and files, each of which, -//! a child, is bound at its own target. Before it mounts anything for a share, the root checks -//! that each target: +//! a child, is bound at its own target. A root has one aggregate at most. Before it mounts +//! anything for a share, the root checks that each target: //! //! - is absolute and canonical: not `/`, without an empty, `.`, or `..` component, and without a //! trailing `/`; @@ -112,6 +112,9 @@ const DISTRO_DIRECTORY: &str = "layers/distro"; const SCRATCH_DIRECTORY: &str = "scratch"; /// The overlay's mount point, the root, below the runtime directory. const ROOT_DIRECTORY: &str = "rootfs"; +/// The mount point of an aggregate live share, below the runtime directory and outside the root, +/// which exists once a root has mounted an aggregate. +const AGGREGATE_DIRECTORY: &str = "shares"; /// The overlay's upper directory, below the scratch. const UPPER_DIRECTORY: &str = "upper"; /// The overlay's work directory, below the scratch. @@ -198,8 +201,8 @@ pub struct Assembly { /// 1. checks the runtime directory; /// 2. creates its `layers/distro`, `scratch`, and `rootfs` directories, as `mkdir -p` does, takes /// the runtime directory's exclusive lock, which the root then holds, and refuses the runtime -/// directory if another root holds it, or if a file system is mounted at one of the three -/// already; +/// directory if another root holds it, or if a file system is mounted already at one of the +/// three, or at `shares`, where a root mounts an aggregate live share; /// 3. finds the distro device, polling for it every 50 ms until `timeout` passes, checks its /// size if the caller expects one, and flushes its buffers; /// 4. checks the expected UUID, if any, in the device's EROFS superblock, and reads nothing else; @@ -309,7 +312,8 @@ impl Root { })?; } self.reservation = Some(reserve(&self.runtime)?); - for directory in [&distro, &scratch, &self.path] { + let shares = self.runtime.join(AGGREGATE_DIRECTORY); + for directory in [&distro, &scratch, &self.path, &shares] { match self.kernel.is_mount_point(directory) { Ok(false) => {} Ok(true) => { @@ -317,6 +321,7 @@ impl Root { path: directory.clone(), }); } + Err(error) if error.kind() == io::ErrorKind::NotFound && *directory == shares => {} Err(error) => { return Err(Error::MountPoint { path: directory.clone(), diff --git a/nvx-rs/src/guest/root/kernel.rs b/nvx-rs/src/guest/root/kernel.rs index f4d459c54..45b11b6fc 100644 --- a/nvx-rs/src/guest/root/kernel.rs +++ b/nvx-rs/src/guest/root/kernel.rs @@ -205,8 +205,8 @@ mod fake { /// needed, except that a write of a process ID to a cgroup's `cgroup.procs` moves the ID /// there from the `cgroup.procs` of the cgroup's parent, as the kernel moves the process. /// The mount points that it reports are the targets of its mounts, other than remounts, that - /// no unmount has undone. Its clones share what they record, so a test keeps one while a - /// root owns another. + /// no unmount has undone, and, as the kernel does, it fails to check a missing directory. Its + /// clones share what they record, so a test keeps one while a root owns another. #[derive(Clone, Default)] pub(in crate::guest::root) struct FakeKernel(Arc>); @@ -394,6 +394,8 @@ mod fake { } fn is_mount_point(&mut self, path: &Path) -> std::io::Result { + // The kernel's answer needs the directory, too. + std::fs::metadata(path)?; Ok(self.with(|state| state.mounted.iter().any(|point| point == path))) } diff --git a/nvx-rs/src/guest/root/share.rs b/nvx-rs/src/guest/root/share.rs index fd89f5bec..2c93add73 100644 --- a/nvx-rs/src/guest/root/share.rs +++ b/nvx-rs/src/guest/root/share.rs @@ -10,7 +10,7 @@ use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; use std::path::{Path, PathBuf}; -use super::{Error, Mount, Root}; +use super::{AGGREGATE_DIRECTORY, Error, Mount, Root}; use crate::guest::sys::{self, MountFlags}; /// The targets that the runtime mounts or binds over in the root, each with everything below it. @@ -24,8 +24,6 @@ const RESERVED_TREES: [&[u8]; 6] = [ ]; /// The directories that the runtime creates or mounts below, without the paths below them. const RESERVED_DIRECTORIES: [&[u8]; 3] = [b"/etc", b"/run", b"/run/nvx"]; -/// The directory, below the runtime directory, at which an aggregate share is mounted. -const AGGREGATE_DIRECTORY: &str = "shares"; /// The file system type of a live share. const VIRTIOFS: &[u8] = b"virtiofs"; @@ -118,6 +116,9 @@ pub enum ShareRefusal { NotDirectory(PathBuf), /// A file child's target exists in the root but is not a regular file. NotFile(PathBuf), + /// The root owns an aggregate already, whose children may or may not all be bound: it mounts + /// every aggregate at the same place. + SecondAggregate, } impl fmt::Display for ShareRefusal { @@ -170,6 +171,9 @@ impl fmt::Display for ShareRefusal { "live share target is not a regular file: {}", target.display() ), + Self::SecondAggregate => { + formatter.write_str("the root has an aggregate live share already") + } } } } @@ -215,12 +219,22 @@ impl Root { /// unless every child passes. The mounts belong to the root, which unmounts them first when /// it is torn down, in reverse mount order, those of an aggregate that failed part of the way /// included. + /// + /// A root has one aggregate at most, since every aggregate is mounted at the same place: once + /// the root has mounted one, whether or not its children were all bound, another is refused. pub fn mount_aggregate( &mut self, tag: &str, access: Access, children: &[ShareChild], ) -> Result<(), Error> { + if self + .shares + .iter() + .any(|(mount, _)| matches!(mount, Mount::Aggregate { .. })) + { + return Err(Error::Share(ShareRefusal::SecondAggregate)); + } check_tag(tag)?; if children.is_empty() { return Err(Error::Share(ShareRefusal::NoChildren)); diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index db7b73a48..694ff25e4 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -516,6 +516,28 @@ fn a_runtime_directory_where_a_file_system_is_mounted_is_refused() { "{error:?}" ); assert_eq!(guest.kernel.mounts().len(), mounts); + + // So does an aggregate share whose unmount alone failed: it is mounted outside the overlay, + // which, like the image and the scratch, was unmounted. + let guest = Guest::new(); + let mut root = guest.assemble(&guest.assembly()).unwrap(); + let rw = Access::ReadWrite; + root.mount_aggregate("all", rw, &[child("a", "/a", rw, ShareKind::Directory)]) + .unwrap(); + let shares = guest.path("run/nvx/shares"); + guest + .kernel + .fail(Operation::Unmount, shares.clone(), Errno::EBUSY); + let error = root.teardown().unwrap_err(); + let failed: Vec<&Path> = error.failures().iter().map(Unmount::path).collect(); + assert_eq!(failed, [shares.as_path()]); + let mounts = guest.kernel.mounts().len(); + let error = guest.assemble(&guest.assembly()).unwrap_err(); + assert!( + matches!(error.error(), Error::Occupied { path } if *path == shares), + "{error:?}" + ); + assert_eq!(guest.kernel.mounts().len(), mounts); } /// Returns a child of an aggregate share. @@ -689,6 +711,61 @@ fn refused_shares_mount_nothing() { assert_eq!(root.claimed, [PathBuf::from("/srv")]); } +#[test] +fn a_root_has_one_aggregate_even_after_one_that_failed_part_of_the_way() { + let rw = Access::ReadWrite; + let directory = ShareKind::Directory; + let rootfs = |guest: &Guest, target: &str| guest.text(&format!("run/nvx/rootfs{target}")); + for bound in [&["/a", "/b"][..], &["/a"]] { + let guest = Guest::new(); + let mut root = guest.assemble(&guest.assembly()).unwrap(); + if bound.len() == 1 { + guest.kernel.fail( + Operation::Mount, + guest.path("run/nvx/rootfs/b"), + Errno::ENOENT, + ); + } + let first = root.mount_aggregate( + "all", + rw, + &[ + child("a", "/a", rw, directory), + child("b", "/b", rw, directory), + ], + ); + assert_eq!(first.is_ok(), bound.len() == 2, "{first:?}"); + let mounts = guest.kernel.mounts().len(); + let error = root + .mount_aggregate("more", rw, &[child("c", "/c", rw, directory)]) + .unwrap_err(); + assert!( + matches!(error, Error::Share(ShareRefusal::SecondAggregate)), + "{error:?}" + ); + assert_eq!( + error.to_string(), + "the root has an aggregate live share already" + ); + assert_eq!(guest.kernel.mounts().len(), mounts); + assert!(!guest.path("run/nvx/rootfs/c").exists()); + // The teardown unmounts the bound children, then the one aggregate. + root.teardown().unwrap(); + let mut unmounts: Vec = bound + .iter() + .rev() + .map(|target| rootfs(&guest, target)) + .collect(); + unmounts.extend([ + guest.text("run/nvx/shares"), + guest.text("run/nvx/rootfs"), + guest.text("run/nvx/layers/distro"), + guest.text("run/nvx/scratch"), + ]); + assert_eq!(guest.kernel.unmounts(), unmounts); + } +} + #[test] fn a_child_that_cannot_be_restricted_stays_mounted_until_teardown() { let guest = Guest::new(); From be72fe488520ec93f0d1947cc4190056f8b420d1 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 07:20:16 -0700 Subject: [PATCH 08/12] nvx-rs: detect a bind mount at a guest root's mount point An assembly refused a runtime directory where something was mounted already at one of its mount points by comparing each mount point's device with its parent directory's. A bind mount of a directory of the parent's own file system shows the parent's device, so the assembly missed it and mounted over it. The kernel's check now compares the IDs of the mounts that the mount point and its parent are on, which procfs shows in a descriptor's fdinfo. A test binds a directory at the distro layer's mount point in a user and mount namespace of its own, where the host allows them, and checks that the assembly refuses it there. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/src/guest/root.rs | 16 +++--- nvx-rs/src/guest/root/kernel.rs | 10 ++-- nvx-rs/src/guest/root/tests.rs | 81 +++++++++++++++++++++++++++++- nvx-rs/src/guest/sys/fd.rs | 4 +- nvx-rs/src/guest/sys/filesystem.rs | 23 ++++++++- nvx-rs/src/guest/sys/mod.rs | 14 +++++- 6 files changed, 131 insertions(+), 17 deletions(-) diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index b76c3d716..74af2754b 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -34,9 +34,10 @@ //! //! A root unmounts its mounts by their paths, so it holds the exclusive lock of its runtime //! directory, from the start of its assembly until it is torn down. An assembly refuses a runtime -//! directory that another root holds, in this process or another, and one where a file system is -//! mounted already at one of these mount points, such as one that a failed teardown left. The -//! lock is advisory: it keeps out the roots of these operations, not other mounts. +//! directory that another root holds, in this process or another, and one where a file system, or +//! a bind mount of a directory, is mounted already at one of these mount points, such as one that +//! a failed teardown left. The lock is advisory: it keeps out the roots of these operations, not +//! other mounts. //! //! # Live shares //! @@ -201,8 +202,9 @@ pub struct Assembly { /// 1. checks the runtime directory; /// 2. creates its `layers/distro`, `scratch`, and `rootfs` directories, as `mkdir -p` does, takes /// the runtime directory's exclusive lock, which the root then holds, and refuses the runtime -/// directory if another root holds it, or if a file system is mounted already at one of the -/// three, or at `shares`, where a root mounts an aggregate live share; +/// directory if another root holds it, or if one of the three, or `shares`, where a root +/// mounts an aggregate live share, is a mount point already, whether of a file system or of a +/// bind mount of a directory; /// 3. finds the distro device, polling for it every 50 ms until `timeout` passes, checks its /// size if the caller expects one, and flushes its buffers; /// 4. checks the expected UUID, if any, in the device's EROFS superblock, and reads nothing else; @@ -613,8 +615,8 @@ pub enum Error { /// The runtime directory. path: PathBuf, }, - /// A file system is mounted already where the assembly mounts, such as one that a failed - /// teardown left. + /// A file system, or a bind mount of a directory, is mounted already where the assembly + /// mounts, such as one that a failed teardown left. Occupied { /// The mount point. path: PathBuf, diff --git a/nvx-rs/src/guest/root/kernel.rs b/nvx-rs/src/guest/root/kernel.rs index 45b11b6fc..334f56cc4 100644 --- a/nvx-rs/src/guest/root/kernel.rs +++ b/nvx-rs/src/guest/root/kernel.rs @@ -7,9 +7,7 @@ //! mount point, whose effects and answers a test simulates, go through a [`Kernel`]. use std::ffi::CStr; -use std::fs; use std::io; -use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -44,8 +42,9 @@ pub(super) trait Kernel: Send + Sync { /// sysctl's, and reports a failure to close it. fn write(&mut self, path: &CStr, value: &[u8]) -> Result<(), Errno>; - /// Returns whether a file system other than its parent directory's is mounted at the - /// directory `path`. + /// Returns whether the directory `path` is a mount point: whether it is on another mount + /// than its parent directory, as it is where a file system is mounted, or where a directory + /// of the parent's own file system is bound. fn is_mount_point(&mut self, path: &Path) -> io::Result; /// Sleeps for `duration`. @@ -94,7 +93,8 @@ impl Kernel for SystemKernel { fn is_mount_point(&mut self, path: &Path) -> io::Result { let parent = path.parent().ok_or(Errno::EINVAL)?; - Ok(fs::metadata(path)?.dev() != fs::metadata(parent)?.dev()) + let mount = sys::mount_id(&sys::path_c_string(path)?)?; + Ok(mount != sys::mount_id(&sys::path_c_string(parent)?)?) } fn sleep(&mut self, duration: Duration) { diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 694ff25e4..b0fb7ba3d 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -1,8 +1,10 @@ //! Tests of the root operations with the fake kernel, and with the kernel's interfaces, a cgroup -//! hierarchy, and the runtime directory in a temporary directory. +//! hierarchy, and the runtime directory in a temporary directory, and of the kernel's own check +//! for a mount point. use std::fs; use std::io; +use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -540,6 +542,83 @@ fn a_runtime_directory_where_a_file_system_is_mounted_is_refused() { assert_eq!(guest.kernel.mounts().len(), mounts); } +#[test] +fn the_kernel_finds_its_mount_points() { + let directory = tempfile::tempdir().unwrap(); + let mut kernel = SystemKernel; + assert!(kernel.is_mount_point(Path::new("/proc")).unwrap()); + assert!(!kernel.is_mount_point(directory.path()).unwrap()); + let error = kernel + .is_mount_point(&directory.path().join("missing")) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); +} + +/// The variable that gives the bind-mount test's inner run the runtime directory where the outer +/// run bound a directory at the distro layer's mount point. +const BOUND_RUNTIME: &str = "NVX_ROOT_TEST_BOUND_RUNTIME"; + +#[test] +fn a_runtime_directory_where_a_directory_is_bound_is_refused() { + // A directory of the same file system, bound at the distro layer's mount point, shows the + // device of the directory's parent, so only their mounts tell it apart. The outer run binds + // it in a user and mount namespace of its own, where it runs the test again, which assembles + // a root there with the kernel itself. + if let Some(runtime) = std::env::var_os(BOUND_RUNTIME) { + let runtime = PathBuf::from(runtime); + let distro = runtime.join(DISTRO_DIRECTORY); + let device = |path: &Path| fs::metadata(path).unwrap().dev(); + assert_eq!(device(&distro), device(distro.parent().unwrap())); + let guest = Guest::new(); + let assembly = Assembly { + runtime, + ..guest.assembly() + }; + let error = assemble_with(Box::new(SystemKernel), &guest.paths, &assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Occupied { path } if *path == distro), + "{error:?}" + ); + assert!(error.cleanup().is_none()); + return; + } + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("source"); + let runtime = directory.path().join("run/nvx"); + let distro = runtime.join(DISTRO_DIRECTORY); + fs::create_dir(&source).unwrap(); + fs::create_dir_all(&distro).unwrap(); + let test = "guest::root::tests::a_runtime_directory_where_a_directory_is_bound_is_refused"; + let output = std::process::Command::new("unshare") + .args(["--user", "--map-root-user", "--mount", "--propagation"]) + .args(["private", "sh", "-c"]) + .arg(r#"mount --bind "$1" "$2" || exit 77; shift 2; exec "$@""#) + .arg("sh") + .args([&source, &distro]) + .arg(std::env::current_exe().unwrap()) + .args(["--exact", test, "--nocapture"]) + .env(BOUND_RUNTIME, &runtime) + .output(); + let output = match output { + Ok(output) => output, + // Without `unshare`, the test stops here. + Err(error) if error.kind() == io::ErrorKind::NotFound => return, + Err(error) => panic!("cannot run unshare: {error}"), + }; + // So it does where the namespaces cannot be created, which fails `unshare` with 1, or where + // they cannot bind a directory, which the script reports with 77. + if matches!(output.status.code(), Some(1 | 77)) { + return; + } + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("test result: ok. 1 passed"), + "{}\n{stdout}{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); +} + /// Returns a child of an aggregate share. fn child(name: &str, target: &str, access: Access, kind: ShareKind) -> ShareChild { ShareChild { diff --git a/nvx-rs/src/guest/sys/fd.rs b/nvx-rs/src/guest/sys/fd.rs index 48989186b..1c09fcd68 100644 --- a/nvx-rs/src/guest/sys/fd.rs +++ b/nvx-rs/src/guest/sys/fd.rs @@ -161,8 +161,8 @@ impl FileDescriptor { self.0.as_fd() } - /// Returns the raw descriptor only to the ioctl wrappers and the forked child in this module - /// tree. + /// Returns the raw descriptor only to the ioctl wrappers, the forked child, and the lookup of + /// a path's mount ID in this module tree. pub(super) fn raw(&self) -> RawFd { self.0.as_raw_fd() } diff --git a/nvx-rs/src/guest/sys/filesystem.rs b/nvx-rs/src/guest/sys/filesystem.rs index fba63ac24..30b08e72a 100644 --- a/nvx-rs/src/guest/sys/filesystem.rs +++ b/nvx-rs/src/guest/sys/filesystem.rs @@ -13,7 +13,8 @@ use nix::sched::CloneFlags; use nix::sys::stat::{FchmodatFlags, Mode}; use nix::unistd::{AccessFlags, Gid, Uid}; -use super::{Errno, errno_of}; +use super::{Errno, OpenFlags, errno_of}; +use crate::guest::parse::parse_decimal; /// Linux's `PATH_MAX`: the size of the longest path that the kernel takes, with its terminating /// NUL. @@ -95,6 +96,26 @@ pub(crate) fn unmount(target: &CStr) -> Result<(), Errno> { nix::mount::umount(target) } +/// Returns the ID of the mount that `path`, after its symbolic links, is on, which procfs, at +/// `/proc`, shows in a descriptor's `fdinfo`, as `mountinfo` numbers the mounts. Two paths are on +/// one mount exactly when their IDs are equal, whatever file systems the mounts show: a bind +/// mount of a directory shows the file system of its source, but has an ID of its own. +pub(crate) fn mount_id(path: &CStr) -> Result { + let descriptor = super::open( + path, + OpenFlags::O_PATH | OpenFlags::O_CLOEXEC, + Mode::empty(), + )?; + let information = read_file(&super::c_string( + format!("/proc/self/fdinfo/{}", descriptor.raw()).as_bytes(), + )?)?; + information + .split(|&byte| byte == b'\n') + .find_map(|line| line.strip_prefix(b"mnt_id:")) + .and_then(|value| parse_decimal(value.trim_ascii())) + .ok_or(Errno::EINVAL) +} + /// Changes the owner and group of `path`, following the final symlink like C `chown`. pub(crate) fn chown(path: &CStr, uid: u32, gid: u32) -> Result<(), Errno> { nix::unistd::chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index ffd78dd11..d9aa6ca18 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -84,7 +84,7 @@ pub(crate) use fd::{ }; pub(crate) use filesystem::{ MountFlags, PATH_MAX, ScratchDirectory, TmpfsMount, change_directory, check_exists, chmod, - chown, current_directory, enter_private_mount_namespace, mount, mount_with_options, + chown, current_directory, enter_private_mount_namespace, mount, mount_id, mount_with_options, path_c_string, path_exists, read_file, read_line, read_number, unmount, write_file, write_number, }; @@ -284,6 +284,18 @@ pub(crate) mod tests { assert!(read_only.truncate(0).is_err()); } + #[test] + fn mount_ids_tell_the_mounts_apart() { + let directory = tempfile::tempdir().unwrap(); + let child = directory.path().join("child"); + std::fs::create_dir(&child).unwrap(); + let id = |path: &std::path::Path| mount_id(&path_c_string(path).unwrap()); + assert_eq!(id(&child).unwrap(), id(directory.path()).unwrap()); + let proc = id(std::path::Path::new("/proc")).unwrap(); + assert_ne!(proc, id(std::path::Path::new("/")).unwrap()); + assert_eq!(id(&child.join("missing")), Err(Errno::ENOENT)); + } + #[test] fn connected_datagrams_wait_for_and_receive_echoes() { let server = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); From 8ac26e347f92509ba9310a107bb478d24d99a47d Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 07:39:14 -0700 Subject: [PATCH 09/12] nvx-rs: keep a guest root's paths from leading elsewhere A root mounts and unmounts at paths below its runtime directory, as given, so they must lead to the same directories whatever the root has mounted. An assembly accepted a runtime directory with a `..` component, or one that a symbolic link led through, and created its mount points through symbolic links. Such a path could lead through a directory that one of the root's own mounts hid later: with a `scratch/probe` directory, the runtime directory `.../scratch/probe/../..` led nowhere once the scratch was mounted, so the assembly failed and could unmount neither the image nor the scratch. An assembly now refuses a runtime directory whose path is not canonical, as a live share's target must be, or that a symbolic link leads through, and a root creates `layers` and the mount points a component at a time, refusing a symbolic link, as the aggregate's mount point, `shares`, is too. The share targets' canonical check moves to a helper that both use. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/README.md | 2 +- nvx-rs/src/guest/root.rs | 115 +++++++++++++++++++--------- nvx-rs/src/guest/root/share.rs | 13 +--- nvx-rs/src/guest/root/tests.rs | 136 +++++++++++++++++++++++++++++---- 4 files changed, 206 insertions(+), 60 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 8c58a727f..6c424bdc8 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -86,7 +86,7 @@ nvx-rs/ | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root, with one aggregate per root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root, with one aggregate per root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 74af2754b..61a27be74 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -32,12 +32,16 @@ //! - and `shares`, an aggregate live share, whose root lists every child, out of the workload's //! reach. //! -//! A root unmounts its mounts by their paths, so it holds the exclusive lock of its runtime -//! directory, from the start of its assembly until it is torn down. An assembly refuses a runtime -//! directory that another root holds, in this process or another, and one where a file system, or -//! a bind mount of a directory, is mounted already at one of these mount points, such as one that -//! a failed teardown left. The lock is advisory: it keeps out the roots of these operations, not -//! other mounts. +//! A root mounts and unmounts by these paths, so they must lead to the same directories whatever +//! the root has mounted. An assembly refuses a runtime directory whose path is not canonical, as +//! a live share's target must be, or leads through a symbolic link, and a root mounts nothing +//! where `layers` or the mount point is a symbolic link, since a `..` component or a link could +//! lead through a directory that one of the root's mounts hides later. A root also holds the +//! exclusive lock of its runtime directory, from the start of its assembly until it is torn down: +//! an assembly refuses a runtime directory that another root holds, in this process or another, +//! and one where a file system, or a bind mount of a directory, is mounted already at one of these +//! mount points, such as one that a failed teardown left. The lock is advisory: it keeps out the +//! roots of these operations, not other mounts. //! //! # Live shares //! @@ -183,8 +187,9 @@ impl fmt::Display for DeviceStep { /// What [`assemble`] mounts, and where. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Assembly { - /// The runtime directory: an absolute path without `,`, `:`, `\`, `=`, or NUL, which the - /// overlay's options reserve. + /// The runtime directory: an absolute path in canonical form, not `/`, without an empty, `.`, + /// or `..` component, or a trailing `/`, that no symbolic link leads through, and without `,`, + /// `:`, `\`, `=`, or NUL, which the overlay's options reserve. pub runtime: PathBuf, /// The device of the EROFS image. pub distro: Device, @@ -199,21 +204,24 @@ pub struct Assembly { /// Assembles the layered root of `assembly`, as the [module](self) describes, and returns the /// root that owns its mounts. In order, it: /// -/// 1. checks the runtime directory; -/// 2. creates its `layers/distro`, `scratch`, and `rootfs` directories, as `mkdir -p` does, takes -/// the runtime directory's exclusive lock, which the root then holds, and refuses the runtime -/// directory if another root holds it, or if one of the three, or `shares`, where a root -/// mounts an aggregate live share, is a mount point already, whether of a file system or of a -/// bind mount of a directory; -/// 3. finds the distro device, polling for it every 50 ms until `timeout` passes, checks its +/// 1. checks the runtime directory's path; +/// 2. creates the runtime directory, as `mkdir -p` does, and refuses it if a symbolic link leads +/// through it; +/// 3. creates its `layers/distro`, `scratch`, and `rootfs` directories, a component at a time, and +/// refuses a component that exists as anything but a directory, a symbolic link included; +/// takes the runtime directory's exclusive lock, which the root holds from then on, and +/// refuses the runtime directory if another root holds it, or if one of the three, or +/// `shares`, where a root mounts an aggregate live share, is a mount point already, whether of +/// a file system or of a bind mount of a directory; +/// 4. finds the distro device, polling for it every 50 ms until `timeout` passes, checks its /// size if the caller expects one, and flushes its buffers; -/// 4. checks the expected UUID, if any, in the device's EROFS superblock, and reads nothing else; -/// 5. mounts the device's EROFS image read-only at `layers/distro`; -/// 6. finds, checks, and flushes the scratch device the same way; -/// 7. mounts its ext4 file system at `scratch` with `nosuid` and `nodev`; -/// 8. creates the scratch's `upper` and `work` directories unless they exist; -/// 9. and mounts the overlay at `rootfs`, `nosuid` and `nodev`, with the distro layer as its -/// lower directory, and `metacopy=on,xino=on`. +/// 5. checks the expected UUID, if any, in the device's EROFS superblock, and reads nothing else; +/// 6. mounts the device's EROFS image read-only at `layers/distro`; +/// 7. finds, checks, and flushes the scratch device the same way; +/// 8. mounts its ext4 file system at `scratch` with `nosuid` and `nodev`; +/// 9. creates the scratch's `upper` and `work` directories unless they exist; +/// 10. and mounts the overlay at `rootfs`, `nosuid` and `nodev`, with the distro layer as its +/// lower directory, and `metacopy=on,xino=on`. /// /// A failure unmounts what the assembly mounted, as [`Root::teardown`] does, and reports both. pub fn assemble(assembly: &Assembly) -> Result { @@ -227,7 +235,7 @@ fn assemble_with( assembly: &Assembly, ) -> Result { let runtime = assembly.runtime.as_os_str().as_bytes(); - if !runtime.starts_with(b"/") || runtime.iter().any(|byte| RUNTIME_RESERVED.contains(byte)) { + if !is_canonical(runtime) || runtime.iter().any(|byte| RUNTIME_RESERVED.contains(byte)) { return Err(AssemblyError { error: Error::Runtime { path: assembly.runtime.clone(), @@ -305,16 +313,28 @@ impl Root { /// Implements [`assemble`]'s steps after the first. fn mount_layers(&mut self, paths: &Paths, assembly: &Assembly) -> Result<(), Error> { - let distro = self.runtime.join(DISTRO_DIRECTORY); - let scratch = self.runtime.join(SCRATCH_DIRECTORY); - for directory in [&distro, &scratch, &self.path] { - fs::create_dir_all(directory).map_err(|error| Error::MountPoint { - path: directory.clone(), - error, - })?; + let runtime = self.runtime.clone(); + fs::create_dir_all(&runtime).map_err(|error| Error::MountPoint { + path: runtime.clone(), + error, + })?; + // The path is in canonical form already, so it differs from the canonical one only + // where a symbolic link leads through it. + match fs::canonicalize(&runtime) { + Ok(canonical) if canonical == runtime => {} + Ok(_) => return Err(Error::Runtime { path: runtime }), + Err(error) => { + return Err(Error::MountPoint { + path: runtime, + error, + }); + } } - self.reservation = Some(reserve(&self.runtime)?); - let shares = self.runtime.join(AGGREGATE_DIRECTORY); + let distro = create_below(&runtime, DISTRO_DIRECTORY)?; + let scratch = create_below(&runtime, SCRATCH_DIRECTORY)?; + create_below(&runtime, ROOT_DIRECTORY)?; + self.reservation = Some(reserve(&runtime)?); + let shares = runtime.join(AGGREGATE_DIRECTORY); for directory in [&distro, &scratch, &self.path, &shares] { match self.kernel.is_mount_point(directory) { Ok(false) => {} @@ -519,6 +539,31 @@ fn reserve(runtime: &Path) -> Result { }) } +/// Returns whether `path` is absolute and canonical: not `/`, without an empty, `.`, or `..` +/// component, and without a trailing `/`. +fn is_canonical(path: &[u8]) -> bool { + // A trailing `/` makes `/` itself, and a path that ends in one, `.`, or `..`, a form below. + let terminated = [path, b"/"].concat(); + path.starts_with(b"/") + && ![b"//".as_slice(), b"/./", b"/../"] + .iter() + .any(|form| terminated.windows(form.len()).any(|part| part == *form)) +} + +/// Creates the directory `relative` below the directory `base`, a component at a time, each +/// unless a directory, and not a symbolic link, is there already, and returns its path. +fn create_below(base: &Path, relative: &str) -> Result { + let mut path = base.to_path_buf(); + for component in relative.split('/') { + path.push(component); + create_directory(&path).map_err(|error| Error::MountPoint { + path: path.clone(), + error, + })?; + } + Ok(path) +} + /// Creates the directory `path` unless a directory, and not a symbolic link, is there already. fn create_directory(path: &Path) -> io::Result<()> { match fs::create_dir(path) { @@ -596,7 +641,8 @@ pub enum Error { /// The step's error. error: io::Error, }, - /// The runtime directory is not an absolute path, or holds a reserved character. + /// The runtime directory's path is not absolute and canonical, holds a reserved character, + /// or leads through a symbolic link. Runtime { /// The runtime directory. path: PathBuf, @@ -696,7 +742,8 @@ impl fmt::Display for Error { Self::Bootstrap { step, error } => write!(formatter, "failed to {step}: {error}"), Self::Runtime { path } => write!( formatter, - "the runtime directory is not an absolute path without `,`, `:`, `\\`, or `=`: {}", + "the runtime directory is not a canonical absolute path without `,`, `:`, `\\`, \ + `=`, or a symbolic link: {}", path.display() ), Self::MountPoint { path, error } => { diff --git a/nvx-rs/src/guest/root/share.rs b/nvx-rs/src/guest/root/share.rs index 2c93add73..c0bf52c1e 100644 --- a/nvx-rs/src/guest/root/share.rs +++ b/nvx-rs/src/guest/root/share.rs @@ -210,8 +210,8 @@ impl Root { /// Mounts the aggregate virtio-fs device `tag`, with `access`, `nosuid`, and `nodev`, at /// `shares` in the runtime directory, outside the root, where the workload cannot list its - /// children. Then binds each child, in order, at its target in the root, and restricts the - /// bind to the child's access, `nosuid`, and `nodev`. + /// children, and which may not be a symbolic link. Then binds each child, in order, at its + /// target in the root, and restricts the bind to the child's access, `nosuid`, and `nodev`. /// /// Every child is checked first: its name, which no other child has, and its target, as the /// [module](super#live-shares) describes, against the other children's and the root's other @@ -257,7 +257,7 @@ impl Root { points.push(mount_point(&self.path, &child.target, child.kind)?); } let directory = self.runtime.join(AGGREGATE_DIRECTORY); - fs::create_dir_all(&directory).map_err(|error| Error::MountPoint { + super::create_directory(&directory).map_err(|error| Error::MountPoint { path: directory.clone(), error, })?; @@ -361,14 +361,9 @@ fn check_name(name: &str) -> Result<(), Error> { /// the runtime reserves. fn check_target(target: &Path) -> Result<(), Error> { let bytes = target.as_os_str().as_bytes(); - // A trailing `/` makes `/` itself, and a target that ends in one, `.`, or `..`, a form below. - let terminated = [bytes, b"/"].concat(); let refusal = if !bytes.starts_with(b"/") { Some(ShareRefusal::Relative(target.to_owned())) - } else if [b"//".as_slice(), b"/./", b"/../"] - .iter() - .any(|form| terminated.windows(form.len()).any(|part| part == *form)) - { + } else if !super::is_canonical(bytes) { Some(ShareRefusal::NotCanonical(target.to_owned())) } else if bytes .iter() diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index b0fb7ba3d..50a27ba51 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -22,8 +22,12 @@ const UUID: [u8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; /// A guest in a temporary directory, and the fake kernel that the operations get clones of. struct Guest { - /// The temporary directory. - directory: tempfile::TempDir, + /// The temporary directory, which the guest holds only so that dropping it removes the + /// directory. + _directory: tempfile::TempDir, + /// The temporary directory's canonical path, which no symbolic link leads through, as none + /// may lead through a runtime directory. + base: PathBuf, /// The kernel's interfaces, below the temporary directory. paths: Paths, /// The fake kernel. @@ -35,8 +39,8 @@ impl Guest { /// attached, and nothing else. fn new() -> Self { let guest = Self::bare(); - attach(guest.directory.path(), 0, "vda", 4096); - attach(guest.directory.path(), 1, "vdb", 8192); + attach(&guest.base, 0, "vda", 4096); + attach(&guest.base, 1, "vdb", 8192); guest } @@ -44,10 +48,11 @@ impl Guest { /// scratch's window 1, without their devices. fn bare() -> Self { let directory = tempfile::tempdir().unwrap(); + let base = directory.path().canonicalize().unwrap(); let paths = Paths { - proc: directory.path().join("proc"), - sys: directory.path().join("sys"), - dev: directory.path().join("dev"), + proc: base.join("proc"), + sys: base.join("sys"), + dev: base.join("dev"), }; fs::create_dir_all(&paths.dev).unwrap(); write( @@ -60,7 +65,8 @@ impl Guest { ), ); Self { - directory, + _directory: directory, + base, paths, kernel: FakeKernel::default(), } @@ -68,7 +74,7 @@ impl Guest { /// Returns `relative`'s path below the temporary directory. fn path(&self, relative: &str) -> PathBuf { - self.directory.path().join(relative) + self.base.join(relative) } /// Returns `relative`'s path below the temporary directory, as the fake kernel records it. @@ -208,8 +214,8 @@ fn assembly_mounts_the_image_and_the_scratch_under_the_overlay() { #[test] fn devices_are_found_when_they_appear_and_missing_ones_time_out() { let guest = Guest::bare(); - attach(guest.directory.path(), 1, "vdb", 8192); - let root_directory = guest.directory.path().to_owned(); + attach(&guest.base, 1, "vdb", 8192); + let root_directory = guest.base.clone(); // The kernel probes the device first, then creates its node. guest.kernel.on_sleep(move |sleeps| match sleeps { 1 => probe(&root_directory, 0, "vda", 4096), @@ -223,7 +229,7 @@ fn devices_are_found_when_they_appear_and_missing_ones_time_out() { // A timeout of 100 ms allows two more polls, 50 ms apart. let guest = Guest::bare(); - attach(guest.directory.path(), 0, "vda", 4096); + attach(&guest.base, 0, "vda", 4096); let error = guest.assemble(&guest.assembly()).unwrap_err(); assert!( matches!( @@ -396,6 +402,103 @@ fn a_runtime_directory_that_the_overlay_options_reserve_is_refused() { assert!(guest.kernel.calls().is_empty()); } +#[test] +fn a_runtime_directory_whose_path_is_not_canonical_is_refused() { + // Once the scratch hid `probe`, the last path would lead nowhere, and the image and the + // scratch could not be unmounted. + let guest = Guest::new(); + fs::create_dir_all(guest.path("run/nvx/scratch/probe")).unwrap(); + let parent = guest.text("run/nvx/scratch/probe/../.."); + let forms = [ + "/", + "/run/nvx/", + "/run//nvx", + "/run/./nvx", + "/run/nvx/.", + "/run/nvx/..", + ]; + for runtime in forms.into_iter().chain([parent.as_str()]) { + let mut assembly = guest.assembly(); + assembly.runtime = runtime.into(); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Runtime { path } if path.as_os_str() == runtime), + "{runtime}: {error:?}" + ); + assert!(error.cleanup().is_none()); + } + assert!(guest.kernel.calls().is_empty()); + assert!(!guest.path("run/nvx/layers").exists()); + let mut assembly = guest.assembly(); + assembly.runtime = parent.clone().into(); + assert_eq!( + guest.assemble(&assembly).unwrap_err().to_string(), + format!( + "the runtime directory is not a canonical absolute path without `,`, `:`, `\\`, `=`, \ + or a symbolic link: {parent}" + ) + ); +} + +#[test] +fn a_root_mounts_nothing_through_a_symbolic_link() { + // A runtime directory that a link leads through, whether the link names it, one of its + // parents, or a path through the scratch's mount point. + let guest = Guest::new(); + fs::create_dir_all(guest.path("run/nvx/scratch/probe")).unwrap(); + std::os::unix::fs::symlink("nvx", guest.path("run/link")).unwrap(); + std::os::unix::fs::symlink("run", guest.path("alias")).unwrap(); + std::os::unix::fs::symlink("nvx/scratch/probe/../..", guest.path("run/up")).unwrap(); + for runtime in ["run/link", "alias/nvx", "run/up"].map(|relative| guest.path(relative)) { + let mut assembly = guest.assembly(); + assembly.runtime = runtime.clone(); + let error = guest.assemble(&assembly).unwrap_err(); + assert!( + matches!(error.error(), Error::Runtime { path } if *path == runtime), + "{}: {error:?}", + runtime.display() + ); + } + assert!(guest.kernel.calls().is_empty()); + assert!(!guest.path("run/nvx/layers").exists()); + + // A link at one of the assembly's mount points, or at the directory above the image's. + for name in ["layers", "layers/distro", "scratch", "rootfs"] { + let guest = Guest::new(); + let link = guest.path("run/nvx").join(name); + fs::create_dir_all(link.parent().unwrap()).unwrap(); + fs::create_dir(guest.path("elsewhere")).unwrap(); + std::os::unix::fs::symlink(guest.path("elsewhere"), &link).unwrap(); + let error = guest.assemble(&guest.assembly()).unwrap_err(); + assert!( + matches!(error.error(), Error::MountPoint { path, error: cause } + if *path == link && cause.raw_os_error() == Some(Errno::ENOTDIR as i32)), + "{name}: {error:?}" + ); + assert!(guest.kernel.calls().is_empty(), "{name}"); + } + + // A link at an aggregate's mount point, which the aggregate refuses. + let guest = Guest::new(); + let shares = guest.path("run/nvx/shares"); + fs::create_dir_all(guest.path("run/nvx")).unwrap(); + fs::create_dir(guest.path("elsewhere")).unwrap(); + std::os::unix::fs::symlink(guest.path("elsewhere"), &shares).unwrap(); + let mut root = guest.assemble(&guest.assembly()).unwrap(); + let mounts = guest.kernel.mounts(); + let rw = Access::ReadWrite; + let error = root + .mount_aggregate("all", rw, &[child("a", "/a", rw, ShareKind::Directory)]) + .unwrap_err(); + assert!( + matches!(&error, Error::MountPoint { path, error: cause } + if *path == shares && cause.raw_os_error() == Some(Errno::ENOTDIR as i32)), + "{error:?}" + ); + assert_eq!(guest.kernel.mounts(), mounts); + root.teardown().unwrap(); +} + #[test] fn a_runtime_directory_that_another_root_holds_is_refused() { let guest = Guest::new(); @@ -436,10 +539,10 @@ fn a_runtime_directory_that_another_root_holds_is_refused() { // An assembly that starts while another one waits for its distro device is refused before // it checks or mounts anything, and the first one goes on. let guest = Guest::bare(); - attach(guest.directory.path(), 1, "vdb", 8192); + attach(&guest.base, 1, "vdb", 8192); let refused = Arc::new(Mutex::new(None)); let (kernel, paths, assembly) = (guest.kernel.clone(), guest.paths.clone(), guest.assembly()); - let (directory, outcome) = (guest.directory.path().to_owned(), Arc::clone(&refused)); + let (directory, outcome) = (guest.base.clone(), Arc::clone(&refused)); guest.kernel.on_sleep(move |sleeps| { if sleeps == 1 { attach(&directory, 0, "vda", 4096); @@ -583,8 +686,9 @@ fn a_runtime_directory_where_a_directory_is_bound_is_refused() { return; } let directory = tempfile::tempdir().unwrap(); - let source = directory.path().join("source"); - let runtime = directory.path().join("run/nvx"); + let base = directory.path().canonicalize().unwrap(); + let source = base.join("source"); + let runtime = base.join("run/nvx"); let distro = runtime.join(DISTRO_DIRECTORY); fs::create_dir(&source).unwrap(); fs::create_dir_all(&distro).unwrap(); From a6e8370a96c0dc27519f8d3b5bfb06dc95de7a49 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 18:04:33 -0700 Subject: [PATCH 10/12] nvx-rs: create a guest root's upper traversable whatever the umask An assembly created the scratch's upper and work directories with the caller's umask, and the overlay's root directory takes the upper's mode. A caller with a restrictive umask, such as 077, so assembled a root whose directory a workload that is not root could not enter. The shell init's mkdir -p created them as 0755 under PID 1's umask. An assembly now gives the upper and work directories that it creates mode 0755 whatever the umask, and leaves the mode of one that the scratch holds already, from an earlier boot. A test checks both under umask 077, in a process of its own, since the umask is the process's. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/src/guest/root.rs | 30 +++++++++++++++++++++------ nvx-rs/src/guest/root/tests.rs | 38 +++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 61a27be74..51161fa8f 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -102,6 +102,7 @@ use std::fmt; use std::fs; use std::io; use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -124,6 +125,10 @@ const AGGREGATE_DIRECTORY: &str = "shares"; const UPPER_DIRECTORY: &str = "upper"; /// The overlay's work directory, below the scratch. const WORK_DIRECTORY: &str = "work"; +/// The mode of the scratch's `upper` and `work` directories that an assembly creates, whatever +/// the process's umask, as the shell init's `mkdir -p` created them under PID 1's: the overlay's +/// root directory takes the upper's mode, and a workload that is not root must enter its root. +const SCRATCH_DIRECTORY_MODE: u32 = 0o755; /// The overlay's options other than its directories: metadata-only copy-up, and inode numbers /// that stay unique across the layers. const OVERLAY_OPTIONS: &[u8] = b"metacopy=on,xino=on"; @@ -219,7 +224,8 @@ pub struct Assembly { /// 6. mounts the device's EROFS image read-only at `layers/distro`; /// 7. finds, checks, and flushes the scratch device the same way; /// 8. mounts its ext4 file system at `scratch` with `nosuid` and `nodev`; -/// 9. creates the scratch's `upper` and `work` directories unless they exist; +/// 9. creates the scratch's `upper` and `work` directories unless they exist, with mode 0755 +/// whatever the process's umask, since the overlay's root directory takes the upper's mode; /// 10. and mounts the overlay at `rootfs`, `nosuid` and `nodev`, with the distro layer as its /// lower directory, and `metacopy=on,xino=on`. /// @@ -403,7 +409,7 @@ impl Root { let upper = scratch.join(UPPER_DIRECTORY); let work = scratch.join(WORK_DIRECTORY); for directory in [&upper, &work] { - create_directory(directory).map_err(|error| Error::MountPoint { + create_scratch_directory(directory).map_err(|error| Error::MountPoint { path: directory.clone(), error, })?; @@ -564,20 +570,32 @@ fn create_below(base: &Path, relative: &str) -> Result { Ok(path) } -/// Creates the directory `path` unless a directory, and not a symbolic link, is there already. -fn create_directory(path: &Path) -> io::Result<()> { +/// Creates the directory `path` unless a directory, and not a symbolic link, is there already, +/// and returns whether it created it. +fn create_directory(path: &Path) -> io::Result { match fs::create_dir(path) { + Ok(()) => Ok(true), Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { if fs::symlink_metadata(path)?.is_dir() { - Ok(()) + Ok(false) } else { Err(Errno::ENOTDIR.into()) } } - result => result, + Err(error) => Err(error), } } +/// Creates the scratch's directory `path` as [`create_directory`] does, with +/// [`SCRATCH_DIRECTORY_MODE`] whatever the process's umask. A directory that is there already, +/// which the scratch keeps across boots, keeps its mode. +fn create_scratch_directory(path: &Path) -> io::Result<()> { + if create_directory(path)? { + fs::set_permissions(path, fs::Permissions::from_mode(SCRATCH_DIRECTORY_MODE))?; + } + Ok(()) +} + /// A mount of a layered root. #[derive(Clone, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 50a27ba51..9bbf70934 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -4,7 +4,7 @@ use std::fs; use std::io; -use std::os::unix::fs::MetadataExt; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -499,6 +499,42 @@ fn a_root_mounts_nothing_through_a_symbolic_link() { root.teardown().unwrap(); } +#[test] +fn new_upper_and_work_directories_have_mode_0755_whatever_the_umask() { + // The umask is the process's, so the test runs alone in a process of its own. + if crate::guest::managed_agent::isolate_process_test( + "NVX_ROOT_UMASK_TEST_CHILD", + "guest::root::tests::new_upper_and_work_directories_have_mode_0755_whatever_the_umask", + ) { + return; + } + nix::sys::stat::umask(sys::Mode::from_bits_truncate(0o077)); + let mode = |path: &Path| fs::metadata(path).unwrap().mode() & 0o7777; + // The overlay's root directory takes the upper's mode, which a workload that is not root + // must be able to enter. + let guest = Guest::new(); + guest + .assemble(&guest.assembly()) + .unwrap() + .teardown() + .unwrap(); + assert_eq!(mode(&guest.path("run/nvx/scratch/upper")), 0o755); + assert_eq!(mode(&guest.path("run/nvx/scratch/work")), 0o755); + + // A directory that the scratch holds already, from an earlier boot, keeps its mode. + let guest = Guest::new(); + let upper = guest.path("run/nvx/scratch/upper"); + fs::create_dir_all(&upper).unwrap(); + fs::set_permissions(&upper, fs::Permissions::from_mode(0o700)).unwrap(); + guest + .assemble(&guest.assembly()) + .unwrap() + .teardown() + .unwrap(); + assert_eq!(mode(&upper), 0o700); + assert_eq!(mode(&guest.path("run/nvx/scratch/work")), 0o755); +} + #[test] fn a_runtime_directory_that_another_root_holds_is_refused() { let guest = Guest::new(); From 6c8cfca26a79eee36dd4f3462a685f2f0382e8bf Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 19:45:08 -0700 Subject: [PATCH 11/12] nvx-rs: build a guest root on the existing share, mapping, and cgroup code The guest root operations repeated components that the guest library and the host already had. They now use those components instead: - A root binds each child of an aggregate live share through guest::mappings::Mapping, the bind and restricting remount with which NVX's managed agent applies its host mapping table, and takes its Access. The root keeps its own checks of the targets, which refuse symbolic links, and creates what is missing of them first. A child whose entry in the aggregate is missing, or of another kind, now fails to mount before its bind. - The root checks its targets and its runtime directory with the canonical-path check of mappings, which also bounds their length. - The guest targets that live shares may not take, and the characters that no target may hold, move to nvx::protocol::share, which the host's sandbox::mount and the guest's root both check. The host keeps its constants' names and its messages. - The bootstrap prepares the workload cgroup with ExecutionCgroup::prepare, as the managed agent prepares its own, and creates the agent's cgroup and enables its memory controller through the same cgroup helpers. BootstrapStep::WorkloadCgroup now covers the workload cgroup's values, and the Controllers, WorkloadMemoryMax, and WorkloadPidsMax steps go. - sys::mount takes the file system's options, so the guest has one mount wrapper. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- doc/project-structure.md | 2 +- nvx-rs/README.md | 13 +- nvx-rs/src/guest/cgroup.rs | 4 +- nvx-rs/src/guest/managed_agent/seam.rs | 2 +- nvx-rs/src/guest/mappings.rs | 29 +++-- nvx-rs/src/guest/root.rs | 45 ++++--- nvx-rs/src/guest/root/bootstrap.rs | 119 ++++++------------ nvx-rs/src/guest/root/kernel.rs | 2 +- nvx-rs/src/guest/root/share.rs | 145 +++++++++------------- nvx-rs/src/guest/root/tests.rs | 161 +++++++++++++++++++++---- nvx-rs/src/guest/sys/filesystem.rs | 14 +-- nvx-rs/src/guest/sys/mod.rs | 5 +- nvx-rs/src/host/sandbox/mount.rs | 42 ++----- nvx-rs/src/lib.rs | 4 +- nvx-rs/src/protocol/mod.rs | 3 + nvx-rs/src/protocol/share.rs | 122 +++++++++++++++++++ 16 files changed, 431 insertions(+), 281 deletions(-) create mode 100644 nvx-rs/src/protocol/share.rs diff --git a/doc/project-structure.md b/doc/project-structure.md index 901be564c..7364a97d4 100644 --- a/doc/project-structure.md +++ b/doc/project-structure.md @@ -100,7 +100,7 @@ nvx/ | `-- build.rs Passes the bundle that nvx staged on to dependents |-- nvx-rs/ Rust crate `nvx` for the NVX mechanics | |-- src/ -| | |-- protocol/ Control-console codec and NVXS session shared by the host and the guest +| | |-- protocol/ Control-console codec, NVXS session, and the guest targets that live shares may not take, shared by the host and the guest | | |-- contract.rs Control-contract versions and release compatibility | | |-- net/ CIDR parsing and subtraction, and hostname and guest-network forms | | |-- host/ Artifacts, launch, network and egress-policy lowering, sessions, VM instances, sandbox stores, the sandbox launch profile, and the `nvx-rs` CLI (feature `cli`) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 6c424bdc8..5a4dfe8b3 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -24,8 +24,9 @@ the nested `openvmm/` workspace. ```text nvx-rs/ ├── src/ -│ ├── protocol/ NVXS session, NVXC frames, EXEC and MAPS payloads, guest features, and -│ │ kernel command-line tokens; no dependencies, shared by the host and the guest +│ ├── protocol/ NVXS session, NVXC frames, EXEC and MAPS payloads, guest features, kernel +│ │ command-line tokens, and the guest targets that live shares may not take; +│ │ no dependencies, shared by the host and the guest │ ├── contract.rs control-contract versions and release-manifest compatibility │ ├── net/ CIDR parsing and subtraction, and hostname and guest-network forms │ ├── uuid.rs UUIDs in their canonical text and in GPT's layout; no dependencies, shared @@ -58,6 +59,7 @@ nvx-rs/ | `nvx::protocol::cmdline` | The kernel command line as Linux 6.18 reads it, for the readers that need the kernel's view of it: `parameters` splits it into `Parameter`s as `parse_args` and `next_arg` do, at the bytes that `is_space` accepts, with double quotes, up to a NUL or `--`, and `Parameter::is_named` compares names as `parameq` does, with dashes and underscores alike. `VirtioMmioDevice::parse` reads a `virtio_mmio.device` value, `SIZE@BASE:IRQ[:ID]`, as the virtio-mmio driver reads it, for a device of any type, and `virtio_mmio_devices` reads every such parameter of a command line | | `nvx::protocol::nvxs` | The NVXS session, independent of `NVXC` and of any other application protocol: `RecordType`, `Record`, `encode_record`, and `RecordParser`, which parses records incrementally across partial reads with buffers bounded by one record; `HostSession` and `GuestSession`, the state machines of the host's and the guest's roles, which validate every record's type, identity, and sequence number, and the guest's receive window and credits, without I/O. An application message may span DATA records, and the application's own framing reassembles it | | `nvx::protocol::root` | The boot tokens of a layered root, which the host renders and guests parse by the same rules: a `RootLayout` is a `RootImage` on the `distro` block device, an EROFS filesystem or a GPT disk with the `nvx::uuid::Uuid` that the guest expects of it, if any, over a `RootUpper`, the ext4 scratch disk or RAM. `RootLayout::tokens` renders `nvx_layer=distro,0xd0003000,FORMAT[:IDENTITY]` and `nvx_scratch=0xd0006000,ext4` or `nvx_overlay_upper=ramfs`, and `RootLayout::parse` reads them from a kernel command line, and refuses malformed, repeated, and conflicting tokens and the earlier form without a format, with no fallback that probes the devices. It reads the virtio-mmio devices of the command line with `nvx::protocol::cmdline`, to refuse a RAM upper beside one at the scratch base. `tests/fixtures/root-tokens.json` holds its cases, which the tests of the guest's shell init replay too | +| `nvx::protocol::share` | The guest targets that a sandbox's live shares may not take, which the host's `sandbox::mount` and the guest's `root` both refuse: `RESERVED_MOUNT_TARGETS`, the trees where the container runtime mounts or binds, `RESERVED_EXACT_MOUNT_TARGETS`, the directories that it creates, `RESERVED_TARGET_CHARACTERS`, and `reservation`, which tells why a target is reserved | | `nvx::contract` | The OpenVMM microVM ABI, broker protocol, and control-contract revision that the crate speaks, and `manifest_is_compatible` for a release's `SOURCE-MANIFEST.json` | | `nvx::net` | `Cidr`: IPv4 and IPv6 networks and their subtraction; `is_valid_hostname` and `is_valid_guest_network`, the forms of a VM's hostname and guest address | | `nvx::uuid` | `Uuid`: a UUID, such as the GUID of a GPT disk or partition or the UUID of a filesystem, in the order of its canonical text, the one text that it displays and parses: 32 lowercase hexadecimal digits in groups of 8, 4, 4, 4, and 12, joined by hyphens. `from_bytes_le` and `to_bytes_le` convert the layout in which GPT and other EFI GUIDs store their first three groups little-endian. It has no dependencies, so the host and the guest share it | @@ -78,7 +80,7 @@ nvx-rs/ | `nvx::testing::fake_openvmm` | A fake OpenVMM that serves the control console and runs scripted workloads without a VM, for direct launches and for the managed and one-shot sandboxes of `nvx-rs sandbox` and `nvx.py sandbox` | | `nvx::host::cli` | Feature `cli`: the `nvx-rs` executable; see [The nvx-rs CLI](#the-nvx-rs-cli) | | `nvx::guest` | The programs that run in Linux guests; see [Guest binaries](#guest-binaries) | -| `nvx::guest::mappings` | `Mapping`: one host mapping for any guest init or agent, a relative source below an export root, where the host's shares appear, and a target below a target root, the tree that the mappings go into, with its `Access`. `Mapping::apply` takes both roots explicitly: it creates the target as the kind of its source, a directory or a regular file, bind-mounts the source on it, and remounts the bind `nosuid` and `nodev`, and read-only when asked. A `MappingFailure` tells which `MappingStage` failed and its `errno`. The caller keeps the order of its mappings and what a failure means for the rest | +| `nvx::guest::mappings` | `Mapping`: one host mapping for any guest init or agent, a relative source below an export root, where the host's shares appear, and a target below a target root, the tree that the mappings go into, with its `Access`. `Mapping::apply` takes both roots explicitly: it creates the target as the kind of its source, a directory or a regular file, bind-mounts the source on it, and remounts the bind `nosuid` and `nodev`, and read-only when asked. A `MappingFailure` tells which `MappingStage` failed and its `errno`. The caller keeps the order of its mappings and what a failure means for the rest. NVX's managed agent applies its host mapping table through it, and a layered `Root` binds the children of its aggregate live share through it | | `nvx::guest::nvxs` | `GuestConnection`: the guest's role of the NVXS session over a byte stream, such as the control console, on which NVX's managed agent and other guest agents run their sessions. It never reads past the record in progress, so a caller that polls the stream between records sees every record that it has not returned | | `nvx::guest::spawn` | `Command`: starts a program in a new process for any guest init or agent, with its arguments, environment, root and working directories, standard streams, inherited descriptors, cgroup, and process group, without NVX's control protocol, managed agent, or workload identity. Its `LaunchReport` tells whether the program started, or which `LaunchStage` failed and why. NVX's managed agent is one caller | | `nvx::guest::process` | `Child`: the status of one child that the caller names, by its pidfd or its process ID. It waits for that child alone, so another child's status stays for the caller; a caller that collects every status itself, such as a PID 1 that adopts orphans, hands each one to its `Child` with `observe`. Statuses are `std::process::ExitStatus`, real-time signals included | @@ -86,7 +88,7 @@ nvx-rs/ | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low` and the workload's `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at checked targets in the root, with one aggregate per root. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, whose children it binds through `mappings::Mapping`. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own @@ -588,7 +590,8 @@ process, `process` collects the status of a child that the caller names, `cgroup` places, kills, and settles the processes of an execution, `exec` drives one program as an execution, `mappings` bind-mounts host mappings below the roots that its caller names, and `root` bootstraps the guest for a layered -root, assembles the root, mounts live shares in it, and tears it down. +root through `cgroup`, assembles the root, mounts live shares in it, binding +the children of an aggregate through `mappings`, and tears it down. `nvx-managed-agent` launches, drives, collects, and settles its workloads through them, and applies its host mapping table through `mappings`, with its mount of the host's export and `/` as the roots, and any other guest init or diff --git a/nvx-rs/src/guest/cgroup.rs b/nvx-rs/src/guest/cgroup.rs index 7c1fb1ad5..b755d3fac 100644 --- a/nvx-rs/src/guest/cgroup.rs +++ b/nvx-rs/src/guest/cgroup.rs @@ -511,7 +511,7 @@ fn controller(file: &str) -> io::Result<&str> { /// Creates the cgroup `path`, which fails with `EEXIST` if it exists and `exclusive` is set. /// Otherwise an existing cgroup is accepted, and a file in its place is `ENOTDIR`. -fn create(path: &Path, exclusive: bool) -> io::Result<()> { +pub(crate) fn create(path: &Path, exclusive: bool) -> io::Result<()> { match sys::create_directory(&sys::path_c_string(path)?, Mode::from_bits_retain(0o755)) { Ok(()) => Ok(()), Err(Errno::EEXIST) if !exclusive && fs::metadata(path)?.is_dir() => Ok(()), @@ -523,7 +523,7 @@ fn create(path: &Path, exclusive: bool) -> io::Result<()> { /// Makes sure that the cgroup `cgroup` has each of the interface files `files`: the controllers /// of those that it lacks are enabled together in `parent`'s `cgroup.subtree_control`, which /// gives them to every child of `parent`. -fn provide<'a>( +pub(crate) fn provide<'a>( cgroup: &Path, parent: &Path, files: impl IntoIterator, diff --git a/nvx-rs/src/guest/managed_agent/seam.rs b/nvx-rs/src/guest/managed_agent/seam.rs index c8a22a46b..7a00723ce 100644 --- a/nvx-rs/src/guest/managed_agent/seam.rs +++ b/nvx-rs/src/guest/managed_agent/seam.rs @@ -117,7 +117,7 @@ impl System for RealSystem { filesystem: Option<&CStr>, flags: MountFlags, ) -> Result<(), Errno> { - sys::mount(source, target, filesystem, flags) + sys::mount(source, target, filesystem, flags, None) } fn chown(&mut self, path: &CStr, uid: u32, gid: u32) -> Result<(), Errno> { diff --git a/nvx-rs/src/guest/mappings.rs b/nvx-rs/src/guest/mappings.rs index a0378fe7b..86dcc5a0a 100644 --- a/nvx-rs/src/guest/mappings.rs +++ b/nvx-rs/src/guest/mappings.rs @@ -19,7 +19,10 @@ //! the file that it created, and, when the remount failed, the bind without its flags. //! [`MappingFailure::stage`] tells which stage failed. NVX's managed agent is one caller: it //! applies the host mapping table that its control session receives, in order, with its mount of -//! the host's export as the export root and `/` as the target root. +//! the host's export as the export root and `/` as the target root. A layered +//! [`Root`](super::root::Root) is another: it binds each child of an aggregate live share from the +//! aggregate's mount into the root, once it has checked that no symbolic link leads through the +//! child's target, and created what is missing of it. //! //! ```no_run //! use std::path::Path; @@ -67,6 +70,18 @@ pub enum Access { ReadWrite, } +impl Access { + /// Returns the flags that restrict a mount with this access: `nosuid` and `nodev`, and `ro` + /// for [`Access::ReadOnly`]. + pub(crate) fn flags(self) -> MountFlags { + let flags = MountFlags::MS_NOSUID | MountFlags::MS_NODEV; + match self { + Self::ReadOnly => flags | MountFlags::MS_RDONLY, + Self::ReadWrite => flags, + } + } +} + /// The path that [`Mapping::new`] refuses, because it does not name a path below its root. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] @@ -238,7 +253,7 @@ impl Mapping { /// documentation describes. pub fn apply(&self, export_root: &Path, target_root: &Path) -> Result<(), MappingFailure> { self.apply_with(export_root, target_root, |source, target, flags| { - sys::mount(source, target, None, flags) + sys::mount(source, target, None, flags, None) }) } @@ -269,13 +284,7 @@ impl Mapping { created.map_err(|error| MappingStage::Target.failure(error))?; mount(Some(&c_source), &c_target, MountFlags::MS_BIND) .map_err(|error| MappingStage::Bind.failure(error))?; - let mut flags = MountFlags::MS_BIND - | MountFlags::MS_REMOUNT - | MountFlags::MS_NOSUID - | MountFlags::MS_NODEV; - if self.access == Access::ReadOnly { - flags |= MountFlags::MS_RDONLY; - } + let flags = MountFlags::MS_BIND | MountFlags::MS_REMOUNT | self.access.flags(); mount(None, &c_target, flags).map_err(|error| MappingStage::Remount.failure(error)) } } @@ -284,7 +293,7 @@ impl Mapping { /// leading `/` of an `absolute` one. It returns `None` unless `path` holds at most /// [`MAX_PATH_BYTES`] bytes and no NUL byte, starts with `/` exactly when it is `absolute`, and /// has a part below its root of at least one component, each neither empty, `.`, nor `..`. -fn below_its_root(path: &Path, absolute: bool) -> Option<&Path> { +pub(crate) fn below_its_root(path: &Path, absolute: bool) -> Option<&Path> { let bytes = path.as_os_str().as_bytes(); let components = if absolute { bytes.strip_prefix(b"/")? diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 51161fa8f..671a0c62a 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -50,25 +50,31 @@ //! a child, is bound at its own target. A root has one aggregate at most. Before it mounts //! anything for a share, the root checks that each target: //! -//! - is absolute and canonical: not `/`, without an empty, `.`, or `..` component, and without a -//! trailing `/`; -//! - holds no `\`, `=`, `,`, or NUL; -//! - is not where a container runtime mounts or binds in the root: `/proc`, `/sys`, `/dev`, -//! `/.nvx-agent`, `/etc/machine-id`, `/run/nvx/time`, or below them, nor `/etc`, `/run`, or -//! `/run/nvx`, below which it mounts; +//! - is absolute and canonical: not `/`, without an empty, `.`, or `..` component, a trailing +//! `/`, or a NUL, and of at most [`MAX_PATH_BYTES`](super::mappings::MAX_PATH_BYTES) bytes, as +//! the target of a [`Mapping`](super::mappings::Mapping) is; +//! - holds no `\`, `=`, or `,`; +//! - is not where a container runtime mounts or binds in the root, as +//! [`protocol::share`](crate::protocol::share) lists them for the host and the guest: `/proc`, +//! `/sys`, `/dev`, `/.nvx-agent`, `/etc/machine-id`, `/run/nvx/time`, or below them, nor `/etc`, +//! `/run`, or `/run/nvx`, below which it mounts; //! - is not another share's target, nor inside one, nor holds one; //! - crosses no symbolic link in the root; //! - and is a directory, or a regular file for a child that is a host file. //! //! It creates each missing component of a target in the root: a directory with mode 0755, or a -//! file child's file with mode 0644. +//! file child's file with mode 0644. It binds each child of an aggregate through a +//! [`Mapping`](super::mappings::Mapping), from the aggregate's mount, as NVX's managed agent +//! applies its host mappings, and restricts the bind to the child's +//! [`Access`](super::mappings::Access), `nosuid`, and `nodev`. //! //! ```no_run //! use std::path::Path; //! use std::time::Duration; //! //! use nvx::guest::cgroup::Layout; -//! use nvx::guest::root::{self, Access, Assembly, Device, Resources}; +//! use nvx::guest::mappings::Access; +//! use nvx::guest::root::{self, Assembly, Device, Resources}; //! //! # fn main() -> Result<(), Box> { //! let layout = Layout::new("/sys/fs/cgroup/agent", "/sys/fs/cgroup/container")?; @@ -107,10 +113,11 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use self::kernel::{Kernel, Paths, SystemKernel}; +use super::mappings::below_its_root; use super::sys::{self, Errno, MountFlags}; pub use self::bootstrap::{Bootstrap, BootstrapStep, Resources, bootstrap}; -pub use self::share::{Access, ShareChild, ShareKind, ShareRefusal}; +pub use self::share::{ShareChild, ShareKind, ShareRefusal}; /// The EROFS image's mount point, below the runtime directory. const DISTRO_DIRECTORY: &str = "layers/distro"; @@ -193,8 +200,9 @@ impl fmt::Display for DeviceStep { #[derive(Clone, Debug, PartialEq, Eq)] pub struct Assembly { /// The runtime directory: an absolute path in canonical form, not `/`, without an empty, `.`, - /// or `..` component, or a trailing `/`, that no symbolic link leads through, and without `,`, - /// `:`, `\`, `=`, or NUL, which the overlay's options reserve. + /// or `..` component, or a trailing `/`, of at most + /// [`MAX_PATH_BYTES`](super::mappings::MAX_PATH_BYTES) bytes, that no symbolic link leads + /// through, and without `,`, `:`, `\`, `=`, or NUL, which the overlay's options reserve. pub runtime: PathBuf, /// The device of the EROFS image. pub distro: Device, @@ -241,7 +249,9 @@ fn assemble_with( assembly: &Assembly, ) -> Result { let runtime = assembly.runtime.as_os_str().as_bytes(); - if !is_canonical(runtime) || runtime.iter().any(|byte| RUNTIME_RESERVED.contains(byte)) { + if below_its_root(&assembly.runtime, true).is_none() + || runtime.iter().any(|byte| RUNTIME_RESERVED.contains(byte)) + { return Err(AssemblyError { error: Error::Runtime { path: assembly.runtime.clone(), @@ -545,17 +555,6 @@ fn reserve(runtime: &Path) -> Result { }) } -/// Returns whether `path` is absolute and canonical: not `/`, without an empty, `.`, or `..` -/// component, and without a trailing `/`. -fn is_canonical(path: &[u8]) -> bool { - // A trailing `/` makes `/` itself, and a path that ends in one, `.`, or `..`, a form below. - let terminated = [path, b"/"].concat(); - path.starts_with(b"/") - && ![b"//".as_slice(), b"/./", b"/../"] - .iter() - .any(|form| terminated.windows(form.len()).any(|part| part == *form)) -} - /// Creates the directory `relative` below the directory `base`, a component at a time, each /// unless a directory, and not a symbolic link, is there already, and returns its path. fn create_below(base: &Path, relative: &str) -> Result { diff --git a/nvx-rs/src/guest/root/bootstrap.rs b/nvx-rs/src/guest/root/bootstrap.rs index f3bb42089..d583215f6 100644 --- a/nvx-rs/src/guest/root/bootstrap.rs +++ b/nvx-rs/src/guest/root/bootstrap.rs @@ -3,12 +3,11 @@ use std::fmt; use std::fs; use std::io; -use std::os::unix::fs::DirBuilderExt; use std::path::Path; use super::Error; use super::kernel::{Kernel, Paths, SystemKernel}; -use crate::guest::cgroup::{ExecutionCgroup, Layout}; +use crate::guest::cgroup::{self, ExecutionCgroup, Layout}; use crate::guest::parse::parse_decimal; use crate::guest::sys::{self, Errno, MountFlags}; @@ -36,13 +35,6 @@ pub struct Resources { pub workload_pids_max: Option, } -impl Resources { - /// Returns whether a value needs the `memory` controller. - fn need_memory(&self) -> bool { - self.agent_memory_low.is_some() || self.workload_memory_max.is_some() - } -} - /// A step of [`bootstrap`], which an [`Error::Bootstrap`] names. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] @@ -57,16 +49,12 @@ pub enum BootstrapStep { AgentCgroup, /// Finding the parent cgroup's processes, and moving them into the agent's cgroup. Placement, - /// Enabling the controllers of the resource values in the parent's `cgroup.subtree_control`. - Controllers, - /// Writing the agent's `memory.low`. + /// Enabling the `memory` controller for the agent's cgroup unless it has it, and writing the + /// agent's `memory.low`. AgentMemoryLow, - /// Creating the workload cgroup. + /// Preparing the workload cgroup with [`ExecutionCgroup::prepare`]: creating it, enabling + /// the controllers that it lacks, and writing its `memory.max` and `pids.max`. WorkloadCgroup, - /// Writing the workload cgroup's `memory.max`. - WorkloadMemoryMax, - /// Writing the workload cgroup's `pids.max`. - WorkloadPidsMax, } impl fmt::Display for BootstrapStep { @@ -78,11 +66,8 @@ impl fmt::Display for BootstrapStep { Self::Hierarchy => "mount the cgroup hierarchy", Self::AgentCgroup => "create the agent's cgroup", Self::Placement => "place the agent's processes in its cgroup", - Self::Controllers => "enable the cgroup controllers", Self::AgentMemoryLow => "set the agent's memory reservation", - Self::WorkloadCgroup => "create the workload cgroup", - Self::WorkloadMemoryMax => "set the workload's memory limit", - Self::WorkloadPidsMax => "set the workload's process limit", + Self::WorkloadCgroup => "prepare the workload cgroup", }) } } @@ -131,15 +116,16 @@ impl Bootstrap { /// which stay: the caller, and every daemon that its prelude started. It repeats until no /// such process is left, for at most 16 passes, so that one that a moved process starts /// meanwhile moves too, and fails with `EBUSY` if one is still left; -/// 5. enables the `pids` controller, and the `memory` controller when a memory value needs it, -/// in the parent's `cgroup.subtree_control`, in one write; -/// 6. writes the agent's `memory.low`; -/// 7. creates the workload cgroup unless it exists; -/// 8. and writes its `memory.max` and `pids.max`. +/// 5. enables the `memory` controller in the parent's `cgroup.subtree_control` unless the +/// agent's cgroup has it, and writes the agent's `memory.low`; +/// 6. and prepares the workload cgroup with [`ExecutionCgroup::prepare`], as the managed agent +/// prepares its own: it creates the cgroup unless it exists, enables the `pids` controller, +/// and that of each value, in the parent's `cgroup.subtree_control`, in one write, unless the +/// cgroup has them, and writes the cgroup's `memory.max` and `pids.max`. /// -/// Each value is written only when `resources` holds one. A failure names its step, and leaves -/// the earlier steps' changes in place. The bootstrap collects no status, and starts and stops -/// no process. +/// Each value is written only when `resources` holds one, and step 5 runs only for the agent's +/// `memory.low`. A failure names its step, and leaves the earlier steps' changes in place. The +/// bootstrap collects no status, and starts and stops no process. pub fn bootstrap(layout: &Layout, resources: &Resources) -> Result { bootstrap_with(&mut SystemKernel, &Paths::default(), layout, resources) } @@ -158,48 +144,33 @@ pub(super) fn bootstrap_with( }; disable_user_namespaces(kernel, paths).map_err(failed(BootstrapStep::UserNamespaces))?; mount_hierarchy(kernel, parent).map_err(failed(BootstrapStep::Hierarchy))?; - create_cgroup(layout.agent()).map_err(failed(BootstrapStep::AgentCgroup))?; + cgroup::create(layout.agent(), false).map_err(failed(BootstrapStep::AgentCgroup))?; let moved = place(kernel, paths, parent, layout.agent()).map_err(failed(BootstrapStep::Placement))?; - let controllers: &[u8] = if resources.need_memory() { - b"+pids +memory" - } else { - b"+pids" - }; - write(kernel, &parent.join("cgroup.subtree_control"), controllers) - .map_err(failed(BootstrapStep::Controllers))?; if let Some(low) = resources.agent_memory_low { - write( - kernel, - &layout.agent().join("memory.low"), - low.to_string().as_bytes(), - ) - .map_err(failed(BootstrapStep::AgentMemoryLow))?; - } - create_cgroup(layout.workload()).map_err(failed(BootstrapStep::WorkloadCgroup))?; - for (value, file, step) in [ - ( - resources.workload_memory_max, - "memory.max", - BootstrapStep::WorkloadMemoryMax, - ), - ( - resources.workload_pids_max, - "pids.max", - BootstrapStep::WorkloadPidsMax, - ), - ] { - if let Some(value) = value { - write( - kernel, - &layout.workload().join(file), - value.to_string().as_bytes(), - ) - .map_err(failed(step))?; - } + cgroup::provide(layout.agent(), parent, ["memory.low"]) + .and_then(|()| { + write( + kernel, + &layout.agent().join("memory.low"), + low.to_string().as_bytes(), + ) + }) + .map_err(failed(BootstrapStep::AgentMemoryLow))?; } - let workload = - ExecutionCgroup::open(layout, None).map_err(failed(BootstrapStep::WorkloadCgroup))?; + let values = [ + ("memory.max", resources.workload_memory_max), + ("pids.max", resources.workload_pids_max), + ] + .into_iter() + .filter_map(|(file, value)| Some((file, value?.to_string()))) + .collect::>(); + let values: Vec<(&str, &str)> = values + .iter() + .map(|(file, value)| (*file, value.as_str())) + .collect(); + let workload = ExecutionCgroup::prepare(layout, None, &values) + .map_err(failed(BootstrapStep::WorkloadCgroup))?; Ok(Bootstrap { layout: layout.clone(), workload, @@ -250,20 +221,6 @@ fn mount_hierarchy(kernel: &mut dyn Kernel, parent: &Path) -> io::Result<()> { )?) } -/// Creates the cgroup `path` unless it exists. A file in its place is `ENOTDIR`. -fn create_cgroup(path: &Path) -> io::Result<()> { - match fs::DirBuilder::new().mode(0o755).create(path) { - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { - if fs::symlink_metadata(path)?.is_dir() { - Ok(()) - } else { - Err(Errno::ENOTDIR.into()) - } - } - result => result, - } -} - /// Moves every process of the cgroup `parent` into the cgroup `agent`, except kernel threads, in /// passes until none is left, and returns the IDs that it moved. fn place( diff --git a/nvx-rs/src/guest/root/kernel.rs b/nvx-rs/src/guest/root/kernel.rs index 334f56cc4..b3f2a19ac 100644 --- a/nvx-rs/src/guest/root/kernel.rs +++ b/nvx-rs/src/guest/root/kernel.rs @@ -64,7 +64,7 @@ impl Kernel for SystemKernel { flags: MountFlags, data: Option<&CStr>, ) -> Result<(), Errno> { - sys::mount_with_options(source, target, filesystem, flags, data) + sys::mount(source, target, filesystem, flags, data) } fn unmount(&mut self, target: &CStr) -> Result<(), Errno> { diff --git a/nvx-rs/src/guest/root/share.rs b/nvx-rs/src/guest/root/share.rs index c0bf52c1e..bbfb40bee 100644 --- a/nvx-rs/src/guest/root/share.rs +++ b/nvx-rs/src/guest/root/share.rs @@ -11,53 +11,12 @@ use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; use std::path::{Path, PathBuf}; use super::{AGGREGATE_DIRECTORY, Error, Mount, Root}; -use crate::guest::sys::{self, MountFlags}; +use crate::guest::mappings::{self, Access, Mapping, MappingStage}; +use crate::protocol::share::{self, RESERVED_TARGET_CHARACTERS}; -/// The targets that the runtime mounts or binds over in the root, each with everything below it. -const RESERVED_TREES: [&[u8]; 6] = [ - b"/proc", - b"/sys", - b"/dev", - b"/.nvx-agent", - b"/etc/machine-id", - b"/run/nvx/time", -]; -/// The directories that the runtime creates or mounts below, without the paths below them. -const RESERVED_DIRECTORIES: [&[u8]; 3] = [b"/etc", b"/run", b"/run/nvx"]; /// The file system type of a live share. const VIRTIOFS: &[u8] = b"virtiofs"; -/// Whether the workload may write a live share. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum Access { - /// The share is mounted read-only. - ReadOnly, - /// The share is mounted read-write. - ReadWrite, -} - -impl Access { - /// Returns the mount flags of a share with this access, without set-user-ID programs or - /// device nodes. - fn flags(self) -> MountFlags { - let flags = MountFlags::MS_NOSUID | MountFlags::MS_NODEV; - match self { - Self::ReadOnly => flags | MountFlags::MS_RDONLY, - Self::ReadWrite => flags, - } - } -} - -impl fmt::Display for Access { - /// Writes `ro` or `rw`, as a mount option names the access. - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::ReadOnly => "ro", - Self::ReadWrite => "rw", - }) - } -} - /// What a child of an aggregate share is: a host directory, or a host file. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub enum ShareKind { @@ -90,18 +49,21 @@ pub enum ShareRefusal { Tag(String), /// An aggregate has no children. NoChildren, - /// A child's name is not a nonempty name of letters, digits, `.`, `_`, and `-`, or is `.` or - /// `..`. + /// A child's name is not a nonempty name of letters, digits, `.`, `_`, and `-` of at most + /// [`MAX_PATH_BYTES`](mappings::MAX_PATH_BYTES) bytes, or is `.` or `..`. ChildName(String), /// Two children of an aggregate have this name. DuplicateChild(String), /// The target is not an absolute path. Relative(PathBuf), - /// The target is `/`, or has an empty, `.`, or `..` component, or a trailing `/`. + /// The target does not name a path below the root: it is `/`, has an empty, `.`, or `..` + /// component, a trailing `/`, or a NUL, or is longer than + /// [`MAX_PATH_BYTES`](mappings::MAX_PATH_BYTES) bytes. NotCanonical(PathBuf), - /// The target holds a reserved character: `\`, `=`, `,`, or NUL. + /// The target holds a reserved character: `\`, `=`, or `,`. ReservedCharacter(PathBuf), - /// The runtime mounts at the target, or below it, or binds over it. + /// The container runtime reserves the target, as [`share::reservation`] tells: it mounts + /// there or below it, binds over it, or creates it. Reserved(PathBuf), /// The target is the target of another share, or one inside it or that holds it. Overlap { @@ -211,14 +173,16 @@ impl Root { /// Mounts the aggregate virtio-fs device `tag`, with `access`, `nosuid`, and `nodev`, at /// `shares` in the runtime directory, outside the root, where the workload cannot list its /// children, and which may not be a symbolic link. Then binds each child, in order, at its - /// target in the root, and restricts the bind to the child's access, `nosuid`, and `nodev`. + /// target in the root through a [`Mapping`], as the managed agent applies its host mappings, + /// which restricts the bind to the child's access, `nosuid`, and `nodev`. /// /// Every child is checked first: its name, which no other child has, and its target, as the /// [module](super#live-shares) describes, against the other children's and the root's other /// shares' targets. Each target is created in the root if it is missing. Nothing is mounted - /// unless every child passes. The mounts belong to the root, which unmounts them first when - /// it is torn down, in reverse mount order, those of an aggregate that failed part of the way - /// included. + /// unless every child passes. A child whose entry in the aggregate is missing, or is not of + /// the child's kind, fails to mount. The mounts belong to the root, which unmounts them first + /// when it is torn down, in reverse mount order, those of an aggregate that failed part of the + /// way included. /// /// A root has one aggregate at most, since every aggregate is mounted at the same place: once /// the root has mounted one, whether or not its children were all bound, another is refused. @@ -240,7 +204,7 @@ impl Root { return Err(Error::Share(ShareRefusal::NoChildren)); } let mut targets: Vec = Vec::with_capacity(children.len()); - let mut points = Vec::with_capacity(children.len()); + let mut binds = Vec::with_capacity(children.len()); for (index, child) in children.iter().enumerate() { check_name(&child.name)?; if children[..index] @@ -254,7 +218,10 @@ impl Root { check_target(&child.target)?; self.check_overlap(&child.target, &targets)?; targets.push(child.target.clone()); - points.push(mount_point(&self.path, &child.target, child.kind)?); + // The checks leave only a name longer than a path to refuse. + let mapping = Mapping::new(&child.name, &child.target, child.access) + .map_err(|_| Error::Share(ShareRefusal::ChildName(child.name.clone())))?; + binds.push((mapping, mount_point(&self.path, &child.target, child.kind)?)); } let directory = self.runtime.join(AGGREGATE_DIRECTORY); super::create_directory(&directory).map_err(|error| Error::MountPoint { @@ -273,36 +240,33 @@ impl Root { None, )?; self.shares.push((aggregate, directory.clone())); - for (child, point) in children.iter().zip(points) { + for (child, (mapping, point)) in children.iter().zip(binds) { + let kernel = &mut self.kernel; + let bound = mapping.apply_with(&directory, &self.path, |source, target, flags| { + kernel.mount(source, target, None, flags, None) + }); let mount = Mount::Child { name: child.name.clone(), target: child.target.clone(), }; - let source = directory.join(&child.name); - self.mount_at( - &mount, - Some(source.as_os_str().as_bytes()), - &point, - None, - MountFlags::MS_BIND, - None, - )?; - self.shares.push((mount.clone(), point.clone())); - self.claimed.push(child.target.clone()); - // A bind takes its source's flags; only a remount of the bind changes them. - let restricted = sys::path_c_string(&point).and_then(|point| { - self.kernel.mount( - None, - &point, - None, - MountFlags::MS_REMOUNT | MountFlags::MS_BIND | child.access.flags(), - None, - ) - }); - restricted.map_err(|error| Error::Restrict { - mount, - error: error.into(), - })?; + // A bind whose restriction failed stays mounted. + let mounted = match &bound { + Ok(()) => true, + Err(failure) => failure.stage() == MappingStage::Remount, + }; + if mounted { + self.shares.push((mount.clone(), point)); + self.claimed.push(child.target.clone()); + } + if let Err(failure) = bound { + let error = io::Error::from_raw_os_error(failure.raw_os_error()); + return Err(match failure.stage() { + MappingStage::Remount => Error::Restrict { mount, error }, + MappingStage::Source | MappingStage::Target | MappingStage::Bind => { + Error::Mount { mount, error } + } + }); + } } Ok(()) } @@ -358,23 +322,19 @@ fn check_name(name: &str) -> Result<(), Error> { } /// Refuses a target that is not canonical and absolute, that holds a reserved character, or that -/// the runtime reserves. +/// the container runtime reserves. fn check_target(target: &Path) -> Result<(), Error> { let bytes = target.as_os_str().as_bytes(); let refusal = if !bytes.starts_with(b"/") { Some(ShareRefusal::Relative(target.to_owned())) - } else if !super::is_canonical(bytes) { + } else if mappings::below_its_root(target, true).is_none() { Some(ShareRefusal::NotCanonical(target.to_owned())) } else if bytes .iter() - .any(|byte| matches!(byte, b'\\' | b'=' | b',' | b'\0')) + .any(|byte| RESERVED_TARGET_CHARACTERS.as_bytes().contains(byte)) { Some(ShareRefusal::ReservedCharacter(target.to_owned())) - } else if RESERVED_DIRECTORIES.contains(&bytes) - || RESERVED_TREES.iter().any(|tree| { - bytes == *tree || (bytes.starts_with(tree) && bytes.get(tree.len()) == Some(&b'/')) - }) - { + } else if share::reservation(bytes).is_some() { Some(ShareRefusal::Reserved(target.to_owned())) } else { None @@ -472,13 +432,20 @@ mod tests { ); } for target in [ - "/", "//a", "/a//b", "/a/", "/a/.", "/a/./b", "/a/..", "/../a", + "/", "//a", "/a//b", "/a/", "/a/.", "/a/./b", "/a/..", "/../a", "/a\0b", ] { assert!( matches!(refusal(target), Some(ShareRefusal::NotCanonical(_))), "{target}" ); } + // A target names a path below the root, which is never longer than a path. + let longest = format!("/{}", "a".repeat(mappings::MAX_PATH_BYTES - 1)); + assert_eq!(refusal(&longest), None); + assert!(matches!( + refusal(&format!("{longest}a")), + Some(ShareRefusal::NotCanonical(_)) + )); for target in ["/a\\b", "/a=b", "/a,b"] { assert!( matches!(refusal(target), Some(ShareRefusal::ReservedCharacter(_))), diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index 9bbf70934..bc6fbbfe6 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -12,6 +12,7 @@ use std::time::Duration; use super::kernel::{Call, FakeKernel, Operation, Paths}; use super::*; use crate::guest::cgroup::Layout; +use crate::guest::mappings::Access; /// The base address of the distro device's MMIO window. const DISTRO: u64 = 0xd000_0000; @@ -662,6 +663,7 @@ fn a_runtime_directory_where_a_file_system_is_mounted_is_refused() { // which, like the image and the scratch, was unmounted. let guest = Guest::new(); let mut root = guest.assemble(&guest.assembly()).unwrap(); + export(&guest, &[("a", ShareKind::Directory)]); let rw = Access::ReadWrite; root.mount_aggregate("all", rw, &[child("a", "/a", rw, ShareKind::Directory)]) .unwrap(); @@ -769,11 +771,27 @@ fn child(name: &str, target: &str, access: Access, kind: ShareKind) -> ShareChil } } +/// Creates the entries that an aggregate mounted at `shares` in `guest`'s runtime directory +/// shows, which the fake kernel does not mount: a directory or a regular file for each child. +fn export(guest: &Guest, children: &[(&str, ShareKind)]) { + let shares = guest.path("run/nvx/shares"); + for &(name, kind) in children { + match kind { + ShareKind::Directory => fs::create_dir_all(shares.join(name)).unwrap(), + ShareKind::File => write(&shares.join(name), "host file"), + } + } +} + #[test] fn shares_are_mounted_restricted_and_torn_down_first_in_reverse_order() { let guest = Guest::new(); let mut root = guest.assemble(&guest.assembly()).unwrap(); let assembled = guest.kernel.calls().len(); + export( + &guest, + &[("a", ShareKind::Directory), ("b", ShareKind::File)], + ); root.mount_share("one", Path::new("/srv/one"), Access::ReadOnly) .unwrap(); root.mount_aggregate( @@ -938,6 +956,7 @@ fn a_root_has_one_aggregate_even_after_one_that_failed_part_of_the_way() { for bound in [&["/a", "/b"][..], &["/a"]] { let guest = Guest::new(); let mut root = guest.assemble(&guest.assembly()).unwrap(); + export(&guest, &[("a", directory), ("b", directory)]); if bound.len() == 1 { guest.kernel.fail( Operation::Mount, @@ -989,6 +1008,10 @@ fn a_root_has_one_aggregate_even_after_one_that_failed_part_of_the_way() { fn a_child_that_cannot_be_restricted_stays_mounted_until_teardown() { let guest = Guest::new(); let mut root = guest.assemble(&guest.assembly()).unwrap(); + export( + &guest, + &[("a", ShareKind::Directory), ("b", ShareKind::Directory)], + ); guest.kernel.fail( Operation::Remount, guest.path("run/nvx/rootfs/data/a"), @@ -1032,6 +1055,56 @@ fn a_child_that_cannot_be_restricted_stays_mounted_until_teardown() { ); } +#[test] +fn a_child_that_the_aggregate_lacks_or_holds_as_another_kind_binds_nothing() { + for (exported, errno) in [ + (None, Errno::ENOENT), + (Some(ShareKind::File), Errno::EINVAL), + ] { + let guest = Guest::new(); + let mut root = guest.assemble(&guest.assembly()).unwrap(); + if let Some(kind) = exported { + export(&guest, &[("a", kind)]); + } + let mounts = guest.kernel.mounts().len(); + let error = root + .mount_aggregate( + "all", + Access::ReadOnly, + &[child( + "a", + "/data/a", + Access::ReadOnly, + ShareKind::Directory, + )], + ) + .unwrap_err(); + assert!( + matches!( + &error, + Error::Mount { mount: Mount::Child { name, .. }, error } + if name == "a" && error.raw_os_error() == Some(errno as i32) + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "failed to mount the live share child a at /data/a: {}", + os_error(errno) + ) + ); + // Only the aggregate is mounted, and the child claims no target. + assert_eq!( + guest.kernel.mounts()[mounts..], + [guest.text("run/nvx/shares")] + ); + assert!(root.claimed.is_empty()); + root.teardown().unwrap(); + assert_eq!(guest.kernel.unmounts()[0], guest.text("run/nvx/shares")); + } +} + #[test] fn a_failed_unmount_is_reported_and_the_later_ones_still_attempted() { let guest = Guest::new(); @@ -1157,6 +1230,15 @@ fn bootstrap_in(guest: &Guest, layout: &Layout, resources: &Resources) -> Result bootstrap::bootstrap_with(&mut guest.kernel.clone(), &guest.paths, layout, resources) } +/// Creates the interface files `files` of the workload cgroup, `cgroup/container`, in `guest`, +/// as the kernel shows them once its parent enables their controllers, which a hierarchy of +/// regular files does not. +fn interface_files(guest: &Guest, files: &[&str]) { + for file in files { + write(&guest.path("cgroup/container").join(file), ""); + } +} + #[test] fn bootstrap_moves_every_process_but_kernel_threads_and_applies_the_resources() { let guest = Guest::bare(); @@ -1167,6 +1249,7 @@ fn bootstrap_moves_every_process_but_kernel_threads_and_applies_the_resources() Process::Ended(99), ]; let layout = hierarchy(&guest, true, &processes); + interface_files(&guest, &["pids.current", "memory.max", "pids.max"]); let resources = Resources { agent_memory_low: Some(16 << 20), workload_memory_max: Some(512 << 20), @@ -1180,16 +1263,26 @@ fn bootstrap_moves_every_process_but_kernel_threads_and_applies_the_resources() ("proc/sys/user/max_user_namespaces", "0"), ("cgroup/agent/cgroup.procs", "1"), ("cgroup/agent/cgroup.procs", "45"), - ("cgroup/cgroup.subtree_control", "+pids +memory"), ("cgroup/agent/memory.low", "16777216"), - ("cgroup/container/memory.max", "536870912"), - ("cgroup/container/pids.max", "65"), ] .into_iter() .map(|(path, value)| (guest.text(path), value.to_owned())) .collect(); assert_eq!(guest.kernel.writes(), writes); assert!(guest.kernel.mounts().is_empty()); + // The agent's `memory.low` enabled the `memory` controller. The workload cgroup has every + // file already, so its preparation enabled nothing, and wrote its values. + assert_eq!( + fs::read_to_string(guest.path("cgroup/cgroup.subtree_control")).unwrap(), + "+memory" + ); + for (file, value) in [("memory.max", "536870912"), ("pids.max", "65")] { + assert_eq!( + fs::read_to_string(guest.path("cgroup/container").join(file)).unwrap(), + value, + "{file}" + ); + } assert_eq!( fs::read_to_string(guest.path("cgroup/cgroup.procs")).unwrap(), "2\n99\n" @@ -1198,7 +1291,6 @@ fn bootstrap_moves_every_process_but_kernel_threads_and_applies_the_resources() fs::read_to_string(guest.path("cgroup/agent/cgroup.procs")).unwrap(), "1\n45\n" ); - assert!(guest.path("cgroup/container").is_dir()); } #[test] @@ -1216,23 +1308,23 @@ fn bootstrap_mounts_the_hierarchy_unless_the_parent_is_a_cgroup() { None, ) ); - // Without a memory value, only the `pids` controller is enabled, and nothing else written. - let writes = guest.kernel.writes(); + // Without a value, the bootstrap writes only the placement, and the preparation of the new + // workload cgroup enables only the `pids` controller. + assert_eq!(guest.kernel.writes().len(), 2); assert_eq!( - writes[2..], - [(guest.text("cgroup/cgroup.subtree_control"), "+pids".into())] + fs::read_to_string(guest.path("cgroup/cgroup.subtree_control")).unwrap(), + "+pids" ); + assert!(guest.path("cgroup/container").is_dir()); } #[test] fn bootstrap_names_the_failed_step_and_keeps_the_earlier_changes() { + // The workload cgroup's `memory.max` cannot be written: a directory takes its place. let guest = Guest::bare(); let layout = hierarchy(&guest, true, &[Process::User(1)]); - guest.kernel.fail( - Operation::Write, - guest.path("cgroup/container/memory.max"), - Errno::EINVAL, - ); + interface_files(&guest, &["pids.current", "pids.max"]); + fs::create_dir(guest.path("cgroup/container/memory.max")).unwrap(); let resources = Resources { agent_memory_low: Some(1 << 20), workload_memory_max: Some(1), @@ -1242,28 +1334,53 @@ fn bootstrap_names_the_failed_step_and_keeps_the_earlier_changes() { assert!( matches!( &error, - Error::Bootstrap { step: BootstrapStep::WorkloadMemoryMax, error } - if error.raw_os_error() == Some(Errno::EINVAL as i32) + Error::Bootstrap { step: BootstrapStep::WorkloadCgroup, error } + if error.raw_os_error() == Some(Errno::EISDIR as i32) ), "{error:?}" ); assert_eq!( error.to_string(), format!( - "failed to set the workload's memory limit: {}", - os_error(Errno::EINVAL) + "failed to prepare the workload cgroup: {}", + os_error(Errno::EISDIR) ) ); - let writes = guest.kernel.writes(); - assert_eq!( - writes.last().unwrap().0, - guest.text("cgroup/container/memory.max") - ); assert_eq!( fs::read_to_string(guest.path("cgroup/agent/memory.low")).unwrap(), "1048576" ); + // The agent's value is refused before the workload cgroup is prepared. + let guest = Guest::bare(); + let layout = hierarchy(&guest, true, &[Process::User(1)]); + guest.kernel.fail( + Operation::Write, + guest.path("cgroup/agent/memory.low"), + Errno::EINVAL, + ); + let error = bootstrap_in(&guest, &layout, &resources).unwrap_err(); + assert!( + matches!( + &error, + Error::Bootstrap { step: BootstrapStep::AgentMemoryLow, error } + if error.raw_os_error() == Some(Errno::EINVAL as i32) + ), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!( + "failed to set the agent's memory reservation: {}", + os_error(Errno::EINVAL) + ) + ); + assert_eq!( + fs::read_to_string(guest.path("cgroup/agent/cgroup.procs")).unwrap(), + "1\n" + ); + assert!(!guest.path("cgroup/container").exists()); + // Cgroups that are not siblings are refused before anything changes. let guest = Guest::bare(); hierarchy(&guest, true, &[Process::User(1)]); diff --git a/nvx-rs/src/guest/sys/filesystem.rs b/nvx-rs/src/guest/sys/filesystem.rs index 30b08e72a..642b1e9b3 100644 --- a/nvx-rs/src/guest/sys/filesystem.rs +++ b/nvx-rs/src/guest/sys/filesystem.rs @@ -69,23 +69,13 @@ pub(crate) fn write_number(path: &CStr, value: impl Display) -> Result<(), Errno write_file(path, format!("{value}\n").as_bytes()) } -/// Mounts `source` on `target` with no data argument. +/// Mounts `source` on `target`, with the file system's own options `data`, if any, such as an +/// overlay's layers. pub(crate) fn mount( source: Option<&CStr>, target: &CStr, filesystem: Option<&CStr>, flags: MountFlags, -) -> Result<(), Errno> { - nix::mount::mount(source, target, filesystem, flags, None::<&CStr>) -} - -/// Mounts `source` on `target`, as [`mount`] does, with the file system's own options `data`, -/// such as an overlay's layers. -pub(crate) fn mount_with_options( - source: Option<&CStr>, - target: &CStr, - filesystem: Option<&CStr>, - flags: MountFlags, data: Option<&CStr>, ) -> Result<(), Errno> { nix::mount::mount(source, target, filesystem, flags, data) diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index d9aa6ca18..efe97cdb1 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -84,9 +84,8 @@ pub(crate) use fd::{ }; pub(crate) use filesystem::{ MountFlags, PATH_MAX, ScratchDirectory, TmpfsMount, change_directory, check_exists, chmod, - chown, current_directory, enter_private_mount_namespace, mount, mount_id, mount_with_options, - path_c_string, path_exists, read_file, read_line, read_number, unmount, write_file, - write_number, + chown, current_directory, enter_private_mount_namespace, mount, mount_id, path_c_string, + path_exists, read_file, read_line, read_number, unmount, write_file, write_number, }; pub(crate) use ioctl::{console_pending, flush_block_buffers, reseed_random}; pub(crate) use mapping::MemoryMapping; diff --git a/nvx-rs/src/host/sandbox/mount.rs b/nvx-rs/src/host/sandbox/mount.rs index 5ec97f304..25b1d13fa 100644 --- a/nvx-rs/src/host/sandbox/mount.rs +++ b/nvx-rs/src/host/sandbox/mount.rs @@ -24,26 +24,14 @@ use crate::host::filesystem::within; use crate::host::python::builtins::{is_space, repr}; use crate::host::python::fs; use crate::host::python::path::{Flavor, PurePath}; +use crate::protocol::share::{self, RESERVED_TARGET_CHARACTERS, Reservation}; + +pub use crate::protocol::share::{RESERVED_EXACT_MOUNT_TARGETS, RESERVED_MOUNT_TARGETS}; /// The tag of OpenVMM's virtio-fs device. pub const MOUNT_TAG: &str = "microvm"; /// The guest directory at which the guest mounts the aggregate of several shares. pub const AGGREGATE_MOUNT_TARGET: &str = "/run/nvx/shares"; -/// The guest trees that the container runtime owns, which a share may neither be in nor contain: -/// it mounts procfs, sysfs, and a private `/dev` over them, stages its tools under -/// `/.nvx-agent`, binds the workload machine ID at `/etc/machine-id`, which it first replaces -/// with a regular file, and binds its time ABI state over `/run/nvx/time`. -pub const RESERVED_MOUNT_TARGETS: [&str; 6] = [ - "/proc", - "/sys", - "/dev", - "/.nvx-agent", - "/etc/machine-id", - "/run/nvx/time", -]; -/// The guest paths at which a share may not be, though it may be below them: `/etc`, and `/run` -/// and `/run/nvx`, which the container runtime creates below the container root. -pub const RESERVED_EXACT_MOUNT_TARGETS: [&str; 3] = ["/etc", "/run", "/run/nvx"]; /// The most denied, allowed, and writable paths each that OpenVMM accepts for a share. pub const MAX_MOUNT_POLICY_PATHS: usize = 128; /// The longest guest target, in UTF-8 bytes. @@ -282,12 +270,12 @@ pub fn parse_mount_value(value: &str) -> Result<(String, String, MountAccess)> { Ok((target.to_owned(), host_path.to_owned(), access)) } -/// Returns whether `target` holds a character that no guest target may: Python whitespace, NUL, -/// `\`, `=`, or `,`. +/// Returns whether `target` holds a character that no guest target may: Python whitespace, or +/// one of the [`RESERVED_TARGET_CHARACTERS`], NUL, `\`, `=`, and `,`. pub(crate) fn has_forbidden_character(target: &str) -> bool { target .chars() - .any(|character| is_space(character) || matches!(character, '\0' | '\\' | '=' | ',')) + .any(|character| is_space(character) || RESERVED_TARGET_CHARACTERS.contains(character)) } /// Returns whether a guest target has an empty, `.`, or `..` component after its leading slash. @@ -317,19 +305,15 @@ pub fn validate_mount_target(target: &str) -> Result<()> { repr(target) ))); } - for reserved in RESERVED_MOUNT_TARGETS { - if within(target, reserved) { - return Err(refused(format!( - "sandbox mount target {target} overlaps the reserved {reserved} tree" - ))); - } - } - if RESERVED_EXACT_MOUNT_TARGETS.contains(&target) { - return Err(refused(format!( + match share::reservation(target.as_bytes()) { + Some(Reservation::Tree(reserved)) => Err(refused(format!( + "sandbox mount target {target} overlaps the reserved {reserved} tree" + ))), + Some(Reservation::Exact) => Err(refused(format!( "sandbox mount target {target} is reserved" - ))); + ))), + None => Ok(()), } - Ok(()) } /// Refuses an owner that this host cannot enforce: `caller` on Windows, which has no diff --git a/nvx-rs/src/lib.rs b/nvx-rs/src/lib.rs index 704fb01f4..8dfe1c99e 100644 --- a/nvx-rs/src/lib.rs +++ b/nvx-rs/src/lib.rs @@ -6,8 +6,8 @@ //! control console. This crate holds everything it takes to drive one: //! //! - [`protocol`]: the codec of the control console, the `NVXS` session that OpenVMM brokers and -//! the `NVXC` frames of the guest agent. It has no dependencies, so the host, the guest, and -//! test doubles share it. +//! the `NVXC` frames of the guest agent, and the guest targets that a sandbox's live shares may +//! not take. It has no dependencies, so the host, the guest, and test doubles share it. //! - [`contract`]: the control-contract versions that this crate speaks, and the compatibility of //! NVX releases with them. //! - [`net`]: IP networks in CIDR notation and their subtraction, and the forms of a VM's diff --git a/nvx-rs/src/protocol/mod.rs b/nvx-rs/src/protocol/mod.rs index d59fb45e1..56cef195a 100644 --- a/nvx-rs/src/protocol/mod.rs +++ b/nvx-rs/src/protocol/mod.rs @@ -20,6 +20,8 @@ //! devices, of any type, that its `virtio_mmio.device` parameters describe. //! - [`root`]: the boot tokens of a layered root, which name its image's format and identity and //! its upper, with the host's renderer and the guest's parser. +//! - [`share`]: the guest targets that a sandbox's live shares may not take, which the host and +//! the guest both refuse. //! - [`nvxs`]: the NVXS session without I/O: validated records, incremental parsing, and the //! state machines of the host's and the guest's roles, which carry application bytes of any //! protocol. @@ -38,6 +40,7 @@ mod maps; pub mod nvxs; mod outer; pub mod root; +pub mod share; #[cfg(test)] mod tests; diff --git a/nvx-rs/src/protocol/share.rs b/nvx-rs/src/protocol/share.rs new file mode 100644 index 000000000..f46fb5c90 --- /dev/null +++ b/nvx-rs/src/protocol/share.rs @@ -0,0 +1,122 @@ +//! The guest targets that a sandbox's live shares may not take: the paths at which the guest's +//! container runtime mounts, binds, or creates directories in the container root, and the +//! characters that no target may hold. +//! +//! The host refuses such a target before it attaches a share, and the guest again before it +//! mounts one, with [`reservation`] and [`RESERVED_TARGET_CHARACTERS`]. + +/// The guest trees that the container runtime owns, which a share may neither be in nor contain: +/// it mounts procfs, sysfs, and a private `/dev` over them, stages its tools under +/// `/.nvx-agent`, binds the workload machine ID at `/etc/machine-id`, which it first replaces +/// with a regular file, and binds its time ABI state over `/run/nvx/time`. +pub const RESERVED_MOUNT_TARGETS: [&str; 6] = [ + "/proc", + "/sys", + "/dev", + "/.nvx-agent", + "/etc/machine-id", + "/run/nvx/time", +]; + +/// The guest paths at which a share may not be, though it may be below them: `/etc`, and `/run` +/// and `/run/nvx`, which the container runtime creates below the container root. +pub const RESERVED_EXACT_MOUNT_TARGETS: [&str; 3] = ["/etc", "/run", "/run/nvx"]; + +/// The characters that a share's guest target may not hold: NUL, `\`, `=`, and `,`. +pub const RESERVED_TARGET_CHARACTERS: &str = "\0\\=,"; + +/// Why the container runtime reserves a guest target, as [`reservation`] tells. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Reservation { + /// The target is in the tree of this one of the [`RESERVED_MOUNT_TARGETS`]: it is that path, + /// or is below it. + Tree(&'static str), + /// The target is one of the [`RESERVED_EXACT_MOUNT_TARGETS`]. + Exact, +} + +/// Returns why the container runtime reserves the guest target `target`, if it does: because it +/// is in one of the [`RESERVED_MOUNT_TARGETS`] trees, or is one of the +/// [`RESERVED_EXACT_MOUNT_TARGETS`]. +/// +/// The target is compared as written, so the caller checks first that it is absolute and +/// canonical: `/proc/` and `/run/./nvx` are not reserved here. +pub fn reservation(target: &[u8]) -> Option { + let tree = RESERVED_MOUNT_TARGETS.into_iter().find(|tree| { + target + .strip_prefix(tree.as_bytes()) + .is_some_and(|rest| rest.is_empty() || rest.starts_with(b"/")) + }); + match tree { + Some(tree) => Some(Reservation::Tree(tree)), + None => RESERVED_EXACT_MOUNT_TARGETS + .iter() + .any(|exact| target == exact.as_bytes()) + .then_some(Reservation::Exact), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_reserved_targets_and_characters_are_pinned() { + assert_eq!( + RESERVED_MOUNT_TARGETS, + [ + "/proc", + "/sys", + "/dev", + "/.nvx-agent", + "/etc/machine-id", + "/run/nvx/time" + ] + ); + assert_eq!(RESERVED_EXACT_MOUNT_TARGETS, ["/etc", "/run", "/run/nvx"]); + assert_eq!(RESERVED_TARGET_CHARACTERS, "\0\\=,"); + } + + #[test] + fn a_target_is_reserved_in_a_tree_or_at_an_exact_path() { + for (target, tree) in [ + ("/proc", "/proc"), + ("/proc/self", "/proc"), + ("/sys/fs/cgroup", "/sys"), + ("/dev/shm", "/dev"), + ("/.nvx-agent/tool", "/.nvx-agent"), + ("/etc/machine-id", "/etc/machine-id"), + ("/etc/machine-id/x", "/etc/machine-id"), + ("/run/nvx/time", "/run/nvx/time"), + ("/run/nvx/time/state", "/run/nvx/time"), + ] { + assert_eq!( + reservation(target.as_bytes()), + Some(Reservation::Tree(tree)), + "{target}" + ); + } + for target in ["/etc", "/run", "/run/nvx"] { + assert_eq!( + reservation(target.as_bytes()), + Some(Reservation::Exact), + "{target}" + ); + } + for target in [ + "/workspace", + "/procs", + "/devices", + "/.nvx-agents", + "/etc/app", + "/etc/machine-ids", + "/run/app", + "/run/nvx/state", + "/run/nvx/timer", + "/", + "", + ] { + assert_eq!(reservation(target.as_bytes()), None, "{target}"); + } + } +} From 0d5f9844255fcc003ea205ef394c48cf007dfa24 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 21:38:37 -0700 Subject: [PATCH 12/12] nvx-rs: name a guest root's image UUID with nvx::uuid nvx::uuid now holds the canonical UUID that the host and the guest share. A guest root's assembly took the EROFS image's expected UUID as 16 raw bytes, and displayed a mismatch with a formatter of its own, a second copy of that type's canonical text. Assembly::distro_uuid and Error::Uuid now hold an nvx::uuid::Uuid, which the superblock check builds from the image's bytes, and a mismatch displays through it, with the same text as before. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59cd1d15-8546-491b-8b77-8f679d03492d --- nvx-rs/README.md | 2 +- nvx-rs/src/guest/root.rs | 27 +++++---------------------- nvx-rs/src/guest/root/block.rs | 9 +++++---- nvx-rs/src/guest/root/tests.rs | 11 ++++++----- 4 files changed, 17 insertions(+), 32 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 5a4dfe8b3..2692aa444 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -88,7 +88,7 @@ nvx-rs/ | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | -| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected UUID it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, whose children it binds through `mappings::Mapping`. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected `nvx::uuid::Uuid` it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, whose children it binds through `mappings::Mapping`. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own diff --git a/nvx-rs/src/guest/root.rs b/nvx-rs/src/guest/root.rs index 671a0c62a..261b29031 100644 --- a/nvx-rs/src/guest/root.rs +++ b/nvx-rs/src/guest/root.rs @@ -115,6 +115,7 @@ use std::time::Duration; use self::kernel::{Kernel, Paths, SystemKernel}; use super::mappings::below_its_root; use super::sys::{self, Errno, MountFlags}; +use crate::uuid::Uuid; pub use self::bootstrap::{Bootstrap, BootstrapStep, Resources, bootstrap}; pub use self::share::{ShareChild, ShareKind, ShareRefusal}; @@ -207,7 +208,7 @@ pub struct Assembly { /// The device of the EROFS image. pub distro: Device, /// The UUID that the EROFS image's superblock must hold, if the caller expects one. - pub distro_uuid: Option<[u8; 16]>, + pub distro_uuid: Option, /// The device of the ext4 scratch. pub scratch: Device, /// How long to wait for each device to appear. @@ -730,9 +731,9 @@ pub enum Error { /// The EROFS image's UUID is not the expected one. Uuid { /// The expected UUID. - expected: [u8; 16], + expected: Uuid, /// The image's UUID. - actual: [u8; 16], + actual: Uuid, }, /// A mount failed. Mount { @@ -811,9 +812,7 @@ impl fmt::Display for Error { ), Self::Uuid { expected, actual } => write!( formatter, - "the distro layer's UUID is {}, expected {}", - Uuid(actual), - Uuid(expected) + "the distro layer's UUID is {actual}, expected {expected}" ), Self::Mount { mount, error } => write!(formatter, "failed to mount {mount}: {error}"), Self::Restrict { mount, error } => { @@ -930,19 +929,3 @@ impl fmt::Display for Unmount { ) } } - -/// A UUID, which displays in its canonical form: 32 lowercase hexadecimal digits, in groups of 8, -/// 4, 4, 4, and 12 that hyphens separate. -struct Uuid<'a>(&'a [u8; 16]); - -impl fmt::Display for Uuid<'_> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - for (index, byte) in self.0.iter().enumerate() { - if matches!(index, 4 | 6 | 8 | 10) { - formatter.write_str("-")?; - } - write!(formatter, "{byte:02x}")?; - } - Ok(()) - } -} diff --git a/nvx-rs/src/guest/root/block.rs b/nvx-rs/src/guest/root/block.rs index 40b2e031e..5d3309deb 100644 --- a/nvx-rs/src/guest/root/block.rs +++ b/nvx-rs/src/guest/root/block.rs @@ -15,6 +15,7 @@ use super::kernel::{Kernel, Paths}; use super::{Device, DeviceStep, Error, Role}; use crate::guest::parse::{parse_decimal, parse_hexadecimal}; use crate::guest::sys; +use crate::uuid::Uuid; /// The interval between two polls for a block device that has not appeared yet. pub(super) const POLL_INTERVAL: Duration = Duration::from_millis(50); @@ -168,7 +169,7 @@ fn parse_count(text: &[u8]) -> io::Result { /// Reads the UUID of the EROFS file system on the block device `device` from its superblock, and /// nothing else, or returns `None` if the device holds no EROFS superblock. -pub(super) fn erofs_uuid(device: &Path) -> io::Result> { +pub(super) fn erofs_uuid(device: &Path) -> io::Result> { let mut superblock = [0; EROFS_UUID.end]; match File::open(device)?.read_exact_at(&mut superblock, EROFS_SUPERBLOCK) { Ok(()) => {} @@ -179,11 +180,11 @@ pub(super) fn erofs_uuid(device: &Path) -> io::Result> { if magic != EROFS_MAGIC { return Ok(None); } - Ok(Some( + Ok(Some(Uuid::from_bytes( superblock[EROFS_UUID] .try_into() .expect("a UUID is 16 bytes"), - )) + ))) } #[cfg(test)] @@ -208,7 +209,7 @@ mod tests { image[1024..1028].copy_from_slice(&EROFS_MAGIC.to_le_bytes()); image[1024 + 48..1024 + 64].copy_from_slice(&uuid); fs::write(&device, &image).unwrap(); - assert_eq!(erofs_uuid(&device).unwrap(), Some(uuid)); + assert_eq!(erofs_uuid(&device).unwrap(), Some(Uuid::from_bytes(uuid))); // Another magic number, or a device too short for a superblock, holds no EROFS. image[1024] ^= 1; fs::write(&device, &image).unwrap(); diff --git a/nvx-rs/src/guest/root/tests.rs b/nvx-rs/src/guest/root/tests.rs index bc6fbbfe6..92589e2da 100644 --- a/nvx-rs/src/guest/root/tests.rs +++ b/nvx-rs/src/guest/root/tests.rs @@ -19,7 +19,7 @@ const DISTRO: u64 = 0xd000_0000; /// The base address of the scratch device's MMIO window. const SCRATCH: u64 = 0x1_d000_1000; /// The UUID of the test's EROFS image. -const UUID: [u8; 16] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; +const UUID: Uuid = Uuid::from_bytes([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); /// A guest in a temporary directory, and the fake kernel that the operations get clones of. struct Guest { @@ -135,10 +135,10 @@ fn probe(root: &Path, index: usize, name: &str, sectors: u64) { } /// Returns an EROFS image's first 4 KiB, whose superblock holds `uuid`. -fn erofs_image(uuid: [u8; 16]) -> Vec { +fn erofs_image(uuid: Uuid) -> Vec { let mut image = vec![0; 4096]; image[1024..1028].copy_from_slice(&0xE0F5_E1E2_u32.to_le_bytes()); - image[1024 + 48..1024 + 64].copy_from_slice(&uuid); + image[1024 + 48..1024 + 64].copy_from_slice(uuid.as_bytes()); image } @@ -297,8 +297,9 @@ fn the_expected_uuid_is_checked_in_the_superblock_before_the_mount() { let guest = Guest::new(); fs::write(guest.path("dev/vda"), erofs_image(UUID)).unwrap(); - let mut expected = UUID; - expected[15] = 0xff; + let mut bytes = *UUID.as_bytes(); + bytes[15] = 0xff; + let expected = Uuid::from_bytes(bytes); assembly = guest.assembly(); assembly.distro_uuid = Some(expected); let error = guest.assemble(&assembly).unwrap_err();