From 36d096fc0bbb813874280984c6e9130193cda5fc Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Thu, 22 Oct 2026 08:48:00 -0700 Subject: [PATCH 01/12] nvx-rs: keep the time locks and records out of the status directory A runtime may bind the time-status directory, /run/nvx/time, read-only into a workload's root. Its lock files and private records are owned by user ID 0, so a workload of user ID 0 could open them through that bind even without capabilities, and hold the exclusive lock of state updates or of the suppression record. That would block the time daemon's discipline and violation reports, and a restore's repair. Keep the state and suppression locks, the suppression record, and the restore record in /run/nvx, beside the port-window lock, so that the status directory holds only what any process may read: the published state, the daemon's process identifier, and their temporary files. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- nvx-rs/src/guest/time/system.rs | 52 ++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/nvx-rs/src/guest/time/system.rs b/nvx-rs/src/guest/time/system.rs index 953ec086d..7ac502f9d 100644 --- a/nvx-rs/src/guest/time/system.rs +++ b/nvx-rs/src/guest/time/system.rs @@ -50,16 +50,19 @@ pub(crate) const PORTB_TIME_WINDOW: u8 = 0x40; /// Snapshot-port value acknowledging completed restore repair. pub(crate) const SNAPSHOT_ACKNOWLEDGE: u8 = 2; -/// Top-level NVX runtime directory. +/// Top-level NVX runtime directory, which also holds the time tools' locks and private records. pub(crate) const RUN_DIR: &CStr = c"/run/nvx"; -/// Time-tool runtime directory. +/// Time-status directory, which holds only what any process may read: the published state and +/// the daemon's process identifier, and their temporary files. A runtime may bind it into a +/// workload's root, where even a workload of user ID 0 must reach no lock that it could hold, +/// so the locks and private records stay in [`RUN_DIR`]. pub(crate) const TIME_DIR: &CStr = c"/run/nvx/time"; /// Published time state file. pub(crate) const STATE_PATH: &CStr = c"/run/nvx/time/state"; /// Temporary state file atomically renamed into place. const STATE_TEMP_PATH: &CStr = c"/run/nvx/time/.state"; /// Advisory lock protecting state updates. -const STATE_LOCK_PATH: &CStr = c"/run/nvx/time/state.lock"; +const STATE_LOCK_PATH: &CStr = c"/run/nvx/time-state.lock"; /// Published time-daemon process identifier. pub(crate) const DAEMON_PID_PATH: &CStr = c"/run/nvx/time/daemon.pid"; /// Temporary daemon-PID file. @@ -68,13 +71,13 @@ pub(crate) const DAEMON_PID_TEMP_PATH: &CStr = c"/run/nvx/time/daemon.pid.tmp"; /// the published process identifier is recognized. pub(crate) const DAEMON_NAME: &CStr = c"nvx-time"; /// Saved stall-detector settings during capture/restore. -pub(crate) const SUPPRESSION_PATH: &CStr = c"/run/nvx/time/suppression"; +pub(crate) const SUPPRESSION_PATH: &CStr = c"/run/nvx/time-suppression"; /// Advisory lock protecting suppression state. -const SUPPRESSION_LOCK_PATH: &CStr = c"/run/nvx/time/suppression.lock"; +const SUPPRESSION_LOCK_PATH: &CStr = c"/run/nvx/time-suppression.lock"; /// Private restore record consumed by the daemon. -pub(crate) const RESTORE_PATH: &CStr = c"/run/nvx/time/restore"; +pub(crate) const RESTORE_PATH: &CStr = c"/run/nvx/time-restore"; /// Temporary restore-record file. -pub(crate) const RESTORE_TEMP_PATH: &CStr = c"/run/nvx/time/restore.tmp"; +pub(crate) const RESTORE_TEMP_PATH: &CStr = c"/run/nvx/time-restore.tmp"; /// Advisory lock serializing port-window transactions. pub(crate) const PORTB_LOCK_PATH: &CStr = c"/run/nvx/portb.lock"; /// Temporary mount point used to force an RCU grace period. @@ -503,11 +506,11 @@ fn describe(prefix: &[u8], error: Errno) -> Vec { /// A missing record succeeds when `missing_ok`. pub(crate) fn suppression_restore(missing_ok: bool) -> Result<(), Vec> { let _lock = lock_file(SUPPRESSION_LOCK_PATH) - .map_err(|error| describe(b"lock /run/nvx/time/suppression.lock: ", error))?; + .map_err(|error| describe(b"lock /run/nvx/time-suppression.lock: ", error))?; let text = match sys::read_file(SUPPRESSION_PATH) { Ok(text) => text, Err(Errno::ENOENT) if missing_ok => return Ok(()), - Err(error) => return Err(describe(b"read /run/nvx/time/suppression: ", error)), + Err(error) => return Err(describe(b"read /run/nvx/time-suppression: ", error)), }; for line in text .split(|byte| *byte == b'\n') @@ -528,7 +531,7 @@ pub(crate) fn suppression_restore(missing_ok: bool) -> Result<(), Vec> { } } sys::unlink(SUPPRESSION_PATH) - .map_err(|error| describe(b"remove /run/nvx/time/suppression: ", error)) + .map_err(|error| describe(b"remove /run/nvx/time-suppression: ", error)) } /// Reads a byte buffer using one scalar four-byte port access per chunk. @@ -1084,4 +1087,33 @@ pub(crate) mod tests { assert_eq!(PUBLIC_FILE_MODE.bits(), 0o644); assert_eq!(PRIVATE_FILE_MODE.bits(), 0o600); } + + #[test] + fn the_status_directory_holds_no_lock_or_private_record() { + let directory = TIME_DIR.to_bytes(); + let within = |path: &CStr| { + path.to_bytes() + .strip_prefix(directory) + .is_some_and(|rest| rest.starts_with(b"/")) + }; + for path in [ + STATE_PATH, + STATE_TEMP_PATH, + DAEMON_PID_PATH, + DAEMON_PID_TEMP_PATH, + ] { + assert!(within(path), "{path:?}"); + } + for path in [ + STATE_LOCK_PATH, + SUPPRESSION_PATH, + SUPPRESSION_LOCK_PATH, + RESTORE_PATH, + RESTORE_TEMP_PATH, + PORTB_LOCK_PATH, + ] { + assert!(!within(path), "{path:?}"); + assert!(path.to_bytes().starts_with(b"/run/nvx/"), "{path:?}"); + } + } } From 8cba325b61d40da7a306da9c14d7c99d30d75c34 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 21 Oct 2026 23:30:47 -0700 Subject: [PATCH 02/12] nvx-rs: enter guest executions with namespaces, a view, and an identity nvx::guest::entry lets any guest init or agent have a program that it spawns enter an execution, from typed inputs: a root, which is a mount, an identity, a hostname, a machine ID mode, a publication state, a time status source, and the device nodes of /dev. The child starts with clone3 in new mount, PID, and UTS namespaces, so that the program is PID 1 of its namespace with no process between it and its caller. It makes its mounts private, sets the hostname, and moves the root over /, never entering it by chroot alone, after it makes the root's mounts nosuid and nodev. It mounts a private procfs, a read-only sysfs, and a /dev tmpfs with only the allowed nodes, its own devpts, and an shm, so that every other mount is nodev. For user ID 0, the kernel's risky procfs entries and /sys/firmware are read-only or masked. It then takes its identity: the supplementary groups, the group and user IDs, and only the capabilities of an allowlist of 13, which only user ID 0 may hold, and of which a workload with published listeners holds neither the binding nor the raw-socket one, applied to every set, with no_new_privs. Every other descriptor of its caller closes as its program runs, and it leads a new session without a controlling terminal. In existing-account mode, the entry reads the identity's passwd and group entries below the root's descriptor, with bounds, and requires one passwd entry with the requested primary group, an existing group, and an absolute home; numeric mode takes the IDs as given. Nothing creates an account. A generated machine ID is one random ID that the caller writes once for the boot, readable by every execution whatever the umask, and the image's own is left alone. Entry::bind_runtime binds a generated ID over /etc/machine-id, and the time status at /run/nvx/time, read-only into the root once, through the mappings module's single bind, in the caller's mount namespace, of which each execution's starts as a copy, so that every later execution sees them; a symbolic link at /etc/machine-id gives way to an empty file first. Each step is a launch stage of nvx::guest::spawn::Command, whose child still makes only system calls over storage prepared before it starts, and reports the stage that failed. A command can now also lead a new session, and hold its child at a barrier until the caller has placed it. The barrier is a connected socket pair, so that dropping it cancels the launch at once by shutting down the caller's end, even while other children that have not run their programs yet hold copies of it. The child now joins its cgroup before its other stages, as an entry must while it still holds the privilege to. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- nvx-rs/README.md | 30 +- nvx-rs/src/guest/entry.rs | 1076 +++++++++++++++++++++ nvx-rs/src/guest/entry/tests.rs | 1098 ++++++++++++++++++++++ nvx-rs/src/guest/managed_agent/launch.rs | 7 +- nvx-rs/src/guest/mod.rs | 21 +- nvx-rs/src/guest/spawn.rs | 373 +++++++- nvx-rs/src/guest/sys/entry.rs | 852 +++++++++++++++++ nvx-rs/src/guest/sys/fd.rs | 24 + nvx-rs/src/guest/sys/mod.rs | 15 +- nvx-rs/src/guest/sys/spawn.rs | 231 ++++- nvx-rs/src/lib.rs | 22 +- 11 files changed, 3639 insertions(+), 110 deletions(-) create mode 100644 nvx-rs/src/guest/entry.rs create mode 100644 nvx-rs/src/guest/entry/tests.rs create mode 100644 nvx-rs/src/guest/sys/entry.rs diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 798678e21..3db8a5db8 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -92,7 +92,8 @@ nvx-rs/ | `nvx::guest::mappings` | `Mapper`: applies host mappings for any guest init or agent beneath the descriptor of a `TargetRoot`, the root of a mount of a whole file system, which `statmount` tells from a bind of a directory below it, such as a layered root's overlay or a guest's `/`, from a `SourceRoot`, such as the guest's mount of the host's export. A `Mapping` names a relative source, an absolute target, its `Access`, and optionally the `Kind` that its source must have. The mapper opens each component with `openat2`, through no symbolic link or magic link, creates what a target lacks only on the root's own mounts, which `statx` tells apart by their unique mount IDs, never inside another mapping, and checks kinds, overlaps, and the targets that the root profile's `Reserved` set holds, with the `/run/nvx/time` tree and its ancestors always. It attaches each mapping as a detached copy of its source's mount, `open_tree`, restricted `nosuid`, `nodev`, and read-only when asked, `mount_setattr`, before `move_mount` puts it at its target, then checks that it is there. Zero mappings do no work; the first that fails is reported with its index as an `ApplyError`, and latches the mapper. `Mapper::teardown` detaches a mapping that a rename moved and that the attach failed to detach, then unmounts the mappings in reverse order, and reports each failure. `bind` applies one runtime bind through the same walk and attach. Linux 6.8 or later. NVX's managed agent applies its host mapping table through it beneath `/`, and a layered `Root` its live shares and mappings | | `nvx::guest::nvxs` | `GuestConnection`: the guest's role of the NVXS session over a byte stream, such as the control console, on which NVX's managed agent and other guest agents run their sessions. It never reads past the record in progress, so a caller that polls the stream between records sees every record that it has not returned | | `nvx::guest::terminal` | `Terminal`: the terminal on which a guest agent serves its session, such as the VM's control console, as a byte stream. It opens the terminal, waiting while the kernel sets it up during boot, puts it in raw mode, so that it passes the session's bytes as they are, and drains it, so that it sends what the agent wrote before the agent ends the VM. NVX's managed agent and other guest agents serve their sessions on it | -| `nvx::guest::spawn` | `Command`: starts a program in a new process for any guest init or agent, with its arguments, environment, root and working directories, standard streams, inherited descriptors, cgroup, and process group, without NVX's control protocol, managed agent, or workload identity. Its `LaunchReport` tells whether the program started, or which `LaunchStage` failed and why. NVX's managed agent is one caller | +| `nvx::guest::spawn` | `Command`: starts a program in a new process for any guest init or agent, with its arguments, environment, root and working directories, standard streams, inherited descriptors, cgroup, process group or new session, a `Barrier` at which the program waits until the caller releases it, and an `entry::Prepared` execution to enter, without NVX's control protocol, managed agent, or workload identity. Its `LaunchReport` tells whether the program started, or which `LaunchStage` failed and why. NVX's managed agent is one caller | +| `nvx::guest::entry` | `Entry`: the execution that a program enters, for any guest init or agent, from typed inputs: a root, which is a mount, moved over `/` in new mount, PID, and UTS namespaces, where the program is PID 1 and leads a new session; a hostname; a fresh procfs, a read-only sysfs, and a `/dev` of its own with only the allowlisted `DeviceNode`s, a devpts, and a `/dev/shm`, every other mount `nodev`; the `MachineId`, the image's own or one generated by `MachineId::generate`, and the time status, which `Entry::bind_runtime` binds read-only into the root once, through `mappings::bind`, for every later execution; and the `Identity`: user and group IDs, supplementary groups, and `Capabilities` from an allowlist, which only user ID 0 may hold, without `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while the execution's listeners are `Publication::Published`, taken with `no_new_privs`. For user ID 0, the risky procfs and sysfs entries are read-only or masked. `Entry::prepare` checks it, reads the existing `Account` of the IDs from the root's account databases unless `AccountMode::Numeric` takes them as they are, and `spawn::Command::enter` runs it, with each step a `LaunchStage`. It reads no kernel command line or boot token and creates no account | | `nvx::guest::process` | `Child`: the status of one child that the caller names, by its pidfd or its process ID. It waits for that child alone, so another child's status stays for the caller; a caller that collects every status itself, such as a PID 1 that adopts orphans, hands each one to its `Child` with `observe`. Statuses are `std::process::ExitStatus`, real-time signals included | | `nvx::guest::cgroup` | `Layout`: the agent's cgroup and the workload cgroup, which the caller names, and which may not hold the agent's. `ExecutionCgroup`: the cgroup of one execution, the workload cgroup itself or a child of it, which it prepares with the caller's resource values, places processes in, checks for occupancy through `pids.current`, kills with `cgroup.kill`, and settles, over the execution's cgroup or the whole workload subtree, as the caller's `Scope` chooses. It holds its cgroups' directories open, and the process's removals and preparations of cgroups take turns, so that it never reaches another cgroup that takes one of their paths. Settling calls the caller's status collector between its checks, and collects no status itself | | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | @@ -888,19 +889,20 @@ The `guest` feature adds the programs that run in NVX's Linux guests: the `nvx::guest` module and one binary per program. The Alpine, Ubuntu, and Azure Linux images ship all nine at their established `/sbin/nvx-*` paths. The module's libraries are not programs: `terminal` opens the terminal on which an -agent serves its session, `spawn` starts a program in a new process, `process` -collects the status of a child that the caller names, `cgroup` places, kills, -and settles the processes of an execution, `exec` drives one program as an -execution, `mappings` mounts host mappings beneath the descriptors of the roots -that its caller names, and `root` bootstraps the guest for a layered root -through `cgroup`, assembles the root, mounts live shares and mappings in it -through `mappings`, and tears it down. `nvx-managed-agent` opens its control -terminal, and launches, drives, collects, and settles its workloads, through -them, and applies its host mapping table through `mappings`, beneath `/`, from -its mount of the host's export, and any other guest init or agent can use them -too. Any guest init or agent can also run the steps of `nvx-time` and -`nvx-reseed` in its own process, through the `nvx::guest::time` and -`nvx::guest::reseed` calls, on which both programs run. +agent serves its session, `spawn` starts a program in a new process, `entry` +has the program enter an execution's namespaces, view, root, and identity, +`process` collects the status of a child that the caller names, `cgroup` +places, kills, and settles the processes of an execution, `exec` drives one +program as an execution, `mappings` mounts host mappings beneath the +descriptors of the roots that its caller names, and `root` bootstraps the +guest for a layered root through `cgroup`, assembles the root, mounts live +shares and mappings in it through `mappings`, and tears it down. +`nvx-managed-agent` opens its control terminal, and launches, drives, collects, +and settles its workloads, through them, and applies its host mapping table +through `mappings`, beneath `/`, from its mount of the host's export, and any +other guest init or agent can use them too. Any guest init or agent can also +run the steps of `nvx-time` and `nvx-reseed` in its own process, through the +`nvx::guest::time` and `nvx::guest::reseed` calls, on which both programs run. Nor is the `gpt` library a program: it reads the partition table of a GPT disk's block device. diff --git a/nvx-rs/src/guest/entry.rs b/nvx-rs/src/guest/entry.rs new file mode 100644 index 000000000..7ff9818b9 --- /dev/null +++ b/nvx-rs/src/guest/entry.rs @@ -0,0 +1,1076 @@ +//! Entering an execution, for any guest init or agent: the namespaces, the view, the root, and +//! the identity in which a program runs, from the adapter's typed inputs. +//! +//! An [`Entry`] describes how an adapter's executions enter its root: +//! +//! - the root: the path of a mount, such as a [`Root`](super::root::Root)'s; +//! - the [`Identity`]: the user and group IDs, the supplementary groups, the [`Capabilities`], +//! from an allowlist of 13, and the [`AccountMode`]; +//! - the hostname, `nvx-sandbox` unless the adapter names another; +//! - the [`MachineId`] mode, the [`Publication`] state, the source of the time status, if any, +//! and the [`DeviceNode`]s of `/dev`. +//! +//! [`Entry::bind_runtime`] binds the runtime's files into the root once, for every execution +//! that enters it later. [`Entry::prepare`] checks the inputs for one execution, and reads and +//! checks the account in existing-account mode; [`Command::enter`] then has the child enter that +//! execution, in launch stages of [`spawn`](super::spawn), in the order of [`LaunchStage`]. +//! Nothing here reads the kernel command line or a boot token, and nothing creates an account. +//! +//! # The runtime's binds +//! +//! [`Entry::bind_runtime`] binds the file of a [generated](MachineId::Generated) machine ID over +//! the root's `/etc/machine-id`, and the time status's directory at the root's `/run/nvx/time`, +//! through [`mappings::bind`]: each is read-only, `nosuid`, and `nodev` before anything sees it, +//! and the walk creates what the root lacks only on the root's own mount, through no symbolic +//! link, and refuses a target of another kind than its source. A symbolic link at +//! `/etc/machine-id` itself, which no bind can cover, is replaced with an empty file first. In +//! [image](MachineId::Image) mode, the image's own `/etc/machine-id` is left alone. The time +//! status's directory must hold only what any execution may use, since one of user ID 0 can +//! open every file in it that user ID 0 owns, and hold its lock: the guest's time tools keep +//! their locks and private records outside it. +//! +//! The binds are made in the caller's mount namespace, of which each execution's starts as a +//! copy, so every execution that enters the root later sees them, and sees their sources as they +//! change, such as a machine ID that the guest rewrites in place after a restore. They last as +//! long as that namespace: a caller whose root is torn down in its own namespace after it, such +//! as an agent that an init outlives, binds them in a mount namespace of its own. +//! +//! # The execution +//! +//! The child starts with `clone3` in new mount, PID, and UTS namespaces: it is PID 1 of its own +//! PID namespace, with no process between it and its caller, so that its end ends every process +//! of the namespace. It then: +//! +//! 1. leads a new session, without a controlling terminal; +//! 2. waits at its [barrier](Command::barrier), if its command has one, until its caller has +//! placed it, then places its descriptors, and has every other descriptor of its caller's +//! close as its program runs, so that no descriptor of the old root, of the caller's control +//! session, or of another namespace or process reaches the program; +//! 3. joins its [cgroup](Command::cgroup), if its command names one; +//! 4. makes every mount of its mount namespace private, recursively, and sets its hostname; +//! 5. moves the root over `/`, after it makes every mount of the root's tree `nosuid` and +//! `nodev`. The root hides the former root, which no path reaches from it, `..` from the top +//! included, and the child never enters the root by `chroot` alone; +//! 6. mounts a procfs of its own PID namespace at `/proc`, `nosuid`, `nodev`, and `noexec`; +//! 7. mounts a read-only sysfs at `/sys`, `nosuid`, `nodev`, and `noexec`; +//! 8. mounts a tmpfs at `/dev`, `nosuid`, which holds only the allowlisted device nodes, the +//! links `fd`, `stdin`, `stdout`, `stderr`, and `ptmx`, a devpts instance of its own at +//! `/dev/pts`, `nosuid` and `noexec`, and a tmpfs at `/dev/shm`, `nosuid` and `nodev`. Every +//! other mount of the execution is `nodev`; +//! 9. takes its [identity](Identity), and sets `no_new_privs`; +//! 10. enters its working directory with that identity, and runs its program. +//! +//! For user ID 0, `/proc/sys`, `/proc/sysrq-trigger`, `/proc/bus`, `/proc/fs`, and `/proc/irq` +//! are read-only, and `/proc/kcore`, `/proc/keys`, `/proc/timer_list`, `/proc/timer_stats`, +//! `/proc/latency_stats`, `/proc/sched_debug`, `/proc/acpi`, `/proc/asound`, `/proc/scsi`, and +//! `/sys/firmware` are masked by an empty, read-only file or directory; an entry that the kernel +//! lacks needs neither. +//! +//! The view creates what the root lacks of `/proc`, `/sys`, and `/dev` on the root's own mount, +//! through no symbolic link and inside no other mount, and attaches its mounts there; a symbolic +//! link or another mount on the way fails the stage. The guest's kernel has no user namespaces, +//! so the entry creates none. +//! +//! ```no_run +//! use std::os::unix::ffi::OsStrExt; +//! +//! use nvx::guest::entry::{Entry, Identity, MachineId}; +//! use nvx::guest::spawn::Command; +//! +//! # fn main() -> Result<(), Box> { +//! let machine_id = MachineId::generate("/run/nvx/workload-machine-id")?; +//! let entry = Entry::new("/run/nvx/rootfs", Identity::new(65534, 65534)) +//! .hostname("sandbox") +//! .machine_id(machine_id) +//! .time_status("/run/nvx/time"); +//! // Once, before the first execution. +//! entry.bind_runtime()?; +//! let prepared = entry.prepare()?; +//! let home = prepared.account().map(|account| account.home().to_vec()); +//! let mut spawned = Command::new(c"/bin/true".to_owned(), vec![c"true".to_owned()]) +//! .barrier() +//! .enter(prepared) +//! .spawn()?; +//! let barrier = spawned.take_barrier().expect("the command has a barrier"); +//! // Place the child, by `spawned.id()`, in its cgroup; then: +//! barrier.release()?; +//! # Ok(()) +//! # } +//! ``` + +use std::fmt; +use std::io::{self, Read}; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; + +use super::mappings::{self, Access, Kind, Mapping, MappingError, SourceRoot, Step, TargetRoot}; +use super::parse::parse_decimal; +#[cfg(doc)] +use super::spawn::{Command, LaunchStage}; +use super::sys::{self, EntryChild, Errno, FileDescriptor, Mode, OpenFlags}; + +/// The hostname that an entry gives its execution unless the adapter names another. +pub const DEFAULT_HOSTNAME: &str = "nvx-sandbox"; +/// The longest hostname, in bytes: Linux's `HOST_NAME_MAX`. +pub const MAX_HOSTNAME_BYTES: usize = 64; +/// The most supplementary groups that an identity holds: Linux's `NGROUPS_MAX`. +pub const MAX_GROUPS: usize = 65536; +/// The largest account database, `/etc/passwd` or `/etc/group`, that existing-account mode reads. +pub const MAX_ACCOUNT_DATABASE_BYTES: u64 = 1 << 20; +/// The path of the passwd database below the root. +const PASSWD: &std::ffi::CStr = c"etc/passwd"; +/// The path of the group database below the root. +const GROUP: &std::ffi::CStr = c"etc/group"; +/// The ID that `setresuid` and `setresgid` read as "unchanged", which no identity may hold. +const UNCHANGED_ID: u32 = u32::MAX; +/// The mode of a generated machine ID's file. +const MACHINE_ID_MODE: u32 = 0o444; +/// Where [`Entry::bind_runtime`] binds a generated machine ID in the root. +const MACHINE_ID_TARGET: &str = "/etc/machine-id"; +/// Where [`Entry::bind_runtime`] binds the time status in the root. +const TIME_STATUS_TARGET: &str = "/run/nvx/time"; + +/// The names of the Linux capabilities, by number. +const CAPABILITY_NAMES: [&str; 41] = [ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_LINUX_IMMUTABLE", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_ADMIN", + "CAP_NET_RAW", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_SYS_MODULE", + "CAP_SYS_RAWIO", + "CAP_SYS_CHROOT", + "CAP_SYS_PTRACE", + "CAP_SYS_PACCT", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_NICE", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_MKNOD", + "CAP_LEASE", + "CAP_AUDIT_WRITE", + "CAP_AUDIT_CONTROL", + "CAP_SETFCAP", + "CAP_MAC_OVERRIDE", + "CAP_MAC_ADMIN", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + "CAP_BLOCK_SUSPEND", + "CAP_AUDIT_READ", + "CAP_PERFMON", + "CAP_BPF", + "CAP_CHECKPOINT_RESTORE", +]; + +/// A Linux capability, by its number. +/// +/// Any of the 64 that a capability set holds can be named, but an entry allows only those of +/// [`Capabilities::ALLOWED`], which have constants here. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Capability(u8); + +impl Capability { + /// `CAP_CHOWN`: changes the owners and groups of files. + pub const CHOWN: Self = Self(0); + /// `CAP_DAC_OVERRIDE`: bypasses file read, write, and execute permission checks. + pub const DAC_OVERRIDE: Self = Self(1); + /// `CAP_FOWNER`: bypasses the checks that require the caller to own a file. + pub const FOWNER: Self = Self(3); + /// `CAP_FSETID`: keeps the set-user-ID and set-group-ID bits of a file that changes. + pub const FSETID: Self = Self(4); + /// `CAP_KILL`: signals processes of other users. + pub const KILL: Self = Self(5); + /// `CAP_SETGID`: changes group IDs and supplementary groups. + pub const SETGID: Self = Self(6); + /// `CAP_SETUID`: changes user IDs. + pub const SETUID: Self = Self(7); + /// `CAP_SETPCAP`: drops capabilities from the bounding set, and changes securebits. + pub const SETPCAP: Self = Self(8); + /// `CAP_NET_BIND_SERVICE`: binds ports below the unprivileged port start. + pub const NET_BIND_SERVICE: Self = Self(10); + /// `CAP_NET_RAW`: uses raw and packet sockets. + pub const NET_RAW: Self = Self(13); + /// `CAP_SYS_CHROOT`: changes the root directory. + pub const SYS_CHROOT: Self = Self(18); + /// `CAP_AUDIT_WRITE`: writes records to the kernel's audit log. + pub const AUDIT_WRITE: Self = Self(29); + /// `CAP_SETFCAP`: sets file capabilities. + pub const SETFCAP: Self = Self(31); + + /// Returns the capability numbered `number`, or `None` above 63. + pub const fn new(number: u8) -> Option { + if number < 64 { + Some(Self(number)) + } else { + None + } + } + + /// Returns the capability's number. + pub const fn number(self) -> u8 { + self.0 + } + + /// Returns whether an entry allows the capability: whether + /// [`Capabilities::ALLOWED`] holds it. + pub const fn is_allowed(self) -> bool { + Capabilities::ALLOWED.contains(self) + } +} + +impl fmt::Display for Capability { + /// Writes the capability's name, such as `CAP_CHOWN`, or `capability N` for a number that + /// Linux does not name. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match CAPABILITY_NAMES.get(usize::from(self.0)) { + Some(name) => formatter.write_str(name), + None => write!(formatter, "capability {}", self.0), + } + } +} + +/// A set of capabilities. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Capabilities(u64); + +impl Capabilities { + /// No capability. + pub const EMPTY: Self = Self(0); + /// The 13 capabilities that an entry allows: `CHOWN`, `DAC_OVERRIDE`, `FOWNER`, `FSETID`, + /// `KILL`, `SETGID`, `SETUID`, `SETPCAP`, `SETFCAP`, `NET_BIND_SERVICE`, `NET_RAW`, + /// `SYS_CHROOT`, and `AUDIT_WRITE`. + pub const ALLOWED: Self = Self::EMPTY + .with(Capability::CHOWN) + .with(Capability::DAC_OVERRIDE) + .with(Capability::FOWNER) + .with(Capability::FSETID) + .with(Capability::KILL) + .with(Capability::SETGID) + .with(Capability::SETUID) + .with(Capability::SETPCAP) + .with(Capability::SETFCAP) + .with(Capability::NET_BIND_SERVICE) + .with(Capability::NET_RAW) + .with(Capability::SYS_CHROOT) + .with(Capability::AUDIT_WRITE); + /// The capabilities that an entry refuses while listeners are + /// [published](Publication::Published): `NET_BIND_SERVICE`, which binds the ports from which + /// the guest accepts announcements, and `NET_RAW`, which forges packets from them. + pub const PUBLISHED_REFUSED: Self = Self::EMPTY + .with(Capability::NET_BIND_SERVICE) + .with(Capability::NET_RAW); + + /// Returns this set and `capability`. + pub const fn with(self, capability: Capability) -> Self { + Self(self.0 | 1 << capability.0) + } + + /// Returns this set without `capability`. + pub const fn without(self, capability: Capability) -> Self { + Self(self.0 & !(1 << capability.0)) + } + + /// Returns whether the set holds `capability`. + pub const fn contains(self, capability: Capability) -> bool { + self.0 & 1 << capability.0 != 0 + } + + /// Returns whether the set is empty. + pub const fn is_empty(self) -> bool { + self.0 == 0 + } + + /// Returns the capabilities of the set, in the order of their numbers. + pub fn iter(self) -> impl Iterator { + (0..64) + .map(Capability) + .filter(move |capability| self.contains(*capability)) + } + + /// Returns the set's bits: bit `n` for capability `n`. + pub const fn bits(self) -> u64 { + self.0 + } +} + +impl FromIterator for Capabilities { + fn from_iter>(capabilities: I) -> Self { + capabilities + .into_iter() + .fold(Self::EMPTY, |set, capability| set.with(capability)) + } +} + +/// How an entry checks its identity's account. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum AccountMode { + /// The root's `/etc/passwd` holds exactly one entry for the user ID, whose primary group is + /// the identity's group ID, which `/etc/group` holds, and whose home is absolute. Each + /// database is read below the root, through no symbolic link, and refused unless it is a + /// regular file of at most [`MAX_ACCOUNT_DATABASE_BYTES`]. + #[default] + Existing, + /// The IDs are used as given, without an account. + Numeric, +} + +/// The identity of an execution: its IDs, its capabilities, and how its account is checked. +/// +/// An identity with user ID 0 keeps exactly its [`Capabilities`], from the allowlist, in its +/// bounding, permitted, and effective sets, so that its program keeps them too, and nothing +/// else. Any other identity gets none: its bounding, permitted, and effective sets are empty. +/// Every identity has empty inheritable and ambient sets and `no_new_privs`, so no program that +/// it runs gains a privilege, set-user-ID ones included. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Identity { + /// The user ID. + uid: u32, + /// The group ID. + gid: u32, + /// The supplementary groups. + groups: Vec, + /// The capabilities, for user ID 0. + capabilities: Capabilities, + /// How the account is checked. + account: AccountMode, +} + +impl Identity { + /// Returns the identity of user ID `uid` and group ID `gid`, without supplementary groups or + /// capabilities, whose account an entry checks in [existing](AccountMode::Existing) mode. + pub fn new(uid: u32, gid: u32) -> Self { + Self { + uid, + gid, + groups: Vec::new(), + capabilities: Capabilities::EMPTY, + account: AccountMode::Existing, + } + } + + /// Gives the identity the supplementary groups `groups`. + pub fn groups(mut self, groups: impl Into>) -> Self { + self.groups = groups.into(); + self + } + + /// Gives the identity the capabilities `capabilities`, which only user ID 0 may hold. + pub fn capabilities(mut self, capabilities: Capabilities) -> Self { + self.capabilities = capabilities; + self + } + + /// Checks the identity's account in `mode`. + pub fn account(mut self, mode: AccountMode) -> Self { + self.account = mode; + self + } + + /// Returns the user ID. + pub fn uid(&self) -> u32 { + self.uid + } + + /// Returns the group ID. + pub fn gid(&self) -> u32 { + self.gid + } + + /// Returns the supplementary groups. + pub fn supplementary_groups(&self) -> &[u32] { + &self.groups + } + + /// Returns the capabilities. + pub fn capability_set(&self) -> Capabilities { + self.capabilities + } + + /// Returns how the account is checked. + pub fn account_mode(&self) -> AccountMode { + self.account + } +} + +/// The machine ID that an execution sees at `/etc/machine-id`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[non_exhaustive] +pub enum MachineId { + /// The image's own `/etc/machine-id`, which the entry leaves untouched. + #[default] + Image, + /// The ID that the regular file at this path holds, such as one that [`MachineId::generate`] + /// wrote once for the boot, which [`Entry::bind_runtime`] binds read-only over + /// `/etc/machine-id`. The bind shows the file as it is, so an ID that the file's owner + /// rewrites in place, as after a restore, reaches every execution. + Generated(PathBuf), +} + +impl MachineId { + /// Generates a machine ID from the kernel's random number generator, writes it to a new file + /// at `path`, with mode 0444 whatever the process's umask, and returns the mode that binds + /// it. + /// + /// The ID is a random version 4 UUID, written as `/etc/machine-id` holds one: 32 lowercase + /// hexadecimal digits and a newline. A file, or a symbolic link, at `path` already fails the + /// call with `AlreadyExists`, so each boot generates its ID once. + pub fn generate(path: impl Into) -> io::Result { + let path = path.into(); + let mut id = [0u8; 16]; + getrandom::fill(&mut id).map_err(|error| { + error + .raw_os_error() + .map_or_else(|| io::Error::other(error), io::Error::from_raw_os_error) + })?; + let mode = Mode::from_bits_retain(MACHINE_ID_MODE); + let file = sys::open( + &sys::path_c_string(&path)?, + OpenFlags::O_WRONLY + | OpenFlags::O_CREAT + | OpenFlags::O_EXCL + | OpenFlags::O_NOFOLLOW + | OpenFlags::O_CLOEXEC, + mode, + )?; + // The umask narrows the mode that creates the file, which could keep executions of + // other users from reading the ID. + file.set_mode(mode)?; + file.write_all(&machine_id_text(id))?; + file.close()?; + Ok(Self::Generated(path)) + } +} + +/// Returns the text of the machine ID made of `id`'s random bits, as a version 4 UUID: 32 +/// lowercase hexadecimal digits and a newline. +fn machine_id_text(mut id: [u8; 16]) -> [u8; 33] { + /// The lowercase hexadecimal digits. + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + id[6] = (id[6] & 0x0f) | 0x40; + id[8] = (id[8] & 0x3f) | 0x80; + let mut text = [b'\n'; 33]; + for (index, byte) in id.iter().enumerate() { + text[2 * index] = DIGITS[usize::from(byte >> 4)]; + text[2 * index + 1] = DIGITS[usize::from(byte & 0x0f)]; + } + text +} + +/// Whether an execution's listeners are published, through private-network ingress or the +/// host's loopback, which decides whether the entry allows the capabilities of +/// [`Capabilities::PUBLISHED_REFUSED`]. +/// +/// The runtime owns `net.ipv4.ip_unprivileged_port_start`, which no execution can lower: its +/// `/proc/sys` is read-only for user ID 0, and no other user may write it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Publication { + /// Listeners are published, so the entry refuses `NET_BIND_SERVICE` and `NET_RAW`. + #[default] + Published, + /// No listener is published. + Unpublished, +} + +/// A device node that an execution's `/dev` may hold, which the character device of the same +/// name backs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DeviceNode { + /// `/dev/null`. + Null, + /// `/dev/zero`. + Zero, + /// `/dev/full`. + Full, + /// `/dev/random`. + Random, + /// `/dev/urandom`. + Urandom, + /// `/dev/tty`, the controlling terminal, of which the execution has none. + Tty, +} + +impl DeviceNode { + /// Every device node, which an execution's `/dev` holds by default. + pub const ALL: [Self; 6] = [ + Self::Null, + Self::Zero, + Self::Full, + Self::Random, + Self::Urandom, + Self::Tty, + ]; + + /// Returns the node's bit in the child's device set, in the order of the entry's device + /// table. + fn bit(self) -> u8 { + 1 << match self { + Self::Null => 0, + Self::Zero => 1, + Self::Full => 2, + Self::Random => 3, + Self::Urandom => 4, + Self::Tty => 5, + } + } +} + +/// How an adapter's executions enter its root: see the [module](self) documentation. +/// +/// An entry holds only the adapter's inputs, so that the adapter builds it once, binds its +/// runtime's files once, and [prepares](Self::prepare) it for each execution. +#[derive(Clone, Debug)] +pub struct Entry { + /// The root: the path of a mount. + root: PathBuf, + /// The identity. + identity: Identity, + /// The hostname. + hostname: Vec, + /// The machine ID mode. + machine_id: MachineId, + /// Whether listeners are published. + publication: Publication, + /// The directory of the time status, if the executions see one. + time_status: Option, + /// The device nodes of `/dev`, as [`DeviceNode::bit`]s. + devices: u8, +} + +impl Entry { + /// Describes executions that enter the mount at `root`, an absolute path, as `identity`, + /// with the hostname `nvx-sandbox`, the image's machine ID, published listeners, no time + /// status, and every [`DeviceNode`]. + pub fn new(root: impl Into, identity: Identity) -> Self { + Self { + root: root.into(), + identity, + hostname: DEFAULT_HOSTNAME.as_bytes().to_vec(), + machine_id: MachineId::Image, + publication: Publication::Published, + time_status: None, + devices: DeviceNode::ALL + .iter() + .fold(0, |devices, node| devices | node.bit()), + } + } + + /// Gives the executions the hostname `hostname`: 1 to 64 bytes, without a NUL. + pub fn hostname(mut self, hostname: impl Into>) -> Self { + self.hostname = hostname.into(); + self + } + + /// Gives the executions the machine ID of `mode`: a generated one once + /// [`bind_runtime`](Self::bind_runtime) has bound it. + pub fn machine_id(mut self, mode: MachineId) -> Self { + self.machine_id = mode; + self + } + + /// Tells whether the executions' listeners are published. + pub fn publication(mut self, publication: Publication) -> Self { + self.publication = publication; + self + } + + /// Has the executions see the directory `source`, which holds the guest's time status, + /// read-only at `/run/nvx/time`, once [`bind_runtime`](Self::bind_runtime) has bound it. + /// + /// The directory must hold only what any execution may use: one of user ID 0 can open, and + /// lock, every file in it that user ID 0 owns. The guest's time tools publish only their + /// state and their daemon's process ID in `/run/nvx/time`, and keep their locks and private + /// records outside it. + pub fn time_status(mut self, source: impl Into) -> Self { + self.time_status = Some(source.into()); + self + } + + /// Gives the executions' `/dev` exactly the device nodes `devices`. + pub fn devices(mut self, devices: impl IntoIterator) -> Self { + self.devices = devices + .into_iter() + .fold(0, |devices, node| devices | node.bit()); + self + } + + /// Returns the identity. + pub fn identity(&self) -> &Identity { + &self.identity + } + + /// Checks the entry's inputs, without reading anything: the root's path, the IDs, the + /// groups, the capabilities, and the hostname. + /// + /// The root's path must be absolute and hold no NUL, and no ID may be `u32::MAX`, which the + /// kernel reads as "unchanged". An identity holds at most [`MAX_GROUPS`] supplementary + /// groups. A capability outside [`Capabilities::ALLOWED`] is refused, as is any capability + /// of an identity whose user ID is not 0, and, while listeners are + /// [published](Publication::Published), `NET_BIND_SERVICE` and `NET_RAW`. + pub fn check(&self) -> Result<(), Error> { + let root = self.root.as_os_str().as_bytes(); + if root.first() != Some(&b'/') || root.contains(&0) { + return Err(Error::RootPath); + } + let identity = &self.identity; + if identity.uid == UNCHANGED_ID + || identity.gid == UNCHANGED_ID + || identity.groups.contains(&UNCHANGED_ID) + { + return Err(Error::InvalidId); + } + if identity.groups.len() > MAX_GROUPS { + return Err(Error::Groups); + } + if let Some(capability) = identity + .capabilities + .iter() + .find(|capability| !capability.is_allowed()) + { + return Err(Error::Capability(capability)); + } + if identity.uid != 0 && !identity.capabilities.is_empty() { + return Err(Error::NonRootCapabilities); + } + if self.publication == Publication::Published + && let Some(capability) = identity + .capabilities + .iter() + .find(|capability| Capabilities::PUBLISHED_REFUSED.contains(*capability)) + { + return Err(Error::Published(capability)); + } + if self.hostname.is_empty() + || self.hostname.len() > MAX_HOSTNAME_BYTES + || self.hostname.contains(&0) + { + return Err(Error::Hostname); + } + Ok(()) + } + + /// Prepares one execution: [checks](Self::check) the inputs, opens the root, and reads and + /// checks the account in [existing](AccountMode::Existing) mode, for [`Command::enter`]. + /// + /// The execution sees the runtime's binds that [`bind_runtime`](Self::bind_runtime) made in + /// the caller's mount namespace before it starts. + pub fn prepare(&self) -> Result { + self.check()?; + let root = sys::path_c_string(&self.root).map_err(|_| Error::RootPath)?; + let directory = sys::open( + &root, + OpenFlags::O_PATH | OpenFlags::O_DIRECTORY | OpenFlags::O_CLOEXEC, + Mode::empty(), + ) + .map_err(|error| Error::Root(error.into()))?; + let account = match self.identity.account { + AccountMode::Existing => { + Some(existing_account(&directory, &self.identity).map_err(Error::Account)?) + } + AccountMode::Numeric => None, + }; + Ok(Prepared { + child: EntryChild { + root, + hostname: self.hostname.clone().into_boxed_slice(), + devices: self.devices, + uid: self.identity.uid, + gid: self.identity.gid, + groups: self.identity.groups.clone().into_boxed_slice(), + capabilities: self.identity.capabilities.bits(), + #[cfg(test)] + simulated: None, + }, + account, + }) + } + + /// Binds the runtime's files into the root, in the caller's mount namespace, for every + /// execution that enters the root later, as the [module](self) describes: the file of a + /// [generated](MachineId::Generated) machine ID over `/etc/machine-id`, then the + /// [time status](Self::time_status) at `/run/nvx/time`. An entry with neither binds nothing. + /// + /// Each call binds again, over the earlier binds, so an adapter calls it once, before its + /// first execution. It needs the privilege to mount, and Linux 6.8 or later. + pub fn bind_runtime(&self) -> Result<(), Error> { + self.check()?; + let machine_id = match &self.machine_id { + MachineId::Image => None, + MachineId::Generated(path) => Some(path), + }; + if machine_id.is_none() && self.time_status.is_none() { + return Ok(()); + } + let root = TargetRoot::open(&self.root).map_err(Error::Root)?; + if let Some(path) = machine_id { + let bind = || bind_runtime_file(&root, path, MACHINE_ID_TARGET, Kind::File); + match bind() { + // No bind covers a link, so an empty file, which the bind creates, replaces it. + Err(MappingError::Link { path: link }) if link == Path::new(MACHINE_ID_TARGET) => { + remove_machine_id_link(&self.root) + .map_err(|_| Error::MachineId(MappingError::Link { path: link }))?; + bind() + } + bound => bound, + } + .map_err(Error::MachineId)?; + } + if let Some(source) = &self.time_status { + bind_runtime_file(&root, source, TIME_STATUS_TARGET, Kind::Directory) + .map_err(Error::TimeStatus)?; + } + Ok(()) + } +} + +/// Binds the file or directory at `path`, which must be of `kind`, read-only at `target` beneath +/// `root`, through [`mappings::bind`] from a source root at `path`'s parent directory. +fn bind_runtime_file( + root: &TargetRoot, + path: &Path, + target: &str, + kind: Kind, +) -> Result<(), MappingError> { + let source_error = |error: i32| MappingError::System { + step: Step::Source, + error, + }; + let (Some(parent), Some(name)) = (path.parent(), path.file_name()) else { + return Err(source_error(Errno::EINVAL as i32)); + }; + let source = SourceRoot::open(parent) + .map_err(|error| source_error(error.raw_os_error().unwrap_or(Errno::EIO as i32)))?; + let mapping = Mapping::new(name, target, Access::ReadOnly) + .map_err(|_| source_error(Errno::EINVAL as i32))? + .with_kind(kind); + mappings::bind(root, &source, &mapping) +} + +/// Removes the symbolic link at `/etc/machine-id` beneath `root`, reached through no other link +/// and on the root's own mount. +fn remove_machine_id_link(root: &Path) -> Result<(), Errno> { + let root = sys::path_c_string(root)?; + let directory = sys::open( + &root, + OpenFlags::O_PATH | OpenFlags::O_DIRECTORY | OpenFlags::O_NOFOLLOW | OpenFlags::O_CLOEXEC, + Mode::empty(), + )?; + let etc = sys::open_beneath( + &directory, + c"etc", + OpenFlags::O_PATH | OpenFlags::O_DIRECTORY, + Mode::empty(), + false, + )?; + let status = nix::sys::stat::fstatat( + etc.borrowed(), + c"machine-id", + nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW, + )?; + if status.st_mode & nix::libc::S_IFMT != nix::libc::S_IFLNK { + return Err(Errno::EEXIST); + } + nix::unistd::unlinkat( + etc.borrowed(), + c"machine-id", + nix::unistd::UnlinkatFlags::NoRemoveDir, + ) +} + +/// One execution that an [`Entry`] prepared, which [`Command::enter`] takes. +#[derive(Debug)] +pub struct Prepared { + /// What the child needs. + child: EntryChild, + /// The account that existing-account mode found. + account: Option, +} + +impl Prepared { + /// Returns the identity's account, in [existing](AccountMode::Existing) mode. + pub fn account(&self) -> Option<&Account> { + self.account.as_ref() + } + + /// Returns what the child needs. + pub(crate) fn into_child(self) -> EntryChild { + self.child + } +} + +/// The account of an identity, as its passwd entry names it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Account { + /// The user name. + name: Vec, + /// The home directory. + home: Vec, +} + +impl Account { + /// Returns the user name, as bytes, since passwd entries need not be UTF-8. + pub fn name(&self) -> &[u8] { + &self.name + } + + /// Returns the home directory, an absolute path. + pub fn home(&self) -> &[u8] { + &self.home + } +} + +/// An account database of existing-account mode. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Database { + /// `/etc/passwd`. + Passwd, + /// `/etc/group`. + Group, +} + +impl fmt::Display for Database { + /// Writes the database's path in the root. + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Passwd => "/etc/passwd", + Self::Group => "/etc/group", + }) + } +} + +/// Why existing-account mode refused an identity. +#[derive(Debug)] +#[non_exhaustive] +pub enum AccountError { + /// The database cannot be opened or read below the root, which a symbolic link on its path + /// fails too. + Read(Database, io::Error), + /// The database is not a regular file. + NotFile(Database), + /// The database is larger than [`MAX_ACCOUNT_DATABASE_BYTES`]. + TooLarge(Database), + /// No passwd entry has the user ID. + Missing, + /// Several passwd entries have the user ID. + Duplicate, + /// The passwd entry of the user ID lacks a field, or a name. + Malformed, + /// The passwd entry's primary group, this group ID, is not the identity's. + PrimaryGroup(u32), + /// No group entry has the group ID. + Group, + /// The passwd entry's home is not absolute. + Home, +} + +impl fmt::Display for AccountError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Read(database, error) => write!(formatter, "cannot read {database}: {error}"), + Self::NotFile(database) => write!(formatter, "{database} is not a regular file"), + Self::TooLarge(database) => write!( + formatter, + "{database} is larger than {MAX_ACCOUNT_DATABASE_BYTES} bytes" + ), + Self::Missing => formatter.write_str("no passwd entry has the user ID"), + Self::Duplicate => formatter.write_str("several passwd entries have the user ID"), + Self::Malformed => formatter.write_str("the passwd entry of the user ID is malformed"), + Self::PrimaryGroup(gid) => write!( + formatter, + "the passwd entry's primary group is {gid}, not the identity's" + ), + Self::Group => formatter.write_str("no group entry has the group ID"), + Self::Home => formatter.write_str("the passwd entry's home is not absolute"), + } + } +} + +impl std::error::Error for AccountError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Read(_, error) => Some(error), + _ => None, + } + } +} + +/// Why an entry refused to prepare an execution. +#[derive(Debug)] +#[non_exhaustive] +pub enum Error { + /// The root's path is not absolute, or holds a NUL. + RootPath, + /// The root cannot be opened as a directory, or, for [`Entry::bind_runtime`], is not the root + /// of a mount. + Root(io::Error), + /// The user ID, the group ID, or a supplementary group is `u32::MAX`, which the kernel reads + /// as "unchanged". + InvalidId, + /// The identity has more than [`MAX_GROUPS`] supplementary groups. + Groups, + /// The capability is outside [`Capabilities::ALLOWED`]. + Capability(Capability), + /// The identity's user ID is not 0, and it has capabilities. + NonRootCapabilities, + /// The capability is refused while listeners are published. + Published(Capability), + /// The hostname is empty, longer than [`MAX_HOSTNAME_BYTES`], or holds a NUL. + Hostname, + /// Existing-account mode refused the identity. + Account(AccountError), + /// [`Entry::bind_runtime`] cannot bind the machine ID's file over the root's + /// `/etc/machine-id`: it is missing or not a regular file, the target cannot hold it, or the + /// caller cannot mount. + MachineId(MappingError), + /// [`Entry::bind_runtime`] cannot bind the time status's directory at the root's + /// `/run/nvx/time`. + TimeStatus(MappingError), +} + +impl fmt::Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::RootPath => formatter.write_str("the root's path is not absolute"), + Self::Root(error) => write!(formatter, "cannot open the root: {error}"), + Self::InvalidId => formatter.write_str("an ID of the identity is invalid"), + Self::Groups => write!( + formatter, + "the identity has more than {MAX_GROUPS} supplementary groups" + ), + Self::Capability(capability) => write!(formatter, "{capability} is not allowed"), + Self::NonRootCapabilities => { + formatter.write_str("an identity other than user ID 0 has capabilities") + } + Self::Published(capability) => write!( + formatter, + "{capability} is not allowed while listeners are published" + ), + Self::Hostname => formatter.write_str("the hostname is invalid"), + Self::Account(error) => write!(formatter, "the account is unavailable: {error}"), + Self::MachineId(error) => write!(formatter, "cannot bind the machine ID: {error}"), + Self::TimeStatus(error) => write!(formatter, "cannot bind the time status: {error}"), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Root(error) => Some(error), + Self::MachineId(error) | Self::TimeStatus(error) => Some(error), + Self::Account(error) => Some(error), + _ => None, + } + } +} + +/// Reads the account databases below `root` and checks `identity`'s account, as +/// [`AccountMode::Existing`] describes. +fn existing_account(root: &FileDescriptor, identity: &Identity) -> Result { + let passwd = read_database(root, Database::Passwd)?; + let account = passwd_account(&passwd, identity.uid, identity.gid)?; + let group = read_database(root, Database::Group)?; + if !has_group(&group, identity.gid) { + return Err(AccountError::Group); + } + Ok(account) +} + +/// Reads `database` below `root`, through no symbolic link, if it is a regular file of at most +/// [`MAX_ACCOUNT_DATABASE_BYTES`]. +fn read_database(root: &FileDescriptor, database: Database) -> Result, AccountError> { + let path = match database { + Database::Passwd => PASSWD, + Database::Group => GROUP, + }; + let read = |error: Errno| AccountError::Read(database, error.into()); + // A FIFO must not block the open, which the file's kind then refuses. + let file = sys::open_beneath( + root, + path, + OpenFlags::O_RDONLY | OpenFlags::O_NONBLOCK | OpenFlags::O_NOCTTY, + Mode::empty(), + true, + ) + .map_err(read)?; + let (regular, size) = sys::file_size(&file).map_err(read)?; + if !regular { + return Err(AccountError::NotFile(database)); + } + if size > MAX_ACCOUNT_DATABASE_BYTES { + return Err(AccountError::TooLarge(database)); + } + let mut bytes = Vec::new(); + file.take(MAX_ACCOUNT_DATABASE_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|error| AccountError::Read(database, error))?; + if bytes.len() as u64 > MAX_ACCOUNT_DATABASE_BYTES { + return Err(AccountError::TooLarge(database)); + } + Ok(bytes) +} + +/// Returns the fields of each entry of an account database, with the ID in its third field: each +/// line that names one. As the C library's files database does, it skips blank lines and +/// comments, whose first character after blanks is `#`. +fn entries(database: &[u8]) -> impl Iterator)> { + database + .split(|&byte| byte == b'\n') + .filter(|line| !matches!(line.trim_ascii_start().first(), None | Some(b'#'))) + .filter_map(|line| { + let fields: Vec<&[u8]> = line.split(|&byte| byte == b':').collect(); + let id = parse_decimal(fields.get(2)?)?; + Some((id, fields)) + }) +} + +/// Finds the one passwd entry of `uid` in `passwd`, and checks that its primary group is `gid` +/// and its home absolute. +fn passwd_account(passwd: &[u8], uid: u32, gid: u32) -> Result { + let mut matching = entries(passwd).filter(|(id, _)| *id == u64::from(uid)); + let (_, fields) = matching.next().ok_or(AccountError::Missing)?; + if matching.next().is_some() { + return Err(AccountError::Duplicate); + } + let [name, _, _, primary, _, home, _] = fields[..] else { + return Err(AccountError::Malformed); + }; + if name.is_empty() { + return Err(AccountError::Malformed); + } + let primary = parse_decimal(primary) + .and_then(|primary| u32::try_from(primary).ok()) + .ok_or(AccountError::Malformed)?; + if primary != gid { + return Err(AccountError::PrimaryGroup(primary)); + } + if home.first() != Some(&b'/') { + return Err(AccountError::Home); + } + Ok(Account { + name: name.to_vec(), + home: home.to_vec(), + }) +} + +/// Returns whether `group` holds an entry of `gid`. +fn has_group(group: &[u8], gid: u32) -> bool { + entries(group).any(|(id, _)| id == u64::from(gid)) +} + +#[cfg(test)] +mod tests; diff --git a/nvx-rs/src/guest/entry/tests.rs b/nvx-rs/src/guest/entry/tests.rs new file mode 100644 index 000000000..f20de17ad --- /dev/null +++ b/nvx-rs/src/guest/entry/tests.rs @@ -0,0 +1,1098 @@ +//! Tests of the entry's checks, accounts, machine IDs, runtime binds, and launch stages. +//! +//! The launch tests run each entry without privileges: the child starts without new namespaces +//! and skips the system calls of every entry stage, but fails the one that a test names, so that +//! each stage reports its own failure. The tests whose names start with `privileged_` run the +//! entry for real, and only where the test process may mount, as root does. + +use std::ffi::CString; +use std::io::Read; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::PermissionsExt; + +use super::*; +use crate::guest::spawn::{Command, LaunchFailure, LaunchStage}; +use crate::guest::sys::ChildStatus; + +/// The passwd database of the tests' roots. +const PASSWD_DATABASE: &[u8] = b"root:x:0:0:root:/root:/bin/sh\n\ +# A comment names no account.\n\ +\n\ +nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin\n\ +workload:x:1000:1001::/home/workload:/bin/sh\n"; +/// The group database of the tests' roots. +const GROUP_DATABASE: &[u8] = b"root:x:0:\nworkload:x:1001:\nnogroup:x:65534:\n"; + +/// Returns an entry of a root at `/run/nvx/rootfs` as `identity`, whose account is numeric, so +/// that checking it reads nothing. +fn entry(identity: Identity) -> Entry { + Entry::new("/run/nvx/rootfs", identity.account(AccountMode::Numeric)) +} + +/// Returns a temporary root whose `etc` holds `passwd` and `group`, when given. +fn root_with(passwd: Option<&[u8]>, group: Option<&[u8]>) -> tempfile::TempDir { + let root = tempfile::tempdir().unwrap(); + let etc = root.path().join("etc"); + std::fs::create_dir(&etc).unwrap(); + if let Some(passwd) = passwd { + std::fs::write(etc.join("passwd"), passwd).unwrap(); + } + if let Some(group) = group { + std::fs::write(etc.join("group"), group).unwrap(); + } + root +} + +/// Returns the account error with which an entry of `root` refuses `identity`. +fn account_error(root: &Path, identity: Identity) -> AccountError { + match Entry::new(root, identity).prepare() { + Err(Error::Account(error)) => error, + other => panic!("expected an account error, not {other:?}"), + } +} + +#[test] +fn the_allowlist_holds_the_thirteen_capabilities() { + let numbers: Vec = Capabilities::ALLOWED + .iter() + .map(Capability::number) + .collect(); + assert_eq!(numbers, [0, 1, 3, 4, 5, 6, 7, 8, 10, 13, 18, 29, 31]); + let names: Vec = Capabilities::ALLOWED + .iter() + .map(|capability| capability.to_string()) + .collect(); + assert_eq!( + names, + [ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_SETGID", + "CAP_SETUID", + "CAP_SETPCAP", + "CAP_NET_BIND_SERVICE", + "CAP_NET_RAW", + "CAP_SYS_CHROOT", + "CAP_AUDIT_WRITE", + "CAP_SETFCAP", + ] + ); + assert_eq!(Capability::new(21).unwrap().to_string(), "CAP_SYS_ADMIN"); + assert_eq!( + Capability::new(40).unwrap().to_string(), + "CAP_CHECKPOINT_RESTORE" + ); + assert_eq!(Capability::new(41).unwrap().to_string(), "capability 41"); + assert_eq!(Capability::new(63).map(Capability::number), Some(63)); + assert_eq!(Capability::new(64), None); + assert!(Capability::SYS_CHROOT.is_allowed()); + assert!(!Capability::new(27).unwrap().is_allowed(), "CAP_MKNOD"); +} + +#[test] +fn capability_sets_add_remove_and_list_their_members() { + let set = Capabilities::EMPTY + .with(Capability::SETFCAP) + .with(Capability::CHOWN) + .with(Capability::CHOWN); + assert_eq!(set.bits(), 1 | 1 << 31); + assert!(set.contains(Capability::CHOWN) && set.contains(Capability::SETFCAP)); + assert!(!set.contains(Capability::KILL)); + assert_eq!( + set.iter().collect::>(), + [Capability::CHOWN, Capability::SETFCAP] + ); + assert_eq!(set.without(Capability::CHOWN).bits(), 1 << 31); + assert_eq!(set.without(Capability::KILL), set); + assert!( + set.without(Capability::CHOWN) + .without(Capability::SETFCAP) + .is_empty() + ); + let highest = Capability::new(63).unwrap(); + assert_eq!(Capabilities::EMPTY.with(highest).bits(), 1 << 63); + assert_eq!( + [Capability::KILL, highest] + .into_iter() + .collect::(), + Capabilities::EMPTY.with(Capability::KILL).with(highest) + ); + assert_eq!(Capabilities::default(), Capabilities::EMPTY); + assert_eq!( + Capabilities::PUBLISHED_REFUSED.iter().collect::>(), + [Capability::NET_BIND_SERVICE, Capability::NET_RAW] + ); +} + +#[test] +fn every_capability_outside_the_allowlist_is_refused() { + for number in 0..64 { + let capability = Capability::new(number).unwrap(); + let identity = Identity::new(0, 0).capabilities(Capabilities::EMPTY.with(capability)); + let result = entry(identity) + .publication(Publication::Unpublished) + .check(); + if Capabilities::ALLOWED.contains(capability) { + assert!(result.is_ok(), "{capability}: {result:?}"); + } else { + assert!( + matches!(result, Err(Error::Capability(refused)) if refused == capability), + "{capability}: {result:?}" + ); + } + } + // The first refused capability is reported, whatever else the set holds. + let identity = Identity::new(0, 0).capabilities( + Capabilities::ALLOWED + .with(Capability::new(21).unwrap()) + .with(Capability::new(27).unwrap()), + ); + let error = entry(identity).check().unwrap_err(); + assert!(matches!(error, Error::Capability(capability) if capability.number() == 21)); + assert_eq!(error.to_string(), "CAP_SYS_ADMIN is not allowed"); +} + +#[test] +fn only_user_id_zero_may_hold_capabilities() { + for uid in [1, 1000, 65534] { + let identity = + Identity::new(uid, 0).capabilities(Capabilities::EMPTY.with(Capability::KILL)); + assert!(matches!( + entry(identity).check(), + Err(Error::NonRootCapabilities) + )); + assert!(entry(Identity::new(uid, 0)).check().is_ok()); + } + // Root with no capability is a valid identity, and so is root with all thirteen while no + // listener is published. + assert!(entry(Identity::new(0, 0)).check().is_ok()); + let identity = Identity::new(0, 0).capabilities(Capabilities::ALLOWED); + assert!( + entry(identity) + .publication(Publication::Unpublished) + .check() + .is_ok() + ); + // A refused capability outranks the non-root rule. + let identity = + Identity::new(1000, 0).capabilities(Capabilities::EMPTY.with(Capability::new(21).unwrap())); + assert!(matches!(entry(identity).check(), Err(Error::Capability(_)))); +} + +#[test] +fn published_listeners_refuse_binding_and_raw_sockets() { + for capability in [Capability::NET_BIND_SERVICE, Capability::NET_RAW] { + let identity = Identity::new(0, 0) + .capabilities(Capabilities::EMPTY.with(capability).with(Capability::CHOWN)); + let published = entry(identity.clone()); + assert_eq!(published.publication, Publication::Published); + let error = published.check().unwrap_err(); + assert!( + matches!(error, Error::Published(refused) if refused == capability), + "{error:?}" + ); + assert_eq!( + error.to_string(), + format!("{capability} is not allowed while listeners are published") + ); + assert!( + entry(identity) + .publication(Publication::Unpublished) + .check() + .is_ok() + ); + } + let identity = Identity::new(0, 0).capabilities( + Capabilities::ALLOWED + .without(Capability::NET_BIND_SERVICE) + .without(Capability::NET_RAW), + ); + assert!(entry(identity).check().is_ok()); +} + +#[test] +fn invalid_roots_ids_groups_and_hostnames_are_refused() { + let check = |entry: Entry| entry.check(); + assert!(matches!( + check(Entry::new("relative", Identity::new(1, 1))), + Err(Error::RootPath) + )); + assert!(matches!( + check(Entry::new( + std::ffi::OsStr::from_bytes(b"/root\0"), + Identity::new(1, 1) + )), + Err(Error::RootPath) + )); + for identity in [ + Identity::new(u32::MAX, 1), + Identity::new(1, u32::MAX), + Identity::new(1, 1).groups([2, u32::MAX]), + ] { + assert!(matches!(check(entry(identity)), Err(Error::InvalidId))); + } + let groups: Vec = (0..u32::try_from(MAX_GROUPS).unwrap()).collect(); + assert!(check(entry(Identity::new(1, 1).groups(groups.clone()))).is_ok()); + let mut too_many = groups; + too_many.push(1 << 20); + assert!(matches!( + check(entry(Identity::new(1, 1).groups(too_many))), + Err(Error::Groups) + )); + for hostname in [b"".to_vec(), vec![b'h'; 65], b"host\0name".to_vec()] { + assert!(matches!( + check(entry(Identity::new(1, 1)).hostname(hostname)), + Err(Error::Hostname) + )); + } + assert!(check(entry(Identity::new(1, 1)).hostname(vec![b'h'; 64])).is_ok()); + assert_eq!( + entry(Identity::new(1, 1)).hostname, + DEFAULT_HOSTNAME.as_bytes() + ); +} + +#[test] +fn passwd_entries_are_parsed_and_checked() { + let account = passwd_account(PASSWD_DATABASE, 65534, 65534).unwrap(); + assert_eq!( + (account.name(), account.home()), + (b"nobody".as_slice(), b"/nonexistent".as_slice()) + ); + let account = passwd_account(PASSWD_DATABASE, 1000, 1001).unwrap(); + assert_eq!(account.home(), b"/home/workload"); + assert!(matches!( + passwd_account(PASSWD_DATABASE, 1001, 1001), + Err(AccountError::Missing) + )); + assert!(matches!( + passwd_account(b"", 0, 0), + Err(AccountError::Missing) + )); + let duplicate = [PASSWD_DATABASE, b"other:x:1000:1001::/home/other:/bin/sh\n"].concat(); + assert!(matches!( + passwd_account(&duplicate, 1000, 1001), + Err(AccountError::Duplicate) + )); + // Leading zeros are decimal, as the shell init's comparison read them. + let padded = b"padded:x:0065534:65534::/home/padded:/bin/sh"; + assert!(passwd_account(padded, 65534, 65534).is_ok()); + assert!(matches!( + passwd_account(PASSWD_DATABASE, 1000, 1000), + Err(AccountError::PrimaryGroup(1001)) + )); + for malformed in [ + b"short:x:1000:1001::/home/short".as_slice(), + b"long:x:1000:1001::/home/long:/bin/sh:extra", + b":x:1000:1001::/home/nameless:/bin/sh", + b"group:x:1000:not-a-number::/home/group:/bin/sh", + b"group:x:1000:-1::/home/group:/bin/sh", + b"group:x:1000:4294967296::/home/group:/bin/sh", + ] { + assert!( + matches!( + passwd_account(malformed, 1000, 1001), + Err(AccountError::Malformed) + ), + "{}", + String::from_utf8_lossy(malformed) + ); + } + for home in [b"".as_slice(), b"relative/home", b"~"] { + let line = [b"workload:x:1000:1001::".as_slice(), home, b":/bin/sh"].concat(); + assert!(matches!( + passwd_account(&line, 1000, 1001), + Err(AccountError::Home) + )); + } + // Blank lines, comments, and lines whose third field is no ID name no account. + assert!(matches!( + passwd_account( + b"# 1000:1000:1000\n #x:x:1000:1000::/home/x:/bin/sh\n\n+::::::\n", + 1000, + 1000 + ), + Err(AccountError::Missing) + )); +} + +#[test] +fn group_entries_are_found_by_id() { + assert!(has_group(GROUP_DATABASE, 1001)); + assert!(has_group(GROUP_DATABASE, 65534)); + assert!(!has_group(GROUP_DATABASE, 1000)); + assert!(!has_group(b"# x:x:1000:\n\n", 1000)); +} + +#[test] +fn existing_accounts_are_read_below_the_root() { + let root = root_with(Some(PASSWD_DATABASE), Some(GROUP_DATABASE)); + let prepared = Entry::new(root.path(), Identity::new(1000, 1001)) + .prepare() + .unwrap(); + let account = prepared.account().unwrap(); + assert_eq!( + (account.name(), account.home()), + (b"workload".as_slice(), b"/home/workload".as_slice()) + ); + let child = prepared.into_child(); + assert_eq!(child.root.as_bytes(), root.path().as_os_str().as_bytes()); + assert_eq!((child.uid, child.gid, child.capabilities), (1000, 1001, 0)); + assert_eq!(child.devices, 0b11_1111); + assert_eq!(&*child.hostname, DEFAULT_HOSTNAME.as_bytes()); + + assert!(matches!( + account_error(root.path(), Identity::new(1000, 1000)), + AccountError::PrimaryGroup(1001) + )); + assert!(matches!( + account_error(root.path(), Identity::new(2000, 2000)), + AccountError::Missing + )); + let missing_group = root_with(Some(PASSWD_DATABASE), Some(b"root:x:0:\n")); + assert!(matches!( + account_error(missing_group.path(), Identity::new(1000, 1001)), + AccountError::Group + )); + // Numeric mode reads no database. + let empty = root_with(None, None); + let prepared = Entry::new( + empty.path(), + Identity::new(1000, 1001).account(AccountMode::Numeric), + ) + .prepare() + .unwrap(); + assert!(prepared.account().is_none()); + assert!(matches!( + account_error(empty.path(), Identity::new(1000, 1001)), + AccountError::Read(Database::Passwd, error) + if error.raw_os_error() == Some(Errno::ENOENT as i32) + )); + let no_group = root_with(Some(PASSWD_DATABASE), None); + assert!(matches!( + account_error(no_group.path(), Identity::new(1000, 1001)), + AccountError::Read(Database::Group, _) + )); +} + +#[test] +fn account_databases_are_bounded_regular_files_reached_through_no_link() { + let identity = || Identity::new(1000, 1001); + // A symbolic link, as the database or on its path, fails the read, even one that leads to a + // valid database inside the root. + let root = root_with(None, Some(GROUP_DATABASE)); + std::fs::write(root.path().join("passwd"), PASSWD_DATABASE).unwrap(); + std::os::unix::fs::symlink("../passwd", root.path().join("etc/passwd")).unwrap(); + assert!(matches!( + account_error(root.path(), identity()), + AccountError::Read(Database::Passwd, error) + if error.raw_os_error() == Some(Errno::ELOOP as i32) + )); + let root = tempfile::tempdir().unwrap(); + let real = root_with(Some(PASSWD_DATABASE), Some(GROUP_DATABASE)); + std::os::unix::fs::symlink(real.path().join("etc"), root.path().join("etc")).unwrap(); + assert!(matches!( + account_error(root.path(), identity()), + AccountError::Read(Database::Passwd, error) + if error.raw_os_error() == Some(Errno::ELOOP as i32) + )); + // A directory or a FIFO is no database, and the FIFO does not block the read. + let root = root_with(None, Some(GROUP_DATABASE)); + std::fs::create_dir(root.path().join("etc/passwd")).unwrap(); + assert!(matches!( + account_error(root.path(), identity()), + AccountError::NotFile(Database::Passwd) + )); + let root = root_with(Some(PASSWD_DATABASE), None); + let fifo = sys::path_c_string(&root.path().join("etc/group")).unwrap(); + sys::create_fifo(&fifo, Mode::from_bits_retain(0o644)).unwrap(); + assert!(matches!( + account_error(root.path(), identity()), + AccountError::NotFile(Database::Group) + )); + // A database of exactly the bound is read, and one byte more is refused. + let mut database = PASSWD_DATABASE.to_vec(); + database.resize(usize::try_from(MAX_ACCOUNT_DATABASE_BYTES).unwrap(), b'\n'); + let root = root_with(Some(&database), Some(GROUP_DATABASE)); + assert!(Entry::new(root.path(), identity()).prepare().is_ok()); + database.push(b'\n'); + std::fs::write(root.path().join("etc/passwd"), &database).unwrap(); + assert!(matches!( + account_error(root.path(), identity()), + AccountError::TooLarge(Database::Passwd) + )); +} + +#[test] +fn an_entry_needs_a_root_directory() { + let file = tempfile::NamedTempFile::new().unwrap(); + let result = Entry::new( + "/nonexistent/root", + Identity::new(1, 1).account(AccountMode::Numeric), + ) + .prepare(); + assert!(matches!(result, Err(Error::Root(_))), "{result:?}"); + let result = Entry::new( + file.path(), + Identity::new(1, 1).account(AccountMode::Numeric), + ) + .prepare(); + assert!( + matches!(&result, Err(Error::Root(error)) + if error.raw_os_error() == Some(Errno::ENOTDIR as i32)), + "{result:?}" + ); +} + +#[test] +fn machine_ids_are_version_4_uuids_in_lowercase_hexadecimal() { + assert_eq!( + &machine_id_text([0; 16]), + b"00000000000040008000000000000000\n" + ); + assert_eq!( + &machine_id_text([0xff; 16]), + b"ffffffffffff4fffbfffffffffffffff\n" + ); + assert_eq!( + &machine_id_text([ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, + 0xcd, 0xef + ]), + b"0123456789ab4def8123456789abcdef\n" + ); +} + +#[test] +fn a_machine_id_is_generated_once_per_file() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("machine-id"); + let mode = MachineId::generate(&path).unwrap(); + assert_eq!(mode, MachineId::Generated(path.clone())); + let text = std::fs::read(&path).unwrap(); + assert_eq!(text.len(), 33); + assert_eq!(text[32], b'\n'); + assert!( + text[..32] + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) + ); + assert_eq!(text[12], b'4'); + assert!(matches!(text[16], b'8' | b'9' | b'a' | b'b')); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o444 + ); + let error = MachineId::generate(&path).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + assert_eq!(std::fs::read(&path).unwrap(), text); + // A link at the path is never followed. + let target = directory.path().join("target"); + let link = directory.path().join("link"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + assert_eq!( + MachineId::generate(&link).unwrap_err().kind(), + io::ErrorKind::AlreadyExists + ); + assert!(!target.exists()); + let other = directory.path().join("other"); + MachineId::generate(&other).unwrap(); + assert_ne!(std::fs::read(&other).unwrap(), text); +} + +#[test] +fn a_generated_machine_id_has_mode_0444_whatever_the_umask() { + // The umask is the process's, so the test runs alone in a process of its own. + if crate::guest::managed_agent::isolate_process_test( + "NVX_ENTRY_UMASK_TEST_CHILD", + "guest::entry::tests::a_generated_machine_id_has_mode_0444_whatever_the_umask", + ) { + return; + } + nix::sys::stat::umask(sys::Mode::from_bits_truncate(0o077)); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("machine-id"); + MachineId::generate(&path).unwrap(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o7777, + 0o444 + ); +} + +#[test] +fn runtime_binds_need_a_mount_root_unless_there_are_none() { + let root = root_with(None, None); + let identity = || Identity::new(1, 1).account(AccountMode::Numeric); + // The image's machine ID and no time status bind nothing, so any root does. + Entry::new(root.path(), identity()).bind_runtime().unwrap(); + let machine_id = MachineId::generate(root.path().join("machine-id")).unwrap(); + let result = Entry::new(root.path(), identity()) + .machine_id(machine_id) + .bind_runtime(); + assert!( + matches!(&result, Err(Error::Root(error)) if error.kind() == io::ErrorKind::InvalidInput), + "{result:?}" + ); + let result = Entry::new(root.path(), identity()) + .time_status(root.path().join("etc")) + .bind_runtime(); + assert!(matches!(result, Err(Error::Root(_))), "{result:?}"); + // The inputs are checked first. + let result = Entry::new(root.path(), identity()) + .hostname("") + .time_status(root.path().join("etc")) + .bind_runtime(); + assert!(matches!(result, Err(Error::Hostname)), "{result:?}"); +} + +#[test] +fn only_a_link_at_the_machine_id_is_removed() { + let root = root_with(None, None); + let target = root.path().join("etc/machine-id"); + let outside = tempfile::NamedTempFile::new().unwrap(); + std::os::unix::fs::symlink(outside.path(), &target).unwrap(); + remove_machine_id_link(root.path()).unwrap(); + assert!(std::fs::symlink_metadata(&target).is_err()); + assert!(outside.path().exists()); + assert_eq!(remove_machine_id_link(root.path()), Err(Errno::ENOENT)); + std::fs::write(&target, b"image\n").unwrap(); + assert_eq!(remove_machine_id_link(root.path()), Err(Errno::EEXIST)); + assert_eq!(std::fs::read(&target).unwrap(), b"image\n"); + // A link on the way to it is not followed. + let linked = tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink(root.path().join("etc"), linked.path().join("etc")).unwrap(); + assert_eq!(remove_machine_id_link(linked.path()), Err(Errno::ELOOP)); + assert_eq!(std::fs::read(&target).unwrap(), b"image\n"); +} + +#[test] +fn device_sets_select_their_nodes() { + let entry = entry(Identity::new(1, 1)); + assert_eq!(entry.devices, 0b11_1111); + assert_eq!(entry.clone().devices([]).devices, 0); + assert_eq!( + entry + .devices([DeviceNode::Null, DeviceNode::Tty, DeviceNode::Null]) + .devices, + 0b10_0001 + ); +} + +/// Returns an execution, prepared without privileges, whose child skips every entry stage's +/// system calls and fails `stage` with `error`. +fn simulated(stage: LaunchStage, error: Errno) -> Prepared { + Prepared { + child: EntryChild { + root: c"/".to_owned(), + hostname: b"simulated".to_vec().into_boxed_slice(), + devices: 0, + uid: 0, + gid: 0, + groups: Box::new([]), + capabilities: 0, + simulated: Some((stage, error)), + }, + account: None, + } +} + +/// Returns a command that runs the shell script `script`. +fn shell(script: &str) -> Command { + Command::new( + c"/bin/sh".to_owned(), + vec![ + c"sh".to_owned(), + c"-c".to_owned(), + CString::new(script).unwrap(), + ], + ) +} + +/// Spawns `command`, which waits at a barrier, releases it, and returns the child's launch +/// failure, status, and standard output. +fn run_released(command: Command) -> (Option, Option, Vec) { + let (mut output, writer) = std::io::pipe().unwrap(); + let mut spawned = command.barrier().stdout(writer).spawn().unwrap(); + let process = spawned.process(); + spawned.take_barrier().unwrap().release().unwrap(); + let failure = spawned.into_report().wait().unwrap(); + let mut bytes = Vec::new(); + output.read_to_end(&mut bytes).unwrap(); + (failure, sys::wait(process), bytes) +} + +#[test] +fn each_entry_stage_reports_its_own_failure_and_runs_nothing() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("ran"); + let script = format!("touch '{}'", marker.display()); + let stages = [ + (LaunchStage::Propagation, Errno::EINVAL), + (LaunchStage::Hostname, Errno::EPERM), + (LaunchStage::MoveRoot, Errno::EBUSY), + (LaunchStage::Proc, Errno::ENODEV), + (LaunchStage::Sys, Errno::EACCES), + (LaunchStage::Devices, Errno::ENOSPC), + (LaunchStage::Bounding, Errno::EPERM), + (LaunchStage::Identity, Errno::EINVAL), + (LaunchStage::Capabilities, Errno::EPERM), + (LaunchStage::NoNewPrivileges, Errno::EINVAL), + ]; + for (stage, error) in stages { + let (failure, status, _) = run_released(shell(&script).enter(simulated(stage, error))); + let failure = failure.unwrap_or_else(|| panic!("{stage} did not fail")); + assert_eq!( + (failure.stage(), failure.raw_os_error()), + (stage, error as i32) + ); + assert_eq!(status, Some(ChildStatus::Exited(125)), "{stage}"); + assert!(!marker.exists(), "{stage} ran the program"); + assert_eq!( + failure.to_string(), + format!( + "the {stage} launch stage failed: {}", + io::Error::from_raw_os_error(error as i32) + ) + ); + } +} + +#[test] +fn an_entered_program_leads_a_session_without_a_terminal_or_the_callers_descriptors() { + // A descriptor that the caller opened without close-on-exec, as another thread of a caller + // might, and one that the command passes on. + let leaked = nix::fcntl::open(c"/dev/null", OpenFlags::O_RDONLY, Mode::empty()).unwrap(); + let passed = tempfile::tempfile().unwrap(); + let leaked_number = std::os::fd::AsRawFd::as_raw_fd(&leaked); + let passed_number = std::os::fd::AsRawFd::as_raw_fd(&passed); + // The simulated stage is no entry stage, so every entry stage succeeds. + let script = format!( + "read -r pid comm state ppid pgrp session tty rest = ids.split(' ').collect(); + // `sh -c` reads its own status file: the entry's child leads its session and process group, + // without a terminal. + assert_eq!(ids[1], ids[0], "{output}"); + assert_eq!(ids[2], ids[0], "{output}"); + assert_eq!(ids[3], "0", "{output}"); + assert_eq!(descriptors, "passed"); +} + +#[test] +fn an_entry_needs_a_placement_and_neither_a_root_nor_a_check() { + let invalid = |command: Command| { + let error = command.spawn().unwrap_err(); + assert_eq!(error.raw_os_error(), Some(Errno::EINVAL as i32)); + }; + let prepared = || simulated(LaunchStage::Credentials, Errno::EPERM); + invalid(shell("exit 0").enter(prepared())); + let root = std::fs::File::open("/").unwrap(); + invalid(shell("exit 0").barrier().root(root).enter(prepared())); + invalid( + shell("exit 0") + .barrier() + .working_directory(c"/".to_owned()) + .check_working_directory_as(1, 1) + .enter(prepared()), + ); + // A cgroup places the child as well as a barrier. + let directory = tempfile::tempdir().unwrap(); + std::fs::write(directory.path().join("cgroup.procs"), b"").unwrap(); + let cgroup = crate::guest::spawn::Cgroup::open(directory.path()).unwrap(); + let spawned = shell("exit 0") + .cgroup(cgroup) + .enter(prepared()) + .spawn() + .unwrap(); + let process = spawned.process(); + assert_eq!(spawned.into_report().wait().unwrap(), None); + assert_eq!(sys::wait(process), Some(ChildStatus::Exited(0))); + let written = std::fs::read_to_string(directory.path().join("cgroup.procs")).unwrap(); + assert_eq!(written, format!("{}\n", process.raw())); +} + +/// A probe that prints what an entered program sees of its identity and view, one `name=value` +/// line each. +const PROBE: &str = r#" +field() { + while IFS= read -r line; do + case "$line" in "$1:"*) set -- ${line#*:}; printf '%s' "$*"; return ;; esac + done /dev/null; then echo "null=writable"; fi +if mknod /tmp/node c 1 3 2>/dev/null; then echo "mknod=allowed"; else echo "mknod=refused"; fi +if (echo 1023 >/proc/sys/net/ipv4/ip_unprivileged_port_start) 2>/dev/null; then + echo "port-start=written" +else + echo "port-start=refused" +fi +echo "kcore=$(wc -c 2>/dev/null /dev/null | tr '\n' ' ')" +echo "leaked=$(test -e /proc/self/fd/$LEAKED && echo open || echo closed)" +echo "root=$(ls -A / | tr '\n' ' ')" +if [ "$(id -u)" = 0 ] && command -v python3 >/dev/null; then + python3 -c ' +import os +os.makedirs("/tmp/jail", exist_ok=True) +try: + os.chroot("/tmp/jail") +except PermissionError: + print("escape=refused") + raise SystemExit +for _ in range(16): + os.chdir("..") +os.chroot(".") +print("escape=" + " ".join(sorted(os.listdir("/"))) + " ") +' +fi +"#; + +/// Returns the values of the probe's lines named `name`, in their order. +fn values<'a>(output: &'a str, name: &str) -> Vec<&'a str> { + output + .lines() + .filter_map(|line| line.strip_prefix(name)?.strip_prefix('=')) + .collect() +} + +/// Returns the probe's one value named `name`. +fn value<'a>(output: &'a str, name: &str) -> &'a str { + match values(output, name)[..] { + [value] => value, + ref found => panic!("{name}: {found:?} in\n{output}"), + } +} + +/// Returns the options of the one mount at `point` that the probe lists. +fn mount_options<'a>(output: &'a str, point: &str) -> Vec<&'a str> { + let mounts: Vec<&str> = values(output, "mount") + .into_iter() + .filter_map(|mount| mount.strip_prefix(point)?.strip_prefix(' ')) + .collect(); + match mounts[..] { + [options] => options.split(',').collect(), + ref found => panic!("mounts at {point}: {found:?} in\n{output}"), + } +} + +/// Runs the probe as an execution of `entry`, with a descriptor of the caller's root leaked to +/// it, and returns its output. +fn probe(entry: &Entry) -> String { + let leaked = nix::fcntl::open( + c"/", + OpenFlags::O_RDONLY | OpenFlags::O_DIRECTORY, + Mode::empty(), + ) + .unwrap(); + let number = std::os::fd::AsRawFd::as_raw_fd(&leaked); + let (mut output, writer) = std::io::pipe().unwrap(); + let mut spawned = shell(PROBE) + .environment(vec![ + c"PATH=/usr/sbin:/usr/bin:/sbin:/bin".to_owned(), + c"LC_ALL=C".to_owned(), + CString::new(format!("LEAKED={number}")).unwrap(), + ]) + .stdout(writer) + .barrier() + .enter(entry.prepare().unwrap()) + .spawn() + .unwrap(); + let process = spawned.process(); + spawned.take_barrier().unwrap().release().unwrap(); + let failure = spawned.into_report().wait().unwrap(); + let mut bytes = Vec::new(); + output.read_to_end(&mut bytes).unwrap(); + drop(leaked); + let output = String::from_utf8(bytes).unwrap(); + assert_eq!(failure, None, "{output}"); + assert_eq!(sys::wait(process), Some(ChildStatus::Exited(0)), "{output}"); + output +} + +/// Mounts a root for the privileged entries in `base`, a tmpfs that holds the account databases +/// and reaches the caller's programs through a read-only bind of `/usr`, and returns its path. +fn mount_test_root(base: &Path) -> PathBuf { + let root = base.join("root"); + std::fs::create_dir(&root).unwrap(); + let path = |path: &Path| sys::path_c_string(path).unwrap(); + sys::mount( + Some(c"tmpfs"), + &path(&root), + Some(c"tmpfs"), + sys::MountFlags::empty(), + Some(c"mode=0755"), + ) + .unwrap(); + for directory in ["usr", "etc", "tmp"] { + std::fs::create_dir(root.join(directory)).unwrap(); + } + std::fs::set_permissions(root.join("tmp"), std::fs::Permissions::from_mode(0o1777)).unwrap(); + for (link, target) in [ + ("bin", "usr/bin"), + ("sbin", "usr/sbin"), + ("lib", "usr/lib"), + ("lib64", "usr/lib64"), + ] { + std::os::unix::fs::symlink(target, root.join(link)).unwrap(); + } + sys::mount( + Some(c"/usr"), + &path(&root.join("usr")), + None, + sys::MountFlags::MS_BIND | sys::MountFlags::MS_REC, + None, + ) + .unwrap(); + sys::mount( + None, + &path(&root.join("usr")), + None, + sys::MountFlags::MS_BIND | sys::MountFlags::MS_REMOUNT | sys::MountFlags::MS_RDONLY, + None, + ) + .unwrap(); + std::fs::write(root.join("etc/passwd"), PASSWD_DATABASE).unwrap(); + std::fs::write(root.join("etc/group"), GROUP_DATABASE).unwrap(); + root +} + +/// Checks what a workload of `entry`, which `output` probed, sees of the view that every +/// identity shares. +fn check_view(output: &str, machine_id: &str) { + assert_eq!(value(output, "pid"), "1", "{output}"); + assert_eq!(value(output, "first-process"), "1", "{output}"); + assert_eq!(value(output, "NoNewPrivs"), "1", "{output}"); + assert_eq!(value(output, "CapInh"), "0000000000000000", "{output}"); + assert_eq!(value(output, "CapAmb"), "0000000000000000", "{output}"); + assert_eq!(value(output, "hostname"), "entry-test", "{output}"); + assert_eq!(value(output, "machine-id"), machine_id, "{output}"); + assert_eq!(value(output, "time"), "time-state", "{output}"); + assert_eq!( + value(output, "dev"), + "fd full null ptmx pts random shm stderr stdin stdout tty urandom zero" + ); + assert_eq!(value(output, "pts"), "ptmx"); + assert_eq!(value(output, "null"), "writable"); + assert_eq!(value(output, "mknod"), "refused"); + assert_eq!(value(output, "port-start"), "refused"); + assert_eq!(value(output, "leaked"), "closed"); + assert_eq!( + value(output, "root"), + "bin dev etc lib lib64 proc run sbin sys tmp usr " + ); + for (point, required) in [ + ("/", &["nosuid", "nodev"][..]), + ("/usr", &["ro", "nosuid", "nodev"]), + ("/proc", &["nosuid", "nodev", "noexec"]), + ("/sys", &["ro", "nosuid", "nodev", "noexec"]), + ("/dev", &["nosuid"]), + ("/dev/pts", &["nosuid", "noexec"]), + ("/dev/shm", &["nosuid", "nodev"]), + ("/etc/machine-id", &["ro", "nosuid", "nodev"]), + ("/run/nvx/time", &["ro", "nosuid", "nodev"]), + ] { + let options = mount_options(output, point); + for option in required { + assert!( + options.contains(option), + "{point} lacks {option}: {options:?}" + ); + } + } + // Only `/dev` and its devpts may hold devices. + for mount in values(output, "mount") { + let (point, options) = mount.split_once(' ').unwrap(); + if !matches!(point, "/dev" | "/dev/pts") { + assert!( + options.split(',').any(|option| option == "nodev"), + "{mount}" + ); + } + } +} + +/// Checks that [`Entry::bind_runtime`] refuses what it cannot bind into `root`, the privileged +/// entries' mounted root, from the sources in `base`, and leaves the root as it was. +fn check_runtime_binds(base: &Path, root: &Path) { + let entry = || Entry::new(root, Identity::new(0, 0).account(AccountMode::Numeric)); + let generated = |path: PathBuf| { + entry() + .machine_id(MachineId::Generated(path)) + .bind_runtime() + }; + let result = generated(base.join("missing")); + assert!( + matches!( + &result, + Err(Error::MachineId(MappingError::System { step: Step::Source, error })) + if *error == Errno::ENOENT as i32 + ), + "{result:?}" + ); + let result = generated(base.join("time")); + assert!( + matches!(result, Err(Error::MachineId(MappingError::SourceKind))), + "{result:?}" + ); + let result = entry().time_status(base.join("machine-id")).bind_runtime(); + assert!( + matches!(result, Err(Error::TimeStatus(MappingError::SourceKind))), + "{result:?}" + ); + // A target that the image holds as another kind, or below a link, is refused. + std::fs::create_dir(root.join("etc/machine-id")).unwrap(); + let result = generated(base.join("machine-id")); + assert!( + matches!(result, Err(Error::MachineId(MappingError::TargetKind))), + "{result:?}" + ); + std::fs::remove_dir(root.join("etc/machine-id")).unwrap(); + std::os::unix::fs::symlink(base, root.join("run")).unwrap(); + let result = entry().time_status(base.join("time")).bind_runtime(); + assert!( + matches!(&result, Err(Error::TimeStatus(MappingError::Link { path })) + if path == Path::new("/run")), + "{result:?}" + ); + std::fs::remove_file(root.join("run")).unwrap(); +} + +#[test] +fn privileged_entries_see_their_own_view_and_identity() { + if crate::guest::managed_agent::isolate_process_test( + "NVX_ENTRY_PRIVILEGED_TEST_CHILD", + "guest::entry::tests::privileged_entries_see_their_own_view_and_identity", + ) { + return; + } + if !sys::holds_capability(sys::CAP_SYS_ADMIN).unwrap() { + return; + } + // The test mounts its root in a mount namespace of its own, which only a process with one + // thread may enter, so a child of the harness's process makes the calls. + match sys::fork_isolated_test().unwrap() { + sys::ForkResult::Child => { + let outcome = std::panic::catch_unwind(|| { + sys::enter_private_mount_namespace().unwrap(); + let base = tempfile::tempdir().unwrap(); + let root = mount_test_root(base.path()); + let machine_id = MachineId::generate(base.path().join("machine-id")).unwrap(); + let id = std::fs::read_to_string(base.path().join("machine-id")).unwrap(); + std::fs::create_dir(base.path().join("time")).unwrap(); + std::fs::write(base.path().join("time/state"), "time-state\n").unwrap(); + let entry = |identity: Identity| { + Entry::new(&root, identity) + .hostname("entry-test") + .machine_id(machine_id.clone()) + .time_status(base.path().join("time")) + }; + check_runtime_binds(base.path(), &root); + // The image's link at the machine ID, which no bind covers, gives way to an empty + // file, and its target is left alone. + std::fs::write(base.path().join("outside"), "outside\n").unwrap(); + std::os::unix::fs::symlink( + base.path().join("outside"), + root.join("etc/machine-id"), + ) + .unwrap(); + entry(Identity::new(65534, 65534)).bind_runtime().unwrap(); + assert_eq!( + std::fs::read_to_string(root.join("etc/machine-id")).unwrap(), + id + ); + assert_eq!( + std::fs::read_to_string(base.path().join("outside")).unwrap(), + "outside\n" + ); + + // The non-root profile, with its existing account. + let output = probe(&entry(Identity::new(65534, 65534))); + check_view(&output, id.trim_end()); + assert_eq!(value(&output, "ids"), "65534 65534 65534"); + for set in ["CapPrm", "CapEff", "CapBnd"] { + assert_eq!(value(&output, set), "0000000000000000", "{set}"); + } + assert_eq!(value(&output, "kcore"), "unreadable"); + + // A machine ID that the guest rewrites in place, as after a restore, is what the + // later executions see. + let id = "0123456789abcdef0123456789abcdef\n"; + std::fs::write(base.path().join("machine-id"), id).unwrap(); + + // Root, with three allowlisted capabilities and its masked and read-only proc. + let capabilities = Capabilities::EMPTY + .with(Capability::CHOWN) + .with(Capability::KILL) + .with(Capability::SYS_CHROOT); + let output = probe( + &entry( + Identity::new(0, 0) + .capabilities(capabilities) + .account(AccountMode::Numeric), + ) + .publication(Publication::Unpublished), + ); + check_view(&output, id.trim_end()); + assert_eq!(value(&output, "ids"), "0 0 0"); + let mask = format!("{:016x}", capabilities.bits()); + for set in ["CapPrm", "CapEff", "CapBnd"] { + assert_eq!(value(&output, set), mask, "{set}"); + } + assert_eq!(value(&output, "kcore"), "0"); + assert_eq!(value(&output, "firmware"), ""); + for point in [ + "/proc/sys", + "/proc/sysrq-trigger", + "/proc/kcore", + "/sys/firmware", + ] { + assert!(mount_options(&output, point).contains(&"ro"), "{point}"); + } + // A nested `chroot` and `..` from it land at the top of the root, never above. + for escape in values(&output, "escape") { + assert_eq!(escape, value(&output, "root")); + } + + // Root with no capability. + let output = probe(&entry(Identity::new(0, 0).account(AccountMode::Numeric))); + for set in ["CapPrm", "CapEff", "CapBnd"] { + assert_eq!(value(&output, set), "0000000000000000", "{set}"); + } + assert_eq!(value(&output, "port-start"), "refused"); + assert_eq!(value(&output, "mknod"), "refused"); + for escape in values(&output, "escape") { + assert_eq!(escape, "refused"); + } + }); + sys::exit_immediately(u8::from(outcome.is_err())); + } + sys::ForkResult::Parent(child) => { + assert_eq!(sys::wait(child), Some(ChildStatus::Exited(0))); + } + } +} diff --git a/nvx-rs/src/guest/managed_agent/launch.rs b/nvx-rs/src/guest/managed_agent/launch.rs index 70ee980b0..bfc91ade4 100644 --- a/nvx-rs/src/guest/managed_agent/launch.rs +++ b/nvx-rs/src/guest/managed_agent/launch.rs @@ -28,9 +28,10 @@ pub(crate) struct LaunchRequest<'a> { /// In both modes the workload's standard input is the null device, its standard output and error /// are the request's pipes, the sealed configuration stays open at its descriptor number, and the /// child leads a new process group. The direct agent runs `setpriv`, which takes the workload's -/// identity and runs the helper; its child first enters the working directory as the workload, -/// records the physical path in `PWD` for a noncanonical name, and joins the execution cgroup. -/// The sandbox agent runs the container launcher, which enters the sandbox's root and identity. +/// identity and runs the helper; its child first joins the execution cgroup, then enters the +/// working directory as the workload and records the physical path in `PWD` for a noncanonical +/// name. The sandbox agent runs the container launcher, which enters the sandbox's root and +/// identity. pub(crate) fn command(paths: &Paths, request: LaunchRequest<'_>) -> Result { let config_fd = request.config_fd.argument(); let directory = c_string(request.execution.cwd.as_deref().unwrap_or(b"/"))?; diff --git a/nvx-rs/src/guest/mod.rs b/nvx-rs/src/guest/mod.rs index ca2da9034..f567e6eb0 100644 --- a/nvx-rs/src/guest/mod.rs +++ b/nvx-rs/src/guest/mod.rs @@ -1,15 +1,17 @@ //! The programs that run in NVX's Linux guests, which ship as `/sbin/nvx-*`, and the libraries //! on which any guest init or agent builds: [`nvxs`], the guest's I/O driver of the NVXS session; //! [`terminal`], the terminal on which it serves that session; [`spawn`], through which it starts -//! a program; [`process`], through which it collects the status of each child that it names, and -//! of no other; [`cgroup`], through which it places, accounts, kills, and settles the processes of -//! its executions; [`mappings`], through which it mounts host mappings beneath the descriptors of -//! the roots that it names; [`exec`], which drives each program as one execution, with its -//! launch, its bounded output and input, its ending, and its outcome, alone or, through its -//! [`binding`](exec::binding), under the [execution engine](crate::exec); [`root`], through which -//! it bootstraps the guest for a layered root, assembles the root, mounts live shares and mappings -//! in it, and tears it down; and [`gpt`], through which it reads the partition table of a GPT disk -//! that the kernel attached as a block device. +//! a program; [`entry`], through which a program that it starts enters an execution's +//! namespaces, view, root, and identity; [`process`], through which it collects the status of +//! each child that it names, and of no other; [`cgroup`], through which it places, accounts, +//! kills, and settles the processes of its executions; [`mappings`], through which it mounts host +//! mappings beneath the descriptors of the roots that it names; [`exec`], which drives each +//! program as one execution, with its launch, its bounded output and input, its ending, and its +//! outcome, alone or, through its [`binding`](exec::binding), under the +//! [execution engine](crate::exec); [`root`], through which it bootstraps the guest for a layered +//! root, assembles the root, mounts live shares and mappings in it, and tears it down; and +//! [`gpt`], through which it reads the partition table of a GPT disk that the kernel attached as +//! a block device. //! //! Each other module holds one program, and the crate's binary of the same name runs its `main`. //! Two of them also offer their steps as typed calls, which any guest init or agent runs in its @@ -83,6 +85,7 @@ pub mod cgroup; pub mod console_pending; #[cfg(target_arch = "x86_64")] pub mod device_io; +pub mod entry; pub mod exec; #[cfg(target_arch = "x86_64")] pub mod exit; diff --git a/nvx-rs/src/guest/spawn.rs b/nvx-rs/src/guest/spawn.rs index f45f17521..31b5879fa 100644 --- a/nvx-rs/src/guest/spawn.rs +++ b/nvx-rs/src/guest/spawn.rs @@ -2,24 +2,27 @@ //! //! A [`Command`] describes a program and how its process starts: its arguments and environment, an //! optional root directory and a working directory in it, the descriptors of its standard streams -//! and of any others that it inherits, the cgroup that it joins, and whether it leads a new -//! process group. [`Command::spawn`] allocates and opens everything that the child needs before it -//! forks; the child then makes only system calls over that storage, in the order of +//! and of any others that it inherits, the cgroup that it joins, whether it leads a new process +//! group or session, whether it waits at a [`Barrier`] until its caller releases it, and the +//! execution that it [enters](Command::enter), if any. [`Command::spawn`] allocates and opens +//! everything that the child needs before it starts the child, with `fork`, or for an entry with +//! `clone3`; the child then makes only system calls over that storage, in the order of //! [`LaunchStage`], and replaces itself with the program. A caller may therefore spawn from any //! thread. //! //! A command owns the descriptors that it takes, and hands each one over once: the child places //! the standard streams and keeps the inherited descriptors open at their numbers, and the -//! caller's process closes all of them as soon as it has forked the child, so that the reader of a -//! workload's pipe sees end of file once the program and its descendants close their ends. Before -//! the fork, a command moves any descriptor that the child uses after it places its standard -//! streams above their numbers and makes it close-on-exec, so the program receives the command's -//! descriptors only as its standard streams and inherited descriptors. Descriptors that the -//! caller opened without close-on-exec stay open in the program, as across any `exec`. +//! caller's process closes all of them as soon as it has started the child, so that the reader +//! of a workload's pipe sees end of file once the program and its descendants close their ends. +//! Before the child starts, a command moves any descriptor that the child uses after it places +//! its standard streams above their numbers and makes it close-on-exec, so the program receives +//! the command's descriptors only as its standard streams and inherited descriptors. Descriptors +//! that the caller opened without close-on-exec stay open in the program, as across any `exec`, +//! except in an entry's, which closes them. //! //! No step needs NVX's control protocol, its managed agent and helper, or its workload identity: -//! the caller chooses the program, which applies its own credentials, and the identity, if any, -//! as which the child checks its working directory. NVX's managed agent is one caller. +//! the caller chooses the program, and the identity, if any, that an entry gives it, or as which +//! the child checks its working directory. NVX's managed agent is one caller. //! //! The caller owns the child. [`Spawned::id`] names it, and the caller collects its status, for //! example through a [`process::Child`](super::process::Child): this module never waits for a @@ -57,42 +60,82 @@ use super::sys::{ REPORT_LENGTH, }; -/// A step that a child takes after it is forked and before its program runs. +/// A step that a child takes after it starts and before its program runs. /// /// The variants are in the order of the steps, and a child skips the steps that its [`Command`] /// does not ask for. A child whose step fails reports it and exits with status 125 without -/// running its program. +/// running its program. The steps from [`Propagation`](Self::Propagation) to +/// [`Devices`](Self::Devices), and from [`Bounding`](Self::Bounding) to +/// [`NoNewPrivileges`](Self::NoNewPrivileges), are an execution's [entry](super::entry), which +/// [`Command::enter`] asks for. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum LaunchStage { /// The child makes itself the leader of a new process group. ProcessGroup, + /// The child makes itself the leader of a new session, and of its process group, without a + /// controlling terminal. + Session, + /// The child waits until its caller releases its [`Barrier`]. + Barrier, /// The child places its standard streams and keeps its inherited descriptors open. Descriptors, + /// The child joins its cgroup. + Cgroup, + /// The child makes every mount of its new mount namespace private. + Propagation, + /// The child sets the hostname of its new UTS namespace. + Hostname, + /// The child moves its root over `/`, and makes it its root and working directory. + MoveRoot, + /// The child mounts its procfs, and restricts it for user ID 0. + Proc, + /// The child mounts its read-only sysfs, and restricts it for user ID 0. + Sys, + /// The child mounts its `/dev`, with its device nodes, devpts, and shm. + Devices, /// The child makes the root directory its root. Root, + /// The child drops every capability but its own from its bounding set. + Bounding, + /// The child takes its supplementary groups, group ID, and user ID. + Identity, + /// The child applies its capabilities, and empties its inheritable and ambient sets. + Capabilities, + /// The child sets `no_new_privs`. + NoNewPrivileges, /// The child takes the identity of its working-directory check, or gives it back after. Credentials, /// The child enters its working directory. WorkingDirectory, /// The child records the physical path of its working directory in `PWD`. PhysicalWorkingDirectory, - /// The child joins its cgroup. - Cgroup, /// The child replaces itself with its program. Exec, } impl LaunchStage { /// Every stage, in order. - const ALL: [Self; 8] = [ + const ALL: [Self; 20] = [ Self::ProcessGroup, + Self::Session, + Self::Barrier, Self::Descriptors, + Self::Cgroup, + Self::Propagation, + Self::Hostname, + Self::MoveRoot, + Self::Proc, + Self::Sys, + Self::Devices, Self::Root, + Self::Bounding, + Self::Identity, + Self::Capabilities, + Self::NoNewPrivileges, Self::Credentials, Self::WorkingDirectory, Self::PhysicalWorkingDirectory, - Self::Cgroup, Self::Exec, ]; @@ -100,13 +143,25 @@ impl LaunchStage { pub(crate) fn code(self) -> i32 { match self { Self::ProcessGroup => 1, - Self::Descriptors => 2, - Self::Root => 3, - Self::Credentials => 4, - Self::WorkingDirectory => 5, - Self::PhysicalWorkingDirectory => 6, - Self::Cgroup => 7, - Self::Exec => 8, + Self::Session => 2, + Self::Barrier => 3, + Self::Descriptors => 4, + Self::Cgroup => 5, + Self::Propagation => 6, + Self::Hostname => 7, + Self::MoveRoot => 8, + Self::Proc => 9, + Self::Sys => 10, + Self::Devices => 11, + Self::Root => 12, + Self::Bounding => 13, + Self::Identity => 14, + Self::Capabilities => 15, + Self::NoNewPrivileges => 16, + Self::Credentials => 17, + Self::WorkingDirectory => 18, + Self::PhysicalWorkingDirectory => 19, + Self::Exec => 20, } } @@ -120,12 +175,24 @@ impl fmt::Display for LaunchStage { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { Self::ProcessGroup => "process group", + Self::Session => "session", + Self::Barrier => "barrier", Self::Descriptors => "descriptors", + Self::Cgroup => "cgroup", + Self::Propagation => "mount propagation", + Self::Hostname => "hostname", + Self::MoveRoot => "root move", + Self::Proc => "procfs", + Self::Sys => "sysfs", + Self::Devices => "devices", Self::Root => "root directory", + Self::Bounding => "bounding set", + Self::Identity => "identity", + Self::Capabilities => "capabilities", + Self::NoNewPrivileges => "no new privileges", Self::Credentials => "credentials", Self::WorkingDirectory => "working directory", Self::PhysicalWorkingDirectory => "physical working directory", - Self::Cgroup => "cgroup", Self::Exec => "exec", }) } @@ -293,8 +360,8 @@ impl Command { self } - /// Makes the child join `cgroup` after its other stages and before its program runs, so that - /// the program and its descendants start in it. + /// Makes the child join `cgroup` once it has placed its descriptors, before the rest of its + /// stages and its program, so that the program and its descendants start in it. pub fn cgroup(mut self, cgroup: Cgroup) -> Self { self.spec.set_cgroup(cgroup.procs); self @@ -306,19 +373,54 @@ impl Command { self } + /// Makes the child the leader of a new session, without a controlling terminal, and of its + /// process group, whose IDs are its process ID. The session replaces a + /// [new process group](Self::new_process_group). + pub fn new_session(mut self) -> Self { + self.spec.new_session(); + self + } + + /// Makes the child wait, before it places its descriptors, until its caller releases the + /// [`Barrier`] that [`Spawned::take_barrier`] returns, for example once it has placed the + /// child in its cgroup. A child whose barrier is dropped unreleased, with the [`Spawned`] that + /// holds it too, fails the [stage](LaunchStage::Barrier) with `ECANCELED` and never runs its + /// program. + pub fn barrier(mut self) -> Self { + self.spec.set_barrier(); + self + } + + /// Makes the child enter the execution that `entry` prepares before its program runs, as the + /// [entry](super::entry) module describes: its namespaces, view, root, and identity. The + /// child then leads a [new session](Self::new_session), and enters its working directory, by + /// default `/`, with that identity. + /// + /// The command must also place the child: it needs a [cgroup](Self::cgroup) or a + /// [barrier](Self::barrier), at which its caller places it. It may name neither a + /// [root](Self::root) nor a [check](Self::check_working_directory_as), which the entry's root + /// and identity replace. A command that breaks either rule fails to spawn. + pub fn enter(mut self, entry: super::entry::Prepared) -> Self { + self.spec.set_entry(entry.into_child()); + self + } + /// Starts the child. /// - /// The call returns once it has forked the child, before the child's launch stages finish; + /// The call returns once it has started the child, before the child's launch stages finish; /// the child's [`LaunchReport`] tells how they end. It fails without a child if the command /// is invalid, or if a resource that the child needs cannot be allocated or opened. On /// x86-64, it also fails, with `PermissionDenied`, while the calling thread may still hold /// the permission for OpenVMM's I/O ports that a time call could not revoke, as /// `TimeError::PortRelease` reports, because the child would inherit it. pub fn spawn(self) -> io::Result { - let (process, report) = PreparedChild::new(self.spec)?.spawn()?; + let started = PreparedChild::new(self.spec)?.spawn()?; Ok(Spawned { - process, - report: LaunchReport { reader: report }, + process: started.process, + report: LaunchReport { + reader: started.report, + }, + barrier: started.barrier.map(|writer| Barrier { writer }), }) } @@ -336,10 +438,13 @@ impl Command { pub(crate) fn spawn_holding_report(self) -> io::Result<(Spawned, OwnedFd)> { let child = PreparedChild::new(self.spec)?; let writer = child.report_writer_copy()?; - let (process, report) = child.spawn()?; + let started = child.spawn()?; let spawned = Spawned { - process, - report: LaunchReport { reader: report }, + process: started.process, + report: LaunchReport { + reader: started.report, + }, + barrier: started.barrier.map(|barrier| Barrier { writer: barrier }), }; Ok((spawned, writer.into())) } @@ -355,18 +460,22 @@ impl Command { /// A child that [`Command::spawn`] started. /// -/// Dropping it neither waits for the child nor stops it. +/// Dropping it neither waits for the child nor stops it, but it drops the child's +/// [`Barrier`], if the caller has not taken it, which cancels the child's launch. #[derive(Debug)] pub struct Spawned { /// The child's process ID. process: ProcessId, /// The report of the child's launch. report: LaunchReport, + /// The barrier at which the child waits, until the caller takes it. + barrier: Option, } impl Spawned { /// Returns the child's process ID, which is also its process group's ID when its command - /// asked for a [new process group](Command::new_process_group). + /// asked for a [new process group](Command::new_process_group) or a + /// [new session](Command::new_session). pub fn id(&self) -> u32 { self.process.raw().unsigned_abs() } @@ -376,12 +485,55 @@ impl Spawned { self.process } + /// Takes the [`Barrier`] at which the child waits, if its command asked for + /// [one](Command::barrier) and it has not been taken. + pub fn take_barrier(&mut self) -> Option { + self.barrier.take() + } + /// Returns the report of the child's launch. pub fn into_report(self) -> LaunchReport { self.report } } +/// The barrier at which a child waits before it places its descriptors, until its caller +/// [releases](Self::release) it. +/// +/// Dropping it unreleased cancels the child's launch: the child reads end of file, fails the +/// [stage](LaunchStage::Barrier) with `ECANCELED`, and never runs its program. So a caller that +/// cannot place the child, or that cancels it or times out first, never releases it by mistake. +/// The drop cancels the launch at once, even while another child that started meanwhile, and has +/// not run its program yet, holds a copy of the barrier's descriptor, which closing it alone +/// would leave open. +#[derive(Debug)] +pub struct Barrier { + /// The caller's end of the barrier's connected socket pair, which the drop shuts down. + writer: FileDescriptor, +} + +impl Barrier { + /// Lets the child continue its launch. + /// + /// A child that has ended already fails the call with `BrokenPipe`, without a `SIGPIPE` for + /// the caller; its launch report tells why it ended. + pub fn release(self) -> io::Result<()> { + match sys::write_without_sigpipe(&self.writer, &[1]) { + Ok(1) => Ok(()), + Ok(_) => Err(io::ErrorKind::WriteZero.into()), + Err(error) => Err(error.into()), + } + } +} + +impl Drop for Barrier { + fn drop(&mut self) { + // The child of a released barrier reads the release before the end of file. A failure + // leaves only the close, which still ends the wait once no copy remains. + let _ = self.writer.shut_down_sending(); + } +} + /// The outcome of a child's launch stages. #[derive(Debug)] pub struct LaunchReport { @@ -661,7 +813,7 @@ mod tests { for (bytes, expected) in [ (Vec::new(), Ok(None)), ( - [5i32.to_ne_bytes(), 2i32.to_ne_bytes()].concat(), + [18i32.to_ne_bytes(), 2i32.to_ne_bytes()].concat(), Ok(Some(LaunchFailure { stage: LaunchStage::WorkingDirectory, error: 2, @@ -669,7 +821,7 @@ mod tests { ), (vec![1, 0, 0], Err(Errno::EPROTO as i32)), ( - [9i32.to_ne_bytes(), 2i32.to_ne_bytes()].concat(), + [21i32.to_ne_bytes(), 2i32.to_ne_bytes()].concat(), Err(Errno::EPROTO as i32), ), ( @@ -691,4 +843,149 @@ mod tests { assert_eq!(LaunchStage::from_code(stage.code()), Some(stage)); } } + + #[test] + fn launch_stages_are_numbered_in_their_order() { + let codes: Vec = LaunchStage::ALL.iter().map(|stage| stage.code()).collect(); + assert_eq!(codes, (1..=20).collect::>()); + // The entry's view comes after the cgroup, which a child joins before it drops its + // privileges, and its identity before the working directory, which it enters as that + // identity. + let position = |stage: LaunchStage| stage.code(); + assert!(position(LaunchStage::Cgroup) < position(LaunchStage::Identity)); + assert!(position(LaunchStage::MoveRoot) < position(LaunchStage::Proc)); + assert!(position(LaunchStage::NoNewPrivileges) < position(LaunchStage::WorkingDirectory)); + assert!(position(LaunchStage::Barrier) < position(LaunchStage::Descriptors)); + } + + /// Returns whether the launch report of `spawned` has arrived, without waiting. + fn report_arrived(spawned: &Spawned) -> bool { + sys::readable_or_hung_up(&[spawned.report.descriptor()], 0).unwrap()[0] + } + + #[test] + fn a_child_waits_at_its_barrier_until_released() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("ran"); + let (stdout, stdout_writer) = std::io::pipe().unwrap(); + let mut spawned = shell(&format!("touch '{}'; printf released", marker.display())) + .barrier() + .stdout(stdout_writer) + .spawn() + .unwrap(); + let process = spawned.process(); + let barrier = spawned.take_barrier().unwrap(); + assert!(spawned.take_barrier().is_none()); + std::thread::sleep(std::time::Duration::from_millis(50)); + assert!(!report_arrived(&spawned)); + assert!(!marker.exists()); + barrier.release().unwrap(); + assert_eq!(spawned.into_report().wait().unwrap(), None); + assert_eq!(read_all(stdout), b"released"); + assert_eq!(sys::wait(process), Some(ChildStatus::Exited(0))); + assert!(marker.exists()); + } + + #[test] + fn a_dropped_barrier_cancels_the_launch() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("ran"); + let script = format!("touch '{}'", marker.display()); + // Dropping the barrier, or the spawned child that still holds it, cancels the launch. + let mut spawned = shell(&script).barrier().spawn().unwrap(); + drop(spawned.take_barrier()); + let cancelled = spawned.process(); + let first = spawned.into_report().wait().unwrap(); + let spawned = shell(&script).barrier().spawn().unwrap(); + let held = spawned.process(); + let second = spawned.into_report().wait().unwrap(); + for (failure, process) in [(first, cancelled), (second, held)] { + let failure = failure.unwrap(); + assert_eq!( + (failure.stage(), failure.raw_os_error()), + (LaunchStage::Barrier, Errno::ECANCELED as i32) + ); + assert_eq!(sys::wait(process), Some(ChildStatus::Exited(125))); + } + assert!(!marker.exists()); + } + + #[test] + fn a_dropped_barrier_cancels_its_launch_while_another_child_waits_at_its_own() { + let directory = tempfile::tempdir().unwrap(); + let marker = |name: &str| directory.path().join(name); + let command = |name: &str| shell(&format!("touch '{}'", marker(name).display())).barrier(); + let mut first = command("first").spawn().unwrap(); + let first_barrier = first.take_barrier().unwrap(); + // The second child starts while the first waits, so it holds a copy of the first's + // barrier until it runs its program, which it does only once released. + let mut second = command("second").spawn().unwrap(); + let second_barrier = second.take_barrier().unwrap(); + let (first_process, second_process) = (first.process(), second.process()); + drop(first_barrier); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !report_arrived(&first) { + assert!( + std::time::Instant::now() < deadline, + "the first child still waits at its dropped barrier" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let failure = first.into_report().wait().unwrap().unwrap(); + assert_eq!( + (failure.stage(), failure.raw_os_error()), + (LaunchStage::Barrier, Errno::ECANCELED as i32) + ); + assert_eq!(sys::wait(first_process), Some(ChildStatus::Exited(125))); + assert!(!report_arrived(&second)); + second_barrier.release().unwrap(); + assert_eq!(second.into_report().wait().unwrap(), None); + assert_eq!(sys::wait(second_process), Some(ChildStatus::Exited(0))); + assert!(!marker("first").exists()); + assert!(marker("second").exists()); + } + + #[test] + fn releasing_the_barrier_of_an_ended_child_fails() { + // Another test's child holds a copy of every descriptor of this process from its fork to + // its `exec`, which would read the release, so the test runs in a process of its own. + if crate::guest::managed_agent::isolate_process_test( + "NVX_SPAWN_ENDED_BARRIER_TEST_CHILD", + "guest::spawn::tests::releasing_the_barrier_of_an_ended_child_fails", + ) { + return; + } + let mut spawned = shell("exit 0").barrier().spawn().unwrap(); + let process = spawned.process(); + let barrier = spawned.take_barrier().unwrap(); + sys::kill_process(process).unwrap(); + assert_eq!( + sys::wait(process), + Some(ChildStatus::Signaled(sys::Signal::SIGKILL as i32)) + ); + let error = barrier.release().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe); + } + + #[test] + fn a_new_session_has_no_controlling_terminal() { + for command in [ + shell(STAT_SCRIPT).new_session(), + // A session replaces a new process group, which the parent then leaves alone. + shell(STAT_SCRIPT).new_process_group().new_session(), + ] { + let (failure, status, output, _) = run(command); + assert_eq!((failure, status), (None, Some(ChildStatus::Exited(0)))); + let output = String::from_utf8(output).unwrap(); + let ids: Vec<&str> = output.split(' ').collect(); + assert_eq!(ids[1], ids[0], "{output}"); + assert_eq!(ids[2], ids[0], "{output}"); + assert_eq!(ids[3], "0", "{output}"); + } + } + + /// A script that prints its process ID, process group ID, session ID, and controlling + /// terminal from its status file. + const STAT_SCRIPT: &str = "read -r pid comm state ppid pgrp session tty rest , + /// The device nodes of the child's `/dev`, as bits of [`DEVICE_NODES`]. + pub(crate) devices: u8, + /// The user ID that the child takes. + pub(crate) uid: u32, + /// The group ID that the child takes. + pub(crate) gid: u32, + /// The supplementary groups that the child takes. + pub(crate) groups: Box<[libc::gid_t]>, + /// The capabilities that the child keeps, bit `n` for capability `n`. + pub(crate) capabilities: u64, + /// Test-only failure of one entry stage: the child skips the system calls of every entry + /// stage, and the stage named here fails with the error. + #[cfg(test)] + pub(crate) simulated: Option<(LaunchStage, Errno)>, +} + +impl EntryChild { + /// Returns the `clone3` flags of the child, which tests that simulate the stages, without + /// privileges, leave empty. + pub(super) fn clone_flags(&self) -> u64 { + #[cfg(test)] + if self.simulated.is_some() { + return 0; + } + NAMESPACES + } + + /// Makes the child's mounts private, sets its hostname, moves its root over `/`, and mounts + /// its view: procfs, sysfs, and `/dev` with its devpts and shm, in the order of their stages. + /// It creates its directories and nodes with exact modes, whatever the process's umask, which + /// it restores before it returns. + /// + /// Each failure names its stage. Only the post-clone child of an entry may call this. + pub(super) fn enter_view(&self) -> Result<(), (LaunchStage, Errno)> { + self.stage(LaunchStage::Propagation, make_mounts_private)?; + self.stage(LaunchStage::Hostname, || set_hostname(&self.hostname))?; + self.stage(LaunchStage::MoveRoot, || move_root(&self.root))?; + let umask = nix::sys::stat::umask(Mode::empty()); + let view = self.mount_view(); + nix::sys::stat::umask(umask); + view + } + + /// Mounts the view's file systems, as [`enter_view`](Self::enter_view) describes. + fn mount_view(&self) -> Result<(), (LaunchStage, Errno)> { + let root_user = self.uid == 0; + let mut mask = None; + self.stage(LaunchStage::Proc, || { + let proc = mount_proc()?; + if root_user { + let source = mask_source()?; + mask = Some(source); + restrict_proc(proc, source)?; + } + Ok(()) + })?; + self.stage(LaunchStage::Sys, || { + let sys = mount_sys()?; + if let Some(source) = mask { + for name in SYS_MASKED { + mask_entry(sys, name, source)?; + } + } + Ok(()) + })?; + self.stage(LaunchStage::Devices, || mount_devices(self.devices)) + } + + /// Takes the child's identity: drops every capability but its own from the bounding set, + /// takes its supplementary groups, group ID, and user ID, applies its capabilities to the + /// permitted and effective sets, with empty inheritable and ambient sets, and sets + /// `no_new_privs`, in the order of their stages. + /// + /// Each failure names its stage. Only the post-clone child of an entry may call this. + pub(super) fn take_identity(&self) -> Result<(), (LaunchStage, Errno)> { + self.stage(LaunchStage::Bounding, || drop_bounding(self.capabilities))?; + self.stage(LaunchStage::Identity, || { + set_identity(self.uid, self.gid, &self.groups) + })?; + self.stage(LaunchStage::Capabilities, || { + set_capabilities(self.capabilities) + })?; + self.stage(LaunchStage::NoNewPrivileges, || { + prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0) + }) + } + + /// Runs the system calls of `stage`, and names it in their failure. A test that simulates + /// the stages skips them, and fails the stage that it names. + fn stage( + &self, + stage: LaunchStage, + calls: impl FnOnce() -> Result<(), Errno>, + ) -> Result<(), (LaunchStage, Errno)> { + #[cfg(test)] + if let Some((failing, error)) = self.simulated { + return if failing == stage { + Err((stage, error)) + } else { + Ok(()) + }; + } + calls().map_err(|error| (stage, error)) + } +} + +/// Starts a child in the namespaces that `flags` names, as `fork` starts one otherwise, and +/// returns its process ID in the parent, or `None` in the child. The child's end sends its +/// parent `SIGCHLD`, and the child keeps a copy of its parent's memory and descriptors. +/// +/// # Safety +/// +/// The child may only make system calls over storage that the parent prepared, as the +/// [module](self) describes, and must end in `execve` or `_exit`. +pub(super) unsafe fn clone_into_namespaces(flags: u64) -> Result, Errno> { + let mut arguments = CloneArgs { + flags, + exit_signal: libc::SIGCHLD as u64, + ..CloneArgs::default() + }; + // SAFETY: the arguments are a valid first-version `clone_args` without a stack, so the child + // runs on its copy of the parent's, as after `fork`; the caller keeps the child's promise. + let result = unsafe { + libc::syscall( + libc::SYS_clone3, + &mut arguments as *mut CloneArgs, + std::mem::size_of::(), + ) + }; + match result { + 0 => Ok(None), + child if child > 0 => Ok(Some(Pid::from_raw(child as libc::pid_t))), + _ => Err(Errno::last()), + } +} + +/// Makes every descriptor from 3 up close on `exec`, for an entry's child, which keeps only its +/// standard streams and the descriptors that its command names. +pub(super) fn close_descriptors_on_exec() -> Result<(), Errno> { + // SAFETY: `close_range` takes three integers and dereferences nothing; with + // `CLOSE_RANGE_CLOEXEC`, it only sets the flag. + unit(unsafe { + libc::syscall( + libc::SYS_close_range, + 3 as c_uint, + c_uint::MAX, + libc::CLOSE_RANGE_CLOEXEC, + ) + }) +} + +/// Returns whether `file` is a regular file, and its size. +pub(crate) fn file_size(file: &FileDescriptor) -> Result<(bool, u64), Errno> { + let status = nix::sys::stat::fstat(file.borrowed())?; + Ok(( + status.st_mode & libc::S_IFMT == libc::S_IFREG, + u64::try_from(status.st_size).unwrap_or(0), + )) +} + +/// Returns the capability data words that give a process exactly `capabilities` as its +/// permitted and effective sets, and an empty inheritable set: bit `n` of the set is bit `n % 32` +/// of word `n / 32`. +fn capability_data(capabilities: u64) -> [CapabilityData; 2] { + let word = |bits: u64| CapabilityData { + effective: bits as u32, + permitted: bits as u32, + inheritable: 0, + }; + [word(capabilities), word(capabilities >> 32)] +} + +/// Makes every mount of the child's new mount namespace private, recursively, so that no mount +/// propagates to or from another namespace. +fn make_mounts_private() -> Result<(), Errno> { + nix::mount::mount( + None::<&CStr>, + c"/", + None::<&CStr>, + MsFlags::MS_REC | MsFlags::MS_PRIVATE, + None::<&CStr>, + ) +} + +/// Sets the hostname of the child's new UTS namespace. +fn set_hostname(hostname: &[u8]) -> Result<(), Errno> { + // SAFETY: the kernel reads `hostname.len()` bytes from the slice. + unit(unsafe { libc::syscall(libc::SYS_sethostname, hostname.as_ptr(), hostname.len()) }) +} + +/// Moves the root mount at `root` over `/` and makes it the child's root and working directory. +/// +/// It first makes every mount of the root's tree `nosuid` and `nodev` in the child's namespace, +/// whatever its options elsewhere. The root then hides the namespace's former root, which no +/// path reaches from it: a `..` from the root's top stays there, as the root's mount point is the +/// top of the former root. +fn move_root(root: &CStr) -> Result<(), Errno> { + nix::unistd::chdir(root)?; + mount_setattr( + libc::AT_FDCWD, + c".", + AT_RECURSIVE, + MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV, + )?; + nix::mount::mount( + Some(c"."), + c"/", + None::<&CStr>, + MsFlags::MS_MOVE, + None::<&CStr>, + )?; + nix::unistd::chroot(c".")?; + nix::unistd::chdir(c"/") +} + +/// Mounts a new procfs instance of the child's PID namespace at `/proc`, `nosuid`, `nodev`, and +/// `noexec`, and returns its mount's descriptor. +fn mount_proc() -> Result { + let target = directory_target(libc::AT_FDCWD, c"proc")?; + let proc = new_mount( + c"proc", + &[(c"source", c"proc")], + MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | MOUNT_ATTR_NOEXEC, + )?; + move_mount(proc, target)?; + Ok(proc) +} + +/// Makes the procfs entries of [`PROC_READ_ONLY`] read-only, and masks those of [`PROC_MASKED`] +/// with `source`, in the procfs whose mount `proc` holds. An entry that this kernel lacks needs +/// neither. +fn restrict_proc(proc: RawFd, source: RawFd) -> Result<(), Errno> { + for name in PROC_READ_ONLY { + let Some(target) = existing_entry(proc, name)? else { + continue; + }; + let tree = open_tree( + target, + c"", + OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC | libc::AT_EMPTY_PATH as c_uint, + )?; + mount_setattr(tree, c"", libc::AT_EMPTY_PATH as c_uint, READ_ONLY_BIND)?; + move_mount(tree, target)?; + } + for name in PROC_MASKED { + mask_entry(proc, name, source)?; + } + Ok(()) +} + +/// Mounts a new sysfs at `/sys`, read-only, `nosuid`, `nodev`, and `noexec`, and returns its +/// mount's descriptor. +fn mount_sys() -> Result { + let target = directory_target(libc::AT_FDCWD, c"sys")?; + let sys = new_mount(c"sysfs", &[(c"source", c"sysfs")], READ_ONLY_BIND)?; + move_mount(sys, target)?; + Ok(sys) +} + +/// Creates the source of the masks of an execution as user ID 0: a tmpfs, `nosuid`, `nodev`, +/// and `noexec`, that holds one empty file and one empty directory, and that is read-only once +/// it holds them. It returns the tmpfs's mount descriptor, of which each mask binds one entry. +fn mask_source() -> Result { + let source = new_mount( + c"tmpfs", + &[(c"source", c"tmpfs"), (c"mode", c"0555")], + MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | MOUNT_ATTR_NOEXEC, + )?; + let file = descriptor( + // SAFETY: the path is a NUL-terminated string, and the mode an integer. + unsafe { + libc::openat( + source, + MASK_FILE.as_ptr(), + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_CLOEXEC, + 0o444 as c_uint, + ) + } + .into(), + )?; + close(file); + make_directory(source, MASK_DIRECTORY, 0o555)?; + mount_setattr( + source, + c"", + libc::AT_EMPTY_PATH as c_uint, + MOUNT_ATTR_RDONLY, + )?; + Ok(source) +} + +/// Masks the entry `name` of the directory `directory` with the empty directory, or the empty +/// file, of the mask source `source`. An entry that does not exist needs no mask. +fn mask_entry(directory: RawFd, name: &CStr, source: RawFd) -> Result<(), Errno> { + let Some(target) = existing_entry(directory, name)? else { + return Ok(()); + }; + let mask = if file_kind(target)? == libc::S_IFDIR { + MASK_DIRECTORY + } else { + MASK_FILE + }; + let tree = open_tree(source, mask, OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC)?; + move_mount(tree, target) +} + +/// Mounts the child's `/dev`: a tmpfs, `nosuid` and with mode 0755, which holds the device +/// nodes that `devices` selects, with mode 0666, the symbolic links of [`DEVICE_LINKS`], a +/// devpts instance of its own at `pts`, `nosuid` and `noexec`, and a tmpfs at `shm`, `nosuid`, +/// `nodev`, and with mode 1777. +fn mount_devices(devices: u8) -> Result<(), Errno> { + let target = directory_target(libc::AT_FDCWD, c"dev")?; + let dev = new_mount( + c"tmpfs", + &[(c"source", c"tmpfs"), (c"mode", c"0755")], + MOUNT_ATTR_NOSUID, + )?; + move_mount(dev, target)?; + for (bit, (name, major, minor)) in DEVICE_NODES.iter().enumerate() { + if devices & (1 << bit) == 0 { + continue; + } + // SAFETY: the name is a NUL-terminated string, and the mode and device integers. + unit( + unsafe { + libc::mknodat( + dev, + name.as_ptr(), + libc::S_IFCHR | DEVICE_MODE, + libc::makedev(*major, *minor), + ) + } + .into(), + )?; + } + for (name, link) in DEVICE_LINKS { + // SAFETY: both strings are NUL-terminated. + unit(unsafe { libc::symlinkat(link.as_ptr(), dev, name.as_ptr()) }.into())?; + } + let pts = new_mount( + c"devpts", + &[ + (c"source", c"devpts"), + (c"ptmxmode", c"0666"), + (c"mode", c"0620"), + ], + MOUNT_ATTR_NOSUID | MOUNT_ATTR_NOEXEC, + )?; + move_mount(pts, directory_target(dev, c"pts")?)?; + let shm = new_mount( + c"tmpfs", + &[(c"source", c"tmpfs"), (c"mode", c"1777")], + MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV, + )?; + move_mount(shm, directory_target(dev, c"shm")?) +} + +/// Drops from the bounding set every capability that `capabilities` does not hold, up to the +/// last capability of the kernel, which then refuses the next with `EINVAL`. +fn drop_bounding(capabilities: u64) -> Result<(), Errno> { + for capability in 0..u64::BITS { + if capabilities & (1 << capability) != 0 { + continue; + } + match prctl(libc::PR_CAPBSET_DROP, c_ulong::from(capability), 0) { + Ok(()) => {} + Err(Errno::EINVAL) => return Ok(()), + Err(error) => return Err(error), + } + } + Ok(()) +} + +/// Takes the supplementary groups `groups`, then the real, effective, and saved group ID `gid`, +/// then the real, effective, and saved user ID `uid`, for the calling thread alone. +fn set_identity(uid: u32, gid: u32, groups: &[libc::gid_t]) -> Result<(), Errno> { + // SAFETY: the kernel reads `groups.len()` group IDs from the slice. + unit(unsafe { libc::syscall(SYS_SETGROUPS, groups.len(), groups.as_ptr()) })?; + // SAFETY: `setresgid` takes three integers. + unit(unsafe { libc::syscall(SYS_SETRESGID, gid, gid, gid) })?; + // SAFETY: `setresuid` takes three integers. + unit(unsafe { libc::syscall(SYS_SETRESUID, uid, uid, uid) }) +} + +/// Makes `capabilities` the permitted and effective sets, empties the inheritable set, and +/// clears the ambient set. +fn set_capabilities(capabilities: u64) -> Result<(), Errno> { + let header = CapabilityHeader { + version: CAPABILITY_VERSION, + pid: 0, + }; + let data = capability_data(capabilities); + // SAFETY: the header names version 3, whose two data words the array holds. + unit(unsafe { libc::syscall(libc::SYS_capset, &header, data.as_ptr()) })?; + prctl( + libc::PR_CAP_AMBIENT, + libc::PR_CAP_AMBIENT_CLEAR_ALL as c_ulong, + 0, + ) +} + +/// Makes the `prctl` call `option` with two arguments, and zeros for the others, which the +/// options here require. +fn prctl(option: c_int, first: c_ulong, second: c_ulong) -> Result<(), Errno> { + // SAFETY: `prctl` takes integers for these options, and dereferences none of them. + unit(unsafe { + libc::syscall( + libc::SYS_prctl, + option, + first, + second, + 0 as c_ulong, + 0 as c_ulong, + ) + }) +} + +/// Returns a descriptor of the directory `name` in `directory`, below it and on its mount, +/// through no symbolic link, and creates the directory, with mode 0755, if it is missing. +fn directory_target(directory: RawFd, name: &CStr) -> Result { + let flags = libc::O_PATH | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC; + match open_target(directory, name, flags) { + Err(Errno::ENOENT) => { + match make_directory(directory, name, DIRECTORY_MODE) { + Ok(()) | Err(Errno::EEXIST) => {} + Err(error) => return Err(error), + } + open_target(directory, name, flags) + } + result => result, + } +} + +/// Returns a descriptor of the entry `name` of `directory`, without following it, or `None` if +/// there is none. +fn existing_entry(directory: RawFd, name: &CStr) -> Result, Errno> { + match open_target( + directory, + name, + libc::O_PATH | libc::O_NOFOLLOW | libc::O_CLOEXEC, + ) { + Ok(target) => Ok(Some(target)), + Err(Errno::ENOENT) => Ok(None), + Err(error) => Err(error), + } +} + +/// Returns the file type bits of `descriptor`'s file. +fn file_kind(descriptor: RawFd) -> Result { + let mut status = MaybeUninit::::uninit(); + // SAFETY: the kernel fills the status, which is initialized once the call succeeds. + unit(unsafe { libc::fstat(descriptor, status.as_mut_ptr()) }.into())?; + // SAFETY: the successful call initialized the status. + Ok(unsafe { status.assume_init() }.st_mode & libc::S_IFMT) +} + +/// Creates the file system `kind` with the string parameters `parameters`, mounts it detached +/// with the mount attributes `attributes`, and returns the mount's descriptor. +fn new_mount(kind: &CStr, parameters: &[(&CStr, &CStr)], attributes: u64) -> Result { + // SAFETY: the name is a NUL-terminated string. + let context = + descriptor(unsafe { libc::syscall(libc::SYS_fsopen, kind.as_ptr(), FSOPEN_CLOEXEC) })?; + let mount = configure_and_mount(context, parameters, attributes); + close(context); + mount +} + +/// Sets `parameters` on the file system context `context`, creates its superblock, and mounts +/// it detached with `attributes`. +fn configure_and_mount( + context: RawFd, + parameters: &[(&CStr, &CStr)], + attributes: u64, +) -> Result { + for (key, value) in parameters { + // SAFETY: the key and the value are NUL-terminated strings. + unit(unsafe { + libc::syscall( + libc::SYS_fsconfig, + context, + FSCONFIG_SET_STRING, + key.as_ptr(), + value.as_ptr(), + 0 as c_int, + ) + })?; + } + // SAFETY: the command takes no key or value. + unit(unsafe { + libc::syscall( + libc::SYS_fsconfig, + context, + FSCONFIG_CMD_CREATE, + std::ptr::null::(), + std::ptr::null::(), + 0 as c_int, + ) + })?; + // SAFETY: `fsmount` takes integers. + descriptor(unsafe { + libc::syscall( + libc::SYS_fsmount, + context, + FSMOUNT_CLOEXEC, + attributes as c_uint, + ) + }) +} + +/// Opens `path` below `directory`, on its mount, and through no symbolic link, with `flags`. +fn open_target(directory: RawFd, path: &CStr, flags: c_int) -> Result { + let how = OpenHow { + flags: flags as u64, + mode: 0, + resolve: BENEATH, + }; + // SAFETY: the path is a NUL-terminated string, and `how` a valid `open_how` of its size. + descriptor(unsafe { + libc::syscall( + libc::SYS_openat2, + directory, + path.as_ptr(), + &how, + std::mem::size_of::(), + ) + }) +} + +/// Opens the mount tree at `path` in `directory` with `open_tree`'s `flags`. +fn open_tree(directory: RawFd, path: &CStr, flags: c_uint) -> Result { + // SAFETY: the path is a NUL-terminated string. + descriptor(unsafe { libc::syscall(libc::SYS_open_tree, directory, path.as_ptr(), flags) }) +} + +/// Sets the mount attributes `attributes` on the mount at `path` in `directory`, with +/// `mount_setattr`'s `flags`. +fn mount_setattr( + directory: RawFd, + path: &CStr, + flags: c_uint, + attributes: u64, +) -> Result<(), Errno> { + let attributes = MountAttr { + attr_set: attributes, + attr_clr: 0, + propagation: 0, + userns_fd: 0, + }; + // SAFETY: the path is a NUL-terminated string, and the attributes a valid `mount_attr` of + // its size. + unit(unsafe { + libc::syscall( + libc::SYS_mount_setattr, + directory, + path.as_ptr(), + flags, + &attributes, + std::mem::size_of::(), + ) + }) +} + +/// Attaches the mount that `source` holds at the file or directory that `target` holds. +fn move_mount(source: RawFd, target: RawFd) -> Result<(), Errno> { + // SAFETY: both paths are empty NUL-terminated strings, which the flags take as the + // descriptors themselves. + unit(unsafe { + libc::syscall( + libc::SYS_move_mount, + source, + c"".as_ptr(), + target, + c"".as_ptr(), + MOVE_MOUNT_F_EMPTY_PATH | MOVE_MOUNT_T_EMPTY_PATH, + ) + }) +} + +/// Creates the directory `name` in `directory` with `mode`. +fn make_directory(directory: RawFd, name: &CStr, mode: u32) -> Result<(), Errno> { + // SAFETY: the name is a NUL-terminated string, and the mode an integer. + unit(unsafe { libc::mkdirat(directory, name.as_ptr(), mode) }.into()) +} + +/// Closes a descriptor that the child opened, ignoring a failure, as the descriptor is gone +/// either way. +fn close(descriptor: RawFd) { + // SAFETY: the child opened `descriptor`, and no owner of it exists. + unsafe { libc::close(descriptor) }; +} + +/// Returns a system call's descriptor, or its `errno`. +fn descriptor(result: c_long) -> Result { + if result < 0 { + Err(Errno::last()) + } else { + // A descriptor fits a `RawFd`. + Ok(result as RawFd) + } +} + +/// Returns a system call's success, or its `errno`. +fn unit(result: c_long) -> Result<(), Errno> { + if result < 0 { + Err(Errno::last()) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capability_sets_split_into_two_data_words() { + assert_eq!( + capability_data(0), + [CapabilityData { + effective: 0, + permitted: 0, + inheritable: 0 + }; 2] + ); + // CAP_CHOWN, CAP_NET_RAW, CAP_SETFCAP (31), and CAP_MAC_OVERRIDE (32) straddle the words. + let set = 1 | 1 << 13 | 1 << 31 | 1 << 32; + let [low, high] = capability_data(set); + assert_eq!( + (low.effective, low.permitted, low.inheritable), + (0x8000_2001, 0x8000_2001, 0) + ); + assert_eq!( + (high.effective, high.permitted, high.inheritable), + (1, 1, 0) + ); + let [low, high] = capability_data(u64::MAX); + assert_eq!((low.permitted, high.permitted), (u32::MAX, u32::MAX)); + assert_eq!((low.inheritable, high.inheritable), (0, 0)); + } + + #[test] + fn the_kernel_structures_have_their_uapi_sizes() { + assert_eq!(std::mem::size_of::(), 64); + assert_eq!(std::mem::size_of::(), 24); + assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::size_of::(), 8); + assert_eq!(std::mem::size_of::(), 12); + } +} diff --git a/nvx-rs/src/guest/sys/fd.rs b/nvx-rs/src/guest/sys/fd.rs index b0c63b323..dd05e21f0 100644 --- a/nvx-rs/src/guest/sys/fd.rs +++ b/nvx-rs/src/guest/sys/fd.rs @@ -8,6 +8,7 @@ pub(crate) use nix::fcntl::OFlag as OpenFlags; use nix::fcntl::{FcntlArg, Flock, FlockArg, SealFlag, fcntl}; use nix::poll::{PollFd, PollFlags}; use nix::sys::memfd::{MFdFlags, memfd_create}; +use nix::sys::socket::{AddressFamily, Shutdown, SockFlag, SockType, shutdown, socketpair}; pub(crate) use nix::sys::stat::Mode; use nix::unistd::Whence; @@ -103,6 +104,18 @@ impl FileDescriptor { nix::unistd::fsync(&self.0) } + /// Sets the file's permission bits to exactly `mode`, which the process's umask, unlike the + /// mode that creates a file, does not narrow. + pub(crate) fn set_mode(&self, mode: Mode) -> Result<(), Errno> { + nix::sys::stat::fchmod(&self.0, mode) + } + + /// Shuts down the sending direction of this connected socket, so that its peer reads end of + /// file, which closing it would not tell while another process holds a copy of it. + pub(crate) fn shut_down_sending(&self) -> Result<(), Errno> { + shutdown(self.0.as_raw_fd(), Shutdown::Write) + } + /// Waits at most `timeout_ms` milliseconds for data to read, with one `poll` call. /// /// Returns whether `poll` reported `POLLIN`. An interrupted call is an error, not a retry. @@ -231,6 +244,17 @@ pub(crate) fn create_pipe() -> Result<(FileDescriptor, FileDescriptor), Errno> { .map(|(reader, writer)| (FileDescriptor(reader), FileDescriptor(writer))) } +/// Creates a connected pair of close-on-exec Unix stream sockets and returns its owned ends. +pub(crate) fn create_socket_pair() -> Result<(FileDescriptor, FileDescriptor), Errno> { + socketpair( + AddressFamily::Unix, + SockType::Stream, + None, + SockFlag::SOCK_CLOEXEC, + ) + .map(|(first, second)| (FileDescriptor(first), FileDescriptor(second))) +} + /// Makes the null device the standard input, output, and error, and closes every other /// descriptor, best effort, in a forked child that may outlive its parent's interest in it. The /// child then holds none of the parent's descriptors, such as the writer of a pipe, whose diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index c0b008b25..561a1f967 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -21,8 +21,12 @@ //! //! The spawn boundary and the managed agent need a few operations that `nix` cannot provide, //! which reach libc through `nix`'s re-export: a spawned child, which may neither allocate nor -//! drop an owner, fills its preallocated `PWD` entry with the `getcwd` system call, and executes -//! and exits through libc; the agent's wait-status decoding, and the waits for one named child, +//! drop an owner, fills its preallocated `PWD` entry with the `getcwd` system call, closes its +//! copy of a barrier's writer, and executes and exits through libc; an execution's entry starts +//! its child with `clone3`, and the child changes its credentials, capabilities, and hostname, +//! and mounts its view with the new mount API, `openat2`, and `close_range`, through libc's +//! `syscall`, as the entry's module describes, because libc does not know of that child; the +//! agent's wait-status decoding, and the waits for one named child, //! keep the real-time signals that `nix`'s `Signal` cannot represent; a pidfd is opened with the //! `pidfd_open` system call, which neither `nix` nor musl wraps; a write to an execution's input //! takes back the `SIGPIPE` that a closed reader raises with `sigtimedwait`, once `sigpending` @@ -44,6 +48,7 @@ mod clock; mod discipline; +mod entry; mod fd; mod filesystem; mod ioctl; @@ -77,13 +82,15 @@ pub(crate) use clock::{Clock, clock_time, monotonic_ms, sleep_once, sleep_until_ #[cfg(target_arch = "x86_64")] pub(crate) use clock::{monotonic_ns, syscall_monotonic_nanoseconds}; pub(crate) use discipline::{ClockAdjustment, ClockDiscipline, adjust_clock}; +pub(crate) use entry::{EntryChild, file_size}; #[cfg(target_arch = "x86_64")] pub(crate) use fd::close_inherited_descriptors; #[cfg(test)] pub(crate) use fd::set_descriptor_limit; pub(crate) use fd::{ CreatedFile, FileDescriptor, FileLock, Mode, OpenFlags, create_directory, create_fifo, - create_memfd, create_pipe, lock_file, open, open_at, page_size, try_lock_directory, unlink, + create_memfd, create_pipe, create_socket_pair, lock_file, open, open_at, page_size, + try_lock_directory, unlink, }; pub(crate) use filesystem::{ MountFlags, PATH_MAX, ScratchDirectory, TmpfsMount, change_directory, check_exists, chmod, @@ -149,7 +156,7 @@ pub(crate) use sock_diag::{encode_inet_diag_done, encode_inet_diag_socket}; pub(crate) use socket::{UdpSocket, connect_datagram}; #[cfg(test)] pub(crate) use spawn::{ - CAP_DAC_OVERRIDE, CAP_DAC_READ_SEARCH, CAP_SETGID, CAP_SETUID, holds_capability, + CAP_DAC_OVERRIDE, CAP_DAC_READ_SEARCH, CAP_SETGID, CAP_SETUID, CAP_SYS_ADMIN, holds_capability, }; pub(crate) use spawn::{ChildSpec, PreparedChild, REPORT_LENGTH}; pub(crate) use terminal::{configure_raw, drain}; diff --git a/nvx-rs/src/guest/sys/spawn.rs b/nvx-rs/src/guest/sys/spawn.rs index 14057d15c..f14d7be3a 100644 --- a/nvx-rs/src/guest/sys/spawn.rs +++ b/nvx-rs/src/guest/sys/spawn.rs @@ -1,20 +1,23 @@ //! The fork/exec boundary of [`crate::guest::spawn`]. //! //! [`PreparedChild::new`] checks a [`ChildSpec`] and allocates or opens everything that the child -//! uses after `fork`, and [`PreparedChild::spawn`] forks it. The child branch performs no -//! allocation, locking, formatting, or Rust cleanup: it makes direct system calls over that -//! storage, in the order of [`LaunchStage`], and ends in `execve`, `execv`, or `_exit`. It makes -//! them through `nix` where its wrappers neither allocate nor consume an owner, and otherwise -//! through libc: its `syscall` fills the preallocated `PWD` entry, because its `getcwd` may walk -//! the file system, and its `execve`, `execv`, and `_exit` end the child. A stage that fails -//! writes its number and `errno` to the launch report, a close-on-exec pipe, and ends the child -//! with status 125, so that the report's reader reaches end of file without a report once the -//! program runs. +//! uses after `fork`, and [`PreparedChild::spawn`] forks it, or, for an execution's +//! [entry](super::EntryChild), starts it with `clone3` in new namespaces. The child branch +//! performs no allocation, locking, formatting, or Rust cleanup: it makes direct system calls +//! over that storage, in the order of [`LaunchStage`], and ends in `execve`, `execv`, or `_exit`. +//! It makes them through `nix` where its wrappers neither allocate nor consume an owner, and +//! otherwise through libc: its `syscall` fills the preallocated `PWD` entry, because its `getcwd` +//! may walk the file system, and makes the entry's calls that the [entry](super::entry) module +//! lists; its `close` drops the child's copy of a barrier's writer; and its `execve`, `execv`, +//! and `_exit` end the child. A stage that fails writes its number and `errno` to the launch +//! report, a close-on-exec pipe, and ends the child with status 125, so that the report's reader +//! reaches end of file without a report once the program runs. //! //! Every descriptor that the child uses after it places its standard streams has a number above //! theirs, so that placing them replaces none of it: [`PreparedChild::new`] moves a stream source, -//! the root, the cgroup's membership file, or the report's writer that has a lower number. It also -//! makes them close-on-exec, so that the program receives them only as its standard streams. +//! the root, the cgroup's membership file, the barrier's ends, or the report's writer that has a +//! lower number. It also makes them close-on-exec, so that the program receives +//! them only as its standard streams. use std::ffi::CString; use std::os::fd::RawFd; @@ -23,7 +26,7 @@ use nix::fcntl::{FcntlArg, FdFlag}; use nix::libc::c_char; use nix::unistd::{ForkResult, Gid, Pid, Uid}; -use super::{Errno, FileDescriptor, ProcessId}; +use super::{EntryChild, Errno, FileDescriptor, ProcessId}; use crate::guest::spawn::LaunchStage; /// Prefix of the mutable physical-working-directory environment entry. @@ -64,6 +67,12 @@ pub(crate) struct ChildSpec { cgroup: Option, /// Whether the child leads a new process group. process_group: bool, + /// Whether the child leads a new session. + session: bool, + /// Whether the child waits at a barrier until its caller releases it. + barrier: bool, + /// The execution that the child enters, if any. + entry: Option, /// Test-only outcome of the credential system calls, which the child then skips: /// `Some(None)` succeeds, and `Some(Some(error))` fails the first one with `error`. #[cfg(test)] @@ -86,6 +95,9 @@ impl ChildSpec { inherited: Vec::new(), cgroup: None, process_group: false, + session: false, + barrier: false, + entry: None, #[cfg(test)] simulated_credentials: None, } @@ -136,6 +148,21 @@ impl ChildSpec { self.process_group = true; } + /// Makes the child the leader of a new session, without a controlling terminal. + pub(crate) fn new_session(&mut self) { + self.session = true; + } + + /// Makes the child wait at a barrier until its caller releases it. + pub(crate) fn set_barrier(&mut self) { + self.barrier = true; + } + + /// Makes the child enter the execution that `entry` prepares. + pub(crate) fn set_entry(&mut self, entry: EntryChild) { + self.entry = Some(entry); + } + /// Replaces the child's credential system calls, which then succeed, or fail with `failure`. #[cfg(test)] pub(crate) fn simulate_credentials(&mut self, failure: Option) { @@ -203,6 +230,15 @@ pub(crate) struct PreparedChild { cgroup: Option, /// Whether the child leads a new process group. process_group: bool, + /// Whether the child leads a new session, which an entry implies. + session: bool, + /// The reader and the writer of the barrier at which the child waits, if any: the ends of a + /// connected socket pair, of which the child keeps the reader, and its caller the writer, + /// whose shutdown the child reads as end of file even while other children hold copies of + /// it. + barrier: Option<(FileDescriptor, FileDescriptor)>, + /// The execution that the child enters, if any. + entry: Option, /// Reader of the launch report, which the caller receives. report_reader: FileDescriptor, /// Writer of the launch report, which only the child keeps. @@ -212,12 +248,24 @@ pub(crate) struct PreparedChild { simulated_credentials: Option>, } +/// A child that [`PreparedChild::spawn`] started. +pub(crate) struct Started { + /// The child's process ID. + pub(crate) process: ProcessId, + /// The reader of the child's launch report. + pub(crate) report: FileDescriptor, + /// The writer of the barrier at which the child waits, if it waits at one. + pub(crate) barrier: Option, +} + impl PreparedChild { /// Checks `spec` and prepares every input of its child. /// /// The working directory must be absolute, credentials need a working directory, the physical /// `PWD` an exact environment, and an inherited descriptor a number above the standard - /// streams'; a command with no argument zero is invalid too. Each fails with `EINVAL`. + /// streams'; a command with no argument zero is invalid too. An entry needs a cgroup or a + /// barrier, at which its caller places it, and takes neither a root directory nor + /// credentials, as its own root and identity replace them. Each fails with `EINVAL`. pub(crate) fn new(spec: ChildSpec) -> Result { if spec.arguments.is_empty() || spec @@ -230,6 +278,10 @@ impl PreparedChild { .inherited .iter() .any(|descriptor| descriptor.raw() < FIRST_NONSTANDARD) + || (spec.entry.is_some() + && ((spec.cgroup.is_none() && !spec.barrier) + || spec.root.is_some() + || spec.credentials.is_some())) { return Err(Errno::EINVAL); } @@ -241,6 +293,16 @@ impl PreparedChild { ]; let root = spec.root.map(above_standard_streams).transpose()?; let cgroup = spec.cgroup.map(above_standard_streams).transpose()?; + let barrier = if spec.barrier { + let (reader, writer) = super::create_socket_pair()?; + Some(( + above_standard_streams(reader)?, + above_standard_streams(writer)?, + )) + } else { + None + }; + let entry = spec.entry; let (report_reader, report_writer) = super::create_pipe()?; let report_writer = above_standard_streams(report_writer)?; let credentials = match spec.credentials { @@ -268,6 +330,9 @@ impl PreparedChild { inherited: spec.inherited, cgroup, process_group: spec.process_group, + session: spec.session || entry.is_some(), + barrier, + entry, report_reader, report_writer, #[cfg(test)] @@ -276,7 +341,9 @@ impl PreparedChild { } /// Forks the child, which runs its launch stages and then its program, and returns its - /// process ID and the reader of its launch report. + /// process ID, the reader of its launch report, and the writer of its barrier, if any. The + /// child of an entry starts with `clone3`, in new mount, PID, and UTS namespaces, of which it + /// is PID 1. /// /// On x86-64, it fails with `EPERM`, before it forks, while the calling thread may still /// hold the permission for OpenVMM's I/O ports that a time call could not revoke, which the @@ -287,7 +354,7 @@ impl PreparedChild { /// value or to this frame, which the child's copied stack keeps, and the child path never /// returns into Rust. `fork` may occur in a multithreaded process: the child makes only /// direct async-signal-safe system calls over storage allocated before the fork. - pub(crate) fn spawn(mut self) -> Result<(ProcessId, FileDescriptor), Errno> { + pub(crate) fn spawn(mut self) -> Result { #[cfg(target_arch = "x86_64")] if super::check_released().is_err() { return Err(Errno::EPERM); @@ -323,10 +390,19 @@ impl PreparedChild { // SAFETY: every child input and pointer vector was allocated above, and the child runs // only the non-returning routine below. - match unsafe { nix::unistd::fork() }? { - ForkResult::Child => { - // SAFETY: this is the just-forked child, and the pointers name storage that `self`, - // the physical `PWD` entry's owner, and the vectors above own. + let child = match self.entry.as_ref().map(EntryChild::clone_flags) { + // SAFETY: as above; the entry's child makes only the system calls that its module + // permits. + Some(flags) => unsafe { super::entry::clone_into_namespaces(flags) }?, + None => match unsafe { nix::unistd::fork() }? { + ForkResult::Child => None, + ForkResult::Parent { child } => Some(child), + }, + }; + match child { + None => { + // SAFETY: this is the just-started child, and the pointers name storage that + // `self`, the physical `PWD` entry's owner, and the vectors above own. unsafe { self.run_child( &argument_pointers, @@ -335,14 +411,19 @@ impl PreparedChild { ) } } - ForkResult::Parent { child } => { - if self.process_group { + Some(child) => { + if self.process_group && !self.session { // The child makes the same call; whichever runs first creates the group, so // that the caller may signal it at once. The child's `exec` or exit makes a - // failure here harmless. + // failure here harmless. A child that starts a session leads the group that + // it creates, which the caller's call would forbid. let _ = nix::unistd::setpgid(child, child); } - Ok((ProcessId::from_pid(child), self.report_reader)) + Ok(Started { + process: ProcessId::from_pid(child), + report: self.report_reader, + barrier: self.barrier.take().map(|(_, writer)| writer), + }) } } } @@ -362,21 +443,56 @@ impl PreparedChild { ) -> ! { let report = &self.report_writer; if self.process_group + && !self.session && let Err(error) = nix::unistd::setpgid(Pid::from_raw(0), Pid::from_raw(0)) { // SAFETY: the caller establishes that this is the post-fork child. unsafe { fail(report, LaunchStage::ProcessGroup, error) } } + if self.session + && let Err(error) = nix::unistd::setsid() + { + // SAFETY: as above. + unsafe { fail(report, LaunchStage::Session, error) } + } + if let Some((reader, writer)) = &self.barrier { + // The child drops its copy of the writer, so that it reads end of file once its + // caller shuts the writer down, or once no other copy of it remains. + // SAFETY: nothing in the child uses the writer, whose owner is never dropped here. + unsafe { nix::libc::close(writer.raw()) }; + if let Err(error) = wait_for_release(reader) { + // SAFETY: as above. + unsafe { fail(report, LaunchStage::Barrier, error) } + } + } if let Err(error) = self.place_descriptors() { // SAFETY: as above. unsafe { fail(report, LaunchStage::Descriptors, error) } } + if let Some(cgroup) = &self.cgroup + && let Err(error) = write_decimal(cgroup, nix::unistd::getpid().as_raw()) + { + // SAFETY: as above. + unsafe { fail(report, LaunchStage::Cgroup, error) } + } + if let Some(entry) = &self.entry + && let Err((stage, error)) = entry.enter_view() + { + // SAFETY: as above. + unsafe { fail(report, stage, error) } + } if let Some(root) = &self.root && let Err(error) = enter_root(root) { // SAFETY: as above. unsafe { fail(report, LaunchStage::Root, error) } } + if let Some(entry) = &self.entry + && let Err((stage, error)) = entry.take_identity() + { + // SAFETY: as above. + unsafe { fail(report, stage, error) } + } if let Some(directory) = &self.directory { match &self.credentials { Some(credentials) => { @@ -424,12 +540,6 @@ impl PreparedChild { unsafe { fail(report, LaunchStage::PhysicalWorkingDirectory, Errno::ENOENT) } } } - if let Some(cgroup) = &self.cgroup - && let Err(error) = write_decimal(cgroup, nix::unistd::getpid().as_raw()) - { - // SAFETY: as above. - unsafe { fail(report, LaunchStage::Cgroup, error) } - } // `nix`'s `exec` wrappers allocate their pointer vectors, so libc takes the ones that the // parent prepared. // SAFETY: the vectors are NUL-terminated, and every pointer names storage retained by the @@ -450,8 +560,13 @@ impl PreparedChild { /// Places the standard streams and clears close-on-exec on the inherited descriptors. /// /// Every source has a number above the standard streams', so `dup2` replaces each stream - /// without closing a source, and clears close-on-exec on its copy. + /// without closing a source, and clears close-on-exec on its copy. The child of an entry + /// first makes every other descriptor close on `exec`, those that its caller opened without + /// close-on-exec too, so that its program holds none of the caller's. fn place_descriptors(&self) -> Result<(), Errno> { + if self.entry.is_some() { + super::entry::close_descriptors_on_exec()?; + } let [stdin, stdout, stderr] = &self.streams; if let Some(source) = stdin { retry(|| nix::unistd::dup2_stdin(source.borrowed()))?; @@ -572,6 +687,21 @@ fn enter_root(root: &FileDescriptor) -> Result<(), Errno> { nix::unistd::chroot(c".") } +/// Waits until the caller releases the barrier whose reader is `reader`, which a byte tells. +/// End of file without a release, once the caller has shut the writer down, or once no copy of +/// the writer remains, is `ECANCELED`. +fn wait_for_release(reader: &FileDescriptor) -> Result<(), Errno> { + let mut byte = [0u8; 1]; + loop { + match reader.read(&mut byte) { + Ok(0) => return Err(Errno::ECANCELED), + Ok(_) => return Ok(()), + Err(Errno::EINTR) => {} + Err(error) => return Err(error), + } + } +} + /// Repeats `call` while it is interrupted. fn retry(mut call: impl FnMut() -> Result<(), Errno>) -> Result<(), Errno> { loop { @@ -643,6 +773,9 @@ pub(crate) const CAP_SETUID: u32 = 7; /// Capability to change the root directory. #[cfg(test)] pub(crate) const CAP_SYS_CHROOT: u32 = 18; +/// Capability to mount, and to enter new namespaces, which an execution's entry needs. +#[cfg(test)] +pub(crate) const CAP_SYS_ADMIN: u32 = 21; /// Returns whether the calling thread's effective set holds `capability`, because tests whose /// outcome depends on a privilege cannot infer it from the user ID: a root in a container may @@ -718,7 +851,11 @@ mod tests { /// Spawns `spec`, reads its launch report to the end, reaps it, and returns both. fn run(spec: ChildSpec) -> (Option<(i32, i32)>, Option) { - let (process, mut reader) = PreparedChild::new(spec).unwrap().spawn().unwrap(); + let Started { + process, + report: mut reader, + .. + } = PreparedChild::new(spec).unwrap().spawn().unwrap(); let mut report = Vec::new(); reader.read_to_end(&mut report).unwrap(); let report = (report.len() == REPORT_LENGTH).then(|| { @@ -843,7 +980,11 @@ mod tests { let mut spec = shell("read -r line"); spec.streams[0] = Some(reader); spec.process_group = process_group; - let (process, _report) = PreparedChild::new(spec).unwrap().spawn().unwrap(); + let Started { + process, + report: _report, + .. + } = PreparedChild::new(spec).unwrap().spawn().unwrap(); let child = Pid::from_raw(process.raw()); let expected = if process_group { child @@ -994,4 +1135,30 @@ mod tests { assert!(FdFlag::from_bits_retain(flags).contains(FdFlag::FD_CLOEXEC)); } } + + #[test] + fn an_entry_leads_a_session_and_keeps_its_barrier_from_the_program() { + if crate::guest::managed_agent::isolate_process_test( + "NVX_SPAWN_ENTRY_BARRIER_TEST_CHILD", + "guest::sys::spawn::tests::an_entry_leads_a_session_and_keeps_its_barrier_from_the_program", + ) { + return; + } + // With the standard streams closed, the barrier's ends take their numbers. + let streams = ClosedStreams::new(&[0, 1, 2]); + let mut spec = shell("exit 0"); + spec.barrier = true; + spec.entry = Some(EntryChild::default()); + let prepared = PreparedChild::new(spec); + drop(streams); + let prepared = prepared.unwrap(); + assert!(prepared.session && !prepared.process_group); + assert!(prepared.entry.is_some()); + let (reader, writer) = prepared.barrier.as_ref().unwrap(); + for descriptor in [reader, writer] { + assert!(descriptor.raw() >= FIRST_NONSTANDARD); + let flags = nix::fcntl::fcntl(descriptor.borrowed(), FcntlArg::F_GETFD).unwrap(); + assert!(FdFlag::from_bits_retain(flags).contains(FdFlag::FD_CLOEXEC)); + } + } } diff --git a/nvx-rs/src/lib.rs b/nvx-rs/src/lib.rs index 03fcb0eda..4acf97948 100644 --- a/nvx-rs/src/lib.rs +++ b/nvx-rs/src/lib.rs @@ -34,16 +34,18 @@ //! - `guest` (feature `guest`, Linux): the programs that run in NVX guests, such as //! `nvx-console-pending`, the guest's NVXS connection, the libraries through which any guest //! init or agent opens the terminal on which it serves its session (`guest::terminal`), starts -//! a program in a new process (`guest::spawn`), collects the status of each child that it names -//! (`guest::process`), places, kills, and settles the processes of its executions in cgroups -//! (`guest::cgroup`), mounts host mappings beneath the descriptors of the roots that it names -//! (`guest::mappings`), drives each program as one execution, with its launch, bounded -//! output, input, ending, and outcome (`guest::exec`), alone or under the execution engine -//! (`guest::exec::binding`), bootstraps the guest for a layered root, assembles the root, mounts -//! live shares and mappings in it, and tears it down (`guest::root`), and reads the partition -//! table of a GPT disk's block device (`guest::gpt`), and the typed calls through which it runs -//! the time ABI's steps (`guest::time`) and reseeds after a restore (`guest::reseed`) in its own -//! process. The crate's `nvx-*` guest binaries run the programs. +//! a program in a new process (`guest::spawn`), has the program enter an execution's +//! namespaces, view, root, and identity (`guest::entry`), collects the status of each child +//! that it names (`guest::process`), places, kills, and settles the processes of its +//! executions in cgroups (`guest::cgroup`), mounts host mappings beneath the descriptors of the +//! roots that it names (`guest::mappings`), drives each program as one execution, with its +//! launch, bounded output, input, ending, and outcome (`guest::exec`), alone or under the +//! execution engine (`guest::exec::binding`), bootstraps the guest for a layered root, +//! assembles the root, mounts live shares and mappings in it, and tears it down +//! (`guest::root`), and reads the partition table of a GPT disk's block device (`guest::gpt`), +//! and the typed calls through which it runs the time ABI's steps (`guest::time`) and reseeds +//! after a restore (`guest::reseed`) in its own process. The crate's `nvx-*` guest binaries run +//! the programs. //! //! # Example //! From cee66a11649d1073552999e0cd33b9e112e8ba6f Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Thu, 22 Oct 2026 02:12:28 -0700 Subject: [PATCH 03/12] tests: enter the runtime adapter's layered root as root profiles The adapter's worker gains an entries probe, which enters its assembled root through nvx::guest::entry and nvx::guest::spawn, as an adapter's root profiles do. The worker copies its own executable into the root and runs the executions in a helper that enters a mount namespace of its own first, as NVX's managed agent does, so that the runtime's binds of a generated machine ID and of the time status end with it before the worker tears the root down. Each execution joins the workload cgroup, receives a descriptor of the guest's own root that its caller leaks without close-on-exec, and runs the staged executable, which reports from inside the root what the execution sees and may do. The root_profiles_enter_the_root scenario checks that user ID 0 with each allowed capability alone holds only that capability, and that only its own attempt succeeds, that root with none and a numeric identity without an account may do none of it, and that the entry refuses the capabilities outside its allowlist, the binding and raw-socket ones while listeners are published, capabilities for another user ID, and an identity without an account in existing-account mode. Every execution is PID 1 of its namespaces with only its standard streams, cannot write /proc/sys or lower the unprivileged port start, and stays in its root after a nested chroot escape. With published listeners, which both publication modes reach the entry as, a root workload can bind no privileged source port and open no raw socket, so it cannot forge a listener announcement. The executions see the root's own machine ID, then a generated one and the time status that the guest's time daemon publishes, read-only, in which a root workload finds no lock of the daemon's, and which leave no mount in the worker's namespace. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- tests/runtime-adapter/Cargo.toml | 9 +- tests/runtime-adapter/README.md | 49 ++ .../src/bin/runtime-adapter-guest.rs | 54 +- tests/runtime-adapter/src/lib.rs | 9 + tests/runtime-adapter/src/worker.rs | 13 +- tests/runtime-adapter/src/worker/entries.rs | 683 ++++++++++++++++++ tests/runtime-adapter/tests/vm_layered.rs | 241 ++++++ 7 files changed, 1048 insertions(+), 10 deletions(-) create mode 100644 tests/runtime-adapter/src/worker/entries.rs diff --git a/tests/runtime-adapter/Cargo.toml b/tests/runtime-adapter/Cargo.toml index 5e2170514..0aebeee74 100644 --- a/tests/runtime-adapter/Cargo.toml +++ b/tests/runtime-adapter/Cargo.toml @@ -28,16 +28,19 @@ serde_json = { version = "1.0.120", optional = true } # The guest agent's PID 1 collects the status of every child and ends the unrelated one, its # worker polls its nonblocking session and executions, its probes read how much space a file -# system has, and its mapping probes mount a procfs in the root: system calls that `nvx` leaves -# to the caller. The guest side of `nvx` depends on the same release, with these features among -# its own. +# system has, its mapping probes mount a procfs in the root, and +# the helper of its `entries` probe enters a mount namespace of its own and leaks a descriptor to +# its executions, which open raw sockets: system calls that `nvx` leaves to the caller. The guest +# side of `nvx` depends on the same release, with these features among its own. [target.'cfg(target_os = "linux")'.dependencies] nix = { version = "0.31.3", default-features = false, features = [ "fs", "mount", "poll", "process", + "sched", "signal", + "socket", ], optional = true } # The guest agent's tests open pseudo-terminals, and trace a child. The guest side of `nvx` diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index e9db3deaa..35de780f0 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -234,6 +234,42 @@ bytes, 2 if standard error did, and 4 if the execution is reusable. Stage 0 is the worker's refusal of the start, and a later stage is the binding's number of the stage of `nvx::guest::spawn` that failed, such as 8 for its `exec`. +### Root-profile entries + +The probe of `worker::entries`, which takes no arguments either, enters the +root as an adapter's root profiles do, through `nvx::guest::entry` and +`nvx::guest::spawn`: each execution in new mount, PID, and UTS namespaces, +with the root moved over `/`, a view of its own, and an identity with only the +capabilities that the entry allows. The worker copies its own executable to +`/.runtime-adapter-guest` at the top of the root, and runs the executions in a +helper, `runtime-adapter-guest entries ROOT CGROUP`, which enters a mount +namespace of its own first, as NVX's managed agent does, so that the runtime's +binds of a generated machine ID and of the time status end with it, before the +worker tears the root down. The helper places each execution in the workload +cgroup, leaks it a descriptor of the guest's own `/` without close-on-exec, +and has it run `runtime-adapter-guest inside DIRECTORY DESCRIPTOR` from the +root, which reports what the execution sees and may do, in its case's +directory below `/tmp/runtime-adapter-entries`. + +| Probe | Facts | +| --- | --- | +| `entries` | The capabilities outside the allowlist that the entry refused, `refused.outside`, and any that it accepted, `refused.accepted`; its refusal of `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while listeners are published, `refused.published.net-bind-service` and `refused.published.net-raw`, of a capability for a user ID other than 0, `refused.non-root`, and of an identity without an account in existing-account mode, `refused.existing`. For each case, `CASE.launch` and `CASE.status`, and what the execution reported, as `CASE.KEY`: the descriptors open in it, `fds`, and whether the leaked one is, `leaked`; `uid`, `gid`, and `groups`; the capability sets, `cap.inh`, `cap.prm`, `cap.eff`, `cap.bnd`, and `cap.amb`, and `no-new-privs`; `pid`; `hostname`; the `machine-id` and a mount's options at `/etc/machine-id`, `mount.machine-id`; the first line of the time status's state, `time.state`, the names in `/run/nvx/time`, `time.files`, and a mount's options there, `mount.time`; the unprivileged port start, `threshold`; the outcome of lowering it, `threshold.write`, of writing the hostname through `/proc/sys`, `hostname.write`, of opening `/proc/sysrq-trigger` for writing, `sysrq.write`, of binding UDP port 1023, a privileged source port, from which the gateway accepts listener announcements, `bind`, of opening a raw socket, which could forge such a datagram, `raw`, of changing a new file's owner, `chown`, of reading a new file of mode 0, `dac-override`, of changing the mode of a file of user ID 1000, `fowner`, and of running a program as user ID 4321, `setuid`, and as group ID 4321, `setgid`; the root's entries, `root`; and, last, a nested `chroot` escape, `escape`: `contained` if climbing with `..` from outside a `chroot` ends in the execution's own root. The generated machine ID, `generated.source`. Once the helper ended, the root's own machine ID, `after.machine-id`, and the mounts below the root, `after.mounts`, in the worker's mount namespace | + +The cases, as user ID 0 in numeric-account mode unless they say otherwise, are +each allowed capability alone, without published listeners, `only-NAME`, such +as `only-net-bind-service`; no capability, `none`; every allowed capability +but those that published listeners refuse, with published listeners, +`published`, which stands for both publication modes, private-network ingress +and the host's loopback, since the entry treats them alike; every allowed +capability, without published listeners, `unpublished`; user and group ID +4242, of which the image has no account, `numeric`; and, once the helper has +bound the runtime's files, a generated machine ID, the time status that the +guest's time daemon publishes at `/run/nvx/time`, and the hostname +`runtime-adapter`, `generated`. The helper first writes a machine ID +in the root, as its own, which the cases before `generated` see. An attempt's +outcome is `allowed`, `refused ERRNO`, such as `refused EPERM`, or `failed +PROBLEM`. + ## Guest agent `guest::serve` serves the host program on a stream to the guest's end of the @@ -416,6 +452,15 @@ credit that it needs, never keeps the others' output from the host. whose status it has not collected, and waits up to a second in all for those programs to end. +The worker's `entries` probe runs two more commands of the guest agent: +`runtime-adapter-guest entries ROOT CGROUP`, its helper, which runs the probe's +executions in the root ROOT, each placed in the cgroup CGROUP, in a mount +namespace of its own, reports their facts on standard output, and exits with 0, +or with 1 after a diagnostic on standard error; and +`runtime-adapter-guest inside DIRECTORY DESCRIPTOR`, which each execution runs +from inside the root, and which reports what the execution sees and may do; +see [Probes](#probes). + ## Host program `host::Client` is the host program's client of the guest agent. It runs over @@ -642,6 +687,9 @@ its own, with a thread of each of two children that it traces, which stop before the worker ends, one with a signal whose report `nix` decodes and one with a real-time signal, and check that PID 1 takes both stops and goes on until it collects the worker's status. +Those of `worker::entries` cover the names of capabilities, the fields of a +process's status, the outcomes of attempts, the entry's refusals, and the +probe's arguments. `tests/guest_session.rs` checks, in the guest-only configuration, that the guest agent serves the control session with the adapter's own framing. A test @@ -832,6 +880,7 @@ own: | `busy_root_fails_the_teardown` | The image over the scratch, which the `hold` probe keeps open | The teardown's failure, which the stop reports, and a VM that ends with 15, leaving the scratch mounted | | `mappings_apply_beneath_the_root` | The image over the scratch, beneath which the `map` and `map-checks` probes map | Through the root: zero mappings that do no work, then a table mounted in order, each mapping with its access, `nosuid`, and `nodev`, whose writes reach the sources of the writable ones only, on targets that the mapper created on the overlay. Through mappers of the adapter's own: a link or a magic link at each component, the wrong kinds, a target that would be created inside another mapping, and the runtime's, the container root's, and the adapter's own reserved targets refused, and a read-only mapping that is never writable. A teardown that unmounts the mappings, then the scratch, cleanly | | `a_refused_mapping_latches_the_root` | The image over the scratch, in which the `map-refused` probe plants a symbolic link | A mapping through the link that the root refuses, nothing mounted or created, and a root that refuses every later mapping, then tears down cleanly | +| `root_profiles_enter_the_root` | The image over the scratch, which the `entries` probe enters | Executions that start as PID 1 of their namespaces, with the root moved over `/`, only their standard streams, the leaked descriptor closed, no inheritable or ambient capability, and no new privileges. User ID 0 with each allowed capability alone holds it alone, and only its own attempt succeeds; with none, nothing does, nor for a numeric identity without an account. No execution writes `/proc/sys`, so none lowers the unprivileged port start, and a nested `chroot` stays in the root. With published listeners, user ID 0 with every other allowed capability can neither bind a privileged source port nor open a raw socket, so it cannot forge a listener announcement in either publication mode; without them, it may. The entry refuses the 51 capabilities outside its allowlist, the binding and raw-socket ones while listeners are published, capabilities for another user ID, and an identity without an account in existing-account mode. The root's own machine ID, then a generated one and the time status that the guest's time daemon publishes, read-only, in which a root workload finds no lock of the daemon's, which the runtime bound in the helper's mount namespace, and which leave the worker's namespace without mounts below the root. A teardown that unmounts the scratch cleanly | A row names the root, the image, whether a GPT disk's top layer is corrupt, the identity that the launch expects of the image, the upper, the runtime diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs index ae0ab1938..da8206ffd 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs @@ -22,6 +22,14 @@ //! session failed or the broker closed the tty, each but the first after a diagnostic on //! standard error. //! - `runtime-adapter-guest idle` sleeps until a signal ends it. +//! - `runtime-adapter-guest entries ROOT CGROUP` runs the executions of the worker's `entries` +//! probe in the root ROOT, each placed in the cgroup CGROUP, in a mount namespace of its own, and +//! reports their facts on standard output; see `runtime_adapter::worker::entries`. It exits with +//! 0, or with 1 after a diagnostic on standard error. +//! - `runtime-adapter-guest inside DIRECTORY DESCRIPTOR` runs in each of those executions, from +//! inside the root: it reports what the execution sees and may do on standard output, with its +//! case's directory DIRECTORY and the number of the descriptor that its caller leaked. It exits +//! with 0, or with 1 after a diagnostic on standard error. //! //! As PID 1, which must not exit, `init` ends the VM with its status instead, through NVX's //! `/sbin/nvx-exit`, as NVX's managed agent does. Anything else is a usage error, with status 2. @@ -31,8 +39,8 @@ use std::process::ExitCode; #[cfg(target_os = "linux")] fn main() -> ExitCode { use runtime_adapter::{ - GUEST_IDLE_COMMAND, GUEST_INIT_COMMAND, GUEST_LAYERED_MODE, GUEST_SERVE_COMMAND, - GUEST_WORKER_COMMAND, + GUEST_ENTRIES_COMMAND, GUEST_IDLE_COMMAND, GUEST_INIT_COMMAND, GUEST_INSIDE_COMMAND, + GUEST_LAYERED_MODE, GUEST_SERVE_COMMAND, GUEST_WORKER_COMMAND, }; use std::path::Path; @@ -58,6 +66,13 @@ fn main() -> ExitCode { } } [command] if command == GUEST_IDLE_COMMAND => runtime_adapter::worker::idle(), + [command, root, workload] if command == GUEST_ENTRIES_COMMAND => { + entries(Path::new(root), Path::new(workload)) + } + [command, directory, descriptor] if command == GUEST_INSIDE_COMMAND => { + let descriptor = descriptor.to_str().and_then(|number| number.parse().ok()); + inside(Path::new(directory), descriptor) + } _ => usage(), } } @@ -66,12 +81,15 @@ fn main() -> ExitCode { #[cfg(target_os = "linux")] fn usage() -> ExitCode { eprintln!( - "usage: runtime-adapter-guest [{} | {} TTY [{}] | {} TTY [SIBLING] | {}]", + "usage: runtime-adapter-guest [{} | {} TTY [{}] | {} TTY [SIBLING] | {} | {} ROOT CGROUP | \ + {} DIRECTORY DESCRIPTOR]", runtime_adapter::GUEST_SERVE_COMMAND, runtime_adapter::GUEST_INIT_COMMAND, runtime_adapter::GUEST_LAYERED_MODE, runtime_adapter::GUEST_WORKER_COMMAND, runtime_adapter::GUEST_IDLE_COMMAND, + runtime_adapter::GUEST_ENTRIES_COMMAND, + runtime_adapter::GUEST_INSIDE_COMMAND, ); ExitCode::from(2) } @@ -146,6 +164,36 @@ fn worker(tty: &std::path::Path, sibling: Option) -> ExitCode { ExitCode::from(status) } +/// Runs the executions of the worker's `entries` probe in the root `root`, each placed in the cgroup +/// `workload`, and reports their facts. +#[cfg(target_os = "linux")] +fn entries(root: &std::path::Path, workload: &std::path::Path) -> ExitCode { + use runtime_adapter::worker::entries; + + match entries::helper(root, workload).and_then(|facts| entries::report(&facts)) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("runtime-adapter-guest: {error}"); + ExitCode::FAILURE + } + } +} + +/// Reports, from inside an execution of the worker's `entries` probe, what it sees and may do, with +/// its case's `directory` and the number of the descriptor that its caller leaked, if any. +#[cfg(target_os = "linux")] +fn inside(directory: &std::path::Path, descriptor: Option) -> ExitCode { + use runtime_adapter::worker::entries; + + match entries::report(&entries::inside(directory, descriptor)) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("runtime-adapter-guest: {error}"); + ExitCode::FAILURE + } + } +} + /// Ends with `status`: as PID 1, by ending the VM through [`NVX_EXIT`]. #[cfg(target_os = "linux")] fn end(status: u8) -> ExitCode { diff --git a/tests/runtime-adapter/src/lib.rs b/tests/runtime-adapter/src/lib.rs index 9763882d7..a0c43e802 100644 --- a/tests/runtime-adapter/src/lib.rs +++ b/tests/runtime-adapter/src/lib.rs @@ -69,6 +69,15 @@ pub const GUEST_WORKER_COMMAND: &str = "worker"; /// signal ends it. pub const GUEST_IDLE_COMMAND: &str = "idle"; +/// Command that runs the executions of the worker's `entries` probe in a mount namespace of its +/// own, with the root and the workload cgroup that follow it; see `worker::entries`. +pub const GUEST_ENTRIES_COMMAND: &str = "entries"; + +/// Command that an execution of the worker's `entries` probe runs inside the root, with its case's +/// directory and the number of the descriptor that its caller leaked to it, and that reports what +/// the execution sees and may do; see `worker::entries`. +pub const GUEST_INSIDE_COMMAND: &str = "inside"; + /// Describes the runtime contract that this build speaks: the control contract revision, and the /// versions of the microVM ABI, the NVXS records, and the NVXC frames. pub fn contract() -> String { diff --git a/tests/runtime-adapter/src/worker.rs b/tests/runtime-adapter/src/worker.rs index 813db0e6a..d8c586b39 100644 --- a/tests/runtime-adapter/src/worker.rs +++ b/tests/runtime-adapter/src/worker.rs @@ -23,9 +23,9 @@ //! 3. It reads the boot layout of the root from the kernel command line with //! `nvx::protocol::root::RootLayout::parse`, then bootstraps the guest and assembles the root. //! 4. It answers `RootReady`, or `RootFailed` with the stage that failed, and then ends. -//! 5. It serves `Ping`, `Echo`, and `Probe`, whose probes the [`probes`] and [`mapping`] modules -//! run, and runs the host's executions. It runs a probe only while it runs no execution, and -//! otherwise answers it with `ProbeFailed` and [`EXECUTING`]. +//! 5. It serves `Ping`, `Echo`, and `Probe`, whose probes the [`probes`], [`mapping`], and +//! [`entries`] modules run, and runs the host's executions. It runs a probe only while it runs +//! no execution, and otherwise answers it with `ProbeFailed` and [`EXECUTING`]. //! 6. On `Stop`, it drops its executions, which ends their programs, tears the root down, reports //! a teardown that failed with `RootFailed`, confirms the stop with `Stopped`, and ends. //! @@ -107,6 +107,7 @@ mod connection; mod executions; +pub mod entries; pub mod mapping; pub mod probes; @@ -593,7 +594,8 @@ impl RootOperations for LibraryRoot { Ok(()) } - /// Runs one of the [`probes`], or of the [`mapping`] probes, which map beneath the root. + /// Runs one of the [`probes`], of the [`mapping`] probes, which map beneath the root, or of the + /// [`entries`] probes, which enter it. fn probe(&mut self, probe: &Probe) -> Result { let (Some(bootstrap), Some(root)) = (&self.bootstrap, &mut self.root) else { return Err("the worker holds no root to probe".to_owned()); @@ -625,6 +627,9 @@ impl RootOperations for LibraryRoot { if mapping::PROBES.contains(&probe.name()) { return mapping::run(probe, &context, root); } + if entries::PROBES.contains(&probe.name()) { + return entries::run(probe, &context); + } probes::run(probe, &context, &mut self.held) } diff --git a/tests/runtime-adapter/src/worker/entries.rs b/tests/runtime-adapter/src/worker/entries.rs new file mode 100644 index 000000000..0509863c3 --- /dev/null +++ b/tests/runtime-adapter/src/worker/entries.rs @@ -0,0 +1,683 @@ +//! The probe that enters the worker's assembled root as an adapter's root profiles do: through +//! `nvx::guest::entry` and `nvx::guest::spawn`, each execution in new mount, PID, and UTS +//! namespaces, with the root moved over `/`, a view of its own, and an identity with only the +//! capabilities that the entry allows. +//! +//! The worker copies its own executable to the top of the root, as [`STAGED`], and runs the +//! executions in a helper, `runtime-adapter-guest entries ROOT CGROUP`, which enters a mount +//! namespace of its own first, as NVX's managed agent does, so that the runtime's binds of a +//! generated machine ID and of the time status end with it, before the worker tears the root +//! down. The helper places each execution in the workload cgroup `CGROUP`, leaks it a descriptor +//! of the guest's own `/` without close-on-exec, and has it run [`STAGED`] as +//! `runtime-adapter-guest inside DIRECTORY DESCRIPTOR`, which reports, from inside the root, what +//! the execution sees and may do, in the case's directory below [`CASES`]. +//! +//! | Probe | Facts | +//! | --- | --- | +//! | `entries` | The capabilities outside the allowlist that the entry refused, `refused.outside`, and those that it accepted, `refused.accepted`; its refusal of `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while listeners are published, `refused.published.net-bind-service` and `refused.published.net-raw`, of a capability for another user ID than 0, `refused.non-root`, and of an identity without an account in existing-account mode, `refused.existing`. For each case: `CASE.launch`, `CASE.status`, and the facts of `inside`, as `CASE.KEY`. The generated machine ID, `generated.source`. Once the helper ended, the root's own machine ID, `after.machine-id`, and the mounts below the root, `after.mounts`, in the worker's mount namespace | +//! +//! The cases, as user ID 0 in numeric-account mode unless they say otherwise, are: each allowed +//! capability alone, unpublished, `only-NAME`, such as `only-net-bind-service`; no capability, +//! `none`; every allowed capability but those that published listeners refuse, published, +//! `published`, which stands for both publication modes, private-network ingress and the host's +//! loopback, since the entry treats them alike; every allowed capability, unpublished, +//! `unpublished`; the user and group ID 4242, of which the image has no account, `numeric`; and, +//! once the helper has bound the runtime's files, a generated machine ID, the time status that +//! the guest's time daemon publishes at `/run/nvx/time`, with the daemon's own layout, and the +//! hostname `runtime-adapter`, `generated`. The cases before `generated` see the machine ID that +//! the root holds as its own. +//! +//! `inside` reports, in this order: the descriptors open in the execution, `fds`, and whether the +//! leaked one is, `leaked`; `uid`, `gid`, and `groups`; the capability sets, `cap.inh`, +//! `cap.prm`, `cap.eff`, `cap.bnd`, and `cap.amb`, and `no-new-privs`; its process ID, `pid`; the +//! `hostname`; the `machine-id`, and the options of a mount at `/etc/machine-id`, if any, +//! `mount.machine-id`; the first line of the time status's state, `time.state`, the names in +//! `/run/nvx/time`, `time.files`, and the options of a mount there, `mount.time`; the +//! unprivileged port start, `threshold`; and the outcome of +//! these attempts: lowering it, `threshold.write`; writing the hostname through `/proc/sys`, +//! `hostname.write`; opening `/proc/sysrq-trigger` for writing, `sysrq.write`; binding UDP port +//! 1023, a privileged source port, from which the gateway accepts listener announcements, +//! `bind`; opening a raw socket, which could forge such a datagram, `raw`; changing the owner of +//! a new file, `chown`; reading a new file of mode 0, `dac-override`; changing the mode of a file +//! of user ID 1000, `fowner`; running a program as user ID 4321, `setuid`, and as group ID 4321, +//! `setgid`. Then it lists the root, `root`, and last tries a nested `chroot` escape, which +//! enters a directory of the root with `chroot`, climbs with `..` from outside it, and enters +//! where it ended with `chroot` again: `escape` is `contained` if it ends in the execution's own +//! root, `escaped: LISTING` otherwise, or the outcome of the first `chroot`. +//! +//! An attempt's outcome is `allowed`, `refused ERRNO`, such as `refused EPERM`, or `failed +//! PROBLEM`. A value that cannot be read is `absent` if its file does not exist, and otherwise +//! `unreadable: PROBLEM`. + +use std::ffi::{CString, OsStr}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::os::fd::{AsFd, AsRawFd, OwnedFd}; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +use nix::errno::Errno; +use nix::mount::MsFlags; +use nix::sched::CloneFlags; +use nix::sys::socket::{AddressFamily, SockFlag, SockProtocol, SockType}; +use nix::sys::wait::{self, WaitStatus}; +use nix::unistd::Pid; +use nvx::guest::entry::{ + AccountError, AccountMode, Capabilities, Capability, Entry, Error as EntryError, Identity, + MachineId, Publication, +}; +use nvx::guest::spawn::{Cgroup, Command}; + +use super::probes::{self, Context, MOUNTINFO}; +use crate::probe::{Facts, Probe}; + +/// The names of the probes of this module, which the worker runs with its root. +pub const PROBES: [&str; 1] = ["entries"]; + +/// Where the worker copies its own executable in the root, which each execution runs. +pub const STAGED: &str = "/.runtime-adapter-guest"; +/// The directory, in the root, of the cases' directories, one for each case, of its name. +pub const CASES: &str = "/tmp/runtime-adapter-entries"; +/// The directory, outside the root, of the generated machine ID. +const SOURCES: &str = "/tmp/runtime-adapter-entry-sources"; +/// The machine ID that the helper writes in the root, as the root's own. +pub const ROOT_MACHINE_ID: &str = "0123456789abcdef0123456789abcdef"; +/// The guest's time status, which NVX's time daemon publishes from the boot, and which the +/// runtime binds into the root at the same path. +const TIME_STATUS: &str = "/run/nvx/time"; +/// The hostname of the `generated` case. +pub const GENERATED_HOSTNAME: &str = "runtime-adapter"; +/// The user and group ID of the `numeric` case, of which the image has no account. +pub const NUMERIC_ID: u32 = 4242; +/// The owner of each case's `owned` file, whose mode only `CAP_FOWNER` lets another user change. +const OWNER: u32 = 1000; +/// The user or group ID that the `setuid` and `setgid` attempts take. +const OTHER_ID: u32 = 4321; +/// The privileged UDP port that the `bind` attempt binds. +const PRIVILEGED_PORT: u16 = 1023; +/// The unprivileged port start, which the runtime owns. +const THRESHOLD: &str = "/proc/sys/net/ipv4/ip_unprivileged_port_start"; +/// How many times the `escape` attempt climbs with `..`. +const CLIMBS: usize = 32; + +/// Runs `probe`, one of [`PROBES`], with `context`, and returns its facts. +/// +/// Fails with the problem for another probe, one with arguments, and one that fails. +pub fn run(probe: &Probe, context: &Context<'_>) -> Result { + let name = probe.name(); + if !PROBES.contains(&name) { + return Err(format!("the worker runs no probe named {name}")); + } + if !probe.arguments().is_empty() { + return Err(format!("the {name} probe takes no arguments")); + } + entries(context).map_err(|error| format!("the {name} probe failed: {error}")) +} + +/// Runs the `entries` probe: stages the worker's executable, runs the helper, and reports what it +/// found, and then what the worker's own mount namespace holds. +fn entries(context: &Context<'_>) -> io::Result { + let program = std::env::current_exe()?; + let staged = context.root.join(below_root(STAGED)); + fs::copy(&program, &staged)?; + fs::set_permissions(&staged, fs::Permissions::from_mode(0o755))?; + let output = std::process::Command::new(&program) + .arg(crate::GUEST_ENTRIES_COMMAND) + .arg(context.root) + .arg(context.workload) + .stdin(Stdio::null()) + .output()?; + if !output.status.success() { + return Err(io::Error::other(format!( + "the helper ended with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ))); + } + let mut facts = Facts::decode(&output.stdout) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.problem().to_owned()))?; + facts.push( + "after.machine-id", + probes::read(&context.root.join("etc/machine-id")), + ); + let table = probes::mount_table(&fs::read_to_string(MOUNTINFO)?)?; + let below = table + .iter() + .filter(|mount| mount.point != context.root && mount.point.starts_with(context.root)) + .count(); + facts.push("after.mounts", below.to_string()); + Ok(facts) +} + +/// Runs the executions of the `entries` probe in the root `root`, which each joins the cgroup +/// `workload`, as the helper `runtime-adapter-guest entries ROOT CGROUP` does, and returns their +/// facts, as the [module documentation](self) describes. +/// +/// It first enters a mount namespace of its own, whose mounts stop propagating, so that the +/// runtime's binds end with the process. It needs the privileges of the guest's root, and must be +/// the only thread of its process. +pub fn helper(root: &Path, workload: &Path) -> io::Result { + nix::sched::unshare(CloneFlags::CLONE_NEWNS)?; + nix::mount::mount( + None::<&str>, + "/", + None::<&str>, + MsFlags::MS_REC | MsFlags::MS_PRIVATE, + None::<&str>, + )?; + let mut facts = Facts::new(); + refusals(root, &mut facts); + + // A descriptor of the guest's own root, which a caller leaks to each execution: it is not + // close-on-exec, unlike the descriptors that the standard library opens. + let leaked = nix::unistd::dup(File::open("/")?)?; + fs::write(root.join("etc/machine-id"), format!("{ROOT_MACHINE_ID}\n"))?; + let profile = |capabilities| { + Identity::new(0, 0) + .capabilities(capabilities) + .account(AccountMode::Numeric) + }; + let unpublished = |identity| Entry::new(root, identity).publication(Publication::Unpublished); + let mut cases = Capabilities::ALLOWED + .iter() + .map(|capability| { + let name = format!("only-{}", capability_name(capability)); + ( + name, + unpublished(profile(Capabilities::EMPTY.with(capability))), + ) + }) + .collect::>(); + let published = Capabilities::ALLOWED + .iter() + .filter(|capability| !Capabilities::PUBLISHED_REFUSED.contains(*capability)) + .collect(); + cases.extend([ + ("none".to_owned(), unpublished(profile(Capabilities::EMPTY))), + ( + "published".to_owned(), + Entry::new(root, profile(published)).publication(Publication::Published), + ), + ( + "unpublished".to_owned(), + unpublished(profile(Capabilities::ALLOWED)), + ), + ( + "numeric".to_owned(), + Entry::new( + root, + Identity::new(NUMERIC_ID, NUMERIC_ID).account(AccountMode::Numeric), + ), + ), + ]); + for (name, entry) in &cases { + execute(root, workload, name, entry, &leaked, &mut facts)?; + } + + // The runtime's files, which the entry binds once, for every later execution: a machine ID + // that it generates, and the time status that the guest's time daemon publishes, with the + // daemon's own layout. + let sources = Path::new(SOURCES); + if sources.exists() { + fs::remove_dir_all(sources)?; + } + fs::create_dir_all(sources)?; + let machine_id = MachineId::generate(sources.join("machine-id"))?; + facts.push( + "generated.source", + probes::read(&sources.join("machine-id")), + ); + let generated = Entry::new(root, profile(Capabilities::EMPTY)) + .hostname(GENERATED_HOSTNAME) + .machine_id(machine_id) + .time_status(TIME_STATUS); + generated.bind_runtime().map_err(io::Error::other)?; + execute(root, workload, "generated", &generated, &leaked, &mut facts)?; + Ok(facts) +} + +/// Adds the entry's refusals of identities in `root` to `facts`. +fn refusals(root: &Path, facts: &mut Facts) { + let check = |identity: Identity, publication| { + refusal( + Entry::new(root, identity.account(AccountMode::Numeric)) + .publication(publication) + .check(), + ) + }; + let outside = (0..64) + .filter_map(Capability::new) + .filter(|capability| !capability.is_allowed()) + .collect::>(); + let accepted = outside + .iter() + .filter(|capability| { + let identity = Identity::new(0, 0).capabilities(Capabilities::EMPTY.with(**capability)); + check(identity, Publication::Unpublished) != format!("capability {capability}") + }) + .map(ToString::to_string) + .collect::>(); + facts.push( + "refused.outside", + (outside.len() - accepted.len()).to_string(), + ); + let accepted = if accepted.is_empty() { + "none".to_owned() + } else { + accepted.join(",") + }; + facts.push("refused.accepted", accepted); + for capability in Capabilities::PUBLISHED_REFUSED.iter() { + let identity = Identity::new(0, 0).capabilities(Capabilities::EMPTY.with(capability)); + facts.push( + format!("refused.published.{}", capability_name(capability)), + check(identity, Publication::Published), + ); + } + let identity = Identity::new(NUMERIC_ID, NUMERIC_ID) + .capabilities(Capabilities::EMPTY.with(Capability::KILL)); + facts.push( + "refused.non-root", + check(identity, Publication::Unpublished), + ); + let existing = Entry::new(root, Identity::new(NUMERIC_ID, NUMERIC_ID)).prepare(); + facts.push("refused.existing", refusal(existing.map(drop))); +} + +/// Returns how the entry took an identity: `accepted`, or what refused it. +fn refusal(result: Result<(), EntryError>) -> String { + match result { + Ok(()) => "accepted".to_owned(), + Err(EntryError::Capability(capability)) => format!("capability {capability}"), + Err(EntryError::NonRootCapabilities) => "non-root-capabilities".to_owned(), + Err(EntryError::Published(capability)) => format!("published {capability}"), + Err(EntryError::Account(AccountError::Missing)) => "account-missing".to_owned(), + Err(error) => format!("other: {error}"), + } +} + +/// Runs the execution of `entry` named `name`, in the cgroup `workload`, which runs [`STAGED`] +/// as `inside`, with `leaked` open in its caller, and adds its facts to `facts`. +fn execute( + root: &Path, + workload: &Path, + name: &str, + entry: &Entry, + leaked: &OwnedFd, + facts: &mut Facts, +) -> io::Result<()> { + let inside = format!("{CASES}/{name}"); + let directory = root.join(below_root(&inside)); + fs::create_dir_all(&directory)?; + fs::set_permissions(&directory, fs::Permissions::from_mode(0o777))?; + let owned = directory.join("owned"); + fs::write(&owned, name)?; + std::os::unix::fs::chown(&owned, Some(OWNER), Some(OWNER))?; + + let prepared = entry.prepare().map_err(io::Error::other)?; + let argument = |text: &str| CString::new(text).map_err(io::Error::other); + let arguments = [ + STAGED, + crate::GUEST_INSIDE_COMMAND, + &inside, + &leaked.as_raw_fd().to_string(), + ] + .into_iter() + .map(argument) + .collect::>>()?; + let (mut reader, writer) = io::pipe()?; + let spawned = Command::new(argument(STAGED)?, arguments) + .environment(Vec::new()) + .stdin(File::open("/dev/null")?) + .stdout(writer) + .stderr(io::stderr().as_fd().try_clone_to_owned()?) + .cgroup(Cgroup::open(workload)?) + .enter(prepared) + .spawn()?; + let process = Pid::from_raw(i32::try_from(spawned.id()).map_err(io::Error::other)?); + let launched = spawned.into_report().wait(); + let mut output = Vec::new(); + let read = reader.read_to_end(&mut output); + let status = wait::waitpid(process, None)?; + let launch = match launched? { + None => "started".to_owned(), + Some(failure) => format!( + "{} {:?}", + failure.stage(), + Errno::from_raw(failure.raw_os_error()) + ), + }; + read?; + facts.push(format!("{name}.launch"), launch); + let status = match status { + WaitStatus::Exited(_, code) => format!("exit {code}"), + WaitStatus::Signaled(_, signal, _) => format!("signal {signal}"), + status => format!("{status:?}"), + }; + facts.push(format!("{name}.status"), status); + let found = Facts::decode(&output) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.problem().to_owned()))?; + for (key, value) in found.iter() { + facts.push(format!("{name}.{key}"), value); + } + Ok(()) +} + +/// Reports, from inside an execution's root, what the execution sees and may do, as the +/// [module documentation](self) describes, as `runtime-adapter-guest inside DIRECTORY DESCRIPTOR` +/// does, with the case's `directory` and the number of the descriptor that its caller leaked, +/// if any. +/// +/// The last attempt, the nested `chroot` escape, changes the process's root. +pub fn inside(directory: &Path, leaked: Option) -> Facts { + let mut facts = Facts::new(); + facts.push("fds", open_descriptors()); + let leaked = leaked.map_or("none", |descriptor| { + if fs::symlink_metadata(format!("/proc/self/fd/{descriptor}")).is_ok() { + "open" + } else { + "closed" + } + }); + facts.push("leaked", leaked); + let status = fs::read_to_string("/proc/self/status").unwrap_or_default(); + let field = |name: &str| status_field(&status, name).unwrap_or("absent").to_owned(); + let effective = |ids: String| ids.split_whitespace().nth(1).unwrap_or("absent").to_owned(); + facts.push("uid", effective(field("Uid"))); + facts.push("gid", effective(field("Gid"))); + facts.push("groups", field("Groups")); + for (key, name) in [ + ("cap.inh", "CapInh"), + ("cap.prm", "CapPrm"), + ("cap.eff", "CapEff"), + ("cap.bnd", "CapBnd"), + ("cap.amb", "CapAmb"), + ("no-new-privs", "NoNewPrivs"), + ] { + facts.push(key, field(name)); + } + facts.push("pid", std::process::id().to_string()); + facts.push( + "hostname", + probes::read(Path::new("/proc/sys/kernel/hostname")), + ); + facts.push("machine-id", probes::read(Path::new("/etc/machine-id"))); + let table = fs::read_to_string(MOUNTINFO) + .and_then(|table| probes::mount_table(&table)) + .unwrap_or_default(); + let options = |point: &str| { + table + .iter() + .rev() + .find(|mount| mount.point == Path::new(point)) + .map_or("none".to_owned(), |mount| mount.options.clone()) + }; + facts.push("mount.machine-id", options("/etc/machine-id")); + let status = Path::new(TIME_STATUS); + let state = probes::read(&status.join("state")); + facts.push("time.state", state.lines().next().unwrap_or_default()); + facts.push("time.files", listing(status)); + facts.push("mount.time", options(TIME_STATUS)); + facts.push("threshold", probes::read(Path::new(THRESHOLD))); + facts.push("threshold.write", outcome(fs::write(THRESHOLD, "1"))); + facts.push( + "hostname.write", + outcome(fs::write("/proc/sys/kernel/hostname", "escaped")), + ); + let sysrq = OpenOptions::new().write(true).open("/proc/sysrq-trigger"); + facts.push("sysrq.write", outcome(sysrq.map(drop))); + let bind = std::net::UdpSocket::bind(("0.0.0.0", PRIVILEGED_PORT)); + facts.push("bind", outcome(bind.map(drop))); + let raw = nix::sys::socket::socket( + AddressFamily::Inet, + SockType::Raw, + SockFlag::SOCK_CLOEXEC, + SockProtocol::Icmp, + ); + facts.push("raw", outcome(raw.map(drop).map_err(io::Error::from))); + facts.push("chown", outcome(change_owner(directory))); + facts.push("dac-override", outcome(read_unreadable(directory))); + let mode = fs::set_permissions(directory.join("owned"), fs::Permissions::from_mode(0o600)); + facts.push("fowner", outcome(mode)); + facts.push("setuid", outcome(run_as(Some(OTHER_ID), None))); + facts.push("setgid", outcome(run_as(None, Some(OTHER_ID)))); + let listing = listing(Path::new("/")); + facts.push("root", listing.clone()); + facts.push("escape", escape(directory, &listing)); + facts +} + +/// Returns the descriptors open in the process, in increasing order, separated by commas. +fn open_descriptors() -> String { + let numbers = match fs::read_dir("/proc/self/fd") { + Ok(entries) => entries + .filter_map(|entry| entry.ok()?.file_name().to_str()?.parse::().ok()) + .collect::>(), + Err(error) => return format!("unreadable: {error}"), + }; + // The directory's own descriptor closed once it was read. + let mut open = numbers + .into_iter() + .filter(|number| fs::symlink_metadata(format!("/proc/self/fd/{number}")).is_ok()) + .collect::>(); + open.sort_unstable(); + open.iter() + .map(ToString::to_string) + .collect::>() + .join(",") +} + +/// Creates a file in `directory`, and gives it to user and group ID 1000. +fn change_owner(directory: &Path) -> io::Result<()> { + let path = directory.join("chown"); + File::create(&path)?; + std::os::unix::fs::chown(&path, Some(OWNER), Some(OWNER)) +} + +/// Creates a file of mode 0 in `directory`, which only `CAP_DAC_OVERRIDE` lets even its owner +/// read, and opens it for reading. +fn read_unreadable(directory: &Path) -> io::Result<()> { + let path = directory.join("dac-override"); + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o000) + .open(&path)?; + File::open(&path).map(drop) +} + +/// Runs [`STAGED`], which reports its contract, as the user ID `uid` or the group ID `gid`. +fn run_as(uid: Option, gid: Option) -> io::Result<()> { + let mut command = std::process::Command::new(STAGED); + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + if let Some(uid) = uid { + command.uid(uid); + } + if let Some(gid) = gid { + command.gid(gid); + } + let status = command.status()?; + if status.success() { + Ok(()) + } else { + Err(io::Error::other(format!("the program ended with {status}"))) + } +} + +/// Tries a nested `chroot` escape from `directory`, and returns where it ended: `contained` if +/// the root then lists as `listing`, the execution's own root did. +fn escape(directory: &Path, listing: &str) -> String { + let jail = directory.join("jail"); + if let Err(error) = fs::create_dir(&jail) { + return format!("failed {error}"); + } + if let Err(error) = std::os::unix::fs::chroot(&jail) { + return outcome(Err(error)); + } + // The working directory stays outside the new root, from where `..` climbs as far as it can. + for _ in 0..CLIMBS { + if let Err(error) = std::env::set_current_dir("..") { + return format!("failed {error}"); + } + } + if let Err(error) = std::os::unix::fs::chroot(".") { + return format!("failed {error}"); + } + let ended = self::listing(Path::new("/")); + if ended == listing { + "contained".to_owned() + } else { + format!("escaped: {ended}") + } +} + +/// Returns the names in the directory `path`, sorted, and separated by commas, or `absent` if it +/// does not exist. +fn listing(path: &Path) -> String { + match fs::read_dir(path) { + Ok(entries) => { + let mut names = entries + .filter_map(|entry| Some(entry.ok()?.file_name().to_string_lossy().into_owned())) + .collect::>(); + names.sort(); + names.join(",") + } + Err(error) if error.kind() == io::ErrorKind::NotFound => "absent".to_owned(), + Err(error) => format!("unreadable: {error}"), + } +} + +/// Returns the outcome of an attempt: `allowed`, `refused ERRNO`, or `failed PROBLEM`. +fn outcome(result: io::Result<()>) -> String { + match result { + Ok(()) => "allowed".to_owned(), + Err(error) => match error.raw_os_error() { + Some(code) => format!("refused {:?}", Errno::from_raw(code)), + None => format!("failed {error}"), + }, + } +} + +/// Returns the value of the field `name` of a process's `/proc/PID/status`, trimmed. +fn status_field<'a>(status: &'a str, name: &str) -> Option<&'a str> { + status + .lines() + .find_map(|line| line.strip_prefix(name)?.strip_prefix(':').map(str::trim)) +} + +/// Returns the name of `capability` in the probe's facts: its kernel name without `CAP_`, in +/// lowercase, with `-` for `_`, such as `net-bind-service`. +pub fn capability_name(capability: Capability) -> String { + let name = capability.to_string(); + name.strip_prefix("CAP_") + .unwrap_or(&name) + .to_ascii_lowercase() + .replace('_', "-") +} + +/// Returns `path`, an absolute path, relative to the root. +fn below_root(path: &str) -> PathBuf { + PathBuf::from(OsStr::from_bytes( + path.as_bytes() + .strip_prefix(b"/") + .unwrap_or(path.as_bytes()), + )) +} + +/// Writes `facts` on standard output, as `inside` and the helper report them. +pub fn report(facts: &Facts) -> io::Result<()> { + let mut output = io::stdout().lock(); + output.write_all(&facts.encode())?; + output.flush() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capabilities_are_named_without_their_prefix() { + assert_eq!(capability_name(Capability::CHOWN), "chown"); + assert_eq!( + capability_name(Capability::NET_BIND_SERVICE), + "net-bind-service" + ); + assert_eq!( + capability_name(Capability::new(63).unwrap()), + "capability 63" + ); + } + + #[test] + fn status_fields_are_found_by_name() { + let status = "Name:\tsh\nUid:\t0\t0\t0\t0\nCapEff:\t0000000000000001\nGroups:\t\n"; + assert_eq!(status_field(status, "Uid"), Some("0\t0\t0\t0")); + assert_eq!(status_field(status, "CapEff"), Some("0000000000000001")); + assert_eq!(status_field(status, "Groups"), Some("")); + assert_eq!(status_field(status, "Cap"), None); + assert_eq!(status_field(status, "Gid"), None); + } + + #[test] + fn outcomes_name_the_error() { + assert_eq!(outcome(Ok(())), "allowed"); + let refused = io::Error::from_raw_os_error(Errno::EROFS as i32); + assert_eq!(outcome(Err(refused)), "refused EROFS"); + assert_eq!(outcome(Err(io::Error::other("odd"))), "failed odd"); + } + + #[test] + fn refusals_name_what_refused_the_identity() { + assert_eq!(refusal(Ok(())), "accepted"); + let capability = Capability::new(21).unwrap(); + assert_eq!( + refusal(Err(EntryError::Capability(capability))), + "capability CAP_SYS_ADMIN" + ); + assert_eq!( + refusal(Err(EntryError::Published(Capability::NET_RAW))), + "published CAP_NET_RAW" + ); + assert_eq!( + refusal(Err(EntryError::NonRootCapabilities)), + "non-root-capabilities" + ); + } + + #[test] + fn unknown_probes_and_arguments_are_refused() { + let lowers = [PathBuf::from("/run/runtime-adapter/layers/distro")]; + let context = Context { + runtime: Path::new("/run/runtime-adapter"), + root: Path::new("/run/runtime-adapter/rootfs"), + lowers: &lowers, + upper: Path::new("/run/runtime-adapter/scratch"), + agent: Path::new("/sys/fs/cgroup/agent"), + workload: Path::new("/sys/fs/cgroup/workload"), + sibling: None, + }; + assert_eq!( + run(&Probe::parse("entry").unwrap(), &context), + Err("the worker runs no probe named entry".to_owned()) + ); + assert_eq!( + run(&Probe::parse("entries now").unwrap(), &context), + Err("the entries probe takes no arguments".to_owned()) + ); + } + + #[test] + fn absolute_paths_lose_their_leading_slash() { + assert_eq!(below_root(STAGED), Path::new(".runtime-adapter-guest")); + assert_eq!( + below_root("/tmp/runtime-adapter-entries/none"), + Path::new("tmp/runtime-adapter-entries/none") + ); + } +} diff --git a/tests/runtime-adapter/tests/vm_layered.rs b/tests/runtime-adapter/tests/vm_layered.rs index 5ab55ba32..2db6d89e8 100644 --- a/tests/runtime-adapter/tests/vm_layered.rs +++ b/tests/runtime-adapter/tests/vm_layered.rs @@ -500,6 +500,19 @@ scenarios! { root: ROOT, expect: Expect::Ready(check_refused_mapping), }; + /// Executions that enter the root as root profiles do, through `nvx::guest::entry`: user ID + /// 0 with each allowed capability alone, with none, and with every one in each publication + /// state, and a numeric identity, each PID 1 of its namespaces, with the root moved over `/`, + /// without the descriptor that its caller leaked, and contained by a nested `chroot`. No root + /// workload lowers the unprivileged port start or binds a privileged port while listeners are + /// published, and the entry refuses the capabilities outside its allowlist. The root's own + /// machine ID, then a generated one, with the time status that the guest's time daemon + /// publishes, in which a root workload finds no lock of the daemon's, both of which the + /// runtime binds in a mount namespace of its own. + root_profiles_enter_the_root: Scenario { + root: ROOT, + expect: Expect::Ready(check_root_profiles), + }; } /// Runs the scenario `name`, `scenario`. @@ -1359,3 +1372,231 @@ fn hexadecimal(bytes: &[u8]) -> String { .map(|digit| char::from(DIGITS[usize::from(digit)])) .collect() } + +/// The capabilities that an entry allows, by their names in the cases of the `entries` probe, each +/// with its number and the attempt of the probe that it alone allows, if any. +const CAPABILITIES: [(&str, u8, Option<&str>); 13] = [ + ("chown", 0, Some("chown")), + ("dac-override", 1, Some("dac-override")), + ("fowner", 3, Some("fowner")), + ("fsetid", 4, None), + ("kill", 5, None), + ("setgid", 6, Some("setgid")), + ("setuid", 7, Some("setuid")), + ("setpcap", 8, None), + ("net-bind-service", 10, Some("bind")), + ("net-raw", 13, Some("raw")), + ("sys-chroot", 18, Some("escape")), + ("audit-write", 29, None), + ("setfcap", 31, None), +]; + +/// The attempts of the `entries` probe that a capability allows, each with its outcome without +/// the capability. +const ATTEMPTS: [(&str, &str); 8] = [ + ("chown", "refused EPERM"), + ("dac-override", "refused EACCES"), + ("fowner", "refused EPERM"), + ("setuid", "refused EPERM"), + ("setgid", "refused EPERM"), + ("bind", "refused EACCES"), + ("raw", "refused EPERM"), + ("escape", "refused EPERM"), +]; + +/// The machine ID that the `entries` probe writes in the root, as the root's own. +const ROOT_MACHINE_ID: &str = "0123456789abcdef0123456789abcdef"; + +/// Returns the outcome of the `entries` probe's `attempt` with the capability that allows it. +fn allowed(attempt: &str) -> &'static str { + if attempt == "escape" { + "contained" + } else { + "allowed" + } +} + +/// Returns a capability set as `/proc/PID/status` writes it: 16 hexadecimal digits. +fn capability_set(mask: u64) -> String { + format!("{mask:016x}") +} + +/// Checks the VM of the `root_profiles_enter_the_root` scenario. +fn check_root_profiles(vm: &mut Vm<'_>) { + let facts = vm.probe(&["entries"]); + let fact = |key: &str| { + facts + .get(&format!("entries.{key}")) + .unwrap_or_else(|| panic!("entries.{key} is missing: {facts:?}")) + .as_str() + }; + + // The entry refuses every capability outside its allowlist, the binding and raw-socket ones + // while listeners are published, any for a user ID other than 0, and an identity that has no + // account in existing-account mode, before an execution starts. + assert_eq!(fact("refused.outside"), "51"); + assert_eq!(fact("refused.accepted"), "none"); + assert_eq!( + fact("refused.published.net-bind-service"), + "published CAP_NET_BIND_SERVICE" + ); + assert_eq!(fact("refused.published.net-raw"), "published CAP_NET_RAW"); + assert_eq!(fact("refused.non-root"), "non-root-capabilities"); + assert_eq!(fact("refused.existing"), "account-missing"); + + let cases = CAPABILITIES + .iter() + .map(|(name, ..)| format!("only-{name}")) + .chain(["none", "published", "unpublished", "numeric", "generated"].map(str::to_owned)) + .collect::>(); + let none = capability_set(0); + for case in &cases { + let fact = |key: &str| fact(&format!("{case}.{key}")); + // Each execution starts, as PID 1 of its namespaces, and keeps only its standard streams: + // the descriptor that its caller leaked closed. It holds no inheritable or ambient + // capability, and no new privileges. + assert_eq!((fact("launch"), fact("status")), ("started", "exit 0")); + assert_eq!(fact("pid"), "1", "{case}"); + assert_eq!(fact("fds"), "0,1,2", "{case}"); + assert_eq!(fact("leaked"), "closed", "{case}"); + assert_eq!(fact("no-new-privs"), "1", "{case}"); + assert_eq!( + (fact("cap.inh"), fact("cap.amb")), + (&*none, &*none), + "{case}" + ); + // Its `/proc/sys` and `/proc/sysrq-trigger` refuse writes, read-only for user ID 0, so no + // execution lowers the unprivileged port start, which the runtime owns. + let refused = if case == "numeric" { + "refused EACCES" + } else { + "refused EROFS" + }; + for write in ["threshold.write", "hostname.write", "sysrq.write"] { + assert_eq!(fact(write), refused, "{case}: {write}"); + } + assert_eq!(fact("threshold"), "1024", "{case}"); + // The root, moved over `/`, holds the staged executable, and a nested `chroot` ends in + // it, when the execution may `chroot` at all. + let root = fact("root").split(',').collect::>(); + assert!(root.contains(".runtime-adapter-guest"), "{case}: {root:?}"); + assert!( + root.contains("etc") && root.contains("usr"), + "{case}: {root:?}" + ); + let hostname = if case == "generated" { + "runtime-adapter" + } else { + "nvx-sandbox" + }; + assert_eq!(fact("hostname"), hostname, "{case}"); + // Before the runtime binds a generated machine ID, every execution sees the root's own. + if case != "generated" { + assert_eq!(fact("machine-id"), ROOT_MACHINE_ID, "{case}"); + assert_eq!(fact("mount.machine-id"), "none", "{case}"); + assert_eq!(fact("time.state"), "absent", "{case}"); + assert_eq!(fact("time.files"), "absent", "{case}"); + assert_eq!(fact("mount.time"), "none", "{case}"); + } + } + + // User ID 0 holds each allowed capability alone in its permitted, effective, and bounding + // sets, and only it: its own attempt succeeds, and every other one fails. + for (name, number, allows) in CAPABILITIES { + let case = format!("only-{name}"); + let fact = |key: &str| fact(&format!("{case}.{key}")); + assert_eq!((fact("uid"), fact("gid"), fact("groups")), ("0", "0", "")); + for set in ["cap.prm", "cap.eff", "cap.bnd"] { + assert_eq!(fact(set), capability_set(1 << number), "{case}: {set}"); + } + for (attempt, refused) in ATTEMPTS { + let expected = if allows == Some(attempt) { + allowed(attempt) + } else { + refused + }; + assert_eq!(fact(attempt), expected, "{case}: {attempt}"); + } + } + // Without capabilities, user ID 0 may do none of it, nor may the numeric identity, which has + // no account in the image. + for case in ["none", "numeric"] { + let fact = |key: &str| fact(&format!("{case}.{key}")); + for set in ["cap.prm", "cap.eff", "cap.bnd"] { + assert_eq!(fact(set), none, "{case}: {set}"); + } + for (attempt, refused) in ATTEMPTS { + assert_eq!(fact(attempt), refused, "{case}: {attempt}"); + } + } + let ids = |case: &str| { + let fact = |key: &str| fact(&format!("{case}.{key}")); + (fact("uid"), fact("gid"), fact("groups")) + }; + assert_eq!(ids("none"), ("0", "0", "")); + assert_eq!(ids("numeric"), ("4242", "4242", "")); + + // In both publication modes, private-network ingress and the host's loopback, which the entry + // treats alike, a root workload with every other allowed capability can neither bind a + // privileged source port, from which the gateway accepts listener announcements, nor open a + // raw socket, which could forge one. Without published listeners, it may. + let every = CAPABILITIES + .iter() + .fold(0u64, |mask, (_, number, _)| mask | 1 << number); + let refused_while_published = 1 << 10 | 1 << 13; + for (case, mask) in [ + ("published", every & !refused_while_published), + ("unpublished", every), + ] { + let fact = |key: &str| fact(&format!("{case}.{key}")); + for set in ["cap.prm", "cap.eff", "cap.bnd"] { + assert_eq!(fact(set), capability_set(mask), "{case}: {set}"); + } + for (attempt, refused) in ATTEMPTS { + let expected = if case == "published" && ["bind", "raw"].contains(&attempt) { + refused + } else { + allowed(attempt) + }; + assert_eq!(fact(attempt), expected, "{case}: {attempt}"); + } + } + + // A generated machine ID and the time status that the guest's time daemon publishes, which the + // runtime bound once, read-only, in the helper's mount namespace, reach the execution, and end + // with the helper: the worker's own namespace keeps the root's machine ID, without mounts + // below the root. + let generated = fact("generated.source"); + assert_eq!(generated.len(), 32, "{generated}"); + assert!( + generated + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "{generated}" + ); + assert_ne!(generated, ROOT_MACHINE_ID); + assert_eq!(fact("generated.machine-id"), generated); + assert_eq!(fact("generated.time.state"), "version=1"); + // The time status holds only what any process may read: no lock of the daemon's, which user + // ID 0 could open and hold to block the daemon, and no private record. + let files = fact("generated.time.files"); + let names = files.split(',').collect::>(); + assert!(names.contains("state"), "{files}"); + assert!( + names.is_subset(&BTreeSet::from([ + "state", + ".state", + "daemon.pid", + "daemon.pid.tmp" + ])), + "{files}" + ); + for mount in ["generated.mount.machine-id", "generated.mount.time"] { + let found = options(fact(mount)); + for flag in ["ro", "nosuid", "nodev"] { + assert!(found.contains(flag), "{mount}: {found:?}"); + } + } + assert_eq!(fact("after.machine-id"), ROOT_MACHINE_ID); + assert_eq!(fact("after.mounts"), "0"); +} From 9627a322ccc5c62c2561ca8be3dd746ff97171a7 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Thu, 22 Oct 2026 19:29:13 -0700 Subject: [PATCH 04/12] nvx-rs: bind runtime files that the current directory holds A runtime file that a relative path of one component names, such as a machine ID that MachineId::generate made at "machine-id", has an empty parent, which Entry::bind_runtime opened as the source root of its bind, and failed with ENOENT. The source root of such a file is now the current directory. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- nvx-rs/src/guest/entry.rs | 14 +++++++++++++- nvx-rs/src/guest/entry/tests.rs | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/nvx-rs/src/guest/entry.rs b/nvx-rs/src/guest/entry.rs index 7ff9818b9..8bddaf5eb 100644 --- a/nvx-rs/src/guest/entry.rs +++ b/nvx-rs/src/guest/entry.rs @@ -749,7 +749,7 @@ fn bind_runtime_file( step: Step::Source, error, }; - let (Some(parent), Some(name)) = (path.parent(), path.file_name()) else { + let Some((parent, name)) = runtime_source(path) else { return Err(source_error(Errno::EINVAL as i32)); }; let source = SourceRoot::open(parent) @@ -760,6 +760,18 @@ fn bind_runtime_file( mappings::bind(root, &source, &mapping) } +/// Returns the directory that holds the runtime file at `path`, the current directory for a +/// relative path of one component, and the file's name in it, unless `path` names no file. +fn runtime_source(path: &Path) -> Option<(&Path, &std::ffi::OsStr)> { + let name = path.file_name()?; + let parent = path.parent()?; + if parent.as_os_str().is_empty() { + Some((Path::new("."), name)) + } else { + Some((parent, name)) + } +} + /// Removes the symbolic link at `/etc/machine-id` beneath `root`, reached through no other link /// and on the root's own mount. fn remove_machine_id_link(root: &Path) -> Result<(), Errno> { diff --git a/nvx-rs/src/guest/entry/tests.rs b/nvx-rs/src/guest/entry/tests.rs index f20de17ad..0c9e82ba1 100644 --- a/nvx-rs/src/guest/entry/tests.rs +++ b/nvx-rs/src/guest/entry/tests.rs @@ -548,6 +548,28 @@ fn runtime_binds_need_a_mount_root_unless_there_are_none() { assert!(matches!(result, Err(Error::Hostname)), "{result:?}"); } +#[test] +fn runtime_sources_are_the_directories_beside_their_files() { + use std::ffi::OsStr; + + // A relative path of one component names a file of the current directory. + assert_eq!( + runtime_source(Path::new("machine-id")), + Some((Path::new("."), OsStr::new("machine-id"))) + ); + assert_eq!( + runtime_source(Path::new("/run/nvx/workload-machine-id")), + Some((Path::new("/run/nvx"), OsStr::new("workload-machine-id"))) + ); + assert_eq!( + runtime_source(Path::new("state/time")), + Some((Path::new("state"), OsStr::new("time"))) + ); + for path in ["", "/", "..", "time/.."] { + assert_eq!(runtime_source(Path::new(path)), None, "{path}"); + } +} + #[test] fn only_a_link_at_the_machine_id_is_removed() { let root = root_with(None, None); From 4a60d67cefd7906960c52be16a0de6d01687c297 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Thu, 22 Oct 2026 20:31:32 -0700 Subject: [PATCH 05/12] nvx-rs: isolate the spawn tests that hold a child at its barrier A child that waits at its barrier holds a copy of every descriptor of the test process until it runs its program, such as the end of another test's pipe, which then stays open for as long. Two tests held a child there for tens of milliseconds while other tests ran. They now run in processes of their own, as the test of an ended child's barrier does. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- nvx-rs/src/guest/spawn.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nvx-rs/src/guest/spawn.rs b/nvx-rs/src/guest/spawn.rs index 31b5879fa..74ef35bfc 100644 --- a/nvx-rs/src/guest/spawn.rs +++ b/nvx-rs/src/guest/spawn.rs @@ -865,6 +865,15 @@ mod tests { #[test] fn a_child_waits_at_its_barrier_until_released() { + // A child that waits at its barrier holds a copy of every descriptor of this process + // until it runs its program, such as the end of another test's pipe that keeps that pipe + // open, so the test runs in a process of its own. + if crate::guest::managed_agent::isolate_process_test( + "NVX_SPAWN_RELEASED_BARRIER_TEST_CHILD", + "guest::spawn::tests::a_child_waits_at_its_barrier_until_released", + ) { + return; + } let directory = tempfile::tempdir().unwrap(); let marker = directory.path().join("ran"); let (stdout, stdout_writer) = std::io::pipe().unwrap(); @@ -912,6 +921,14 @@ mod tests { #[test] fn a_dropped_barrier_cancels_its_launch_while_another_child_waits_at_its_own() { + // The children that wait at their barriers hold copies of every descriptor of this + // process, so the test runs in a process of its own, as the test above does. + if crate::guest::managed_agent::isolate_process_test( + "NVX_SPAWN_WAITING_BARRIER_TEST_CHILD", + "guest::spawn::tests::a_dropped_barrier_cancels_its_launch_while_another_child_waits_at_its_own", + ) { + return; + } let directory = tempfile::tempdir().unwrap(); let marker = |name: &str| directory.path().join(name); let command = |name: &str| shell(&format!("touch '{}'", marker(name).display())).barrier(); From a35b72d0328d7d124f2e9f6b88039fe4f5ef4e06 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Fri, 23 Oct 2026 01:19:48 -0700 Subject: [PATCH 06/12] nvx-rs: close a waiting child's copies of other children's descriptors A child that waits at its barrier was forked with copies of every descriptor of its caller, close-on-exec ones included, and kept them until it ran its program, which it does only once released. Among them are the launch report's writer and the output pipes of any child that the caller started or prepared meanwhile, so an execution that waits for its pipes to hang up could not end while another child waited. Before it waits, the child now closes every descriptor that it does not use, with close_range over the gaps between the sorted numbers that PreparedChild::new records: its stream sources, root, cgroup file, barrier reader, report writer, and inherited descriptors. Its copy of the barrier's writer is among them, which an explicit close dropped before. So the program of a child with a barrier no longer receives descriptors that its caller opened without close-on-exec, which an entry's program never received either. The barrier tests no longer need processes of their own, and a new test checks that another child's pipes hang up while a child waits. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- nvx-rs/src/guest/spawn.rs | 66 ++++++++++++++++++--------- nvx-rs/src/guest/sys/mod.rs | 13 +++--- nvx-rs/src/guest/sys/spawn.rs | 85 +++++++++++++++++++++++++++++------ 3 files changed, 124 insertions(+), 40 deletions(-) diff --git a/nvx-rs/src/guest/spawn.rs b/nvx-rs/src/guest/spawn.rs index 74ef35bfc..e08314127 100644 --- a/nvx-rs/src/guest/spawn.rs +++ b/nvx-rs/src/guest/spawn.rs @@ -18,7 +18,9 @@ //! its standard streams above their numbers and makes it close-on-exec, so the program receives //! the command's descriptors only as its standard streams and inherited descriptors. Descriptors //! that the caller opened without close-on-exec stay open in the program, as across any `exec`, -//! except in an entry's, which closes them. +//! except in the program of an entry, which closes them, or of a child that waits at a +//! [`Barrier`]: such a child closes every descriptor that it does not use before it waits, so +//! that it holds no copy of another child's pipes while it waits. //! //! No step needs NVX's control protocol, its managed agent and helper, or its workload identity: //! the caller chooses the program, and the identity, if any, that an entry gives it, or as which @@ -76,7 +78,8 @@ pub enum LaunchStage { /// The child makes itself the leader of a new session, and of its process group, without a /// controlling terminal. Session, - /// The child waits until its caller releases its [`Barrier`]. + /// The child closes every descriptor that it does not use, then waits until its caller + /// releases its [`Barrier`]. Barrier, /// The child places its standard streams and keeps its inherited descriptors open. Descriptors, @@ -386,6 +389,10 @@ impl Command { /// child in its cgroup. A child whose barrier is dropped unreleased, with the [`Spawned`] that /// holds it too, fails the [stage](LaunchStage::Barrier) with `ECANCELED` and never runs its /// program. + /// + /// Before it waits, the child closes every descriptor that it does not use, those that the + /// caller opened without close-on-exec too, so that it holds no copy of the descriptors of a + /// child that the caller starts meanwhile, which would keep that child's pipes open. pub fn barrier(mut self) -> Self { self.spec.set_barrier(); self @@ -865,15 +872,6 @@ mod tests { #[test] fn a_child_waits_at_its_barrier_until_released() { - // A child that waits at its barrier holds a copy of every descriptor of this process - // until it runs its program, such as the end of another test's pipe that keeps that pipe - // open, so the test runs in a process of its own. - if crate::guest::managed_agent::isolate_process_test( - "NVX_SPAWN_RELEASED_BARRIER_TEST_CHILD", - "guest::spawn::tests::a_child_waits_at_its_barrier_until_released", - ) { - return; - } let directory = tempfile::tempdir().unwrap(); let marker = directory.path().join("ran"); let (stdout, stdout_writer) = std::io::pipe().unwrap(); @@ -921,21 +919,14 @@ mod tests { #[test] fn a_dropped_barrier_cancels_its_launch_while_another_child_waits_at_its_own() { - // The children that wait at their barriers hold copies of every descriptor of this - // process, so the test runs in a process of its own, as the test above does. - if crate::guest::managed_agent::isolate_process_test( - "NVX_SPAWN_WAITING_BARRIER_TEST_CHILD", - "guest::spawn::tests::a_dropped_barrier_cancels_its_launch_while_another_child_waits_at_its_own", - ) { - return; - } let directory = tempfile::tempdir().unwrap(); let marker = |name: &str| directory.path().join(name); let command = |name: &str| shell(&format!("touch '{}'", marker(name).display())).barrier(); let mut first = command("first").spawn().unwrap(); let first_barrier = first.take_barrier().unwrap(); - // The second child starts while the first waits, so it holds a copy of the first's - // barrier until it runs its program, which it does only once released. + // The second child starts while the first waits, with a copy of the first's barrier, + // which it closes before it waits at its own; the drop cancels the first's launch + // whichever copies remain. let mut second = command("second").spawn().unwrap(); let second_barrier = second.take_barrier().unwrap(); let (first_process, second_process) = (first.process(), second.process()); @@ -962,6 +953,39 @@ mod tests { assert!(marker("second").exists()); } + #[test] + fn another_childs_pipes_hang_up_while_a_child_waits_at_its_barrier() { + // The first child's launch report and output pipe exist before the waiting child + // starts, which therefore starts with copies of their writers, and closes them before it + // waits: the first child's pipes hang up once it ends, while the other still waits. + let (stdout, stdout_writer) = std::io::pipe().unwrap(); + let first = shell("printf done").stdout(stdout_writer).prepare(); + let mut waiting = shell("exit 0").barrier().spawn().unwrap(); + let barrier = waiting.take_barrier().unwrap(); + let first = first.spawn().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !(sys::hung_up(first.report.borrowed()).unwrap() + && sys::hung_up(std::os::fd::AsFd::as_fd(&stdout)).unwrap()) + { + assert!( + std::time::Instant::now() < deadline, + "the waiting child holds the first child's pipes" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!(!report_arrived(&waiting)); + let report = LaunchReport { + reader: first.report, + }; + assert_eq!(report.wait().unwrap(), None); + assert_eq!(read_all(stdout), b"done"); + assert_eq!(sys::wait(first.process), Some(ChildStatus::Exited(0))); + let process = waiting.process(); + barrier.release().unwrap(); + assert_eq!(waiting.into_report().wait().unwrap(), None); + assert_eq!(sys::wait(process), Some(ChildStatus::Exited(0))); + } + #[test] fn releasing_the_barrier_of_an_ended_child_fails() { // Another test's child holds a copy of every descriptor of this process from its fork to diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index 561a1f967..a7340587e 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -21,12 +21,13 @@ //! //! The spawn boundary and the managed agent need a few operations that `nix` cannot provide, //! which reach libc through `nix`'s re-export: a spawned child, which may neither allocate nor -//! drop an owner, fills its preallocated `PWD` entry with the `getcwd` system call, closes its -//! copy of a barrier's writer, and executes and exits through libc; an execution's entry starts -//! its child with `clone3`, and the child changes its credentials, capabilities, and hostname, -//! and mounts its view with the new mount API, `openat2`, and `close_range`, through libc's -//! `syscall`, as the entry's module describes, because libc does not know of that child; the -//! agent's wait-status decoding, and the waits for one named child, +//! drop an owner, fills its preallocated `PWD` entry with the `getcwd` system call, closes the +//! descriptors that it does not use with the `close_range` system call, and executes and exits +//! through libc; an execution's entry starts its child with `clone3`, and the child changes its +//! credentials, capabilities, and hostname, and mounts its view with the new mount API, +//! `openat2`, and `close_range`, through libc's `syscall`, as the entry's module describes, +//! because libc does not know of that child; the agent's wait-status decoding, and the waits for +//! one named child, //! keep the real-time signals that `nix`'s `Signal` cannot represent; a pidfd is opened with the //! `pidfd_open` system call, which neither `nix` nor musl wraps; a write to an execution's input //! takes back the `SIGPIPE` that a closed reader raises with `sigtimedwait`, once `sigpending` diff --git a/nvx-rs/src/guest/sys/spawn.rs b/nvx-rs/src/guest/sys/spawn.rs index f14d7be3a..688a8eb79 100644 --- a/nvx-rs/src/guest/sys/spawn.rs +++ b/nvx-rs/src/guest/sys/spawn.rs @@ -7,23 +7,24 @@ //! over that storage, in the order of [`LaunchStage`], and ends in `execve`, `execv`, or `_exit`. //! It makes them through `nix` where its wrappers neither allocate nor consume an owner, and //! otherwise through libc: its `syscall` fills the preallocated `PWD` entry, because its `getcwd` -//! may walk the file system, and makes the entry's calls that the [entry](super::entry) module -//! lists; its `close` drops the child's copy of a barrier's writer; and its `execve`, `execv`, -//! and `_exit` end the child. A stage that fails writes its number and `errno` to the launch -//! report, a close-on-exec pipe, and ends the child with status 125, so that the report's reader -//! reaches end of file without a report once the program runs. +//! may walk the file system, closes the descriptors that the child does not use with +//! `close_range`, and makes the entry's calls that the [entry](super::entry) module lists; and +//! its `execve`, `execv`, and `_exit` end the child. A stage that fails writes its number and +//! `errno` to the launch report, a close-on-exec pipe, and ends the child with status 125, so +//! that the report's reader reaches end of file without a report once the program runs. //! //! Every descriptor that the child uses after it places its standard streams has a number above //! theirs, so that placing them replaces none of it: [`PreparedChild::new`] moves a stream source, //! the root, the cgroup's membership file, the barrier's ends, or the report's writer that has a //! lower number. It also makes them close-on-exec, so that the program receives -//! them only as its standard streams. +//! them only as its standard streams. A child that waits at a barrier first closes every other +//! descriptor, so that it holds no copy of another child's while it waits. use std::ffi::CString; use std::os::fd::RawFd; use nix::fcntl::{FcntlArg, FdFlag}; -use nix::libc::c_char; +use nix::libc::{c_char, c_uint}; use nix::unistd::{ForkResult, Gid, Pid, Uid}; use super::{EntryChild, Errno, FileDescriptor, ProcessId}; @@ -243,6 +244,11 @@ pub(crate) struct PreparedChild { report_reader: FileDescriptor, /// Writer of the launch report, which only the child keeps. report_writer: FileDescriptor, + /// The numbers of every descriptor that the child uses before its program runs, or that its + /// program inherits, in ascending order: the stream sources, the root, the cgroup's + /// membership file, the barrier's reader, the report's writer, and the inherited + /// descriptors. A child that waits at a barrier closes every other. + used: Box<[RawFd]>, /// Test-only outcome of the credential system calls. #[cfg(test)] simulated_credentials: Option>, @@ -318,6 +324,16 @@ impl PreparedChild { (Some(environment), true) => Some(PhysicalPwd::new(environment)), _ => None, }; + let used = ascending_numbers( + streams + .iter() + .flatten() + .chain(&root) + .chain(&cgroup) + .chain(barrier.as_ref().map(|(reader, _)| reader)) + .chain([&report_writer]) + .chain(&spec.inherited), + ); Ok(Self { program: spec.program, arguments: spec.arguments, @@ -335,6 +351,7 @@ impl PreparedChild { entry, report_reader, report_writer, + used, #[cfg(test)] simulated_credentials: spec.simulated_credentials, }) @@ -455,12 +472,14 @@ impl PreparedChild { // SAFETY: as above. unsafe { fail(report, LaunchStage::Session, error) } } - if let Some((reader, writer)) = &self.barrier { - // The child drops its copy of the writer, so that it reads end of file once its - // caller shuts the writer down, or once no other copy of it remains. - // SAFETY: nothing in the child uses the writer, whose owner is never dropped here. - unsafe { nix::libc::close(writer.raw()) }; - if let Err(error) = wait_for_release(reader) { + if let Some((reader, _)) = &self.barrier { + // Before it waits, the child closes every descriptor that it does not use, its copy + // of the barrier's writer among them. It then holds no copy of another child's pipes + // or launch report, which would stay open while it waits, and reads end of file once + // its caller shuts the writer down, or once no other copy of it remains. + if let Err(error) = + close_other_descriptors(&self.used).and_then(|()| wait_for_release(reader)) + { // SAFETY: as above. unsafe { fail(report, LaunchStage::Barrier, error) } } @@ -681,6 +700,46 @@ fn above_standard_streams(descriptor: FileDescriptor) -> Result( + descriptors: impl IntoIterator, +) -> Box<[RawFd]> { + let mut numbers: Vec = descriptors.into_iter().map(FileDescriptor::raw).collect(); + numbers.sort_unstable(); + numbers.dedup(); + numbers.into_boxed_slice() +} + +/// Closes, in a child, every descriptor from 3 up but those numbered in `kept`, which ascend +/// from 3 and hold each number once, through `close_range` over the gaps between them: those +/// that the caller opened without close-on-exec too, and the child's copies of other children's +/// descriptors, which would otherwise stay open until the child runs its program. +fn close_other_descriptors(kept: &[RawFd]) -> Result<(), Errno> { + let mut first = FIRST_NONSTANDARD as c_uint; + for &number in kept { + // A kept descriptor's number is at least 3, so it converts. + let number = number as c_uint; + if number > first { + close_range(first, number - 1)?; + } + first = number + 1; + } + close_range(first, c_uint::MAX) +} + +/// Closes the descriptors numbered from `first` to `last`, with libc's `syscall`, which neither +/// allocates nor relies on the state that libc keeps for a thread. +fn close_range(first: c_uint, last: c_uint) -> Result<(), Errno> { + // SAFETY: `close_range` takes three integers and dereferences nothing. + let result = + unsafe { nix::libc::syscall(nix::libc::SYS_close_range, first, last, 0 as c_uint) }; + if result < 0 { + Err(Errno::last()) + } else { + Ok(()) + } +} + /// Makes `root` the calling process's root directory and working directory. fn enter_root(root: &FileDescriptor) -> Result<(), Errno> { nix::unistd::fchdir(root.borrowed())?; From 90a37a6d7ac0120aaf17026a164071f1ddf701e8 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Fri, 23 Oct 2026 01:20:21 -0700 Subject: [PATCH 07/12] nvx-rs: enter an execution's working directory through no magic link An entry's child only made its other descriptors close on exec, so the descriptors that its caller leaked and those that its view's stages opened, such as the O_PATH ones below procfs, stayed open until its program ran. Its working directory, which chdir resolved before then, could name one of them through /proc/self/fd, or a working directory or root through another magic link, and so place the program outside its root, or in the writable /proc/sys below its read-only bind, where the program stayed after exec. The child now closes every descriptor that it does not use when it places its descriptors, rather than marking it, and, once its stages have run, every descriptor but its program's and its launch report's. It then opens its working directory, / unless its command names another, with openat2 and RESOLVE_NO_MAGICLINKS, and enters it with fchdir, so that neither a magic link nor a symbolic link to one resolves. Either failure is the working-directory stage's. The privileged entry test checks that / and an ordinary directory are entered, and that the links of a leaked descriptor, of each descriptor up to 63, of the child's working directory and root, and of the working directory of its namespace's first process, and a symbolic link to one of them, each fail the stage before the program runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- nvx-rs/src/guest/entry.rs | 11 +++-- nvx-rs/src/guest/entry/tests.rs | 83 +++++++++++++++++++++++++++++---- nvx-rs/src/guest/spawn.rs | 20 +++++--- nvx-rs/src/guest/sys/entry.rs | 44 ++++++++++------- nvx-rs/src/guest/sys/mod.rs | 8 ++-- nvx-rs/src/guest/sys/spawn.rs | 32 ++++++++++--- 6 files changed, 152 insertions(+), 46 deletions(-) diff --git a/nvx-rs/src/guest/entry.rs b/nvx-rs/src/guest/entry.rs index 8bddaf5eb..c4d0fd602 100644 --- a/nvx-rs/src/guest/entry.rs +++ b/nvx-rs/src/guest/entry.rs @@ -43,9 +43,9 @@ //! //! 1. leads a new session, without a controlling terminal; //! 2. waits at its [barrier](Command::barrier), if its command has one, until its caller has -//! placed it, then places its descriptors, and has every other descriptor of its caller's -//! close as its program runs, so that no descriptor of the old root, of the caller's control -//! session, or of another namespace or process reaches the program; +//! placed it, then places its descriptors, and closes every other descriptor of its caller's +//! before it waits or places them, so that no descriptor of the old root, of the caller's +//! control session, or of another namespace or process reaches the program; //! 3. joins its [cgroup](Command::cgroup), if its command names one; //! 4. makes every mount of its mount namespace private, recursively, and sets its hostname; //! 5. moves the root over `/`, after it makes every mount of the root's tree `nosuid` and @@ -58,7 +58,10 @@ //! `/dev/pts`, `nosuid` and `noexec`, and a tmpfs at `/dev/shm`, `nosuid` and `nodev`. Every //! other mount of the execution is `nodev`; //! 9. takes its [identity](Identity), and sets `no_new_privs`; -//! 10. enters its working directory with that identity, and runs its program. +//! 10. closes every descriptor that its stages used or opened but its program's, enters its +//! working directory, `/` unless its command names another, with that identity, through no +//! magic link, so that neither a `/proc` link to a descriptor, a working directory, or a root, +//! nor a symbolic link to one, places it outside the root, and runs its program. //! //! For user ID 0, `/proc/sys`, `/proc/sysrq-trigger`, `/proc/bus`, `/proc/fs`, and `/proc/irq` //! are read-only, and `/proc/kcore`, `/proc/keys`, `/proc/timer_list`, `/proc/timer_stats`, diff --git a/nvx-rs/src/guest/entry/tests.rs b/nvx-rs/src/guest/entry/tests.rs index 0c9e82ba1..7ba46f544 100644 --- a/nvx-rs/src/guest/entry/tests.rs +++ b/nvx-rs/src/guest/entry/tests.rs @@ -1004,6 +1004,73 @@ fn check_runtime_binds(base: &Path, root: &Path) { std::fs::remove_file(root.join("run")).unwrap(); } +/// Checks that an execution of `entry`, whose root is the privileged entries' mounted `root`, +/// enters its working directory, by default `/`, through no magic link: neither a `/proc` link to +/// a descriptor, such as one of the caller's root that leaks to the child or one that the view's +/// stages opened, a working directory, or a root, nor a symbolic link to one, which could name a +/// directory outside the root. +fn check_working_directories(entry: &Entry, root: &Path) { + let run = |directory: Option<&str>| { + let command = shell("pwd -P"); + let command = match directory { + Some(directory) => command.working_directory(CString::new(directory).unwrap()), + None => command, + }; + run_released(command.enter(entry.prepare().unwrap())) + }; + for (directory, expected) in [(None, "/\n"), (Some("/tmp"), "/tmp\n")] { + let (failure, status, output) = run(directory); + assert_eq!( + (failure, status), + (None, Some(ChildStatus::Exited(0))), + "{directory:?}" + ); + assert_eq!(String::from_utf8(output).unwrap(), expected); + } + let leaked = nix::fcntl::open( + c"/", + OpenFlags::O_RDONLY | OpenFlags::O_DIRECTORY, + Mode::empty(), + ) + .unwrap(); + let number = std::os::fd::AsRawFd::as_raw_fd(&leaked); + std::os::unix::fs::symlink("/proc/self/cwd", root.join("tmp/cwd-link")).unwrap(); + let mut refused: Vec<(String, &[Errno])> = vec![ + (format!("/proc/self/fd/{number}"), &[Errno::ENOENT][..]), + ("/proc/self/cwd".to_owned(), &[Errno::ELOOP][..]), + ("/proc/self/root".to_owned(), &[Errno::ELOOP][..]), + ("/proc/1/cwd".to_owned(), &[Errno::ELOOP][..]), + ("/tmp/cwd-link".to_owned(), &[Errno::ELOOP][..]), + ]; + // The child closes every other descriptor first, so a link names a standard stream or the + // launch report's writer, or nothing. + refused.extend((0..64).map(|number| { + ( + format!("/proc/self/fd/{number}"), + &[Errno::ELOOP, Errno::ENOENT][..], + ) + })); + for (directory, errors) in refused { + let (failure, status, output) = run(Some(&directory)); + let failure = failure.unwrap_or_else(|| panic!("{directory} was entered: {output:?}")); + assert_eq!( + failure.stage(), + LaunchStage::WorkingDirectory, + "{directory}" + ); + assert!( + errors + .iter() + .any(|error| failure.raw_os_error() == *error as i32), + "{directory}: {failure}" + ); + assert_eq!(status, Some(ChildStatus::Exited(125)), "{directory}"); + assert!(output.is_empty(), "{directory}"); + } + std::fs::remove_file(root.join("tmp/cwd-link")).unwrap(); + drop(leaked); +} + #[test] fn privileged_entries_see_their_own_view_and_identity() { if crate::guest::managed_agent::isolate_process_test( @@ -1071,14 +1138,13 @@ fn privileged_entries_see_their_own_view_and_identity() { .with(Capability::CHOWN) .with(Capability::KILL) .with(Capability::SYS_CHROOT); - let output = probe( - &entry( - Identity::new(0, 0) - .capabilities(capabilities) - .account(AccountMode::Numeric), - ) - .publication(Publication::Unpublished), - ); + let privileged = entry( + Identity::new(0, 0) + .capabilities(capabilities) + .account(AccountMode::Numeric), + ) + .publication(Publication::Unpublished); + let output = probe(&privileged); check_view(&output, id.trim_end()); assert_eq!(value(&output, "ids"), "0 0 0"); let mask = format!("{:016x}", capabilities.bits()); @@ -1099,6 +1165,7 @@ fn privileged_entries_see_their_own_view_and_identity() { for escape in values(&output, "escape") { assert_eq!(escape, value(&output, "root")); } + check_working_directories(&privileged, &root); // Root with no capability. let output = probe(&entry(Identity::new(0, 0).account(AccountMode::Numeric))); diff --git a/nvx-rs/src/guest/spawn.rs b/nvx-rs/src/guest/spawn.rs index e08314127..84ca64c91 100644 --- a/nvx-rs/src/guest/spawn.rs +++ b/nvx-rs/src/guest/spawn.rs @@ -18,9 +18,10 @@ //! its standard streams above their numbers and makes it close-on-exec, so the program receives //! the command's descriptors only as its standard streams and inherited descriptors. Descriptors //! that the caller opened without close-on-exec stay open in the program, as across any `exec`, -//! except in the program of an entry, which closes them, or of a child that waits at a -//! [`Barrier`]: such a child closes every descriptor that it does not use before it waits, so -//! that it holds no copy of another child's pipes while it waits. +//! except in the program of a child that waits at a [`Barrier`] or enters an execution: such a +//! child closes every descriptor that it does not use before it waits or enters, so that it holds +//! no copy of another child's pipes while it waits, and the program of an entry none of its +//! caller's. //! //! No step needs NVX's control protocol, its managed agent and helper, or its workload identity: //! the caller chooses the program, and the identity, if any, that an entry gives it, or as which @@ -81,7 +82,8 @@ pub enum LaunchStage { /// The child closes every descriptor that it does not use, then waits until its caller /// releases its [`Barrier`]. Barrier, - /// The child places its standard streams and keeps its inherited descriptors open. + /// The child places its standard streams and keeps its inherited descriptors open. The child + /// of an entry first closes every descriptor that it does not use. Descriptors, /// The child joins its cgroup. Cgroup, @@ -109,7 +111,9 @@ pub enum LaunchStage { NoNewPrivileges, /// The child takes the identity of its working-directory check, or gives it back after. Credentials, - /// The child enters its working directory. + /// The child enters its working directory. The child of an entry first closes every + /// descriptor but its program's and its launch report's, and resolves the directory through + /// no magic link. WorkingDirectory, /// The child records the physical path of its working directory in `PWD`. PhysicalWorkingDirectory, @@ -400,8 +404,10 @@ impl Command { /// Makes the child enter the execution that `entry` prepares before its program runs, as the /// [entry](super::entry) module describes: its namespaces, view, root, and identity. The - /// child then leads a [new session](Self::new_session), and enters its working directory, by - /// default `/`, with that identity. + /// child then leads a [new session](Self::new_session), closes every descriptor but those that + /// its program receives, and enters its working directory, by default `/`, with that + /// identity, through no magic link, such as `/proc/self/cwd` or a `/proc/self/fd` entry, which + /// could name a directory outside the root. /// /// The command must also place the child: it needs a [cgroup](Self::cgroup) or a /// [barrier](Self::barrier), at which its caller places it. It may name neither a diff --git a/nvx-rs/src/guest/sys/entry.rs b/nvx-rs/src/guest/sys/entry.rs index d16391e53..280608f35 100644 --- a/nvx-rs/src/guest/sys/entry.rs +++ b/nvx-rs/src/guest/sys/entry.rs @@ -12,12 +12,11 @@ //! still names its parent's thread, and libc's wrappers that change credentials signal every //! thread that libc believes the process has. So the child calls no libc function that relies //! on that state. It changes its credentials, capabilities, and hostname, and uses the new mount -//! API (`fsopen`, `fsconfig`, `fsmount`, `open_tree`, `move_mount`, and `mount_setattr`), -//! `openat2`, and `close_range`, through libc's `syscall`, with the kernel's structures that the -//! libc crate does not define laid out here from the kernel's UAPI. It makes its other calls -//! through `nix`, or libc's plain wrappers of one system call each, as a forked child does. No -//! step can panic: a panic would abort through libc's `raise`, which signals the thread that libc -//! names, its parent's. +//! API (`fsopen`, `fsconfig`, `fsmount`, `open_tree`, `move_mount`, and `mount_setattr`) and +//! `openat2`, through libc's `syscall`, with the kernel's structures that the libc crate does not +//! define laid out here from the kernel's UAPI. It makes its other calls through `nix`, or libc's +//! plain wrappers of one system call each, as a forked child does. No step can panic: a panic +//! would abort through libc's `raise`, which signals the thread that libc names, its parent's. use std::ffi::CStr; use std::mem::MaybeUninit; @@ -337,19 +336,30 @@ pub(super) unsafe fn clone_into_namespaces(flags: u64) -> Result, Er } } -/// Makes every descriptor from 3 up close on `exec`, for an entry's child, which keeps only its -/// standard streams and the descriptors that its command names. -pub(super) fn close_descriptors_on_exec() -> Result<(), Errno> { - // SAFETY: `close_range` takes three integers and dereferences nothing; with - // `CLOSE_RANGE_CLOEXEC`, it only sets the flag. - unit(unsafe { +/// Makes the directory at `path` the child's working directory, resolving it in the child's root +/// through no magic link: neither a `/proc` link to a descriptor, a working directory, or a root, +/// nor a symbolic link to one, may then place the child outside its root, or in a directory that +/// a mount of the view covers, such as the writable `/proc/sys` below its read-only bind. +pub(super) fn enter_directory(path: &CStr) -> Result<(), Errno> { + let how = OpenHow { + flags: (libc::O_PATH | libc::O_DIRECTORY | libc::O_CLOEXEC) as u64, + mode: 0, + resolve: libc::RESOLVE_NO_MAGICLINKS, + }; + // SAFETY: the path is a NUL-terminated string, and `how` a valid `open_how` of its size. + let directory = descriptor(unsafe { libc::syscall( - libc::SYS_close_range, - 3 as c_uint, - c_uint::MAX, - libc::CLOSE_RANGE_CLOEXEC, + libc::SYS_openat2, + libc::AT_FDCWD, + path.as_ptr(), + &how, + std::mem::size_of::(), ) - }) + })?; + // SAFETY: `fchdir` takes a descriptor, which the call above opened. + let entered = unit(unsafe { libc::fchdir(directory) }.into()); + close(directory); + entered } /// Returns whether `file` is a regular file, and its size. diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index a7340587e..28591c641 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -24,10 +24,10 @@ //! drop an owner, fills its preallocated `PWD` entry with the `getcwd` system call, closes the //! descriptors that it does not use with the `close_range` system call, and executes and exits //! through libc; an execution's entry starts its child with `clone3`, and the child changes its -//! credentials, capabilities, and hostname, and mounts its view with the new mount API, -//! `openat2`, and `close_range`, through libc's `syscall`, as the entry's module describes, -//! because libc does not know of that child; the agent's wait-status decoding, and the waits for -//! one named child, +//! credentials, capabilities, and hostname, and mounts its view and enters its working directory +//! with the new mount API and `openat2`, through libc's `syscall`, as the entry's module +//! describes, because libc does not know of that child; the agent's wait-status decoding, and +//! the waits for one named child, //! keep the real-time signals that `nix`'s `Signal` cannot represent; a pidfd is opened with the //! `pidfd_open` system call, which neither `nix` nor musl wraps; a write to an execution's input //! takes back the `SIGPIPE` that a closed reader raises with `sigtimedwait`, once `sigpending` diff --git a/nvx-rs/src/guest/sys/spawn.rs b/nvx-rs/src/guest/sys/spawn.rs index 688a8eb79..d142e7479 100644 --- a/nvx-rs/src/guest/sys/spawn.rs +++ b/nvx-rs/src/guest/sys/spawn.rs @@ -17,8 +17,10 @@ //! theirs, so that placing them replaces none of it: [`PreparedChild::new`] moves a stream source, //! the root, the cgroup's membership file, the barrier's ends, or the report's writer that has a //! lower number. It also makes them close-on-exec, so that the program receives -//! them only as its standard streams. A child that waits at a barrier first closes every other -//! descriptor, so that it holds no copy of another child's while it waits. +//! them only as its standard streams. A child that waits at a barrier, or enters an execution, +//! first closes every other descriptor, so that it holds no copy of another child's while it +//! waits; an entry's child also closes, before it enters its working directory, every descriptor +//! that its stages used or opened but those that its program inherits. use std::ffi::CString; use std::os::fd::RawFd; @@ -247,8 +249,11 @@ pub(crate) struct PreparedChild { /// The numbers of every descriptor that the child uses before its program runs, or that its /// program inherits, in ascending order: the stream sources, the root, the cgroup's /// membership file, the barrier's reader, the report's writer, and the inherited - /// descriptors. A child that waits at a barrier closes every other. + /// descriptors. A child that waits at a barrier, or enters an execution, closes every other. used: Box<[RawFd]>, + /// The numbers of the descriptors that an entry's program inherits, and of the report's + /// writer, in ascending order: all that its child keeps once its stages have run. + kept: Box<[RawFd]>, /// Test-only outcome of the credential system calls. #[cfg(test)] simulated_credentials: Option>, @@ -334,6 +339,7 @@ impl PreparedChild { .chain([&report_writer]) .chain(&spec.inherited), ); + let kept = ascending_numbers(spec.inherited.iter().chain([&report_writer])); Ok(Self { program: spec.program, arguments: spec.arguments, @@ -352,6 +358,7 @@ impl PreparedChild { report_reader, report_writer, used, + kept, #[cfg(test)] simulated_credentials: spec.simulated_credentials, }) @@ -512,7 +519,20 @@ impl PreparedChild { // SAFETY: as above. unsafe { fail(report, stage, error) } } - if let Some(directory) = &self.directory { + if self.entry.is_some() { + // An entry's child closes every descriptor that its stages used or opened, but its + // program's, and resolves its working directory, `/` unless the command names + // another, through no magic link: no `/proc` link to a descriptor, a working + // directory, or a root then reaches outside the root, or a directory that a mount of + // the view covers. + let directory = self.directory.as_deref().unwrap_or(c"/"); + if let Err(error) = close_other_descriptors(&self.kept) + .and_then(|()| super::entry::enter_directory(directory)) + { + // SAFETY: as above. + unsafe { fail(report, LaunchStage::WorkingDirectory, error) } + } + } else if let Some(directory) = &self.directory { match &self.credentials { Some(credentials) => { let saved = (nix::unistd::geteuid(), nix::unistd::getegid()); @@ -580,11 +600,11 @@ impl PreparedChild { /// /// Every source has a number above the standard streams', so `dup2` replaces each stream /// without closing a source, and clears close-on-exec on its copy. The child of an entry - /// first makes every other descriptor close on `exec`, those that its caller opened without + /// first closes every descriptor that it does not use, those that its caller opened without /// close-on-exec too, so that its program holds none of the caller's. fn place_descriptors(&self) -> Result<(), Errno> { if self.entry.is_some() { - super::entry::close_descriptors_on_exec()?; + close_other_descriptors(&self.used)?; } let [stdin, stdout, stderr] = &self.streams; if let Some(source) = stdin { From a450c819773412c2cea6fd1ceb55d5a84509861a Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Fri, 23 Oct 2026 01:20:35 -0700 Subject: [PATCH 08/12] tests: check that executions enter no directory through a magic link The adapter's entries probe now reports each execution's working directory, the top of the root unless the execution names another, and runs working-directory cases with the root profile of every allowed capability: its own case directory, which it enters, and, through magic links, the directory of the descriptor that its caller leaked, its own working directory and root, the working directory of its namespace's first process, a symbolic link to its working directory, and the link of each descriptor up to 63, whose distinct outcomes it reports. The layered VM scenario checks that every execution without a working directory runs at the top of its root, that the case directory is entered, and that each magic link fails the working-directory stage before the program runs: with ENOENT for the leaked descriptor, which the child closed, and ELOOP otherwise. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- tests/runtime-adapter/README.md | 13 +- tests/runtime-adapter/src/worker/entries.rs | 161 ++++++++++++++++---- tests/runtime-adapter/tests/vm_layered.rs | 37 ++++- 3 files changed, 176 insertions(+), 35 deletions(-) diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index 35de780f0..b90bee66a 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -253,7 +253,7 @@ directory below `/tmp/runtime-adapter-entries`. | Probe | Facts | | --- | --- | -| `entries` | The capabilities outside the allowlist that the entry refused, `refused.outside`, and any that it accepted, `refused.accepted`; its refusal of `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while listeners are published, `refused.published.net-bind-service` and `refused.published.net-raw`, of a capability for a user ID other than 0, `refused.non-root`, and of an identity without an account in existing-account mode, `refused.existing`. For each case, `CASE.launch` and `CASE.status`, and what the execution reported, as `CASE.KEY`: the descriptors open in it, `fds`, and whether the leaked one is, `leaked`; `uid`, `gid`, and `groups`; the capability sets, `cap.inh`, `cap.prm`, `cap.eff`, `cap.bnd`, and `cap.amb`, and `no-new-privs`; `pid`; `hostname`; the `machine-id` and a mount's options at `/etc/machine-id`, `mount.machine-id`; the first line of the time status's state, `time.state`, the names in `/run/nvx/time`, `time.files`, and a mount's options there, `mount.time`; the unprivileged port start, `threshold`; the outcome of lowering it, `threshold.write`, of writing the hostname through `/proc/sys`, `hostname.write`, of opening `/proc/sysrq-trigger` for writing, `sysrq.write`, of binding UDP port 1023, a privileged source port, from which the gateway accepts listener announcements, `bind`, of opening a raw socket, which could forge such a datagram, `raw`, of changing a new file's owner, `chown`, of reading a new file of mode 0, `dac-override`, of changing the mode of a file of user ID 1000, `fowner`, and of running a program as user ID 4321, `setuid`, and as group ID 4321, `setgid`; the root's entries, `root`; and, last, a nested `chroot` escape, `escape`: `contained` if climbing with `..` from outside a `chroot` ends in the execution's own root. The generated machine ID, `generated.source`. Once the helper ended, the root's own machine ID, `after.machine-id`, and the mounts below the root, `after.mounts`, in the worker's mount namespace | +| `entries` | The capabilities outside the allowlist that the entry refused, `refused.outside`, and any that it accepted, `refused.accepted`; its refusal of `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while listeners are published, `refused.published.net-bind-service` and `refused.published.net-raw`, of a capability for a user ID other than 0, `refused.non-root`, and of an identity without an account in existing-account mode, `refused.existing`. For each case, `CASE.launch` and `CASE.status`, and what the execution reported, as `CASE.KEY`: the descriptors open in it, `fds`, and whether the leaked one is, `leaked`; `uid`, `gid`, and `groups`; the capability sets, `cap.inh`, `cap.prm`, `cap.eff`, `cap.bnd`, and `cap.amb`, and `no-new-privs`; `pid`; its working directory, `cwd`; `hostname`; the `machine-id` and a mount's options at `/etc/machine-id`, `mount.machine-id`; the first line of the time status's state, `time.state`, the names in `/run/nvx/time`, `time.files`, and a mount's options there, `mount.time`; the unprivileged port start, `threshold`; the outcome of lowering it, `threshold.write`, of writing the hostname through `/proc/sys`, `hostname.write`, of opening `/proc/sysrq-trigger` for writing, `sysrq.write`, of binding UDP port 1023, a privileged source port, from which the gateway accepts listener announcements, `bind`, of opening a raw socket, which could forge such a datagram, `raw`, of changing a new file's owner, `chown`, of reading a new file of mode 0, `dac-override`, of changing the mode of a file of user ID 1000, `fowner`, and of running a program as user ID 4321, `setuid`, and as group ID 4321, `setgid`; the root's entries, `root`; and, last, a nested `chroot` escape, `escape`: `contained` if climbing with `..` from outside a `chroot` ends in the execution's own root. For the working-directory descriptors, only their distinct outcomes, sorted and separated by commas, `cwd-descriptors.launch` and `cwd-descriptors.status`. The generated machine ID, `generated.source`. Once the helper ended, the root's own machine ID, `after.machine-id`, and the mounts below the root, `after.mounts`, in the worker's mount namespace | The cases, as user ID 0 in numeric-account mode unless they say otherwise, are each allowed capability alone, without published listeners, `only-NAME`, such @@ -262,7 +262,14 @@ but those that published listeners refuse, with published listeners, `published`, which stands for both publication modes, private-network ingress and the host's loopback, since the entry treats them alike; every allowed capability, without published listeners, `unpublished`; user and group ID -4242, of which the image has no account, `numeric`; and, once the helper has +4242, of which the image has no account, `numeric`; with every allowed +capability, without published listeners, the working-directory cases, each of +which names a working directory: the case's own, `cwd-directory`, and, through +a magic link, which the entry refuses before the program runs, the leaked +descriptor's, `cwd-leaked`, the execution's own working directory and root, +`cwd-self` and `cwd-root`, that of the first process of its namespace, +`cwd-init`, a symbolic link to its working directory, `cwd-link`, and the link +of each descriptor from 0 to 63, `cwd-descriptors`; and, once the helper has bound the runtime's files, a generated machine ID, the time status that the guest's time daemon publishes at `/run/nvx/time`, and the hostname `runtime-adapter`, `generated`. The helper first writes a machine ID @@ -880,7 +887,7 @@ own: | `busy_root_fails_the_teardown` | The image over the scratch, which the `hold` probe keeps open | The teardown's failure, which the stop reports, and a VM that ends with 15, leaving the scratch mounted | | `mappings_apply_beneath_the_root` | The image over the scratch, beneath which the `map` and `map-checks` probes map | Through the root: zero mappings that do no work, then a table mounted in order, each mapping with its access, `nosuid`, and `nodev`, whose writes reach the sources of the writable ones only, on targets that the mapper created on the overlay. Through mappers of the adapter's own: a link or a magic link at each component, the wrong kinds, a target that would be created inside another mapping, and the runtime's, the container root's, and the adapter's own reserved targets refused, and a read-only mapping that is never writable. A teardown that unmounts the mappings, then the scratch, cleanly | | `a_refused_mapping_latches_the_root` | The image over the scratch, in which the `map-refused` probe plants a symbolic link | A mapping through the link that the root refuses, nothing mounted or created, and a root that refuses every later mapping, then tears down cleanly | -| `root_profiles_enter_the_root` | The image over the scratch, which the `entries` probe enters | Executions that start as PID 1 of their namespaces, with the root moved over `/`, only their standard streams, the leaked descriptor closed, no inheritable or ambient capability, and no new privileges. User ID 0 with each allowed capability alone holds it alone, and only its own attempt succeeds; with none, nothing does, nor for a numeric identity without an account. No execution writes `/proc/sys`, so none lowers the unprivileged port start, and a nested `chroot` stays in the root. With published listeners, user ID 0 with every other allowed capability can neither bind a privileged source port nor open a raw socket, so it cannot forge a listener announcement in either publication mode; without them, it may. The entry refuses the 51 capabilities outside its allowlist, the binding and raw-socket ones while listeners are published, capabilities for another user ID, and an identity without an account in existing-account mode. The root's own machine ID, then a generated one and the time status that the guest's time daemon publishes, read-only, in which a root workload finds no lock of the daemon's, which the runtime bound in the helper's mount namespace, and which leave the worker's namespace without mounts below the root. A teardown that unmounts the scratch cleanly | +| `root_profiles_enter_the_root` | The image over the scratch, which the `entries` probe enters | Executions that start as PID 1 of their namespaces, with the root moved over `/`, only their standard streams, the leaked descriptor closed, no inheritable or ambient capability, and no new privileges, at the top of the root unless they name a working directory. A working directory that the execution names is entered, but none through a magic link, such as the leaked descriptor's, its own working directory's or root's, or any descriptor's, nor through a symbolic link to one: each fails the launch before the program runs. User ID 0 with each allowed capability alone holds it alone, and only its own attempt succeeds; with none, nothing does, nor for a numeric identity without an account. No execution writes `/proc/sys`, so none lowers the unprivileged port start, and a nested `chroot` stays in the root. With published listeners, user ID 0 with every other allowed capability can neither bind a privileged source port nor open a raw socket, so it cannot forge a listener announcement in either publication mode; without them, it may. The entry refuses the 51 capabilities outside its allowlist, the binding and raw-socket ones while listeners are published, capabilities for another user ID, and an identity without an account in existing-account mode. The root's own machine ID, then a generated one and the time status that the guest's time daemon publishes, read-only, in which a root workload finds no lock of the daemon's, which the runtime bound in the helper's mount namespace, and which leave the worker's namespace without mounts below the root. A teardown that unmounts the scratch cleanly | A row names the root, the image, whether a GPT disk's top layer is corrupt, the identity that the launch expects of the image, the upper, the runtime diff --git a/tests/runtime-adapter/src/worker/entries.rs b/tests/runtime-adapter/src/worker/entries.rs index 0509863c3..1a91fb54b 100644 --- a/tests/runtime-adapter/src/worker/entries.rs +++ b/tests/runtime-adapter/src/worker/entries.rs @@ -14,26 +14,31 @@ //! //! | Probe | Facts | //! | --- | --- | -//! | `entries` | The capabilities outside the allowlist that the entry refused, `refused.outside`, and those that it accepted, `refused.accepted`; its refusal of `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while listeners are published, `refused.published.net-bind-service` and `refused.published.net-raw`, of a capability for another user ID than 0, `refused.non-root`, and of an identity without an account in existing-account mode, `refused.existing`. For each case: `CASE.launch`, `CASE.status`, and the facts of `inside`, as `CASE.KEY`. The generated machine ID, `generated.source`. Once the helper ended, the root's own machine ID, `after.machine-id`, and the mounts below the root, `after.mounts`, in the worker's mount namespace | +//! | `entries` | The capabilities outside the allowlist that the entry refused, `refused.outside`, and those that it accepted, `refused.accepted`; its refusal of `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while listeners are published, `refused.published.net-bind-service` and `refused.published.net-raw`, of a capability for another user ID than 0, `refused.non-root`, and of an identity without an account in existing-account mode, `refused.existing`. For each case: `CASE.launch`, `CASE.status`, and the facts of `inside`, as `CASE.KEY`; for the working-directory descriptors, only their distinct `cwd-descriptors.launch` and `cwd-descriptors.status`. The generated machine ID, `generated.source`. Once the helper ended, the root's own machine ID, `after.machine-id`, and the mounts below the root, `after.mounts`, in the worker's mount namespace | //! //! The cases, as user ID 0 in numeric-account mode unless they say otherwise, are: each allowed //! capability alone, unpublished, `only-NAME`, such as `only-net-bind-service`; no capability, //! `none`; every allowed capability but those that published listeners refuse, published, //! `published`, which stands for both publication modes, private-network ingress and the host's //! loopback, since the entry treats them alike; every allowed capability, unpublished, -//! `unpublished`; the user and group ID 4242, of which the image has no account, `numeric`; and, -//! once the helper has bound the runtime's files, a generated machine ID, the time status that -//! the guest's time daemon publishes at `/run/nvx/time`, with the daemon's own layout, and the -//! hostname `runtime-adapter`, `generated`. The cases before `generated` see the machine ID that -//! the root holds as its own. +//! `unpublished`; the user and group ID 4242, of which the image has no account, `numeric`; with +//! every allowed capability, unpublished, the working-directory cases, which name a working +//! directory: the case's own, `cwd-directory`, and, through a magic link, which the entry refuses +//! before the program runs, the leaked descriptor's, `cwd-leaked`, the execution's own working +//! directory and root, `cwd-self` and `cwd-root`, that of its namespace's first process, +//! `cwd-init`, a symbolic link to its working directory, `cwd-link`, and each descriptor's link +//! from 0 to 63, `cwd-descriptors`; and, once the helper has bound the runtime's files, a +//! generated machine ID, the time status that the guest's time daemon publishes at +//! `/run/nvx/time`, with the daemon's own layout, and the hostname `runtime-adapter`, +//! `generated`. The cases before `generated` see the machine ID that the root holds as its own. //! //! `inside` reports, in this order: the descriptors open in the execution, `fds`, and whether the //! leaked one is, `leaked`; `uid`, `gid`, and `groups`; the capability sets, `cap.inh`, -//! `cap.prm`, `cap.eff`, `cap.bnd`, and `cap.amb`, and `no-new-privs`; its process ID, `pid`; the -//! `hostname`; the `machine-id`, and the options of a mount at `/etc/machine-id`, if any, -//! `mount.machine-id`; the first line of the time status's state, `time.state`, the names in -//! `/run/nvx/time`, `time.files`, and the options of a mount there, `mount.time`; the -//! unprivileged port start, `threshold`; and the outcome of +//! `cap.prm`, `cap.eff`, `cap.bnd`, and `cap.amb`, and `no-new-privs`; its process ID, `pid`; its +//! working directory, `cwd`; the `hostname`; the `machine-id`, and the options of a mount at +//! `/etc/machine-id`, if any, `mount.machine-id`; the first line of the time status's state, +//! `time.state`, the names in `/run/nvx/time`, `time.files`, and the options of a mount there, +//! `mount.time`; the unprivileged port start, `threshold`; and the outcome of //! these attempts: lowering it, `threshold.write`; writing the hostname through `/proc/sys`, //! `hostname.write`; opening `/proc/sysrq-trigger` for writing, `sysrq.write`; binding UDP port //! 1023, a privileged source port, from which the gateway accepts listener announcements, @@ -49,6 +54,7 @@ //! PROBLEM`. A value that cannot be read is `absent` if its file does not exist, and otherwise //! `unreadable: PROBLEM`. +use std::collections::BTreeSet; use std::ffi::{CString, OsStr}; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; @@ -214,8 +220,15 @@ pub fn helper(root: &Path, workload: &Path) -> io::Result { ), ]); for (name, entry) in &cases { - execute(root, workload, name, entry, &leaked, &mut facts)?; + execute(root, workload, name, entry, &leaked, None, &mut facts)?; } + working_directories( + root, + workload, + &unpublished(profile(Capabilities::ALLOWED)), + &leaked, + &mut facts, + )?; // The runtime's files, which the entry binds once, for every later execution: a machine ID // that it generates, and the time status that the guest's time daemon publishes, with the @@ -235,10 +248,73 @@ pub fn helper(root: &Path, workload: &Path) -> io::Result { .machine_id(machine_id) .time_status(TIME_STATUS); generated.bind_runtime().map_err(io::Error::other)?; - execute(root, workload, "generated", &generated, &leaked, &mut facts)?; + execute( + root, + workload, + "generated", + &generated, + &leaked, + None, + &mut facts, + )?; Ok(facts) } +/// Runs the working-directory cases of `entry` in `root`, which each join the cgroup `workload`, +/// with `leaked` open in their caller, and adds their facts to `facts`: `cwd-directory`, in its +/// own directory; and, through a magic link, which the entry refuses, `cwd-leaked`, in the +/// directory of the leaked descriptor, `cwd-self` and `cwd-root`, in the execution's own working +/// directory and root, `cwd-init`, in that of its namespace's first process, and `cwd-link`, +/// through a symbolic link to the execution's working directory. Then `cwd-descriptors` tries each +/// descriptor's link from 0 to 63, and adds only the distinct outcomes, `cwd-descriptors.launch` +/// and `cwd-descriptors.status`, each sorted and separated by commas. +fn working_directories( + root: &Path, + workload: &Path, + entry: &Entry, + leaked: &OwnedFd, + facts: &mut Facts, +) -> io::Result<()> { + let own = format!("{CASES}/cwd-directory"); + let descriptor = format!("/proc/self/fd/{}", leaked.as_raw_fd()); + let link = format!("{CASES}/cwd-link/link"); + let target = root.join(below_root(&link)); + fs::create_dir_all(target.parent().unwrap_or(root))?; + if fs::symlink_metadata(&target).is_ok() { + fs::remove_file(&target)?; + } + std::os::unix::fs::symlink("/proc/self/cwd", &target)?; + for (name, directory) in [ + ("cwd-directory", own.as_str()), + ("cwd-leaked", descriptor.as_str()), + ("cwd-self", "/proc/self/cwd"), + ("cwd-root", "/proc/self/root"), + ("cwd-init", "/proc/1/cwd"), + ("cwd-link", link.as_str()), + ] { + execute(root, workload, name, entry, leaked, Some(directory), facts)?; + } + let mut launches = BTreeSet::new(); + let mut statuses = BTreeSet::new(); + for number in 0..64 { + let directory = format!("/proc/self/fd/{number}"); + let (launched, status, _) = spawn_case( + root, + workload, + "cwd-descriptors", + entry, + leaked, + Some(&directory), + )?; + launches.insert(launched); + statuses.insert(status); + } + let joined = |outcomes: BTreeSet| outcomes.into_iter().collect::>().join(","); + facts.push("cwd-descriptors.launch", joined(launches)); + facts.push("cwd-descriptors.status", joined(statuses)); + Ok(()) +} + /// Adds the entry's refusals of identities in `root` to `facts`. fn refusals(root: &Path, facts: &mut Facts) { let check = |identity: Identity, publication| { @@ -300,20 +376,43 @@ fn refusal(result: Result<(), EntryError>) -> String { } /// Runs the execution of `entry` named `name`, in the cgroup `workload`, which runs [`STAGED`] -/// as `inside`, with `leaked` open in its caller, and adds its facts to `facts`. +/// as `inside`, with `leaked` open in its caller, in the working `directory`, if any, and adds +/// its facts to `facts`. fn execute( root: &Path, workload: &Path, name: &str, entry: &Entry, leaked: &OwnedFd, + directory: Option<&str>, facts: &mut Facts, ) -> io::Result<()> { + let (launched, status, output) = spawn_case(root, workload, name, entry, leaked, directory)?; + facts.push(format!("{name}.launch"), launched); + facts.push(format!("{name}.status"), status); + let found = Facts::decode(&output) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.problem().to_owned()))?; + for (key, value) in found.iter() { + facts.push(format!("{name}.{key}"), value); + } + Ok(()) +} + +/// Runs the execution of `entry` named `name`, as [`execute`] does, and returns how it launched, +/// `started` or its failed stage and error, how it ended, and what it reported. +fn spawn_case( + root: &Path, + workload: &Path, + name: &str, + entry: &Entry, + leaked: &OwnedFd, + directory: Option<&str>, +) -> io::Result<(String, String, Vec)> { let inside = format!("{CASES}/{name}"); - let directory = root.join(below_root(&inside)); - fs::create_dir_all(&directory)?; - fs::set_permissions(&directory, fs::Permissions::from_mode(0o777))?; - let owned = directory.join("owned"); + let case = root.join(below_root(&inside)); + fs::create_dir_all(&case)?; + fs::set_permissions(&case, fs::Permissions::from_mode(0o777))?; + let owned = case.join("owned"); fs::write(&owned, name)?; std::os::unix::fs::chown(&owned, Some(OWNER), Some(OWNER))?; @@ -329,20 +428,22 @@ fn execute( .map(argument) .collect::>>()?; let (mut reader, writer) = io::pipe()?; - let spawned = Command::new(argument(STAGED)?, arguments) + let mut command = Command::new(argument(STAGED)?, arguments) .environment(Vec::new()) .stdin(File::open("/dev/null")?) .stdout(writer) .stderr(io::stderr().as_fd().try_clone_to_owned()?) - .cgroup(Cgroup::open(workload)?) - .enter(prepared) - .spawn()?; + .cgroup(Cgroup::open(workload)?); + if let Some(directory) = directory { + command = command.working_directory(argument(directory)?); + } + let spawned = command.enter(prepared).spawn()?; let process = Pid::from_raw(i32::try_from(spawned.id()).map_err(io::Error::other)?); let launched = spawned.into_report().wait(); let mut output = Vec::new(); let read = reader.read_to_end(&mut output); let status = wait::waitpid(process, None)?; - let launch = match launched? { + let launched = match launched? { None => "started".to_owned(), Some(failure) => format!( "{} {:?}", @@ -351,19 +452,12 @@ fn execute( ), }; read?; - facts.push(format!("{name}.launch"), launch); let status = match status { WaitStatus::Exited(_, code) => format!("exit {code}"), WaitStatus::Signaled(_, signal, _) => format!("signal {signal}"), status => format!("{status:?}"), }; - facts.push(format!("{name}.status"), status); - let found = Facts::decode(&output) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error.problem().to_owned()))?; - for (key, value) in found.iter() { - facts.push(format!("{name}.{key}"), value); - } - Ok(()) + Ok((launched, status, output)) } /// Reports, from inside an execution's root, what the execution sees and may do, as the @@ -400,6 +494,11 @@ pub fn inside(directory: &Path, leaked: Option) -> Facts { facts.push(key, field(name)); } facts.push("pid", std::process::id().to_string()); + let working = std::env::current_dir().map_or_else( + |error| format!("unreadable: {error}"), + |path| path.display().to_string(), + ); + facts.push("cwd", working); facts.push( "hostname", probes::read(Path::new("/proc/sys/kernel/hostname")), diff --git a/tests/runtime-adapter/tests/vm_layered.rs b/tests/runtime-adapter/tests/vm_layered.rs index 2db6d89e8..0c0888cb7 100644 --- a/tests/runtime-adapter/tests/vm_layered.rs +++ b/tests/runtime-adapter/tests/vm_layered.rs @@ -503,7 +503,8 @@ scenarios! { /// Executions that enter the root as root profiles do, through `nvx::guest::entry`: user ID /// 0 with each allowed capability alone, with none, and with every one in each publication /// state, and a numeric identity, each PID 1 of its namespaces, with the root moved over `/`, - /// without the descriptor that its caller leaked, and contained by a nested `chroot`. No root + /// without the descriptor that its caller leaked, and contained by a nested `chroot`. No + /// working directory reached through a magic link starts a program. No root /// workload lowers the unprivileged port start or binds a privileged port while listeners are /// published, and the entry refuses the capabilities outside its allowlist. The root's own /// machine ID, then a generated one, with the time status that the guest's time daemon @@ -1459,6 +1460,8 @@ fn check_root_profiles(vm: &mut Vm<'_>) { assert_eq!(fact("pid"), "1", "{case}"); assert_eq!(fact("fds"), "0,1,2", "{case}"); assert_eq!(fact("leaked"), "closed", "{case}"); + // Without a working directory, it runs in its root's top. + assert_eq!(fact("cwd"), "/", "{case}"); assert_eq!(fact("no-new-privs"), "1", "{case}"); assert_eq!( (fact("cap.inh"), fact("cap.amb")), @@ -1562,6 +1565,38 @@ fn check_root_profiles(vm: &mut Vm<'_>) { } } + // An execution enters the working directory that it names in the root, but through no magic + // link: neither the link of the descriptor that its caller leaked, which it closed, nor that + // of its own or its first process's working directory or root, nor a symbolic link to one, + // nor that of any descriptor, may place it outside the root, or run its program. + assert_eq!( + (fact("cwd-directory.launch"), fact("cwd-directory.status")), + ("started", "exit 0") + ); + assert_eq!( + fact("cwd-directory.cwd"), + "/tmp/runtime-adapter-entries/cwd-directory" + ); + for (case, error) in [ + ("cwd-leaked", "ENOENT"), + ("cwd-self", "ELOOP"), + ("cwd-root", "ELOOP"), + ("cwd-init", "ELOOP"), + ("cwd-link", "ELOOP"), + ] { + let fact = |key: &str| fact(&format!("{case}.{key}")); + assert_eq!( + (fact("launch"), fact("status")), + (&*format!("working directory {error}"), "exit 125"), + "{case}" + ); + } + assert_eq!( + fact("cwd-descriptors.launch"), + "working directory ELOOP,working directory ENOENT" + ); + assert_eq!(fact("cwd-descriptors.status"), "exit 125"); + // A generated machine ID and the time status that the guest's time daemon publishes, which the // runtime bound once, read-only, in the helper's mount namespace, reach the execution, and end // with the helper: the worker's own namespace keeps the root's machine ID, without mounts From 8398fc08372b9a8b7e055e4ee2430b67673b41ac Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Fri, 23 Oct 2026 02:57:17 -0700 Subject: [PATCH 09/12] nvx-rs: clear an entry's umask without nix's panicking wrapper An entry's child cleared its umask before it mounted its view, and restored it after, with nix's umask, which panics when the system call fails, as a seccomp policy can make it. The child that clone3 starts must not panic: the panic would neither report the failed stage nor end the child cleanly, and could unwind into its caller's code. The child now makes the system call itself. A failure to clear the umask is the proc stage's, and one to restore it the devices stage's, so the launch reports it and runs nothing. A test denies umask with a seccomp filter, in a process of its own, and checks that a simulated entry then fails its proc stage with EPERM. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e06da46b-cc03-40f6-84e4-199397657333 --- nvx-rs/src/guest/entry/tests.rs | 28 +++++++++++++++ nvx-rs/src/guest/spawn.rs | 5 +-- nvx-rs/src/guest/sys/entry.rs | 39 ++++++++++++++------ nvx-rs/src/guest/sys/mod.rs | 13 +++---- nvx-rs/src/guest/sys/process.rs | 64 +++++++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 19 deletions(-) diff --git a/nvx-rs/src/guest/entry/tests.rs b/nvx-rs/src/guest/entry/tests.rs index 7ba46f544..317000717 100644 --- a/nvx-rs/src/guest/entry/tests.rs +++ b/nvx-rs/src/guest/entry/tests.rs @@ -682,6 +682,34 @@ fn each_entry_stage_reports_its_own_failure_and_runs_nothing() { } } +#[test] +fn a_denied_umask_fails_the_proc_stage_and_runs_nothing() { + // A seccomp filter lasts as long as the thread that installs it, and passes to the processes + // that the thread starts, so the test runs in a process of its own. + if crate::guest::managed_agent::isolate_process_test( + "NVX_ENTRY_DENIED_UMASK_TEST_CHILD", + "guest::entry::tests::a_denied_umask_fails_the_proc_stage_and_runs_nothing", + ) { + return; + } + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("ran"); + // The simulated stage is no entry stage, so every entry stage succeeds, and the child's own + // call, which clears its umask before it mounts the view, is the one that fails. + sys::deny_system_call(nix::libc::SYS_umask); + let (failure, status, _) = run_released( + shell(&format!("touch '{}'", marker.display())) + .enter(simulated(LaunchStage::Credentials, Errno::EPERM)), + ); + let failure = failure.expect("the denied umask fails the launch"); + assert_eq!( + (failure.stage(), failure.raw_os_error()), + (LaunchStage::Proc, Errno::EPERM as i32) + ); + assert_eq!(status, Some(ChildStatus::Exited(125))); + assert!(!marker.exists()); +} + #[test] fn an_entered_program_leads_a_session_without_a_terminal_or_the_callers_descriptors() { // A descriptor that the caller opened without close-on-exec, as another thread of a caller diff --git a/nvx-rs/src/guest/spawn.rs b/nvx-rs/src/guest/spawn.rs index 84ca64c91..66ba4ab22 100644 --- a/nvx-rs/src/guest/spawn.rs +++ b/nvx-rs/src/guest/spawn.rs @@ -93,11 +93,12 @@ pub enum LaunchStage { Hostname, /// The child moves its root over `/`, and makes it its root and working directory. MoveRoot, - /// The child mounts its procfs, and restricts it for user ID 0. + /// The child clears its umask, then mounts its procfs, and restricts it for user ID 0. Proc, /// The child mounts its read-only sysfs, and restricts it for user ID 0. Sys, - /// The child mounts its `/dev`, with its device nodes, devpts, and shm. + /// The child mounts its `/dev`, with its device nodes, devpts, and shm, then restores its + /// umask. Devices, /// The child makes the root directory its root. Root, diff --git a/nvx-rs/src/guest/sys/entry.rs b/nvx-rs/src/guest/sys/entry.rs index 280608f35..9ddde5f31 100644 --- a/nvx-rs/src/guest/sys/entry.rs +++ b/nvx-rs/src/guest/sys/entry.rs @@ -11,12 +11,13 @@ //! libc does not know of a child that `clone3` starts: the child's copy of libc's thread state //! still names its parent's thread, and libc's wrappers that change credentials signal every //! thread that libc believes the process has. So the child calls no libc function that relies -//! on that state. It changes its credentials, capabilities, and hostname, and uses the new mount -//! API (`fsopen`, `fsconfig`, `fsmount`, `open_tree`, `move_mount`, and `mount_setattr`) and -//! `openat2`, through libc's `syscall`, with the kernel's structures that the libc crate does not -//! define laid out here from the kernel's UAPI. It makes its other calls through `nix`, or libc's -//! plain wrappers of one system call each, as a forked child does. No step can panic: a panic -//! would abort through libc's `raise`, which signals the thread that libc names, its parent's. +//! on that state. It changes its credentials, capabilities, hostname, and umask, and uses the new +//! mount API (`fsopen`, `fsconfig`, `fsmount`, `open_tree`, `move_mount`, and `mount_setattr`) +//! and `openat2`, through libc's `syscall`, with the kernel's structures that the libc crate does +//! not define laid out here from the kernel's UAPI. It makes its other calls through `nix`, or +//! libc's plain wrappers of one system call each, as a forked child does. No step can panic: a +//! panic would abort through libc's `raise`, which signals the thread that libc names, its +//! parent's. use std::ffi::CStr; use std::mem::MaybeUninit; @@ -24,7 +25,6 @@ use std::os::fd::RawFd; use nix::libc::{self, c_int, c_long, c_uint, c_ulong}; use nix::mount::MsFlags; -use nix::sys::stat::Mode; use nix::unistd::Pid; use super::{Errno, FileDescriptor}; @@ -230,17 +230,19 @@ impl EntryChild { /// Makes the child's mounts private, sets its hostname, moves its root over `/`, and mounts /// its view: procfs, sysfs, and `/dev` with its devpts and shm, in the order of their stages. /// It creates its directories and nodes with exact modes, whatever the process's umask, which - /// it restores before it returns. + /// it clears before it mounts the view and restores before it returns: a failure to clear it + /// is the [`Proc`](LaunchStage::Proc) stage's, and one to restore it the + /// [`Devices`](LaunchStage::Devices) stage's. /// /// Each failure names its stage. Only the post-clone child of an entry may call this. pub(super) fn enter_view(&self) -> Result<(), (LaunchStage, Errno)> { self.stage(LaunchStage::Propagation, make_mounts_private)?; self.stage(LaunchStage::Hostname, || set_hostname(&self.hostname))?; self.stage(LaunchStage::MoveRoot, || move_root(&self.root))?; - let umask = nix::sys::stat::umask(Mode::empty()); + let umask = set_umask(0).map_err(|error| (LaunchStage::Proc, error))?; let view = self.mount_view(); - nix::sys::stat::umask(umask); - view + let restored = set_umask(umask).map_err(|error| (LaunchStage::Devices, error)); + view.and(restored.map(drop)) } /// Mounts the view's file systems, as [`enter_view`](Self::enter_view) describes. @@ -426,6 +428,21 @@ fn move_root(root: &CStr) -> Result<(), Errno> { nix::unistd::chdir(c"/") } +/// Sets the calling process's umask to `mask`, and returns the umask that it replaces. +/// +/// The child makes the system call itself, because `nix`'s `umask` panics when the call fails, +/// as a seccomp policy can make it. +fn set_umask(mask: libc::mode_t) -> Result { + // SAFETY: `umask` takes an integer and dereferences nothing. + let previous = unsafe { libc::syscall(libc::SYS_umask, mask) }; + if previous < 0 { + Err(Errno::last()) + } else { + // A umask fits a `mode_t`. + Ok(previous as libc::mode_t) + } +} + /// Mounts a new procfs instance of the child's PID namespace at `/proc`, `nosuid`, `nodev`, and /// `noexec`, and returns its mount's descriptor. fn mount_proc() -> Result { diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index 28591c641..06c3b5abd 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -24,10 +24,10 @@ //! drop an owner, fills its preallocated `PWD` entry with the `getcwd` system call, closes the //! descriptors that it does not use with the `close_range` system call, and executes and exits //! through libc; an execution's entry starts its child with `clone3`, and the child changes its -//! credentials, capabilities, and hostname, and mounts its view and enters its working directory -//! with the new mount API and `openat2`, through libc's `syscall`, as the entry's module -//! describes, because libc does not know of that child; the agent's wait-status decoding, and -//! the waits for one named child, +//! credentials, capabilities, hostname, and umask, and mounts its view and enters its working +//! directory with the new mount API and `openat2`, through libc's `syscall`, as the entry's +//! module describes, because libc does not know of that child; the agent's wait-status decoding, +//! and the waits for one named child, //! keep the real-time signals that `nix`'s `Signal` cannot represent; a pidfd is opened with the //! `pidfd_open` system call, which neither `nix` nor musl wraps; a write to an execution's input //! takes back the `SIGPIPE` that a closed reader raises with `sigtimedwait`, once `sigpending` @@ -40,7 +40,8 @@ //! shortcuts. The guest mappings need `statx`, for the kinds and unique mount IDs of open files, //! and the mount API's `open_tree`, `mount_setattr`, and `move_mount`, which neither `nix` nor //! musl wraps. The tests trace a child with libc's `ptrace`, and lower the process's descriptor -//! limit with libc's `setrlimit`, as the crate enables neither for `nix`. Each such call says why. +//! limit with libc's `setrlimit`, as the crate enables neither for `nix`, and deny a system call +//! to a thread with libc's `prctl`, as `nix` does not wrap seccomp. Each such call says why. //! //! The x86 instructions of the time tools are `core::arch` intrinsics, except the port //! instructions, which only `port` executes, as inline assembly. The constants of the kernel's @@ -125,7 +126,7 @@ pub(crate) use process::{ with_sigpipe_ignored, write_port_diagnostic, write_standard_error, }; #[cfg(test)] -pub(crate) use process::{continue_traced, spawn_traced_child}; +pub(crate) use process::{continue_traced, deny_system_call, spawn_traced_child}; pub(crate) use rooted::{ FileKind, FileStatus, MountAttributes, change_mode, create_directory_at, create_file_at, detach_mount, file_status, is_whole_file_system, move_mount, open_beneath, open_tree, diff --git a/nvx-rs/src/guest/sys/process.rs b/nvx-rs/src/guest/sys/process.rs index 12da6e368..1c93e45d3 100644 --- a/nvx-rs/src/guest/sys/process.rs +++ b/nvx-rs/src/guest/sys/process.rs @@ -445,6 +445,70 @@ pub(crate) fn continue_traced(process: ProcessId) { assert_eq!(result, 0, "PTRACE_CONT: {}", Errno::last()); } +/// Makes the system call `number` fail with `EPERM` in the calling thread, and in every process +/// that it starts from now on, through a seccomp filter that allows every other call, for a test +/// that runs in a process of its own, since no filter can be removed. It first sets the thread's +/// `no_new_privs`, without which a filter needs privileges. +/// +/// libc's `prctl` is used because `nix` does not wrap seccomp. +#[cfg(test)] +pub(crate) fn deny_system_call(number: nix::libc::c_long) { + use nix::libc::{self, c_ulong, sock_filter}; + + let statement = |code: u32, if_equal: u8, otherwise: u8, k: u32| sock_filter { + // Every BPF code fits 16 bits. + code: code as u16, + jt: if_equal, + jf: otherwise, + k, + }; + // The filter loads the call's number, the first word of the kernel's `seccomp_data`, and + // returns `EPERM` for `number`, or allows the call. + let mut filter = [ + statement(libc::BPF_LD | libc::BPF_W | libc::BPF_ABS, 0, 0, 0), + // A system call number fits 32 bits. + statement( + libc::BPF_JMP | libc::BPF_JEQ | libc::BPF_K, + 0, + 1, + number as u32, + ), + statement( + libc::BPF_RET | libc::BPF_K, + 0, + 0, + libc::SECCOMP_RET_ERRNO | libc::EPERM as u32, + ), + statement(libc::BPF_RET | libc::BPF_K, 0, 0, libc::SECCOMP_RET_ALLOW), + ]; + let program = libc::sock_fprog { + len: filter.len() as u16, + filter: filter.as_mut_ptr(), + }; + // SAFETY: `PR_SET_NO_NEW_PRIVS` reads only its integer arguments. + let result = unsafe { + libc::prctl( + libc::PR_SET_NO_NEW_PRIVS, + 1 as c_ulong, + 0 as c_ulong, + 0 as c_ulong, + 0 as c_ulong, + ) + }; + assert_eq!(result, 0, "PR_SET_NO_NEW_PRIVS: {}", Errno::last()); + // SAFETY: the program names its four statements, which the kernel copies before it returns. + let result = unsafe { + libc::prctl( + libc::PR_SET_SECCOMP, + libc::SECCOMP_MODE_FILTER as c_ulong, + &program as *const libc::sock_fprog, + 0 as c_ulong, + 0 as c_ulong, + ) + }; + assert_eq!(result, 0, "PR_SET_SECCOMP: {}", Errno::last()); +} + /// Opens a close-on-exec pidfd that refers to `process`, which polls readable once the process /// ends, and keeps naming it after it is reaped. /// From 6e344490430fef127b95f2adddad54666f5a85e7 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 23 Oct 2026 10:19:06 -0300 Subject: [PATCH 10/12] tests: format the runtime adapter's combined worker modules Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/runtime-adapter/src/worker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runtime-adapter/src/worker.rs b/tests/runtime-adapter/src/worker.rs index d8c586b39..a7087b065 100644 --- a/tests/runtime-adapter/src/worker.rs +++ b/tests/runtime-adapter/src/worker.rs @@ -106,8 +106,8 @@ //! end. mod connection; -mod executions; pub mod entries; +mod executions; pub mod mapping; pub mod probes; From 422fca53add9def54db4acfd6c998974342a131d Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 23 Oct 2026 10:41:15 -0300 Subject: [PATCH 11/12] tests: adapt entry probes to the layered root context Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/runtime-adapter/src/worker/entries.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/runtime-adapter/src/worker/entries.rs b/tests/runtime-adapter/src/worker/entries.rs index 1a91fb54b..142c654c0 100644 --- a/tests/runtime-adapter/src/worker/entries.rs +++ b/tests/runtime-adapter/src/worker/entries.rs @@ -757,6 +757,7 @@ mod tests { root: Path::new("/run/runtime-adapter/rootfs"), lowers: &lowers, upper: Path::new("/run/runtime-adapter/scratch"), + first_partition: None, agent: Path::new("/sys/fs/cgroup/agent"), workload: Path::new("/sys/fs/cgroup/workload"), sibling: None, From d2bbbc45a5bbefe7e34c637810b6465589eab3ee Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 23 Oct 2026 10:51:20 -0300 Subject: [PATCH 12/12] tests: follow the extended guest launch stages Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/runtime-adapter/README.md | 2 +- tests/runtime-adapter/tests/guest_session.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index b90bee66a..0370d5016 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -232,7 +232,7 @@ or 0 for none; whether the launch failed, 0 or 1, then its stage, a `u32`, and its `errno`, an `i32`, or 0 for each; and flags: 1 if standard output lost bytes, 2 if standard error did, and 4 if the execution is reusable. Stage 0 is the worker's refusal of the start, and a later stage is the binding's number of -the stage of `nvx::guest::spawn` that failed, such as 8 for its `exec`. +the stage of `nvx::guest::spawn` that failed, such as 20 for its `exec`. ### Root-profile entries diff --git a/tests/runtime-adapter/tests/guest_session.rs b/tests/runtime-adapter/tests/guest_session.rs index 4b7df2820..2bdc6fadd 100644 --- a/tests/runtime-adapter/tests/guest_session.rs +++ b/tests/runtime-adapter/tests/guest_session.rs @@ -2487,7 +2487,7 @@ fn the_worker_refuses_starts_beyond_its_limits_with_a_failed_launch() { let memory = LaunchFailure::new(0, Errno::ENOMEM as i32); assert_eq!(host.outcome(20).unwrap().launch_failure(), Some(memory)); let missing = host.outcome(21).unwrap(); - let exec = LaunchFailure::new(8, Errno::ENOENT as i32); + let exec = LaunchFailure::new(20, Errno::ENOENT as i32); assert_eq!(missing.launch_failure(), Some(exec)); assert_eq!(missing.status(), Some(Status::Exited(125))); let (_, worked) = host.stop();