From bacc4a073a64d5e120cd7a0095efc41fd2dc6508 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 30 Oct 2026 18:47:54 -0700 Subject: [PATCH 1/2] nvx-rs: count a process's reads and what an endpoint has not read Process::bytes_read and own_bytes_read report the bytes that a process read through the operating system's read calls, the cache's included: rchar of /proc//io on Linux, read only while the process still runs with its reference's identity, and ReadTransferCount of GetProcessIoCounters on Windows. Transport::unread reports what the endpoint has not read of the transport's writes, where the platform tells: SIOCOUTQ of the Unix socket on Linux, and None on Windows, whose named pipes tell their writer nothing of the kind. An adapter, which reaches NVX through its public API alone, needs both to check from the host that a clone's restore reads no large content, and that OpenVMM reads no host input before the guest's repair acknowledges the restore. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- nvx-rs/README.md | 21 ++++++ nvx-rs/src/host/platform/linux.rs | 95 +++++++++++++++++++++++ nvx-rs/src/host/platform/mod.rs | 13 ++++ nvx-rs/src/host/platform/other.rs | 10 +++ nvx-rs/src/host/platform/windows.rs | 39 ++++++++-- nvx-rs/src/host/process.rs | 113 ++++++++++++++++++++++++++++ nvx-rs/src/host/sys/mod.rs | 2 +- nvx-rs/src/host/sys/socket.rs | 21 ++++++ 8 files changed, 308 insertions(+), 6 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 6aed10f2f..8e183e5fa 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -523,6 +523,18 @@ and a process whose identity this user cannot check, such as another user's, makes `Process::open` fail rather than report an exit. Linux hosts without pidfds poll the reference instead, and cannot terminate the process. +`Process::bytes_read` reports how many bytes the process has read through the +operating system's read calls, whether the host's cache served them or not: +`rchar` of `/proc//io` on Linux, which it reads only while the process +still runs with its reference's identity, and the `ReadTransferCount` of +`GetProcessIoCounters` on Windows. `own_bytes_read` reports the calling +process's. Neither counts what a process maps from a file, as OpenVMM maps the +memory of a snapshot that it restores, nor, on Linux, what it reads through +io_uring, as OpenVMM reads a sandbox's block devices. On Linux, the count also +holds what the process's children read once it reaped them, as the kernel +accounts them to their parent, while Windows counts the process's own reads +alone. + ### `nvx::host::supervisor` The supervisor of a sandbox's VMM, beneath the CLI, `Instance`, and external @@ -927,6 +939,15 @@ The authenticated control session, which carries NVX's control protocol, `NVXC`, one frame per DATA record of a `HostConnection`; Linux and Windows primitives; and OpenVMM's outcome report. +A `Transport` to an OpenVMM control endpoint reads and writes with deadlines, +and tells, through `Transport::unread`, what the endpoint has not read yet of +what it wrote, where the platform tells: on Linux, what `SIOCOUTQ` reports of +its Unix socket, the kernel's charge for the buffers of each write until +OpenVMM read each whole, which a read of part of a buffer leaves as it was, and +which is zero once OpenVMM read everything. A named pipe tells its writer +nothing of the kind, so on Windows, and for a transport that does not tell, it +is `None`. + ### `nvx::host::sandbox` The launch profile of `nvx-rs sandbox`, with native input validation: diff --git a/nvx-rs/src/host/platform/linux.rs b/nvx-rs/src/host/platform/linux.rs index 435e4820a..9447a7dd4 100644 --- a/nvx-rs/src/host/platform/linux.rs +++ b/nvx-rs/src/host/platform/linux.rs @@ -188,6 +188,53 @@ impl ProcessHandle { result => super::unless_exited(result, || pidfd.wait_exit(Duration::ZERO)), } } + + /// Returns how many bytes process `pid`, which the handle holds, and which started at + /// `start_time`, has read through read calls: the `rchar` of its `/proc//io`. + /// + /// The file is opened before the process's identity is checked, so a process that still runs + /// with that start time held the ID when the file was opened, and the count is its own, with + /// or without a pidfd. Fails with [`io::ErrorKind::NotFound`] once the process has exited. + pub(crate) fn bytes_read(&self, pid: u32, start_time: u64) -> io::Result { + let path = format!("/proc/{pid}/io"); + let exited = || io::Error::new(io::ErrorKind::NotFound, format!("process {pid} exited")); + let gone = |error: &io::Error| { + error.kind() == io::ErrorKind::NotFound + || error.raw_os_error() == Some(Errno::ESRCH as i32) + }; + let mut file = match File::open(&path) { + Ok(file) => file, + Err(error) if gone(&error) => return Err(exited()), + Err(error) => return Err(error), + }; + if process_start_time(pid)? != Some(start_time) { + return Err(exited()); + } + let mut text = String::new(); + match file.read_to_string(&mut text) { + Ok(_) => read_count(&path, &text), + Err(error) if gone(&error) => Err(exited()), + Err(error) => Err(error), + } + } +} + +/// Returns how many bytes this process has read through read calls: the `rchar` of +/// `/proc/self/io`. +pub(crate) fn own_bytes_read() -> io::Result { + let path = "/proc/self/io"; + read_count(path, &fs::read_to_string(path)?) +} + +/// Returns the `rchar` count of the text `text` of the file at `path`, a process's `/proc` file +/// `io`: the bytes that the process, and the children that it reaped, passed to read calls, from +/// any file, whether the page cache served them or not, which counts no read through io_uring nor +/// a file's mapped pages. +fn read_count(path: &str, text: &str) -> io::Result { + text.lines() + .find_map(|line| line.strip_prefix("rchar:")) + .and_then(|count| count.trim().parse().ok()) + .ok_or_else(|| io::Error::other(format!("{path} has no rchar count"))) } /// Opens the process `pid` if it still runs with the start time `start_time`, and returns `None` @@ -1097,6 +1144,10 @@ impl Transport for UnixTransport { } Ok(()) } + + fn unread(&self) -> io::Result> { + sys::unread_by_peer(&self.stream).map(Some) + } } #[cfg(test)] @@ -1212,4 +1263,48 @@ mod tests { drop(client); assert_eq!(receive_all(&mut listener), b""); } + + #[test] + fn a_unix_transport_reports_what_its_peer_has_not_read() { + let (stream, mut peer) = UnixStream::pair().unwrap(); + let mut transport = UnixTransport { stream }; + assert_eq!(transport.unread().unwrap(), Some(0)); + let data = b"an attachment"; + transport.write_all(data, None).unwrap(); + // The kernel charges the socket for the write's buffer, more than its bytes, until the + // peer read it whole: a read of a prefix leaves the charge as it was. + let queued = transport.unread().unwrap().unwrap(); + assert!(queued >= data.len(), "{queued}"); + let mut received = [0; 6]; + peer.read_exact(&mut received).unwrap(); + assert_eq!(transport.unread().unwrap(), Some(queued)); + let mut rest = vec![0; data.len() - received.len()]; + peer.read_exact(&mut rest).unwrap(); + assert_eq!(transport.unread().unwrap(), Some(0)); + + // Writes of a byte each queue a buffer each, which the kernel charges the same for, and + // which the peer's read of any prefix frees. + let charges = data + .chunks(1) + .map(|byte| { + transport.write_all(byte, None).unwrap(); + transport.unread().unwrap().unwrap() + }) + .collect::>(); + let step = charges[0]; + assert!(step > 0, "{charges:?}"); + assert!( + charges + .iter() + .enumerate() + .all(|(index, charge)| *charge == (index + 1) * step), + "{charges:?}" + ); + peer.read_exact(&mut received[..1]).unwrap(); + let queued = charges[charges.len() - 1]; + assert_eq!(transport.unread().unwrap(), Some(queued - step)); + let mut rest = vec![0; data.len() - 1]; + peer.read_exact(&mut rest).unwrap(); + assert_eq!(transport.unread().unwrap(), Some(0)); + } } diff --git a/nvx-rs/src/host/platform/mod.rs b/nvx-rs/src/host/platform/mod.rs index c1886f7dd..e60d9b954 100644 --- a/nvx-rs/src/host/platform/mod.rs +++ b/nvx-rs/src/host/platform/mod.rs @@ -54,6 +54,19 @@ pub trait Transport: Send { /// [`io::ErrorKind::TimedOut`] error if it has not by then, however much of it the endpoint /// reads meanwhile. fn write_all(&mut self, data: &[u8], timeout: Option) -> io::Result<()>; + + /// Returns what the platform tells of the data that this end wrote, and that the endpoint has + /// not read yet, or `None` where it tells nothing. + /// + /// On Linux, it is what `SIOCOUTQ` reports of the Unix socket: the memory that the kernel + /// charges the socket for the buffers that hold that data, each write's buffers of their own, + /// until the endpoint read each buffer whole. The charge exceeds the data's bytes, and a read + /// of part of a buffer leaves it as it was, so only data written a byte at a time shows the + /// read of each byte. It is zero once the endpoint read everything. A named pipe tells its + /// writer nothing of the kind, so on Windows, and by default, it is `None`. + fn unread(&self) -> io::Result> { + Ok(None) + } } /// Returns the time left until `deadline`, `None` without one, or an [`io::ErrorKind::TimedOut`] diff --git a/nvx-rs/src/host/platform/other.rs b/nvx-rs/src/host/platform/other.rs index 06fc9ce1a..b4c8586b1 100644 --- a/nvx-rs/src/host/platform/other.rs +++ b/nvx-rs/src/host/platform/other.rs @@ -56,6 +56,16 @@ impl ProcessHandle { pub(crate) fn kill(&self, _pid: u32) -> io::Result<()> { match *self {} } + + /// Never called: no handle exists. + pub(crate) fn bytes_read(&self, _pid: u32, _start_time: u64) -> io::Result { + match *self {} + } +} + +/// Fails: this host has no read counters that NVX reads. +pub(crate) fn own_bytes_read() -> io::Result { + Err(unsupported()) } /// Reports every process as gone, as [`process_start_time`] does. diff --git a/nvx-rs/src/host/platform/windows.rs b/nvx-rs/src/host/platform/windows.rs index c38c9d180..4ac02b957 100644 --- a/nvx-rs/src/host/platform/windows.rs +++ b/nvx-rs/src/host/platform/windows.rs @@ -66,11 +66,12 @@ use windows_sys::Win32::System::Threading::GetExitCodeProcess; use windows_sys::Win32::System::Threading::{ CREATE_BREAKAWAY_FROM_JOB, CREATE_NEW_PROCESS_GROUP, CREATE_NO_WINDOW, CREATE_UNICODE_ENVIRONMENT, CreateEventW, CreateProcessW, DeleteProcThreadAttributeList, - EXTENDED_STARTUPINFO_PRESENT, GetCurrentProcess, GetProcessTimes, INFINITE, - InitializeProcThreadAttributeList, LPPROC_THREAD_ATTRIBUTE_LIST, OpenProcess, OpenProcessToken, - PROC_THREAD_ATTRIBUTE_HANDLE_LIST, PROC_THREAD_ATTRIBUTE_JOB_LIST, PROCESS_CREATION_FLAGS, - PROCESS_INFORMATION, PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE, PROCESS_TERMINATE, - ResetEvent, STARTF_USESTDHANDLES, STARTUPINFOEXW, TerminateProcess, UpdateProcThreadAttribute, + EXTENDED_STARTUPINFO_PRESENT, GetCurrentProcess, GetProcessIoCounters, GetProcessTimes, + INFINITE, IO_COUNTERS, InitializeProcThreadAttributeList, LPPROC_THREAD_ATTRIBUTE_LIST, + OpenProcess, OpenProcessToken, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + PROC_THREAD_ATTRIBUTE_JOB_LIST, PROCESS_CREATION_FLAGS, PROCESS_INFORMATION, + PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE, PROCESS_TERMINATE, ResetEvent, + STARTF_USESTDHANDLES, STARTUPINFOEXW, TerminateProcess, UpdateProcThreadAttribute, WaitForSingleObject, }; use windows_sys::Win32::System::WindowsProgramming::QueryInterruptTimePrecise; @@ -150,6 +151,27 @@ fn has_exited(process: &OwnedHandle) -> io::Result { } } +/// Returns how many bytes the process of `process`, a handle with query access, has read through +/// read calls: the `ReadTransferCount` of `GetProcessIoCounters`, which counts every `ReadFile`, +/// whatever it reads, and whether the cache served it or not. +fn read_transfer(process: HANDLE) -> io::Result { + let mut counters = IO_COUNTERS::default(); + // SAFETY: the handle has query access, and the output pointer is valid. + if unsafe { GetProcessIoCounters(process, &mut counters) } != 0 { + Ok(counters.ReadTransferCount) + } else { + Err(io::Error::last_os_error()) + } +} + +/// Returns how many bytes this process has read through read calls, as [`read_transfer`] counts +/// them. +pub(crate) fn own_bytes_read() -> io::Result { + // SAFETY: GetCurrentProcess has no preconditions, and returns a pseudo-handle with every + // access, which needs no closing. + read_transfer(unsafe { GetCurrentProcess() }) +} + /// Returns the outcome of a termination that Windows refused with `error` through a handle with /// terminate access: success if the process is exiting or has exited, and `error` otherwise. /// @@ -273,6 +295,13 @@ impl ProcessHandle { ) } + /// Returns how many bytes the process has read through read calls, as [`read_transfer`] + /// counts them: once the process has exited, those that it read until then, which the handle + /// keeps. + pub(crate) fn bytes_read(&self, _pid: u32, _start_time: u64) -> io::Result { + read_transfer(raw(&self.process)) + } + /// Closes the handle of process `pid` that has the value of this process's handle `ours`, /// which the process inherited from this one: a handle keeps its value in a child that /// inherits it, and a child that never uses it, as a VMM never uses the guards of the checked diff --git a/nvx-rs/src/host/process.rs b/nvx-rs/src/host/process.rs index 12b8ae1f1..519bac297 100644 --- a/nvx-rs/src/host/process.rs +++ b/nvx-rs/src/host/process.rs @@ -15,6 +15,10 @@ //! Linux 5.3 or in sandboxes that refuse them, check the reference at a short interval instead, //! until the deadline of the wait, and cannot terminate a process that they cannot hold. //! +//! [`Process::bytes_read`] reports how many bytes the process has read through the operating +//! system's read calls, and [`own_bytes_read`] how many the calling process has, so that a caller +//! can tell how much of its files a VMM, or the caller itself, read. +//! //! The [`supervisor`](super::supervisor) launches OpenVMM, and its //! [`Launched::process`](super::supervisor::Launched::process) holds the child that it started //! through a handle of its own. @@ -240,6 +244,27 @@ impl Process { self.handle.kill(self.reference.pid) } + /// Returns how many bytes the process has read through the operating system's read calls, + /// from files, pipes, sockets, and devices alike, whether the host's cache served them or not: + /// the `rchar` of `/proc//io` on Linux, and the `ReadTransferCount` of + /// `GetProcessIoCounters` on Windows. Neither counts what the process reads otherwise: the + /// pages that it maps from a file, as OpenVMM maps the memory of a snapshot that it restores, + /// nor, on Linux, what it reads through io_uring, as OpenVMM reads a sandbox's block devices. + /// On Linux, the count also holds what the process's children read, once the process reaped + /// them, as the kernel accounts them to their parent, while Windows counts the process's own + /// reads alone. + /// + /// On Linux, the count is read from the process's `/proc` file, which is opened before the + /// process's identity is checked, so that the count is the process's own, and the read fails + /// with [`io::ErrorKind::NotFound`] once the process has exited. On Windows, the handle keeps + /// the count of a process that exited. Fails with [`io::ErrorKind::PermissionDenied`] when + /// this user may not read the process's counters, and with [`io::ErrorKind::Unsupported`] on + /// hosts other than Linux and Windows. + pub fn bytes_read(&self) -> io::Result { + self.handle + .bytes_read(self.reference.pid, self.reference.start_time) + } + /// Closes the process's handle that has the value of this process's handle `ours`, which the /// process inherited from this one, and never uses, as a VMM never uses the guards of the /// checked files that it inherits on Windows. @@ -266,6 +291,15 @@ impl fmt::Debug for Process { } } +/// Returns how many bytes the calling process has read through the operating system's read +/// calls, as [`Process::bytes_read`] counts them: on Linux, with what the children that it reaped +/// read. +/// +/// Fails with [`io::ErrorKind::Unsupported`] on hosts other than Linux and Windows. +pub fn own_bytes_read() -> io::Result { + platform::own_bytes_read() +} + /// Checks `reference` against the boot that `current_boot` returns, and during the same boot, /// opens the process with `open`, as [`Process::open`] does. fn open_reference( @@ -546,6 +580,85 @@ mod reference_tests { .unwrap() ); } + + /// Returns a command that reads the first line of the file at `path`, prints `read`, then + /// waits for a line of its standard input, which reaches its end once the test closes it, and + /// exits. The shell reads the line itself, through read calls of its own. + #[cfg(windows)] + fn reader(path: &std::path::Path) -> std::process::Command { + use std::os::windows::process::CommandExt; + + let mut command = std::process::Command::new("cmd"); + // `cmd` takes the rest of its command line as it is, the quotes around the path included. + command.args(["/d", "/q", "/c"]).raw_arg(format!( + "set /p line=<\"{}\" & echo read & set /p wait=", + path.display() + )); + command + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + command + } + + /// Returns a command that reads the first line of the file at `path`, prints `read`, then + /// waits for a line of its standard input, which reaches its end once the test closes it, and + /// exits. The shell reads the line itself, through read calls of its own. + #[cfg(target_os = "linux")] + fn reader(path: &std::path::Path) -> std::process::Command { + let mut command = std::process::Command::new("sh"); + command + .args(["-c", "read -r line < \"$0\"; echo read; read -r wait"]) + .arg(path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()); + command + } + + #[cfg(any(target_os = "linux", windows))] + #[test] + fn processes_count_the_bytes_that_their_read_calls_read() { + use std::io::{BufRead, BufReader, Write}; + + // A line short enough for `set /p`, which reads at most 1023 characters of one. + const LINE: usize = 512; + let path = std::env::temp_dir().join(format!("nvx-bytes-read-{}", std::process::id())); + let mut contents = vec![b'x'; LINE]; + contents.push(b'\n'); + std::fs::write(&path, &contents).unwrap(); + + // This process counts what it reads, and what other tests' threads read meanwhile. + let before = own_bytes_read().unwrap(); + let read = std::fs::read(&path).unwrap(); + assert!(own_bytes_read().unwrap() >= before + read.len() as u64); + + // Another process counts what it read once it says so, while it runs. + let mut child = platform::inherit::spawn_command(&mut reader(&path)).unwrap(); + let mut said = String::new(); + let stdout = child.stdout.take().unwrap(); + BufReader::new(stdout).read_line(&mut said).unwrap(); + let reference = ProcessRef { + pid: child.id(), + start_time: platform::process_start_time(child.id()).unwrap().unwrap(), + boot: BootId::current().unwrap(), + }; + let process = Process::open(&reference).unwrap().unwrap(); + let counted = process.bytes_read(); + let mut stdin = child.stdin.take().unwrap(); + stdin.write_all(b"x\n").unwrap(); + drop(stdin); + child.wait().unwrap(); + std::fs::remove_file(&path).unwrap(); + assert_eq!(said.trim_end(), "read"); + assert!(counted.unwrap() >= LINE as u64); + // Once the process exited, Linux keeps no count of it, and Windows keeps its count in the + // handle. + let exited = process.bytes_read(); + if cfg!(windows) { + assert!(exited.unwrap() >= LINE as u64); + } else { + assert_eq!(exited.unwrap_err().kind(), io::ErrorKind::NotFound); + } + } } #[cfg(all(test, target_os = "linux"))] diff --git a/nvx-rs/src/host/sys/mod.rs b/nvx-rs/src/host/sys/mod.rs index bd580b635..4d30dd199 100644 --- a/nvx-rs/src/host/sys/mod.rs +++ b/nvx-rs/src/host/sys/mod.rs @@ -35,7 +35,7 @@ pub(crate) use process::{ pub(crate) use process::{clear_close_on_exec, mark_listed_descriptors_on_spawn}; #[cfg(target_os = "linux")] pub(crate) use socket::{ - connect_nonblocking, peer_pid, send_nonblocking, wait_readable, wait_writable, + connect_nonblocking, peer_pid, send_nonblocking, unread_by_peer, wait_readable, wait_writable, }; /// Returns the effective user and group IDs of this process. diff --git a/nvx-rs/src/host/sys/socket.rs b/nvx-rs/src/host/sys/socket.rs index de9ba558d..52420a979 100644 --- a/nvx-rs/src/host/sys/socket.rs +++ b/nvx-rs/src/host/sys/socket.rs @@ -1,5 +1,6 @@ //! Unix stream connections to OpenVMM's control socket. +use std::ffi::c_int; use std::io; use std::os::fd::{AsFd, AsRawFd}; use std::os::unix::net::UnixStream; @@ -13,6 +14,14 @@ use nix::sys::socket::{ use super::Errno; +nix::ioctl_read_bad!( + /// Reads what a socket's peer has not read of what the socket sent: Linux `SIOCOUTQ`, which + /// is `TIOCOUTQ`, whose number `nix` takes from its `libc` dependency, by architecture. + siocoutq, + nix::libc::TIOCOUTQ, + c_int +); + /// Connects to the Unix socket at `path` without waiting in a full listen backlog, and returns /// the connected stream in blocking mode. /// @@ -54,6 +63,18 @@ pub(crate) fn send_nonblocking(stream: &UnixStream, data: &[u8]) -> io::Result io::Result { + let mut queued: c_int = 0; + // SAFETY: `stream` keeps the descriptor open for the call, and `queued` is writable for the + // one C integer that `SIOCOUTQ` writes, and outlives the call. + unsafe { siocoutq(stream.as_raw_fd(), &raw mut queued) }?; + usize::try_from(queued).map_err(io::Error::other) +} + /// Waits until `descriptor` has something to read, or its peer closed it, a signal arrives, or /// `timeout` passes, and returns whether it is readable. pub(crate) fn wait_readable(descriptor: &impl AsFd, timeout: Duration) -> io::Result { From ed5e24bcf7855524253097d1c971ce54e0da39fb Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 30 Oct 2026 18:48:02 -0700 Subject: [PATCH 2/2] runtime-adapter: check two clone-restore invariants from the host side A template may delay the repair of each of its clones, for tests: the host program's create and refresh take the delay from RUNTIME_ADAPTER_REPAIR_DELAY, write it into the capture's initramfs, at /etc/runtime-adapter-repair-delay, and name it as the class's one invariant setting. The kernel command line cannot carry it, since OpenVMM's capture of a platform snapshot refuses any parameter beyond the tier's allowlist. The guest reads the file before the capture, waits that long before each clone's repair, and reports the wait in its activation probe. vm_clone then checks from the host that OpenVMM reads none of the host's input until the repair acknowledges the restore: it starts a clone of a delayed template through NVX's public API, over transports that watch the session's attachment, which the broker answers only after the delay, and which waits unread at the control socket meanwhile on Linux, as SIOCOUTQ reports it. A named pipe tells its writer nothing of the kind, so on Windows the check is the timing alone. A reattach, which no gate holds, is answered at once, as the positive control. The scenario of a clone's disk reads also counts, from the host, the bytes that the start and the clone's VMM read, which stage and start now print as they exit: the start reads less than an eighth of the image, which the staging's hash reads whole, and the VMM less than an eighth of the disks' files once the clone is ready. The guest's read of each disk to its end raises the VMM's count by what it asked of the files on Windows, while on Linux, where OpenVMM reads them through io_uring, rchar misses those reads, as the README documents. Fixes #965 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/runtime-adapter/README.md | 149 +++- tests/runtime-adapter/scenarios.txt | 2 + .../src/bin/runtime-adapter-guest.rs | 24 +- .../src/bin/runtime-adapter-host.rs | 69 +- tests/runtime-adapter/src/capture.rs | 59 +- tests/runtime-adapter/src/initramfs.rs | 32 +- tests/runtime-adapter/src/template.rs | 96 ++- tests/runtime-adapter/src/worker.rs | 6 +- .../runtime-adapter/src/worker/activation.rs | 22 +- tests/runtime-adapter/src/worker/capture.rs | 22 +- tests/runtime-adapter/tests/common/mod.rs | 13 +- tests/runtime-adapter/tests/guest_session.rs | 4 + tests/runtime-adapter/tests/initramfs.rs | 39 +- tests/runtime-adapter/tests/staged_images.rs | 16 +- .../runtime-adapter/tests/template_capture.rs | 58 +- tests/runtime-adapter/tests/vm_clone.rs | 645 ++++++++++++++++-- 16 files changed, 1125 insertions(+), 131 deletions(-) diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index f6a4e2878..32a5a4bf6 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -245,13 +245,15 @@ arguments, itself, at any time once the clone is ready, executions running or not, with when the events of the clone's activation and order happened, in microseconds after the repair's acknowledgement of the restore: whether the repair acknowledged the restore, `acknowledged`, `yes`, or `none` for a restore -whose host did not gate its input until the acknowledgement; and when the clone -took the reset that the restore queued, `reset`, when the host's first message -came, `message`, which must be the clone's `Activate`, when the clone was -ready, `ready`, when its sandbox started, `started`, and when the start of the -host's first execution came, `execution`. Each event is `none` until it -happened, `before` if it happened before the acknowledgement, and `unknown` -without one. The worker of a VM that boots cold runs no such probe. +whose host did not gate its input until the acknowledgement; how long the worker +waited before the repair, `repair-delay`, in microseconds, as its template's +[repair delay](#template-mode) asked, for tests, or `none` without one; and +when the clone took the reset that the restore queued, `reset`, when the host's +first message came, `message`, which must be the clone's `Activate`, when the +clone was ready, `ready`, when its sandbox started, `started`, and when the +start of the host's first execution came, `execution`. Each event is `none` +until it happened, `before` if it happened before the acknowledgement, and +`unknown` without one. The worker of a VM that boots cold runs no such probe. The probes of `worker::mapping` inspect the mappings beneath the root, and map from the host export, as an adapter maps the host paths of its sandboxes. `map` @@ -794,6 +796,22 @@ the ABI fails the repair, rather than stopping the VM, and the worker ends the VM with the failure's status. A failure of the deferred time work stops the VM with the time ABI's own status, since the daemon enforces the ABI. +For tests, a template may delay the repair of each of its clones: once +`template::boot` took a template's command line, the worker reads +`template::REPAIR_DELAY_FILE`, `/etc/runtime-adapter-repair-delay`, which the +initramfs of the template's capture holds when the host program's creation +asks for a delay, and which `template::parse_repair_delay` takes as a positive +number of milliseconds, of at most `template::MAX_REPAIR_DELAY`, 30 seconds, +in decimal digits, with or without a newline. Each clone of the template then +waits that long before its repair, so that its restore stays unacknowledged, +and OpenVMM gates the host's input, while the host's attachment reaches +OpenVMM. The snapshot holds the delay, which the worker read before the +capture. The kernel command line cannot carry it: OpenVMM's capture of a +platform snapshot refuses a parameter that the tier's allowlist does not hold, +as `template::boot` does. A template without the file delays nothing, and a +file that the worker cannot read, or that holds anything else, ends the +template's VM with status 20, before the capture. + Once it repaired a clone, the worker activates it, on the same tty, with `worker::work_clone`, through the clone's `nvx::guest::clone::Activation`, whose deadline ends `worker::activation::ACTIVATION_TIMEOUT`, 120 seconds, @@ -898,7 +916,7 @@ ends with one of these statuses, after a diagnostic on standard error: | 1 | The control tty cannot be opened or put in raw mode, the attachment or the acknowledgement of the broker's reset cannot be sent, or the broker closes the session or sends anything but a reset first, or, in a clone, its session failed or the broker closed it | | 13 | The bootstrap of the guest for the template's root failed, before the attachment | | 15 | In a clone, the root's teardown failed once the host stopped the worker | -| 20 | The kernel command line names the platform tier, but holds another parameter, a double quote, `--`, or a gateway that OpenVMM does not render, or the worker cannot read it | +| 20 | The kernel command line names the platform tier, but holds another parameter, a double quote, `--`, or a gateway that OpenVMM does not render, or the worker cannot read it, or the template's repair delay file holds no delay, or the worker cannot read it | | 21 | The fence of workload admission cannot be closed | | 22 | Something is mounted from the template's devices or from an overlay, or the mount table cannot be checked | | 23 | The time ABI's boot checks did not complete | @@ -1110,7 +1128,10 @@ archive, in the `newc` format, which Linux unpacks over the first: `runtime-adapter-guest init /dev/hvc2 layered`, which names the control tty itself, since `/init` passes NVX's init agent none; - a symbolic link from `/sbin/nvx-init-agent` to that script, so that NVX's - init agent never assembles the root in the adapter's VMs. + init agent never assembles the root in the adapter's VMs; +- for the capture of a template whose clones delay their repair, for tests, the + delay's file, `/etc/runtime-adapter-repair-delay`, in the `/etc` directory of + NVX's initramfs; see [Template mode](#template-mode). Linux removes a file whose type an archive changes before it creates the new one, so each link replaces NVX's agent without writing to its file, which could @@ -1416,14 +1437,24 @@ command's options choose of the template, and `Request::capture` returns NVX's processors, those online at boot, 1 unless the command names more, the fewest that the template's clones take, the RAM and its capacity, the geometries of the placeholder and of the dormant scratch, as OpenVMM presents - a sandbox block device, which `capture::geometry` gives, and the network, as - the capture attaches its NIC, which no network's policy changes; + a sandbox block device, which `capture::geometry` gives, the network, as + the capture attaches its NIC, which no network's policy changes, and, for a + template whose clones delay their repair, for tests, the delay, in + milliseconds, as its one invariant setting, + `runtime-adapter.repair-delay-ms`, so that the class is one of its own; - NVX's artifacts, with the initramfs that puts the guest agent in place of - NVX's agents, which the command writes into a directory of its own in the + NVX's agents, and holds the file of the delay of each clone's repair, if the + template has one, which the command writes into a directory of its own in the system's temporary directory, private to the current user, as a launch writes it into the sandbox's private store, and removes once the creation returns. +For tests, `create` and `refresh` take that delay from the variable +`RUNTIME_ADAPTER_REPAIR_DELAY`, `capture::REPAIR_DELAY_VARIABLE`: a positive +number of milliseconds, of at most 30000, in decimal digits. Any other value is +a usage error, before the command creates anything, and without the variable, +the template delays nothing; see [Template mode](#template-mode). + NVX's store launches the capture's VM from the capture's transaction, in the class's slot, whose VMM listens there, beside the snapshot's destination, as OpenVMM requires. On Linux, the path of that socket, @@ -1769,7 +1800,10 @@ Each command of `runtime-adapter-host` but `stage`, `list`, `inspect`, and an EROFS file system, `erofs`, or a GPT disk, `gpt`, of which the guest expects the identity UUID, and the ext4 scratch image PATH, if given. It verifies and binds the images before it takes the sandbox's lock, which it - holds while it records them, and prints nothing. + holds while it records them, and prints how many bytes it read through the + operating system's read calls, the cache's included, `read-bytes=BYTES`, as + `nvx::host::process::own_bytes_read` counts them, as it exits: a staging + that hashes the image reads it whole. - `launch STORE --guest-agent PATH [--hypervisor NAME] [LAYERED]` launches a VM in the sandbox store STORE, which it creates if needed, with the guest agent PATH, a static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, @@ -1902,7 +1936,10 @@ Each command of `runtime-adapter-host` but `stage`, `list`, `inspect`, and Once the restore sequence sent the clone's mapping table, if it has one, and started the clone's sandbox, it exchanges a `Ping` and the longest `Echo` with the guest agent, commits the launch, and exits, - leaving the VM running, and the sandbox holding the lease. A clone that + leaving the VM running, and the sandbox holding the lease, once it printed + how many bytes it read through the operating system's read calls, + `read-bytes=BYTES`, as `stage` prints them: a start never hashes, nor copies, + the clone's images. A clone that fails, such as one whose activation failed before it was ready, and whose VM ended with the status of the failure's class, from 50 to 58, ends as [Clones](#clones) describes, with the record of its failure. A selection @@ -2066,7 +2103,9 @@ unless the teardown ended with 4, 8, or 9, which `run` then exits with, after the diagnostic. A value of `RUNTIME_ADAPTER_HOST_TIME` that is not a host's time is a usage error, as is a value of `RUNTIME_ADAPTER_STEP_HOOK` that sets no step -hook, or that sets one at another step than `checked` for a start, with +hook, or that sets one at another step than `checked` for a start, and a value +of `RUNTIME_ADAPTER_REPAIR_DELAY` that sets no delay of a template's clones' +repair, for `create` and `refresh`, with status 2. Without arguments, the program reports the contract that it was built against. @@ -2108,7 +2147,10 @@ parameter outside the allowlist at its start, in its middle, or at its end, or within a platform token's value, after a vertical tab or the byte 0xa0, which the kernel takes for whitespace and a split at ASCII whitespace does not, for a platform token that the no-break space's first byte leaves without UTF-8 text, -for a double quote, and for `--`. In the host-only configuration, those of +for a double quote, and for `--`, and the file of a template's delay of its +clones' repair: a positive number of milliseconds, of at most 30 seconds, with +or without a newline, and the contents that it refuses. In the host-only +configuration, those of `generations` check which generations an eviction retires: those of the served generation's class that it supersedes, and, whatever their class, the tombstones that interrupted retirements left alone, but not those of the @@ -2139,8 +2181,9 @@ of the session, and keeps the session, active in the broker's first epoch, and that no capture runs once the attachment cannot be written, the broker closes the stream or sends anything but a reset first, or the acknowledgement cannot be sent. Those of `worker::activation` check the status of -each class of a clone's activation's failures, and that the activation's -timeout covers the wait for both block devices. +each class of a clone's activation's failures, that the activation's +timeout covers the wait for both block devices, and the facts of the +`activation` probe, the delay of the repair, if any, among them. `tests/guest_session.rs` checks, in the guest-only configuration, that the guest agent serves the control session with the adapter's own framing. A test @@ -2431,7 +2474,8 @@ for every execution, each of which a test may fail. The tests cover: - the `activation` probe, which the worker answers itself, with no root operation, executions running or not: the restore's reset, the host's first message, the clone's `Ready`, and its start, in that order, after the - repair's acknowledgement, and the start of the first execution after them; + repair's acknowledgement, and the start of the first execution after them, + and no delay of the repair, which the clone's template did not ask for; - the mapping table of the record's export, which the host sends in parts once the clone is ready, and which the worker applies in order, from the export that the application mounted, before it starts the sandbox, and the endpoint @@ -2493,10 +2537,12 @@ configuration, on Linux and Windows: - the archive that puts the guest agent in place of NVX's agents: its entries, their owners and modes, both launchers, and the padding of its headers and - data, for guest agents of every length modulo 4; + data, for guest agents of every length modulo 4, and a template's, which + holds the file of the delay of its clones' repair after the launchers, for + delays of every length modulo 4; - the copy of NVX's initramfs, which stays as it is, of every length modulo 4, - with the archive after it at a multiple of 4 bytes, and the files that a - failure names; + with the archive after it at a multiple of 4 bytes, a template's with the + delay's file, and the files that a failure names; - the runtime identity of a checkout and of a release, which differs from NVX's in the initramfs alone, another build of the guest agent, which differs in the initramfs too, and missing install metadata, which fails as NVX @@ -2512,8 +2558,10 @@ Windows: images, which it does not; - the record of the staged images, as written and read, and the forms that a reader refuses; -- `runtime-adapter-host stage`, which records the staged images, a staging that - fails, which leaves them, and a staging that replaces them. +- `runtime-adapter-host stage`, which records the staged images, and prints the + bytes that it read, at least the image's length once it hashed the image, a + staging that fails, which leaves them, and prints nothing, and a staging that + replaces them. `tests/layered_launch.rs` checks, in the host-only configuration, on Linux and Windows: @@ -2582,11 +2630,16 @@ and Windows: those online at boot, which the request names, 1 by default, whatever the capacity, and which name the class, the RAM and its capacity, the geometries that OpenVMM presents, of 512-byte logical blocks, and the network, which - names the class: a class without it is another; + names the class: a class without it is another, as is the class of a + template whose clones delay their repair, whose one invariant setting is the + delay; - `runtime-adapter-host create` with a network that the plan refuses, or that it cannot read, which it refuses with status 1 before it needs NVX's artifacts or creates the store, and with one that the plan takes, or none, - which goes on to need NVX's artifacts. + which goes on to need NVX's artifacts; +- `create` and `refresh` with a value of `RUNTIME_ADAPTER_REPAIR_DELAY` that + sets no delay, which they refuse as a usage error, with status 2, before + they create anything. `tests/template_operations.rs` checks the template operations, in the host-only configuration, on Linux and Windows, without a hypervisor: @@ -2892,8 +2945,9 @@ directory that the starts authorize: | `clones_at_once_keep_their_own_uppers_and_machine_ids` | Three at once: over a scratch image and in RAM, each with a generated machine ID, and in RAM with the image's own; two executions in each, as user ID 0, that write a marker in the root, and try to write `/etc/machine-id` | In every execution, the hostname `runtime-adapter`, and no marker of another clone; a generated machine ID that differs between the clones, the same in each execution of a clone, held by one bind that is read-only, `nosuid`, and `nodev`, and that the write cannot change; with the image's own machine ID, none, since the image has none, no bind, and in the later execution, the ID that the first wrote; then each stop releases its lease | | `a_clone_whose_view_cannot_be_set_ends_before_it_is_ready` | Of a template of the EROFS image, a GPT disk whose layer holds a regular file `etc`, with the image's own machine ID, then with a generated one | The first started, with the disk's root; then status 8, since no bind can hold the machine ID below a file, and the record of a failure of the request, at the activation, once the VM exited, by the evidence of invalid settings, with the guest's status 55; nothing quarantined, and the lease released, with the records | | `a_clone_runs_its_executions_as_its_workload_identity_once_it_is_ready` | Over a scratch image, as nobody | An execution that runs as nobody, one that names an identity refused with `EINVAL`, and the first execution after the clone's start | +| `openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore` | Of a template whose clones wait 8 seconds before their repair, in RAM, which the host starts through NVX's public API, with the host program's clone order, over transports that watch their attachments, each of which they write a byte at a time; then a reattach through NVX's public API | The template's key, which names the delay; the attachment written within half the delay of the launch, the broker's first answer, and the attachment of the host, only after the delay; on Linux, `SIOCOUTQ` at the control socket risen by the same step with each byte that the host wrote, then as the write left it at every record before then, and zero once the broker answered, while a listener of the test's own process sees it fall once it read the first byte of such an attachment, and on Windows no record, since a named pipe tells its writer nothing; then the reattach's attachment answered within a quarter of the delay, and on Linux read whole by then; the guest's `repair-delay` of 8 seconds, and the events of its activation in order; and a stop that releases the lease | | `a_generation_is_eligible_by_the_clock_of_its_capture` | At the host's time, then with clock fixtures: UTC a day behind, another boot an hour after the capture by UTC, and the restore window passed by each clock, or UTC before the capture | The host's monotonic clock on the boot of the capture, whatever UTC says, UTC after a reboot, with the fixture's downtime, and, outside the window, status 10, with the record of the store's refusal of the selection, `expired` once the window passed, and `ineligible`, by the evidence `early`, before the capture, without a lease | -| `a_clone_reads_no_more_of_its_disks_than_its_root_needs` | Over a scratch image, and in RAM | Less than an eighth of each disk read once the clone is ready, and each disk's whole length once the guest read it to its end | +| `a_clone_reads_no_more_of_its_disks_than_its_root_needs` | Over a scratch image, and in RAM | Less than an eighth of each disk read once the clone is ready, by the guest's statistics, and each disk's whole length once the guest read it to its end; on the host, the staging's count of the bytes that it read at least the image's length, which it hashed, the start's less than an eighth of it, and the VMM's less than an eighth of the disks whose files back the clone's, those of the image and of a scratch image, once the clone is ready; then, once the guest read each disk to its end, the VMM's count raised by at least what the guest asked of those files on Windows, and by less than an eighth of it on Linux, whose `rchar` misses OpenVMM's reads through io_uring | | `a_clone_s_time_settles_after_it_is_ready` | In RAM | The time daemon's restore check `ok`, after the restore's generation, without a violation, after a downtime that the host's monotonic clock measured, and a clone that still serves | | `each_command_reattaches_to_the_clone_from_a_new_process` | Over a scratch image, then `reattach`, `probe`, `exec`, and `reattach` again, each from a new process | The launch's records, committed, with the clone's VM and no failure; each command `live`, with the clone's VMM, in a new session of the same broker, in a later epoch; the events of the activation as they were; an execution as nobody; and a stop that releases the lease, with the records | | `a_host_program_of_another_runtime_ends_the_clone_through_its_verified_identity` | In RAM, then a `reattach` with another guest agent's build | Status 8, `reattached=failed`, without a session, and the record of a failure of the host, at the reattach, by the evidence of the runtime's mismatch, once NVX terminated the VMM, which quarantines nothing; the lease released, with the records; then another clone that starts and stops | @@ -2930,6 +2984,39 @@ them. A clone's start names its network by a file that the test writes in the template's directory. +Two of the scenarios check from the host what the guest cannot tell it, and +each backend observes part of it alone: + +- That OpenVMM reads none of the host's input until the clone's repair + acknowledges its restore, which NVX's restore sequence relies on, and which + the guest's `activation` probe cannot show, since its worker serves the + session only once its repair returned, so input that OpenVMM let through + early would wait in the guest. On Linux, `SIOCOUTQ` of the host's end of the + control socket shows that the attachment waited there, unread, until the + repair could acknowledge the restore. `SIOCOUTQ` charges each buffer of the + socket's writes until the reader read it whole, and a read of part of a + buffer leaves it as it was, so the host writes the attachment a byte at a + time, a buffer for each byte, whose charge falls as soon as OpenVMM reads any + byte. The kernel charges the same for each byte's buffer, so the charge rises + by the same step with each byte that the host writes, while OpenVMM reads + none. A named pipe tells its writer nothing of what its reader has not read, + so on Windows the check is the timing alone: the broker, which answers a host + that it authenticated at once, answered only after the delay. +- That a clone's restore reads no large content of its disks. The host + program's own count of the bytes that it read, and the VMM's, which + `nvx::host::process::Process::bytes_read` reads through its process, count + every read call, the cache's included, and on Linux those of the children + that the process reaped, but no file that the process maps, so they leave + out the memory of the snapshot, which OpenVMM maps copy-on-write. + On Windows, OpenVMM reads the clone's disks with `ReadFile`, which + `ReadTransferCount` counts, so the guest's read of each disk to its end + raises the VMM's count by at least what it asked of the disks' files. On + Linux, OpenVMM reads them through io_uring, which `rchar` does not count, so + the VMM's count covers its read calls alone, as a hash or a copy of an image + through them would show, and not its reads of the disks: the scenario checks + that the guest's read of each disk to its end raises it by less than an + eighth of what the guest asked. + A test that fails keeps its template's directory, with the template store and the platform's image, as long as it keeps the directory of a clone that leases the template, whose VM might still run. @@ -2955,7 +3042,8 @@ layered roots, captures of templates, and the starts and runs of their clones. | Target | Scenario | Proves | | --- | --- | --- | -| `template_capture` | `a_capture_takes_its_class_and_inputs_from_the_request` | A capture's request: the plan of the placeholder over RAM, with the request's network, the placeholder's binding, the policy revision, the artifacts, and the class, which the network names | +| `template_capture` | `a_capture_takes_its_class_and_inputs_from_the_request` | A capture's request: the plan of the placeholder over RAM, with the request's network, the placeholder's binding, the policy revision, the artifacts, and the class, which the network names, and the delay of a template's clones' repair, as its invariant setting | +| `template_capture` | `the_host_program_refuses_a_repair_delay_that_it_cannot_read` | `create` and `refresh` refuse a value of `RUNTIME_ADAPTER_REPAIR_DELAY` that sets no delay with status 2 before they create anything | | `template_capture` | `the_host_program_refuses_a_template_before_it_creates_anything` | `create` refuses a network that the plan refuses, or that it cannot read, with status 1 before it needs NVX's artifacts or creates the store | | `template_operations` | `a_refresh_refuses_a_template_before_it_creates_anything` | `refresh` refuses a network that the plan refuses with status 1 before it needs NVX's artifacts or creates the store, as `create` does | | `template_operations` | `an_empty_template_store_lists_no_generation` | `list` of a store without generations prints none, and `inspect` and `remove` of a generation that it never published fail with status 1, and tombstone nothing | @@ -3000,7 +3088,7 @@ layered roots, captures of templates, and the starts and runs of their clones. | `host_session` | `the_host_sends_nothing_more_to_a_guest_agent_of_another_build_or_whose_root_failed` | The host sends nothing more to a guest agent of another build, or whose root failed | | `staged_images` | `a_launch_takes_the_bindings_that_the_store_stages_for_its_images` | A launch takes the bindings that the store staged for its images, and not those of other images | | `staged_images` | `staging_verifies_the_image_and_binds_the_scratch` | Staging verifies an image against the manifest beside it, refuses another image's, stages a GPT disk over RAM, and refuses images that no launch could attach | -| `staged_images` | `the_host_program_stages_images_in_the_store` | `stage` records the staged images, a staging that fails leaves them, and another replaces them | +| `staged_images` | `the_host_program_stages_images_in_the_store` | `stage` records the staged images, and prints the bytes that it read, the image's once it hashed it, a staging that fails leaves them, and another replaces them | | `staged_images` | `the_store_records_the_staged_images` | The record of the staged images, as written and read, and the forms that a reader refuses | | `layered_launch` | `a_launch_claims_its_scratch_image_until_it_releases_its_checked_files` | A launch claims its scratch image, which a second launch waits for, then fails to take, until the first releases its checked files | | `layered_launch` | `a_launch_reads_only_the_networks_that_the_plan_takes` | A launch refuses the network files that the plan refuses, each for the plan's reason | @@ -3078,7 +3166,7 @@ layered roots, captures of templates, and the starts and runs of their clones. | `vm_clone` | `a_clone_binds_the_resolver_of_its_policy_s_dns_server_read_only` | A clone of a network with egress allowed binds a resolver file that names the gateway's DNS server read-only at `/etc/resolv.conf`, and a TCP connection to the server's DNS port opens, which the gateway's DNS proxy accepts; one whose resolver cannot be bound ends with status 57 | | `vm_clone` | `a_clone_maps_an_aggregate_of_directories_and_files_beneath_its_root_before_its_sandbox_starts` | A clone maps an aggregate of host directories and files beneath its root before its sandbox starts, each with its access, a hidden file, and a hidden directory that shows a file again, and writes reach the host's directories and file through the writable ones alone | | `vm_clone` | `a_clone_of_a_template_with_a_nic_maps_its_export_beside_its_resolver` | A clone of a template with a NIC maps its export below the binds of its machine ID and of its resolver configuration | -| `vm_clone` | `a_clone_reads_no_more_of_its_disks_than_its_root_needs` | A clone's restore reads less than an eighth of each disk, before the guest reads each to its end | +| `vm_clone` | `a_clone_reads_no_more_of_its_disks_than_its_root_needs` | A clone's restore reads less than an eighth of each disk, before the guest reads each to its end, by the guest's statistics, and as the host counts the reads of the start, which hashes and copies nothing, and of the VMM, whose count on Windows rises by what the guest asked of the disks' files, and on Linux, where it misses io_uring's reads, does not | | `vm_clone` | `a_clone_refuses_a_scratch_image_whose_upper_an_earlier_clone_used` | A scratch image whose upper an earlier clone used ends the clone before it is ready, with status 54 | | `vm_clone` | `a_clone_runs_its_executions_as_its_workload_identity_once_it_is_ready` | A clone runs its executions as its record's workload identity once it is ready, and refuses one that names an identity with `EINVAL` | | `vm_clone` | `a_clone_s_executions_find_the_proxy_of_its_policy` | The executions of a clone of a proxy-only network find its proxy's URL in their environment, and the clone keeps the image's own `/etc/resolv.conf` | @@ -3108,6 +3196,7 @@ layered roots, captures of templates, and the starts and runs of their clones. | `vm_clone` | `one_generation_serves_clones_of_each_upper_and_memory_target` | One generation serves clones over a scratch image and in RAM, at each memory target up to its capacity, each with its own lease, and leaves its snapshot unchanged | | `vm_clone` | `one_generation_serves_clones_of_either_format_over_either_upper` | A template of a GPT placeholder serves EROFS and GPT clones of other layouts, shorter than the placeholder, at its length, over either upper, those in RAM with no scratch file, and a first partition over its bound ends a clone with status 54 | | `vm_clone` | `one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address` | One template with a NIC serves the clones of every network policy at its guest's address, whose restores install their own policies on its NIC, which lets through what each allows, publishes the guest's listeners, through the clone's announcer, and forwards ports where it says, and NVX's plan refuses another address, or no NIC, with status 8 before any VMM starts | +| `vm_clone` | `openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore` | OpenVMM reads none of the host's attachment until the repair of a clone, whose template delays it, acknowledges the restore: the broker answers it only after the delay, and on Linux, where the host writes it a byte at a time, `SIOCOUTQ` rises with each byte and stays as the write left it until then, while a reattach's is answered at once | | `vm_clone` | `the_lease_of_a_start_that_ended_before_its_record_is_released_through_its_template_store` | The lease of a start that ended before it recorded its clone is released only through its own template store, which `--template` names | ### Running the VM tests diff --git a/tests/runtime-adapter/scenarios.txt b/tests/runtime-adapter/scenarios.txt index 88b9f3aac..a79cf517f 100644 --- a/tests/runtime-adapter/scenarios.txt +++ b/tests/runtime-adapter/scenarios.txt @@ -7,6 +7,7 @@ # run and pass. The README's "Stage scenarios" says what each scenario proves. template_capture a_capture_takes_its_class_and_inputs_from_the_request +template_capture the_host_program_refuses_a_repair_delay_that_it_cannot_read template_capture the_host_program_refuses_a_template_before_it_creates_anything template_operations a_refresh_refuses_a_template_before_it_creates_anything @@ -169,4 +170,5 @@ vm_clone one_generation_serves_clones_of_each_processor_count_from_its_boot_onli vm_clone one_generation_serves_clones_of_each_upper_and_memory_target vm_clone one_generation_serves_clones_of_either_format_over_either_upper vm_clone one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address +vm_clone openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore vm_clone the_lease_of_a_start_that_ended_before_its_record_is_released_through_its_template_store diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs index fe242e039..305a0c01c 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs @@ -188,13 +188,14 @@ fn worker(tty: &std::path::Path, sibling: Option) -> ExitCode { /// standard error, or `None` in a VM that boots cold, whose worker serves the host. /// /// A command line that the worker cannot read, or that `runtime_adapter::template::boot` refuses, -/// ends the worker with `COMMAND_LINE_STATUS`. +/// ends the worker with `COMMAND_LINE_STATUS`, as does a template's repair delay, for tests, that +/// the worker cannot read, or that `runtime_adapter::template::parse_repair_delay` refuses. #[cfg(all(target_os = "linux", target_arch = "x86_64"))] fn capture_template( tty: &std::path::Path, operations: &mut runtime_adapter::worker::LibraryRoot, ) -> Option { - use runtime_adapter::template::{self, Boot, COMMAND_LINE_STATUS}; + use runtime_adapter::template::{self, Boot, COMMAND_LINE_STATUS, REPAIR_DELAY_FILE}; use runtime_adapter::worker::capture; use runtime_adapter::worker::{RootOperations, SESSION_STATUS}; @@ -209,8 +210,25 @@ fn capture_template( return Some(COMMAND_LINE_STATUS); } }; + let repair_delay = match std::fs::read(REPAIR_DELAY_FILE) { + Ok(contents) => match template::parse_repair_delay(&contents) { + Some(delay) => Some(delay), + None => { + eprintln!( + "runtime-adapter-guest: {REPAIR_DELAY_FILE} holds no repair delay: {}", + contents.escape_ascii() + ); + return Some(COMMAND_LINE_STATUS); + } + }, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => { + eprintln!("runtime-adapter-guest: cannot read {REPAIR_DELAY_FILE}: {error}"); + return Some(COMMAND_LINE_STATUS); + } + }; Some( - match capture::capture_on_terminal(tty, operations, gateway) { + match capture::capture_on_terminal(tty, operations, gateway, repair_delay) { Ok(templated) => { eprintln!("runtime-adapter-guest: {templated}"); templated.status() diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs index a751a6e29..0436ac512 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs @@ -9,7 +9,8 @@ //! identity UUID, and the scratch image the ext4 image PATH, if given. Staging verifies the //! image explicitly, which hashes the image only when a manifest is beside it, binds the scratch //! image without reading it, and records both bindings in STORE, in place of those staged -//! earlier; see `runtime_adapter::staged::Staged`. It needs no artifacts, and prints nothing. +//! earlier; see `runtime_adapter::staged::Staged`. It needs no artifacts, and prints the bytes +//! that it read as it exits, `read-bytes`. //! - `runtime-adapter-host launch STORE --guest-agent PATH [--hypervisor NAME] [LAYERED]` launches //! a VM in the sandbox store STORE, which it creates if needed, through NVX's supervisor, with //! the guest agent PATH, a static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, @@ -195,8 +196,8 @@ //! table, nor `Start`, and ends the clone, whose guest the clone order asks to stop, which tears //! down the root, and which then fails as below. Once every mapping applied, the sequence //! starts the clone's sandbox, and the command exchanges a `Ping` and the longest `Echo` with -//! the guest agent, commits the launch, and exits, leaving the VM running, and the sandbox -//! holding the lease, which a stop releases. +//! the guest agent, commits the launch, prints the bytes that it read, `read-bytes`, and exits, +//! leaving the VM running, and the sandbox holding the lease, which a stop releases. //! A clone that fails, from its plan to its commit, fails with NVX's //! `nvx::host::template::CloneFailure`, which attributes the failure by its evidence, and records //! it in STORE from the clone's launch on: the command prints the failure's record, releases the @@ -395,7 +396,10 @@ //! - `length.state` and `length.memory`: the lengths of the snapshot's saved state and of its RAM, //! in bytes; //! - `removed`: what `remove` did: `retired`, once the generation is gone, or `draining`, once it -//! tombstoned the generation, which the leases that it reports keep. +//! tombstoned the generation, which the leases that it reports keep; +//! - `read-bytes`: how many bytes `stage`, or a `start` that started its clone, read through the +//! operating system's read calls, the cache's included, as it exits, as +//! `nvx::host::process::own_bytes_read` counts them. //! //! Values escape `\`, newlines, and carriage returns as `\\`, `\n`, and `\r`, and each byte of a //! program's output that is not part of UTF-8 text as `\xHH`, with two hexadecimal digits. @@ -409,7 +413,11 @@ //! `run` take the host's time, at which the store checks the generation's eligibility, from the //! variable `RUNTIME_ADAPTER_HOST_TIME`, if it is set, as a clock fixture, such as another boot's; //! see -//! `runtime_adapter::clone::HOST_TIME_VARIABLE`. +//! `runtime_adapter::clone::HOST_TIME_VARIABLE`. `create` and `refresh` take the delay of the +//! repair of each clone of the template that a capture makes from the variable +//! `RUNTIME_ADAPTER_REPAIR_DELAY`, in milliseconds, if it is set, which they write into the +//! capture's initramfs, and name as the class's invariant setting; see +//! `runtime_adapter::capture::REPAIR_DELAY_VARIABLE`. //! //! They exit with 0 once they did what they were asked, `exec` once every execution finished, //! whatever its outcome, and `run` once its execution finished so, and the clone stopped, as @@ -426,7 +434,8 @@ //! or 9, as when the clone's VM exited during the execution, which `run` then exits with, after //! the diagnostic. Anything else is a usage error, with status 2, as is a value of //! `RUNTIME_ADAPTER_STEP_HOOK` that sets no step hook, or a step hook of a start at another step -//! than `checked`, or a value of `RUNTIME_ADAPTER_HOST_TIME` that is not a host's time. +//! than `checked`, a value of `RUNTIME_ADAPTER_HOST_TIME` that is not a host's time, or a value of +//! `RUNTIME_ADAPTER_REPAIR_DELAY` that sets no delay. use std::env; use std::error::Error; @@ -462,7 +471,7 @@ use nvx::{ErrorCode, ErrorKind, Hypervisor}; use runtime_adapter::GUEST_BUILD; use runtime_adapter::capture::{ DEFAULT_BOOT_PROCESSORS, DEFAULT_CPU_PROFILE, DEFAULT_PROCESSORS, DEFAULT_SCRATCH_LENGTH, - Request, + Request, repair_delay_from_env, }; use runtime_adapter::clone::{CloneRecord, host_time_from_env}; use runtime_adapter::exec::{self, AccountMode, HostExecution, Start}; @@ -852,6 +861,8 @@ struct Template { scratch_length: u64, /// The file of the template's network, if it has one. network: Option, + /// The delay of each clone's repair, for tests, if `RUNTIME_ADAPTER_REPAIR_DELAY` sets one. + repair_delay: Option, } /// How `create` and `refresh` serve a template. @@ -1000,9 +1011,10 @@ fn main() -> ExitCode { format, identity, scratch, - } => { - stage(&store, &distro, format, identity, scratch.as_deref()).map(|()| ExitCode::SUCCESS) - } + } => stage(&store, &distro, format, identity, scratch.as_deref()).map(|()| { + print_read_bytes(); + ExitCode::SUCCESS + }), Command::Launch { store, guest_agent, @@ -1233,6 +1245,7 @@ fn parse(arguments: Vec) -> Result { memory_capacity: options.memory_capacity.unwrap_or(memory), scratch_length: options.scratch_length.unwrap_or(DEFAULT_SCRATCH_LENGTH), network: options.network, + repair_delay: repair_delay_from_env()?, }), serving: if name == "refresh" { Serving::Refresh @@ -1794,6 +1807,19 @@ fn stage( staged.write(store.dir()).map_err(|error| describe(&error)) } +/// Prints how many bytes this process has read through the operating system's read calls, +/// `nvx::host::process::own_bytes_read`, as `read-bytes=BYTES`, as `stage`, or a `start` that +/// started its clone, exits, so that a test can tell how much of its files the command read. A +/// count that cannot be read is left out, after a diagnostic on standard error. +fn print_read_bytes() { + match nvx::host::process::own_bytes_read() { + Ok(read) => println!("read-bytes={read}"), + Err(error) => { + eprintln!("runtime-adapter-host: cannot count the bytes that it read: {error}"); + } + } +} + /// Launches a VM of the guest agent `guest_agent` in the sandbox store `store`, on `hypervisor` /// or else the one that `nvx::Hypervisor::from_env_or_default` selects, with the guest agent's /// build `guest_build`, and with the layered root `layered`, if any. The step hook `hook`, if any, @@ -1874,9 +1900,11 @@ fn launch( let command = |endpoint: &str| -> nvx::Result> { // The launch has checked that no VM of the sandbox runs, which could still read the // initramfs of an earlier launch. - initramfs::write(&runtime.artifacts.initrd, guest_agent, &initramfs).map_err(|error| { - nvx::Error::io("cannot write the initramfs of the guest agent").with_source(error) - })?; + initramfs::write(&runtime.artifacts.initrd, guest_agent, None, &initramfs).map_err( + |error| { + nvx::Error::io("cannot write the initramfs of the guest agent").with_source(error) + }, + )?; let outcome = store.outcome_path(); let (arguments, kernel_command_line) = match layered.as_ref().zip(prepared.as_ref()) { Some((layered, prepared)) => ( @@ -2145,6 +2173,7 @@ fn create( memory_capacity: template.memory_capacity, scratch_length: template.scratch_length, network, + repair_delay: template.repair_delay, }; // The placeholder is staged as `stage` stages an image: its explicit verification may hash it, // which no capture does. @@ -2154,8 +2183,13 @@ fn create( // launch's VM boots it from the sandbox's store, so that no one else may change it first. let directory = TemporaryDirectory::new(CAPTURE_DIRECTORY_PREFIX)?; let initramfs = directory.path().join(INITRAMFS_NAME); - initramfs::write(&runtime.artifacts.initrd, guest_agent, &initramfs) - .map_err(|error| format!("cannot write the initramfs of the guest agent: {error}"))?; + initramfs::write( + &runtime.artifacts.initrd, + guest_agent, + template.repair_delay, + &initramfs, + ) + .map_err(|error| format!("cannot write the initramfs of the guest agent: {error}"))?; // The capture creates the dormant scratch in the store, on the file system of its directory, // or, before the store exists, of its parent's. let store = absolute(store).map_err(|error| describe(&error))?; @@ -2403,7 +2437,10 @@ fn start( // lock, as once a launch committed. Ok( match start_clone(store, guest_build, clone, planned, time, hook)? { - Ok(_running) => ExitCode::SUCCESS, + Ok(_running) => { + print_read_bytes(); + ExitCode::SUCCESS + } Err(status) => status, }, ) diff --git a/tests/runtime-adapter/src/capture.rs b/tests/runtime-adapter/src/capture.rs index 6e7c7aa3b..37cdf8bf6 100644 --- a/tests/runtime-adapter/src/capture.rs +++ b/tests/runtime-adapter/src/capture.rs @@ -22,16 +22,22 @@ //! block device, which [`geometry`] gives; //! - the network, of which the class takes the NIC alone, at the guest's address, under the //! complete deny-all policy, whatever the network's own, which each clone's restore replaces -//! with the clone's, and no other invariant setting. +//! with the clone's; +//! - for tests, the delay of each clone's repair, if [`REPAIR_DELAY_VARIABLE`] sets one, as the +//! invariant setting [`REPAIR_DELAY_SETTING`], and no other invariant setting, so that a +//! template whose clones delay their repair is of a class of its own. //! //! The capture's VM boots NVX's kernel with the initramfs of the adapter's VMs, which puts the -//! guest agent in place of NVX's agents; see [`crate::initramfs`]. NVX gives it the kernel command +//! guest agent in place of NVX's agents, and holds the file of the delay of each clone's repair, +//! if any; see [`crate::initramfs`]. NVX gives it the kernel command //! line `nvx_sandbox=1`, with which NVX's `/init` hands it to the guest agent's PID 1 of a layered //! root, and for a template whose processors are not all online at boot, `maxcpus=` with those //! that are, and OpenVMM adds the platform tier's tokens, `nvx_snapshot_tier=platform` among them: //! the worker then captures the VM before it assembles any root, and the capture ends the VM. +use std::env; use std::path::{Path, PathBuf}; +use std::time::Duration; use nvx::host::artifacts::Artifacts; use nvx::host::sandbox::distro::{DISTRO_POLICY_REVISION, DistroBinding}; @@ -44,6 +50,17 @@ use nvx::{Hypervisor, Result}; use crate::layered::plan; use crate::runtime::Runtime; +use crate::template::{MAX_REPAIR_DELAY, parse_repair_delay}; + +/// The variable from which `runtime-adapter-host create` and `refresh` take the delay of the +/// repair of each clone of the template that a capture makes, for tests: a positive number of +/// milliseconds, of at most `template::MAX_REPAIR_DELAY`, as `template::parse_repair_delay` +/// reads the file of the delay that the capture's initramfs holds. +pub const REPAIR_DELAY_VARIABLE: &str = "RUNTIME_ADAPTER_REPAIR_DELAY"; + +/// The invariant setting of the class of a template whose clones delay their repair, for tests, +/// which holds the delay, in milliseconds. +pub const REPAIR_DELAY_SETTING: &str = "runtime-adapter.repair-delay-ms"; /// The CPU profile of a template, unless the host program names another: `auto`, which OpenVMM's /// `--cpu-profile` takes for the built-in profile of the host's CPU, and which OpenVMM also @@ -99,6 +116,8 @@ pub struct Request { /// The network at whose guest's address the template has its fixed NIC, under the complete /// deny-all policy, or `None` for a template without a NIC. pub network: Option, + /// How long each clone of the template delays its repair, for tests, or `None` for no delay. + pub repair_delay: Option, } impl Request { @@ -118,8 +137,9 @@ impl Request { /// Returns NVX's capture of the template that this requests, with the placeholder's binding /// `binding`, the runtime of the adapter's VMs `runtime`, with the guest agent's build /// `guest_build`, and the initramfs `initramfs`, which puts that guest agent in place of NVX's - /// agents, and whose dormant scratch is on the file system of the directory `scratch`, the - /// template store's, as the [module documentation](self) describes. + /// agents, and holds the file of the delay of each clone's repair, if this requests one, and + /// whose dormant scratch is on the file system of the directory `scratch`, the template + /// store's, as the [module documentation](self) describes. /// /// Fails as [`plan`](Self::plan) and [`geometry`] do, and with /// [`ErrorKind::Unsupported`](nvx::ErrorKind::Unsupported) if the guest agent's build is not @@ -134,6 +154,12 @@ impl Request { scratch: &Path, ) -> Result { let adapter = self.plan()?; + let invariants = match self.repair_delay { + None => Invariants::new(), + Some(delay) => { + Invariants::new().with(REPAIR_DELAY_SETTING, &delay.as_millis().to_string())? + } + }; let class = CaptureClass { runtime: runtime.identity.clone(), guest_build: ArtifactIdentity::new(format!("runtime-adapter-guest/{guest_build}"))?, @@ -146,7 +172,7 @@ impl Request { distro: geometry(adapter.distro().path(), binding.image().length())?, scratch: geometry(scratch, self.scratch_length)?, network: self.network.clone(), - invariants: Invariants::new(), + invariants, }; Ok(Capture { class, @@ -162,6 +188,29 @@ impl Request { } } +/// Returns the delay of each clone's repair that [`REPAIR_DELAY_VARIABLE`] sets, for tests, or +/// `None` without the variable, or says why its value is not one: a positive number of +/// milliseconds, in decimal digits, of at most [`MAX_REPAIR_DELAY`]. +pub fn repair_delay_from_env() -> std::result::Result, String> { + let Some(value) = env::var_os(REPAIR_DELAY_VARIABLE) else { + return Ok(None); + }; + let delay = value.to_str().and_then(|text| { + // The file's newline is no part of the variable. + (!text.ends_with('\n')) + .then(|| parse_repair_delay(text.as_bytes())) + .flatten() + }); + match delay { + Some(delay) => Ok(Some(delay)), + None => Err(format!( + "{REPAIR_DELAY_VARIABLE} takes a positive number of milliseconds, of at most {}, \ + not {value:?}", + MAX_REPAIR_DELAY.as_millis() + )), + } +} + /// Returns the geometry that OpenVMM presents for a sandbox block device of `length` bytes, whose /// file is `path`, or is in the directory `path`: logical blocks of [`LOGICAL_BLOCK_SIZE`] bytes, /// and physical blocks of the size that the file system prefers for I/O, `st_blksize`, on Linux, diff --git a/tests/runtime-adapter/src/initramfs.rs b/tests/runtime-adapter/src/initramfs.rs index f4e866376..67384d158 100644 --- a/tests/runtime-adapter/src/initramfs.rs +++ b/tests/runtime-adapter/src/initramfs.rs @@ -25,7 +25,9 @@ //! `runtime-adapter-guest init /dev/hvc2 layered`, PID 1 of a VM with a layered root, which //! names the control tty, [`CONTROL_TTY`], itself, since `/init` passes NVX's init agent none; //! - and a symbolic link from [`INIT_AGENT`] to that script, so that NVX's init agent never -//! assembles the root in the adapter's VMs. +//! assembles the root in the adapter's VMs; +//! - for the capture of a template whose clones delay their repair, for tests, the delay's file, +//! [`REPAIR_DELAY_FILE`], in the `/etc` directory of NVX's initramfs. //! //! Linux removes a file whose type an archive changes before it creates the new one, so each link //! replaces NVX's agent without writing to its file, which could have other names. @@ -37,6 +39,9 @@ use std::fs::{self, File}; use std::io::{self, Write}; use std::path::Path; +use std::time::Duration; + +use crate::template::{REPAIR_DELAY_FILE, repair_delay_contents}; /// Path of the guest agent in the VM. pub const GUEST_AGENT: &str = "/sbin/runtime-adapter-guest"; @@ -84,13 +89,19 @@ const SYMLINK: u32 = 0o120_000; /// Writes to `output` a copy of NVX's initramfs `base` with the guest agent `guest_agent`, a /// static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, in place of NVX's managed -/// agent and init agent, as the [module documentation](self) describes. +/// agent and init agent, and the file that delays the repair of each clone of a template by +/// `repair_delay`, if any, as the [module documentation](self) describes. /// /// Fails if a file cannot be read or written, or if the guest agent does not fit an archive /// entry, whose size has 32 bits. The error names the file. -pub fn write(base: &Path, guest_agent: &Path, output: &Path) -> io::Result<()> { +pub fn write( + base: &Path, + guest_agent: &Path, + repair_delay: Option, + output: &Path, +) -> io::Result<()> { let agent = fs::read(guest_agent).map_err(naming(guest_agent))?; - let archive = archive(&agent)?; + let archive = archive(&agent, repair_delay)?; let mut file = File::create(output).map_err(naming(output))?; let mut source = File::open(base).map_err(naming(base))?; let length = io::copy(&mut source, &mut file).map_err(naming(output))?; @@ -106,14 +117,16 @@ fn naming(path: &Path) -> impl Fn(io::Error) -> io::Error + '_ { move |error| io::Error::new(error.kind(), format!("{}: {error}", path.display())) } -/// Returns the archive that [`write`](fn@write) appends, with the guest agent `guest_agent`: -/// every entry that the [module documentation](self) lists, owned by root, then the trailer. +/// Returns the archive that [`write`](fn@write) appends, with the guest agent `guest_agent`, and +/// the file that delays the repair of each clone of a template by `repair_delay`, if any: every +/// entry that the [module documentation](self) lists, owned by root, then the trailer. /// /// Fails if the guest agent does not fit an archive entry, whose size has 32 bits. -pub fn archive(guest_agent: &[u8]) -> io::Result> { +pub fn archive(guest_agent: &[u8], repair_delay: Option) -> io::Result> { let script = launcher_script(); let layered_script = layered_launcher_script(); - let entries: [(&str, u32, &[u8]); 5] = [ + let delay = repair_delay.map(repair_delay_contents); + let mut entries: Vec<(&str, u32, &[u8])> = vec![ (GUEST_AGENT, REGULAR | 0o755, guest_agent), (LAUNCHER, REGULAR | 0o755, script.as_bytes()), // A link's data is its target. @@ -121,6 +134,9 @@ pub fn archive(guest_agent: &[u8]) -> io::Result> { (LAYERED_LAUNCHER, REGULAR | 0o755, layered_script.as_bytes()), (INIT_AGENT, SYMLINK | 0o777, LAYERED_LAUNCHER.as_bytes()), ]; + if let Some(delay) = &delay { + entries.push((REPAIR_DELAY_FILE, REGULAR | 0o644, delay.as_bytes())); + } let mut archive = Vec::new(); for (inode, (path, mode, data)) in (1..).zip(entries) { // Archives name their entries relative to the root. diff --git a/tests/runtime-adapter/src/template.rs b/tests/runtime-adapter/src/template.rs index 7ed9e6c5c..56e2bf1d0 100644 --- a/tests/runtime-adapter/src/template.rs +++ b/tests/runtime-adapter/src/template.rs @@ -2,8 +2,9 @@ //! so that every clone that a restore of the snapshot starts attaches a root of its own. This //! module holds what the host program and the guest agent share of it: how the guest agent tells //! a template's VM by its kernel command line, the platform tier's allowlist of that command line, -//! and the statuses with which the guest agent of a template's VM ends. The worker captures the -//! VM, in `worker::capture`. +//! the file of a template's VM that delays its clones' repair, for tests, and the statuses with +//! which the guest agent of a template's VM ends. The worker captures the VM, in +//! `worker::capture`. //! //! OpenVMM adds [`PLATFORM_TIER`], `nvx_snapshot_tier=platform`, to the kernel command line of a //! VM that captures a platform snapshot. [`boot`] reads the command line as the kernel splits it, @@ -19,17 +20,25 @@ //! has one, with `nvx::protocol::cmdline::gateway`, which refuses a gateway that OpenVMM would not //! render: the announcer of each clone that publishes its listeners announces them to it. //! +//! For tests, the initramfs of a template's VM may hold [`REPAIR_DELAY_FILE`], which +//! [`parse_repair_delay`] reads: each clone of the template then waits that long before its repair +//! acknowledges its restore, so that a test can check that OpenVMM reads none of the host's input +//! meanwhile. Only the initramfs can carry the delay, since the platform tier's allowlist holds no +//! parameter for it. +//! //! The worker of a template's VM, and with it the VM, ends with [`COMMAND_LINE_STATUS`] once -//! `boot` refused its command line, with the bootstrap's status, 13, once the bootstrap of the -//! guest for the template's root failed, with [`CAPTURE_STATUS_BASE`] plus the number of the stage -//! of the capture that failed, and with [`CONTINUED_STATUS`] once the capture ended without a -//! snapshot. In a clone, which it repairs, it ends with the status of the repair's failure, which -//! `nvx::protocol::status` documents, and once it repaired the clone, as the clone's session ends: -//! with the status of the class of the clone's activation's failure, which `nvx::protocol::status` -//! documents too, or, once the clone was ready, as the worker of a VM that boots cold does. A -//! committed capture ends the template's VM before the worker returns. +//! `boot` refused its command line, or [`parse_repair_delay`] its delay file, with the bootstrap's +//! status, 13, once the bootstrap of the guest for the template's root failed, with +//! [`CAPTURE_STATUS_BASE`] plus the number of the stage of the capture that failed, and with +//! [`CONTINUED_STATUS`] once the capture ended without a snapshot. In a clone, which it repairs, +//! it ends with the status of the repair's failure, which `nvx::protocol::status` documents, and +//! once it repaired the clone, as the clone's session ends: with the status of the class of the +//! clone's activation's failure, which `nvx::protocol::status` documents too, or, once the clone +//! was ready, as the worker of a VM that boots cold does. A committed capture ends the template's +//! VM before the worker returns. use std::net::Ipv4Addr; +use std::time::Duration; use std::{fmt, str}; use nvx::protocol::ProtocolError; @@ -81,7 +90,8 @@ mod openvmm { pub const PLATFORM_TIER: &str = "nvx_snapshot_tier=platform"; /// The status with which the worker of a template's VM, and with it the VM, ends once [`boot`] -/// refused its kernel command line, or the worker could not read it. +/// refused its kernel command line, or the worker could not read it, or [`parse_repair_delay`] +/// refused its [`REPAIR_DELAY_FILE`], or the worker could not read that. pub const COMMAND_LINE_STATUS: u8 = 20; /// The base of the statuses with which the worker of a template's VM ends once a stage of its @@ -190,6 +200,39 @@ fn token(parameter: &Parameter<'_>) -> Vec { token } +/// The file of a template's VM, in the initramfs that the host program writes for its capture, +/// that sets how long each clone of the template delays its repair, for tests: the +/// [contents](repair_delay_contents) of the delay. A clone whose repair has not acknowledged its +/// restore yet must read no host input, so a test captures a template with a delay to hold that +/// state for as long as it needs. The template's kernel command line cannot set the delay: +/// OpenVMM's capture of a platform snapshot refuses a parameter that the platform tier's allowlist +/// does not hold, as [`boot`] does. +pub const REPAIR_DELAY_FILE: &str = "/etc/runtime-adapter-repair-delay"; + +/// The longest delay of a clone's repair that a template may set: half of OpenVMM's default +/// deadline of the acknowledgement of a restore that gates the host's input, and of NVX's default +/// restore-to-Ready deadline, each 60 seconds, which the delay counts against. +pub const MAX_REPAIR_DELAY: Duration = Duration::from_secs(30); + +/// Returns the contents of [`REPAIR_DELAY_FILE`] that set `delay`: its whole milliseconds, in +/// decimal, and a newline. +pub fn repair_delay_contents(delay: Duration) -> String { + format!("{}\n", delay.as_millis()) +} + +/// Returns the delay that `contents`, those of [`REPAIR_DELAY_FILE`], set: a positive number of +/// milliseconds, in decimal digits, of at most [`MAX_REPAIR_DELAY`], with or without a newline +/// after it, or `None` for anything else. +pub fn parse_repair_delay(contents: &[u8]) -> Option { + let digits = contents.strip_suffix(b"\n").unwrap_or(contents); + if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) { + return None; + } + let milliseconds = str::from_utf8(digits).ok()?.parse::().ok()?; + let delay = Duration::from_millis(milliseconds); + (milliseconds > 0 && delay <= MAX_REPAIR_DELAY).then_some(delay) +} + #[cfg(test)] mod tests { use nvx::protocol::status; @@ -596,4 +639,35 @@ mod tests { ); } } + + #[test] + fn a_template_s_repair_delay_is_positive_milliseconds_of_at_most_half_a_minute() { + for (delay, contents) in [ + (Duration::from_millis(1), "1\n"), + (Duration::from_millis(8_000), "8000\n"), + (MAX_REPAIR_DELAY, "30000\n"), + ] { + assert_eq!(repair_delay_contents(delay), contents); + assert_eq!(parse_repair_delay(contents.as_bytes()), Some(delay)); + let line = contents.trim_end(); + assert_eq!(parse_repair_delay(line.as_bytes()), Some(delay)); + } + // The milliseconds are whole. + assert_eq!(repair_delay_contents(Duration::from_micros(2_500)), "2\n"); + for contents in [ + "", "\n", "0\n", "30001\n", "+5\n", "-5\n", " 5\n", "5 \n", "5\n\n", "5\r\n", "5s\n", + "0x10\n", "1.5\n", + ] { + assert_eq!( + parse_repair_delay(contents.as_bytes()), + None, + "{contents:?}" + ); + } + assert_eq!(parse_repair_delay(&[b'9'; 40]), None); + assert_eq!(parse_repair_delay(b"5\xff"), None); + // A file of the initramfs's `/etc`, which NVX's initramfs holds. + assert!(REPAIR_DELAY_FILE.starts_with("/etc/")); + assert_eq!(REPAIR_DELAY_FILE.matches('/').count(), 2); + } } diff --git a/tests/runtime-adapter/src/worker.rs b/tests/runtime-adapter/src/worker.rs index 9c810cc15..adc75f003 100644 --- a/tests/runtime-adapter/src/worker.rs +++ b/tests/runtime-adapter/src/worker.rs @@ -586,7 +586,8 @@ pub fn work( /// assembly that the template captured, with which the worker [applies](RootOperations::apply) /// the record of the host's `Activate`. `acknowledged` is when the repair acknowledged the /// restore, `nvx::guest::clone::Repaired::acknowledged_at`, if it did, from which the -/// `activation::ACTIVATION_PROBE` reports the events of the clone. +/// `activation::ACTIVATION_PROBE` reports the events of the clone, with `repair_delay`, how long +/// the worker waited before the repair, for tests, if the template set a delay. /// /// It ends as [`work`] does, and with [`Worked::Unactivated`] once the activation failed, or /// [`Worked::Abandoned`] once the broker closed the stream before the clone was ready, after which @@ -600,8 +601,9 @@ pub fn work_clone( activation: Activation, template: RootTemplate, acknowledged: Option, + repair_delay: Option, ) -> Result { - let state = CloneState::new(activation, template, acknowledged); + let state = CloneState::new(activation, template, acknowledged, repair_delay); serve( Connection::resume(stream, session)?, operations, diff --git a/tests/runtime-adapter/src/worker/activation.rs b/tests/runtime-adapter/src/worker/activation.rs index f1057d7ef..b425acbcb 100644 --- a/tests/runtime-adapter/src/worker/activation.rs +++ b/tests/runtime-adapter/src/worker/activation.rs @@ -15,6 +15,8 @@ //! - `acknowledged`: `yes` once the repair acknowledged the restore, and `none` for a restore //! whose host did not gate its input until the acknowledgement, whose repair acknowledged //! nothing; +//! - `repair-delay`: how long, in microseconds, the worker waited before the repair, as the +//! template's `template::REPAIR_DELAY_FILE` asked, for tests, and `none` without the file; //! - `reset`: when the clone took the reset that the restore queued; //! - `message`: when the host's first message came, which must be the clone's `Activate`; //! - `ready`: when the clone was ready, once the record applied; @@ -153,12 +155,14 @@ struct Applied { impl CloneState { /// Returns the state of the clone whose activation is `activation`, in its transport - /// bootstrap, whose template captured `template`, and whose repair acknowledged the restore at + /// bootstrap, whose template captured `template`, whose worker waited `repair_delay` before + /// the repair, if the template set a delay, and whose repair acknowledged the restore at /// `acknowledged`, if it did. pub(super) fn new( activation: Activation, template: RootTemplate, acknowledged: Option, + repair_delay: Option, ) -> Self { Self { activation, @@ -167,6 +171,7 @@ impl CloneState { order: None, timeline: Timeline { acknowledged, + repair_delay, ..Timeline::default() }, } @@ -374,6 +379,8 @@ fn ready(activation: ReadyActivation) -> Message { struct Timeline { /// When the repair acknowledged the restore, if it did. acknowledged: Option, + /// How long the worker waited before the repair, if the template set a delay. + repair_delay: Option, /// When the clone took the reset that the restore queued. reset: Option, /// When the host's first message came. @@ -414,6 +421,10 @@ impl Timeline { "none" }; facts.push("acknowledged", acknowledged); + let repair_delay = self + .repair_delay + .map_or("none".to_owned(), |delay| delay.as_micros().to_string()); + facts.push("repair-delay", repair_delay); let events = [ ("reset", self.reset), ("message", self.message), @@ -714,6 +725,7 @@ mod tests { let after = |micros| acknowledged + Duration::from_micros(micros); let timeline = Timeline { acknowledged: Some(acknowledged), + repair_delay: None, reset: Some(after(10)), message: Some(after(250)), ready: Some(after(4_000)), @@ -723,6 +735,7 @@ mod tests { let mut expected = Facts::new(); let facts = [ ("acknowledged", "yes"), + ("repair-delay", "none"), ("reset", "10"), ("message", "250"), ("ready", "4000"), @@ -766,5 +779,12 @@ mod tests { assert_eq!(facts.get("acknowledged"), Some("none")); assert_eq!(facts.get("reset"), Some("unknown")); assert_eq!(facts.get("execution"), Some("none")); + + // A template that delayed the repair of its clones, for tests. + let delayed = Timeline { + repair_delay: Some(Duration::from_millis(8_000)), + ..timeline + }; + assert_eq!(delayed.facts().get("repair-delay"), Some("8000000")); } } diff --git a/tests/runtime-adapter/src/worker/capture.rs b/tests/runtime-adapter/src/worker/capture.rs index cd0050170..7a504879a 100644 --- a/tests/runtime-adapter/src/worker/capture.rs +++ b/tests/runtime-adapter/src/worker/capture.rs @@ -32,7 +32,9 @@ //! worker repairs the restore next, with `nvx::guest::clone::repair`, behind its fence, which //! stays closed: the repair activates the restore's memory target, reseeds the kernel's random //! number generator, and finishes the restore, with the acknowledgement that the host awaits, -//! which hands the deferred time work to the time daemon. NVX's `/init` started the daemon in the +//! which hands the deferred time work to the time daemon. For tests, a template whose initramfs +//! holds `template::REPAIR_DELAY_FILE` has the worker of each of its clones wait that long first, +//! while OpenVMM gates the host's input. NVX's `/init` started the daemon in the //! template, with the time ABI's boot step, before the worker, so the snapshot holds it, and PID 1 //! collects its status. A repair that failed ends the clone with the //! [status](nvx::guest::clone::RepairError::status) of its failure, which `nvx::protocol::status` @@ -55,7 +57,8 @@ use std::fmt; use std::io::{self, Read, Write}; use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; -use std::time::Instant; +use std::thread; +use std::time::{Duration, Instant}; use nvx::guest::clone::{ self, Activation, AdmissionFence, CaptureBarrier, CaptureCause, CaptureError, @@ -310,10 +313,11 @@ enum Capture { /// Captures the template's VM on its control tty at `tty`, with the root operations `operations`, /// whose NIC's IPv4 gateway is `gateway`, if it has a NIC, as its kernel command line names it, -/// as the [module documentation](self) describes, and returns how the worker ended: once the -/// bootstrap failed, or the capture ended without a snapshot, or failed, or in a clone, which a -/// restore of the snapshot started, once its repair failed, or the session that the worker served -/// once it repaired the clone ended. +/// and in each clone of which the worker waits `repair_delay` before the repair, if the template +/// sets a delay, for tests, as the [module documentation](self) describes, and returns how the +/// worker ended: once the bootstrap failed, or the capture ended without a snapshot, or failed, +/// or in a clone, which a restore of the snapshot started, once its repair failed, or the session +/// that the worker served once it repaired the clone ended. /// /// It opens the tty through [`guest::on_terminal_within`], as the worker of a VM that boots cold /// does, and fails as that does if the tty cannot be opened or put in raw mode. It fails with @@ -326,6 +330,7 @@ pub fn capture_on_terminal( tty: &Path, operations: &mut dyn RootOperations, gateway: Option, + repair_delay: Option, ) -> Result { guest::on_terminal_within(tty, |terminal| { let config = template_config(); @@ -349,6 +354,10 @@ pub fn capture_on_terminal( Ok(Captured::Restored(restored)) => { // The repair runs behind the fence, which stays closed. let (restore, fence) = restored.into_parts(); + // Until the repair acknowledges the restore, OpenVMM reads no host input. + if let Some(delay) = repair_delay { + thread::sleep(delay); + } match clone::repair(restore) { Ok(repaired) => Capture::Repaired { repaired, fence }, Err(error) => Capture::Ended(Templated::Unrepaired(error)), @@ -372,6 +381,7 @@ pub fn capture_on_terminal( activation, template, acknowledged, + repair_delay, )?; // A clone that ends before it is ready, or whose activation failed, ends even if the tty // sends nothing. diff --git a/tests/runtime-adapter/tests/common/mod.rs b/tests/runtime-adapter/tests/common/mod.rs index 8b13f765d..f27089851 100644 --- a/tests/runtime-adapter/tests/common/mod.rs +++ b/tests/runtime-adapter/tests/common/mod.rs @@ -192,10 +192,21 @@ impl Sandbox { /// hypervisor that `NVX_HYPERVISOR` selects, and with `options` too, checks that the creation /// exits with `status`, and returns what it printed. pub(crate) fn create(&mut self, options: &[&OsStr], status: i32) -> Printed { + self.create_with(options, &[], status) + } + + /// Creates a template of the guest agent in the sandbox's store, as [`create`](Self::create) + /// does, with the variables `environment` too. + pub(crate) fn create_with( + &mut self, + options: &[&OsStr], + environment: &[(&str, &OsStr)], + status: i32, + ) -> Printed { let guest = guest_options(); let mut arguments: Vec<&OsStr> = guest.iter().map(OsString::as_os_str).collect(); arguments.extend(options); - let started = self.start("create", &arguments, &[]); + let started = self.start("create", &arguments, environment); self.finish(started, status) } diff --git a/tests/runtime-adapter/tests/guest_session.rs b/tests/runtime-adapter/tests/guest_session.rs index 6d4002665..db18e9195 100644 --- a/tests/runtime-adapter/tests/guest_session.rs +++ b/tests/runtime-adapter/tests/guest_session.rs @@ -4302,6 +4302,7 @@ mod clones { activation, template(), Some(acknowledged), + None, ); (root, worked) }) @@ -5063,6 +5064,7 @@ mod clones { activation, gatewayless, Some(acknowledged), + None, ); (root, worked) }); @@ -5102,6 +5104,8 @@ mod clones { host.run_until(|host| !host.replies.is_empty()); let facts = reported(&host.replies.remove(0)); assert_eq!(facts.get("acknowledged"), Some("yes")); + // Its template delayed no repair. + assert_eq!(facts.get("repair-delay"), Some("none")); assert_eq!(facts.get("execution"), Some("none")); let events = ["reset", "message", "ready", "started"].map(|name| after(&facts, name)); assert!(events.is_sorted(), "{facts:?}"); diff --git a/tests/runtime-adapter/tests/initramfs.rs b/tests/runtime-adapter/tests/initramfs.rs index da118a9db..22eea4537 100644 --- a/tests/runtime-adapter/tests/initramfs.rs +++ b/tests/runtime-adapter/tests/initramfs.rs @@ -1,14 +1,17 @@ //! The initramfs that boots the guest agent is a copy of NVX's initramfs, with an archive after it //! that Linux unpacks over NVX's: the guest agent, the scripts that run it, and the links that -//! replace NVX's managed agent and init agent with the scripts. +//! replace NVX's managed agent and init agent with the scripts, and for the capture of a template +//! whose clones delay their repair, for tests, the delay's file. #![cfg(feature = "host")] use std::fs; use std::path::PathBuf; +use std::time::Duration; use runtime_adapter::initramfs::{ self, CONTROL_TTY, GUEST_AGENT, INIT_AGENT, LAUNCHER, LAYERED_LAUNCHER, MANAGED_AGENT, }; +use runtime_adapter::template::{REPAIR_DELAY_FILE, parse_repair_delay}; /// An entry of a `newc` archive. #[derive(Debug, PartialEq, Eq)] @@ -109,7 +112,7 @@ fn the_archive_replaces_nvx_s_agents_with_links_to_the_guest_agent_launchers() { // Guest agents of every length modulo 4 pad their data differently. for length in 0..8 { let agent = (1..=length).collect::>(); - let archive = initramfs::archive(&agent).unwrap(); + let archive = initramfs::archive(&agent, None).unwrap(); assert_eq!( parse(&archive), expected(&agent), @@ -118,6 +121,26 @@ fn the_archive_replaces_nvx_s_agents_with_links_to_the_guest_agent_launchers() { } } +#[test] +fn a_template_s_archive_holds_the_delay_of_its_clones_repair_after_the_launchers() { + // Delays of every length modulo 4 pad their data differently. + for milliseconds in [1, 10, 100, 1_000, 30_000] { + let delay = Duration::from_millis(milliseconds); + let archive = initramfs::archive(b"guest agent", Some(delay)).unwrap(); + let mut entries = expected(b"guest agent"); + let contents = format!("{milliseconds}\n"); + entries.push(Entry { + name: REPAIR_DELAY_FILE.trim_start_matches('/').to_owned(), + inode: 6, + mode: 0o100_644, + owner: (0, 0, 1), + data: contents.clone().into_bytes(), + }); + assert_eq!(parse(&archive), entries, "{delay:?}"); + assert_eq!(parse_repair_delay(contents.as_bytes()), Some(delay)); + } +} + #[test] fn the_launchers_run_the_guest_agent_on_the_control_tty() { // `/init` passes the managed agent the control tty first. @@ -149,13 +172,13 @@ fn the_initramfs_is_a_copy_of_nvx_s_with_the_archive_at_a_multiple_of_4_bytes() directory.join("output.cpio.gz"), ); fs::write(&agent, b"guest agent").unwrap(); - let archive = initramfs::archive(b"guest agent").unwrap(); + let archive = initramfs::archive(b"guest agent", None).unwrap(); // An earlier, longer output is replaced. fs::write(&output, vec![0xee; 4096]).unwrap(); for length in 0..8 { let contents = vec![0xab; length]; fs::write(&base, &contents).unwrap(); - initramfs::write(&base, &agent, &output).unwrap(); + initramfs::write(&base, &agent, None, &output).unwrap(); let written = fs::read(&output).unwrap(); let start = length.next_multiple_of(4); assert_eq!(&written[..length], contents, "a base of {length} bytes"); @@ -164,13 +187,19 @@ fn the_initramfs_is_a_copy_of_nvx_s_with_the_archive_at_a_multiple_of_4_bytes() } // NVX's initramfs stays as it is. assert_eq!(fs::read(&base).unwrap(), vec![0xab; 7]); + // A template's initramfs appends the archive with the delay of its clones' repair. + let delay = Some(Duration::from_secs(8)); + initramfs::write(&base, &agent, delay, &output).unwrap(); + let written = fs::read(&output).unwrap(); + let archive = initramfs::archive(b"guest agent", delay).unwrap(); + assert_eq!(&written[8..], archive); // A failure names the file. for (base, agent) in [ (&directory.join("missing"), &agent), (&base, &directory.join("missing")), ] { - let error = initramfs::write(base, agent, &output).unwrap_err(); + let error = initramfs::write(base, agent, None, &output).unwrap_err(); let missing = directory.join("missing").display().to_string(); assert!(error.to_string().contains(&missing), "{error}"); } diff --git a/tests/runtime-adapter/tests/staged_images.rs b/tests/runtime-adapter/tests/staged_images.rs index 6b65c062c..a52149957 100644 --- a/tests/runtime-adapter/tests/staged_images.rs +++ b/tests/runtime-adapter/tests/staged_images.rs @@ -267,10 +267,18 @@ fn the_host_program_stages_images_in_the_store() { command.output().unwrap() }; - // Staging creates the store, records both bindings in it, and prints nothing. + // Staging creates the store, records both bindings in it, and prints the bytes that it read. + let read_bytes = |output: &std::process::Output| -> u64 { + let stdout = String::from_utf8_lossy(&output.stdout); + stdout + .strip_prefix("read-bytes=") + .and_then(|line| line.strip_suffix('\n')) + .and_then(|bytes| bytes.parse().ok()) + .unwrap_or_else(|| panic!("{output:?}")) + }; let output = stage(&[scratch.as_path()]); assert!(output.status.success(), "{output:?}"); - assert!(output.stdout.is_empty(), "{output:?}"); + read_bytes(&output); let expected = Staged::stage(&distro, ImageFormat::Erofs, identity(), Some(&scratch)).unwrap(); assert_eq!(Staged::read(&store).unwrap(), Some(expected.clone())); @@ -278,6 +286,7 @@ fn the_host_program_stages_images_in_the_store() { manifest(&distro, "distro.erofs", &"0".repeat(64)); let output = stage(&[]); assert_eq!(output.status.code(), Some(1), "{output:?}"); + assert!(output.stdout.is_empty(), "{output:?}"); let stderr = String::from_utf8_lossy(&output.stderr); assert!( stderr.contains("sandbox distro manifest does not match its artifact"), @@ -286,10 +295,11 @@ fn the_host_program_stages_images_in_the_store() { assert_eq!(Staged::read(&store).unwrap(), Some(expected)); // Staging again replaces the staged images: an image whose manifest matches, without a scratch - // image. + // image. It hashed the image, whose bytes it read. manifest(&distro, "distro.erofs", ZERO_KIB_SHA256); let output = stage(&[]); assert!(output.status.success(), "{output:?}"); + assert!(read_bytes(&output) >= 1024, "{output:?}"); let staged = Staged::read(&store).unwrap().unwrap(); assert!(staged.distro.binding.manifest().is_some()); assert_eq!(staged.scratch, None); diff --git a/tests/runtime-adapter/tests/template_capture.rs b/tests/runtime-adapter/tests/template_capture.rs index 78845e3a8..961a78eae 100644 --- a/tests/runtime-adapter/tests/template_capture.rs +++ b/tests/runtime-adapter/tests/template_capture.rs @@ -17,7 +17,7 @@ use nvx::protocol::root::{ImageFormat, RootImage}; use nvx::uuid::Uuid; use runtime_adapter::capture::{ DEFAULT_BOOT_PROCESSORS, DEFAULT_CPU_PROFILE, DEFAULT_PROCESSORS, DEFAULT_SCRATCH_LENGTH, - LOGICAL_BLOCK_SIZE, Request, geometry, + LOGICAL_BLOCK_SIZE, REPAIR_DELAY_SETTING, REPAIR_DELAY_VARIABLE, Request, geometry, }; use runtime_adapter::layered::read_network; use runtime_adapter::runtime::Runtime; @@ -88,6 +88,7 @@ fn template(distro: &Path, network: Option) -> Request { memory_capacity: 256 * MIB, scratch_length: DEFAULT_SCRATCH_LENGTH, network, + repair_delay: None, } } @@ -207,7 +208,7 @@ fn a_capture_takes_its_class_and_inputs_from_the_request() { more.boot_processors = 2; let capture = more .capture( - staged.distro.binding, + staged.distro.binding.clone(), &runtime, "build", &initramfs, @@ -220,6 +221,23 @@ fn a_capture_takes_its_class_and_inputs_from_the_request() { ); assert_ne!(capture.class.identity().unwrap(), fewer); + // A template whose clones delay their repair, for tests, is of a class of its own, which + // holds the delay as its one invariant setting. + let mut delayed = template(&distro, Some(network)); + delayed.repair_delay = Some(std::time::Duration::from_secs(8)); + let capture = delayed + .capture( + staged.distro.binding, + &runtime, + "build", + &initramfs, + &directory, + ) + .unwrap(); + let invariants = capture.class.invariants.iter().collect::>(); + assert_eq!(invariants, [(REPAIR_DELAY_SETTING, "8000")]); + assert_ne!(capture.class.identity().unwrap(), with_nic); + // OpenVMM presents logical blocks of 512 bytes. let presented = geometry(&distro, 4096).unwrap(); assert_eq!(presented.length, 4096); @@ -285,3 +303,39 @@ fn the_host_program_refuses_a_template_before_it_creates_anything() { ); } } + +#[test] +fn the_host_program_refuses_a_repair_delay_that_it_cannot_read() { + // A delay of each clone's repair, for tests, is a positive number of milliseconds, of at most + // half a minute: anything else is a usage error of `create` and `refresh`, which creates + // nothing. + let directory = directory("template-capture-repair-delay"); + let store = directory.join("templates"); + for command in ["create", "refresh"] { + for value in ["", "0", "30001", "8s", "-1", " 8000", "8000\n", "1.5"] { + let output = Command::new(env!("CARGO_BIN_EXE_runtime-adapter-host")) + .arg(command) + .arg(&store) + .args(["--guest-agent", "agent", "--distro", "erofs:image"]) + .args(["--distro-identity", &identity().to_string()]) + .env(REPAIR_DELAY_VARIABLE, value) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(2), + "{command} {value:?}: {output:?}" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + let problem = format!( + "{REPAIR_DELAY_VARIABLE} takes a positive number of milliseconds, of at most \ + 30000, not {value:?}" + ); + assert!( + stderr.contains(&problem) && stderr.contains("usage: runtime-adapter-host"), + "{command} {value:?}: {stderr}" + ); + assert!(!store.exists(), "{command} {value:?}: {output:?}"); + } + } +} diff --git a/tests/runtime-adapter/tests/vm_clone.rs b/tests/runtime-adapter/tests/vm_clone.rs index 34067bd60..f5bc2acd5 100644 --- a/tests/runtime-adapter/tests/vm_clone.rs +++ b/tests/runtime-adapter/tests/vm_clone.rs @@ -82,6 +82,22 @@ //! - A clone runs its executions as its record's workload identity, once it is ready: the repair //! acknowledged the restore before the clone took its reset, the host's first message came //! after that, and the clone's Ready, its start, and its first execution came in that order. +//! - OpenVMM reads none of the host's input until the clone's repair acknowledges its restore, as +//! the host sees it: a template whose initramfs delays each clone's repair, for tests, holds a +//! clone's restore unacknowledged for that long, and is of a class of its own, whose key names +//! the delay. The host starts a clone of it through NVX's public API, with the host program's +//! clone order, over transports that watch the session's attachment: the host writes the +//! attachment well before the repair can acknowledge the restore, but the broker, which answers +//! a host that it authenticated at once, answers only after the delay, and then attaches the +//! host. On Linux, the host writes the attachment a byte at a time, which the kernel queues in +//! a buffer for each byte, and charges the same for, so that what `SIOCOUTQ` reports of it falls +//! as soon as OpenVMM reads any byte: it rises by the same step with each byte that the host +//! writes, stays as the write left it until the delay passed, and is zero once the broker +//! answered, while a listener of the test's own process, which reads the first byte of such an +//! attachment, shows that it would fall. A named pipe tells its writer nothing of the kind, so +//! on Windows the check is the timing alone. A reattach once the clone is ready, which no gate +//! holds, gets its answer within a fraction of the delay, as the positive control, and the guest +//! reports that it waited the delay before its repair. //! - Each command after the start reattaches to the clone from a new process: the clone answers //! the Activate of its record again, in a new session of the same broker, with its Ready, without //! activating again, and serves the command. @@ -117,7 +133,15 @@ //! which clock fixtures stand for. It refuses the selection of a generation that is no longer //! eligible, as expired, or not yet, as ineligible, without a lease, with the refusal's record. //! - A clone reads no more of its disks on its restore than its root needs, which a read of each -//! disk to its end shows the guest's statistics count. +//! disk to its end shows the guest's statistics count. The host's processes count the bytes +//! that their read calls read, the cache's included, as `nvx::host::process` tells them: the +//! staging of the clone's images, which hashes the image, counts at least its length, while the +//! start reads less than an eighth of it, and the clone's VMM less than an eighth of the disks +//! whose files it reads, once the clone is ready. Its count rises by what the guest's read of +//! each disk to its end asked of those files on Windows, where OpenVMM reads them with +//! `ReadFile`, but not on Linux, where OpenVMM reads them through io_uring, which `rchar` does +//! not count: there, the VMM's count covers its read calls alone. Neither counts the memory of +//! the snapshot, which OpenVMM maps. //! - A clone's time settles after it is ready, with no violation, after the downtime that the //! host's monotonic clock measured. //! - A template without a NIC serves clones without a network, and of a network that lets nothing @@ -191,8 +215,8 @@ use std::net::{ }; use std::num::NonZeroU64; use std::path::{Path, PathBuf}; -use std::sync::LazyLock; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, LazyLock, Mutex}; use std::thread; use std::time::{Duration, Instant, SystemTime}; @@ -210,19 +234,23 @@ use nvx::host::store::{CallerFiles, LOG_NAME, SandboxStore}; use nvx::host::template::{ CAPTURE_RECORD_NAME, CloneControl, ClonePlan, CloneRecords, Evidence, GENERATIONS_NAME, Generation, HostTime, KEY_NAME, Lease, MAX_DOWNTIME_NS, MEMORY_BLOCK_SIZE, Outcome, Progress, - RestoreDeadlines, SNAPSHOT_NAME, Stage, Store, Teardown, + ReattachedClone, RestoreDeadlines, SNAPSHOT_NAME, Stage, Store, Teardown, reattach_clone, }; use nvx::protocol::root::ImageFormat; use nvx::protocol::status; use nvx::protocol::{CAPABILITY_LEN, HostMap}; use nvx::uuid::Uuid; use runtime_adapter::GUEST_BUILD; -use runtime_adapter::capture::DEFAULT_SCRATCH_LENGTH; -use runtime_adapter::clone::{CLONE_HOSTNAME, HOST_TIME_VARIABLE, Request, host_time_text}; +use runtime_adapter::capture::{ + DEFAULT_SCRATCH_LENGTH, REPAIR_DELAY_SETTING, REPAIR_DELAY_VARIABLE, +}; +use runtime_adapter::clone::{ + CLONE_HOSTNAME, CloneRecord, HOST_TIME_VARIABLE, Request, host_time_text, +}; use runtime_adapter::exec::{HostMessage, Start}; use runtime_adapter::frame::Message; use runtime_adapter::hook::{Action, STEP_HOOK, Step, StepHook}; -use runtime_adapter::host::Client; +use runtime_adapter::host::{Client, CloneOrder, OrderError}; use runtime_adapter::layered::PROXY_VARIABLES; use runtime_adapter::root::DEFAULT_RUNTIME; use runtime_adapter::runtime::Runtime; @@ -259,6 +287,18 @@ const STRANGER: &str = "4242:4242"; const SECTOR: u64 = 512; /// How long a clone's time may take to settle once the clone is ready. const TIME_SETTLEMENT: Duration = Duration::from_secs(30); +/// How long each clone of the template of the scenario of OpenVMM's gate of the host's input +/// waits before its repair: long enough for the host's attachment, which a restore of a fraction +/// of a second lets the host write, to reach OpenVMM well before the repair acknowledges the +/// clone's restore, even on a host that other scenarios load. +const REPAIR_DELAY: Duration = Duration::from_secs(8); +/// How often a watched attachment records what the broker has not read of it, while it awaits +/// the broker's answer, on a host whose transport tells. +const UNREAD_INTERVAL: Duration = Duration::from_millis(50); +/// Whether the count of the bytes that a clone's VMM read, `Process::bytes_read`, counts its +/// reads of the clone's disks: on Windows, OpenVMM reads them with `ReadFile`, which +/// `ReadTransferCount` counts, and on Linux through io_uring, which `rchar` does not count. +const VMM_COUNTS_DISK_READS: bool = cfg!(windows); /// Nanoseconds in an hour. const HOUR_NS: u64 = 3_600_000_000_000; /// The error number with which the guest refuses an execution that names an identity in a @@ -942,7 +982,15 @@ impl Template { boot.iter() .flat_map(|boot| ["--boot-processors", boot.as_str()]), ); - Self::captured(name, None, None, &options) + Self::captured(name, None, None, &options, &[]) + } + + /// Creates the sandbox of the scenario `name`, stages the platform's copy of the image, and + /// captures the template of the image, as [`create`](Self::create) does, each of whose clones + /// waits `delay` before its repair, as [`REPAIR_DELAY_VARIABLE`] asks of the capture. + fn delayed(name: &str, delay: Duration) -> Self { + let delay = OsString::from(delay.as_millis().to_string()); + Self::captured(name, None, None, &[], &[(REPAIR_DELAY_VARIABLE, &delay)]) } /// Creates the sandbox of the scenario `name`, stages the platform's copy of the image, and @@ -950,15 +998,17 @@ impl Template { /// than the image, and of the image otherwise, with the network `network`, if any, with the /// guest at [`GUEST_NETWORK`]. fn capture(name: &str, placeholder: Option<&Disk>, network: Option) -> Self { - Self::captured(name, placeholder, network, &[]) + Self::captured(name, placeholder, network, &[], &[]) } - /// Captures the template as [`capture`](Self::capture) does, with the options `extra` too. + /// Captures the template as [`capture`](Self::capture) does, with the options `extra` too, and + /// the variables `environment`. fn captured( name: &str, placeholder: Option<&Disk>, network: Option, extra: &[&str], + environment: &[(&str, &OsStr)], ) -> Self { let mut sandbox = Sandbox::new(name); let fixtures = Fixtures::find(); @@ -990,7 +1040,7 @@ impl Template { options.extend([OsStr::new("--network"), network.as_os_str()]); } options.extend(extra.iter().map(OsStr::new)); - let created = sandbox.create(&options, 0); + let created = sandbox.create_with(&options, environment, 0); assert_eq!(created["created"], "captured", "{created:?}"); assert_eq!(created["generation"], GENERATION.to_string(), "{created:?}"); Self { @@ -1118,10 +1168,11 @@ impl Template { execution: &[&str], status: i32, ) -> Cloned { - let (sandbox, started) = self.begin_in(name, command, start, execution); + let (sandbox, staged, started) = self.begin_in(name, command, start, execution); let started = sandbox.finish(started, status); Cloned { sandbox, + staged, started, ran: command == "run", } @@ -1129,17 +1180,17 @@ impl Template { /// Begins the host program's `command`, `start` or `run`, of a clone in a new sandbox of the /// scenario `name`, as `start` asks, with the arguments `execution` after the start's options, - /// and returns the sandbox, with the command, which may still run. Before a start, the - /// sandbox's store stages the clone's images, unless `start` says otherwise. A start that - /// `start` asks to act at `checked` pauses there, through the step hook, until the scenario - /// acted. + /// and returns the sandbox, what its staging printed, if it staged, and the command, which may + /// still run. Before a start, the sandbox's store stages the clone's images, unless `start` + /// says otherwise. A start that `start` asks to act at `checked` pauses there, through the + /// step hook, until the scenario acted. fn begin_in( &self, name: &str, command: &str, start: &StartOf<'_>, execution: &[&str], - ) -> (Sandbox, Started) { + ) -> (Sandbox, Option, Started) { let mut sandbox = Sandbox::new(name); let (distro, identity) = match start.image { None => (erofs(&self.image), self.identity.as_str()), @@ -1172,6 +1223,7 @@ impl Template { }; let identity = OsStr::new(identity); let run = command == "run"; + let mut staged = None; if !run && !start.unstaged { let mut staging = vec![ OsStr::new("--distro"), @@ -1182,7 +1234,7 @@ impl Template { if start.upper != Upper::Ram { staging.extend([OsStr::new("--upper"), &upper]); } - sandbox.run("stage", &staging, 0); + staged = Some(sandbox.run("stage", &staging, 0)); } let generation = GENERATION.to_string(); let mut options = vec![ @@ -1222,7 +1274,7 @@ impl Template { act(); fs::remove_file(&pause).unwrap(); } - (sandbox, started) + (sandbox, staged, started) } } @@ -1312,6 +1364,8 @@ struct StartOf<'a> { struct Cloned { /// The clone's sandbox. sandbox: Sandbox, + /// What the staging of the clone's images printed, before a start that staged them. + staged: Option, /// What the start, or the run, printed. started: Printed, /// Whether a run, whose preflight staged the clone's images, rather than a start, started @@ -2305,21 +2359,7 @@ fn refuses_early(template: &Template, early: Message) { let store = template.store(); let artifacts = env::var_os(Artifacts::ENV_DIR).unwrap(); let runtime = Runtime::from_dir(artifacts, GUEST_BUILD).unwrap(); - let request = Request { - distro: template.image.clone(), - format: ImageFormat::Erofs, - identity: Uuid::parse(&template.identity).unwrap(), - scratch: None, - processors: None, - memory: None, - workload_identity: None, - overlay_mode: None, - scratch_journal: None, - first_partition_bound: None, - machine_id: MachineIdMode::Image, - sharing: HostFsSharing::default(), - network: None, - }; + let request = in_ram(template); let plan = request.plan().unwrap(); let staged = Staged::read(&sandbox.store).unwrap().unwrap(); let key = store.read_key(&template.generation()).unwrap(); @@ -2389,6 +2429,220 @@ fn refuses_early(template: &Template, early: Message) { assert!(template.leases().is_empty()); } +/// Returns the request of a clone of `template` in RAM, of the platform's image, which the +/// template's processors online at boot and RAM serve, without a setting of its own, as a start +/// without options makes it. +fn in_ram(template: &Template) -> Request { + Request { + distro: template.image.clone(), + format: ImageFormat::Erofs, + identity: Uuid::parse(&template.identity).unwrap(), + scratch: None, + processors: None, + memory: None, + workload_identity: None, + overlay_mode: None, + scratch_journal: None, + first_partition_bound: None, + machine_id: MachineIdMode::Image, + sharing: HostFsSharing::default(), + network: None, + } +} + +/// What the host saw of an attachment of a session to a clone's control endpoint, through a +/// [`WatchedTransport`]. +#[derive(Clone, Debug, Default)] +struct Attachment { + /// When the host had written the attachment, the first bytes of the session. + written: Option, + /// What the broker had not read of what the host wrote, as the transport tells it, or `None` + /// where it does not, before the host wrote the attachment, and once it had written each of + /// the attachment's bytes, which it writes one at a time. + writing: Vec>, + /// What the broker had not read of what the host wrote, as the transport tells it, or `None` + /// where it does not, and when: once the host wrote the attachment, every + /// [`UNREAD_INTERVAL`] while it awaited the broker's answer, and once the answer came. + unread: Vec<(Instant, Option)>, + /// When the broker's first answer came. + answered: Option, + /// When the broker attached the host, if it did. + attached: Option, +} + +/// A transport to a clone's control endpoint that records what the host saw of the attachment +/// that it writes first, as an [`Attachment`], and writes it a byte at a time. On Linux, the +/// kernel then queues a buffer for each byte, and charges the same for each until the broker read +/// it, so what `SIOCOUTQ` reports of the attachment rises by the same step with each byte that the +/// broker has not read, and falls as soon as the broker reads any byte, while a buffer that the +/// broker read in part would leave it as it was. +struct WatchedTransport { + /// The transport that NVX connected. + transport: Box, + /// What the host saw of the attachment. + attachment: Arc>, +} + +impl WatchedTransport { + /// Records what the broker has not read of what the host wrote, now. + fn record_unread(&self) -> io::Result<()> { + let unread = self.transport.unread()?; + let mut attachment = self.attachment.lock().unwrap(); + attachment.unread.push((Instant::now(), unread)); + Ok(()) + } +} + +impl Transport for WatchedTransport { + fn read(&mut self, buffer: &mut [u8], timeout: Option) -> io::Result { + let (awaiting, tells) = { + let attachment = self.attachment.lock().unwrap(); + let awaiting = attachment.written.is_some() && attachment.answered.is_none(); + let tells = attachment.unread.iter().any(|(_, unread)| unread.is_some()); + (awaiting, tells) + }; + if !awaiting { + return self.transport.read(buffer, timeout); + } + // Until the broker answers, a transport that tells what the broker has not read is asked + // between waits of at most `UNREAD_INTERVAL`, until the read's own timeout. + let deadline = timeout.map(|timeout| Instant::now() + timeout); + let read = loop { + let left = deadline.map(|deadline| deadline.saturating_duration_since(Instant::now())); + let wait = if tells { + Some(left.map_or(UNREAD_INTERVAL, |left| left.min(UNREAD_INTERVAL))) + } else { + left + }; + match self.transport.read(buffer, wait) { + Err(error) + if error.kind() == io::ErrorKind::TimedOut + && tells + && left.is_none_or(|left| left > UNREAD_INTERVAL) => + { + self.record_unread()?; + } + read => break read, + } + }; + if let Ok(1..) = read { + self.attachment.lock().unwrap().answered = Some(Instant::now()); + self.record_unread()?; + } + read + } + + fn write_all(&mut self, data: &[u8], timeout: Option) -> io::Result<()> { + let first = self.attachment.lock().unwrap().written.is_none(); + if !first { + return self.transport.write_all(data, timeout); + } + // The attachment goes out a byte at a time, each in a write of its own, which the kernel + // queues in a buffer of its own on Linux, and charges for until the broker read the + // buffer whole: `SIOCOUTQ` rises with each byte that the broker has not read, and falls as + // soon as the broker reads any byte of it. + let deadline = timeout.map(|timeout| Instant::now() + timeout); + let mut writing = vec![self.transport.unread()?]; + for byte in data.chunks(1) { + let left = deadline.map(|deadline| deadline.saturating_duration_since(Instant::now())); + self.transport.write_all(byte, left)?; + writing.push(self.transport.unread()?); + } + { + let mut attachment = self.attachment.lock().unwrap(); + attachment.written = Some(Instant::now()); + attachment.writing = writing; + } + self.record_unread() + } + + fn unread(&self) -> io::Result> { + self.transport.unread() + } +} + +/// The adapter's framing of a clone's control session, the host program's own clone order, over +/// transports that watch their attachments, as [`WatchedTransport`]s. +struct Watched { + /// The host program's clone order. + order: CloneOrder<'static>, + /// What the host saw of the attachment of each session, in order. + attachments: Vec>>, +} + +impl Watched { + /// Returns the framing of the host program's clone order, of the guest agent's build that the + /// host ships, which has attached no session yet. + fn new() -> Self { + Self { + order: CloneOrder { build: GUEST_BUILD }, + attachments: Vec::new(), + } + } + + /// Returns what the host saw of the attachment of the framing's one session. + fn attachment(&self) -> Attachment { + let [attachment] = self.attachments.as_slice() else { + panic!("the host attached {} sessions", self.attachments.len()); + }; + attachment.lock().unwrap().clone() + } +} + +impl CloneControl for Watched { + type Session = Client; + type Error = OrderError; + + fn attach( + &mut self, + transport: Box, + capability: &[u8; CAPABILITY_LEN], + deadline: Instant, + ) -> Result { + let attachment = Arc::new(Mutex::new(Attachment::default())); + self.attachments.push(Arc::clone(&attachment)); + let watched = WatchedTransport { + transport, + attachment: Arc::clone(&attachment), + }; + let attached = self.order.attach(Box::new(watched), capability, deadline); + if attached.is_ok() { + attachment.lock().unwrap().attached = Some(Instant::now()); + } + attached + } + + fn activate( + &mut self, + session: &mut Client, + record: &[u8], + deadline: Instant, + ) -> Result, OrderError> { + self.order.activate(session, record, deadline) + } + + fn map( + &mut self, + session: &mut Client, + table: &[HostMap], + deadline: Instant, + ) -> Result<(), OrderError> { + self.order.map(session, table, deadline) + } + + fn start(&mut self, session: &mut Client, deadline: Instant) -> Result<(), OrderError> { + self.order.start(session, deadline) + } + + fn lost(&self, error: &OrderError) -> bool { + self.order.lost(error) + } + + fn refused(&self, error: &OrderError) -> bool { + self.order.refused(error) + } +} + /// Returns the limits of the executions of the scenarios that the host starts itself. fn limits() -> Limits { Limits::new(16 * 1024, Window::frames(4), Window::frames(8)) @@ -3074,6 +3328,255 @@ fn a_clone_runs_its_executions_as_its_workload_identity_once_it_is_ready() { cloned.stop(); } +/// Checks what the host saw, through its transport, of what the broker had not read of +/// `attachment` once the broker answered: on Linux, as `SIOCOUTQ` reports it, nothing, since the +/// broker read the attachment whole first. A named pipe tells its writer nothing of the kind, so +/// on Windows the transport told nothing at all. +fn assert_read_once_answered(attachment: &Attachment) { + let unread = &attachment.unread; + if !cfg!(target_os = "linux") { + assert!( + attachment.writing.iter().all(Option::is_none) + && unread.iter().all(|(_, unread)| unread.is_none()), + "{attachment:?}" + ); + return; + } + let answered = attachment.answered.expect("the broker answered"); + let last = unread + .last() + .expect("the host recorded what the broker read"); + assert!(last.0 >= answered && last.1 == Some(0), "{attachment:?}"); +} + +/// Returns what the host saw of `attachment` once it had written it, on Linux, and checks that +/// the broker read none of it meanwhile: the host wrote it a byte at a time, which the kernel +/// charges the same for, so what `SIOCOUTQ` reported rose by the same step with each byte, from +/// what it reported before the write. +fn charge_once_written(attachment: &Attachment) -> usize { + let writing = attachment + .writing + .iter() + .map(|unread| unread.expect("the transport tells what the broker has not read")) + .collect::>(); + let [before, first, ..] = writing[..] else { + panic!("the host wrote no attachment: {attachment:?}"); + }; + let step = first.saturating_sub(before); + assert!(step > 0, "{attachment:?}"); + assert!( + writing + .iter() + .enumerate() + .all(|(bytes, unread)| *unread == before + bytes * step), + "{attachment:?}" + ); + writing[writing.len() - 1] +} + +/// Checks, on Linux, that the records of a [`WatchedTransport`] show the read of a prefix of its +/// attachment, as they would show OpenVMM's: NVX connects the transport, as it connects one to a +/// control endpoint, to a listener of this process at `socket`, which plays the broker, and reads +/// the first byte of the attachment that the transport writes, then the rest. What `SIOCOUTQ` +/// reports rises by the same step with each byte that the transport writes, falls once the +/// listener read the first byte, and is zero once it read the rest. +#[cfg(target_os = "linux")] +fn assert_a_prefix_read_shows(socket: &Path) { + let listener = std::os::unix::net::UnixListener::bind(socket).unwrap(); + let pid = std::process::id(); + let start_time = nvx::host::platform::process_start_time(pid) + .unwrap() + .unwrap(); + let endpoint = socket.to_str().unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let transport = nvx::host::session::connect(endpoint, pid, start_time, deadline).unwrap(); + let (mut broker, _) = listener.accept().unwrap(); + let attachment = Arc::new(Mutex::new(Attachment::default())); + let mut watched = WatchedTransport { + transport, + attachment: Arc::clone(&attachment), + }; + let sent = [0x5a; 64]; + watched + .write_all(&sent, Some(Duration::from_secs(10))) + .unwrap(); + let whole = charge_once_written(&attachment.lock().unwrap()); + let mut received = [0; 64]; + broker.read_exact(&mut received[..1]).unwrap(); + watched.record_unread().unwrap(); + broker.read_exact(&mut received[1..]).unwrap(); + watched.record_unread().unwrap(); + let unread = attachment + .lock() + .unwrap() + .unread + .iter() + .map(|(_, unread)| *unread) + .collect::>(); + let [Some(written), Some(prefix), Some(rest)] = unread[..] else { + panic!("{unread:?}"); + }; + assert!( + written == whole && 0 < prefix && prefix < whole && rest == 0, + "{whole}: {unread:?}" + ); + assert_eq!(received, sent); +} + +#[test] +#[ignore = "boots VMs; see the module documentation for what they need"] +fn openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore() { + // A template whose clones wait before their repair, of a class of its own. + let template = Template::delayed("cl-gate", REPAIR_DELAY); + let store = template.store(); + let key = store.read_key(&template.generation()).unwrap(); + let delay = REPAIR_DELAY.as_millis().to_string(); + assert_eq!( + key.invariants().get(REPAIR_DELAY_SETTING), + Some(delay.as_str()) + ); + + // The host starts a clone in RAM through NVX's public API, as the host program starts one, + // with the record that the host program's later commands read, and the host program's clone + // order, over transports that watch their attachments. + let mut sandbox = Sandbox::new("cl-g"); + template.stage(&mut sandbox); + let artifacts = env::var_os(Artifacts::ENV_DIR).unwrap(); + let runtime = Runtime::from_dir(artifacts, GUEST_BUILD).unwrap(); + let request = in_ram(&template); + let plan = request.plan().unwrap(); + let staged = Staged::read(&sandbox.store).unwrap().unwrap(); + let bindings = staged.bindings(&plan); + let clone = request + .clone_request( + plan.clone(), + None, + &bindings, + key.processors(), + key.memory(), + ) + .unwrap(); + let now = HostTime::now().unwrap(); + let mut acquired = store + .acquire(GENERATION, &sandbox.store, &CallerFiles::none(), &now, None) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let planned = ClonePlan::new(&store, &mut acquired, &clone, deadline).unwrap(); + let record = CloneRecord { + template: template.dir(), + generation: GENERATION, + kernel_command_line: planned.key().command_line(), + plan: &plan, + bindings: &bindings, + }; + record.write(&sandbox.store).unwrap(); + // The clone's VMM starts after this, and its guest waits the delay once the restored VM runs, + // so the repair acknowledges the restore no sooner than the delay after it. + let launched = Instant::now(); + let earliest = launched + REPAIR_DELAY; + let restoring = planned + .launch(&mut acquired, &runtime.artifacts.openvmm, &runtime.identity) + .unwrap(); + let pid = restoring.starting().process().reference().pid; + let mut gated = Watched::new(); + let activated = restoring + .activate(&mut gated, RestoreDeadlines::default()) + .unwrap(); + let (vmm, session) = activated.start(&mut gated).unwrap().commit().unwrap(); + drop(session); + + // The host wrote its attachment once the restore was ready, well before the repair could + // acknowledge the restore, but the broker, which answers a host that it authenticated at + // once, answered only after then, and then attached the host. + let attachment = gated.attachment(); + let written = attachment.written.expect("the host wrote its attachment"); + let answered = attachment.answered.expect("the broker answered"); + assert!( + written + REPAIR_DELAY / 2 <= earliest, + "the host wrote its attachment {:?} after the launch, too late to see the gate: \ + {attachment:?}", + written - launched + ); + assert!( + answered >= earliest, + "the broker answered {:?} after the launch, before the repair could acknowledge the \ + restore: {attachment:?}", + answered - launched + ); + assert!( + attachment + .attached + .is_some_and(|attached| attached >= answered), + "{attachment:?}" + ); + // On Linux, OpenVMM read none of the attachment until then: what `SIOCOUTQ` reports of it, + // which falls as soon as OpenVMM reads any byte of it, rose by the same step with each byte + // that the host wrote, and stayed as the write left it until the repair could acknowledge the + // restore, and OpenVMM had read it whole once the broker answered. A listener of this process + // shows that the records would show the read of a prefix. + if cfg!(target_os = "linux") { + let charge = charge_once_written(&attachment); + let held = attachment + .unread + .iter() + .filter(|(at, _)| *at < earliest) + .collect::>(); + assert!(!held.is_empty(), "{attachment:?}"); + assert!( + held.iter().all(|(_, unread)| *unread == Some(charge)), + "{attachment:?}" + ); + } + assert_read_once_answered(&attachment); + #[cfg(target_os = "linux")] + assert_a_prefix_read_shows(&sandbox.root.join("prefix.sock")); + + // Once the repair acknowledged the restore, OpenVMM reads the host's input at once: the + // broker answers the attachment of a reattach, which no gate holds, within a fraction of the + // delay, and on Linux, OpenVMM read it whole by then. + let mut open = Watched::new(); + let deadline = Instant::now() + Duration::from_secs(30); + let reattached = reattach_clone( + &mut acquired.locked, + &runtime.identity, + &mut open, + deadline, + Duration::from_secs(10), + ) + .unwrap(); + let ReattachedClone::Live { vmm: held, session } = reattached else { + panic!("the reattach did not find the clone running"); + }; + drop(session); + drop(held); + let attachment = open.attachment(); + let written = attachment.written.expect("the host wrote its attachment"); + let answered = attachment.answered.expect("the broker answered"); + assert!(answered - written < REPAIR_DELAY / 4, "{attachment:?}"); + assert_read_once_answered(&attachment); + drop(vmm); + let lease = acquired.lease.id().to_string(); + drop(acquired); + + // The guest waited the delay before its repair, which acknowledged the restore before the + // clone took its reset and the host's first message. + let started = Printed::from([ + ("pid".to_owned(), pid.to_string()), + ("lease".to_owned(), lease), + ]); + let mut cloned = Cloned { + sandbox, + staged: None, + started, + ran: false, + }; + let facts = cloned.probe(&["activation"]); + let delay = REPAIR_DELAY.as_micros().to_string(); + assert_eq!(facts["activation.repair-delay"], delay, "{facts:?}"); + assert_timeline(&facts); + cloned.stop(); +} + #[test] #[ignore = "boots VMs; see the module documentation for what they need"] fn a_generation_is_eligible_by_the_clock_of_its_capture() { @@ -3167,10 +3670,35 @@ fn a_generation_is_eligible_by_the_clock_of_its_capture() { } } +/// Returns the bytes that a command of the host program read through its read calls, as it +/// printed them in `printed` as it exited. +fn read_bytes(printed: &Printed) -> u64 { + printed + .get("read-bytes") + .and_then(|bytes| bytes.parse().ok()) + .unwrap_or_else(|| panic!("{printed:?}")) +} + +/// Returns the names of the disks of a clone whose files its VMM reads, from the `disks` fact of +/// its `devices` probe, `NAME:SECTORS:ACCESS` each: its image's, read-only, and its scratch +/// device, writable, unless its upper is in RAM, whose scratch no file backs. +fn backed_disks(fact: &str, upper: Upper<'_>) -> Vec { + fact.split(' ') + .filter_map(|disk| { + let parts = disk.split(':').collect::>(); + let &[name, _, access] = parts.as_slice() else { + panic!("{disk:?} in {fact:?}"); + }; + (access == "ro" || upper != Upper::Ram).then(|| name.to_owned()) + }) + .collect() +} + #[test] #[ignore = "boots VMs; see the module documentation for what they need"] fn a_clone_reads_no_more_of_its_disks_than_its_root_needs() { let template = Template::create("cl-reads"); + let image = fs::metadata(&template.image).unwrap().len(); for upper in [Upper::Scratch, Upper::Ram] { let start = StartOf { upper, @@ -3178,8 +3706,25 @@ fn a_clone_reads_no_more_of_its_disks_than_its_root_needs() { }; let mut cloned = template.start("cl-d", start, 0); cloned.assert_started(); + // The host's processes count the bytes that their read calls read, the cache's included: + // the staging hashed the image, which its count shows, while the start read less than an + // eighth of it. The clone's VMM, which the start left running, is counted below. + let staged = cloned + .staged + .as_ref() + .expect("the clone's images were staged"); + assert!(read_bytes(staged) >= image, "{upper:?}: {staged:?}"); + let started = read_bytes(&cloned.started); + assert!( + started * 8 < image, + "{upper:?}: the start read {started} bytes" + ); + let vmm = cloned.sandbox.vmm().expect("the clone's VMM runs"); + let ready = vmm.bytes_read().unwrap(); + let facts = cloned.probe(&["devices", "reads"]); let sizes = disks(&facts["devices.disks"]); + let backed = backed_disks(&facts["devices.disks"], upper); let restored = disks(&facts["reads.disks"]); assert_eq!( sizes.keys().collect::>(), @@ -3192,10 +3737,7 @@ fn a_clone_reads_no_more_of_its_disks_than_its_root_needs() { .map(|sectors| sectors * SECTOR) .collect::>(); lengths.sort_unstable(); - let mut expected = [ - fs::metadata(&template.image).unwrap().len(), - DEFAULT_SCRATCH_LENGTH, - ]; + let mut expected = [image, DEFAULT_SCRATCH_LENGTH]; expected.sort_unstable(); assert_eq!(lengths, expected, "{facts:?}"); for (disk, sectors) in &sizes { @@ -3205,6 +3747,12 @@ fn a_clone_reads_no_more_of_its_disks_than_its_root_needs() { restored[disk] ); } + // Nor did the VMM read more than an eighth of the disks whose files it reads by then. + let length = backed.iter().map(|disk| sizes[disk] * SECTOR).sum::(); + assert!( + ready * 8 < length, + "{upper:?}: the VMM read {ready} bytes once the clone was ready, of disks of {length}" + ); // A read of each disk to its end, which the statistics count. let facts = cloned.probe(&["read-disks"]); @@ -3224,6 +3772,27 @@ fn a_clone_reads_no_more_of_its_disks_than_its_root_needs() { after[disk] ); } + // The VMM read from the disks' files what the guest's reads asked of them, which the + // VMM's count shows on Windows, but not on Linux, where OpenVMM reads them through + // io_uring, which `rchar` misses: there, the count covers the VMM's read calls alone. + let asked = backed + .iter() + .map(|disk| (after[disk] - restored[disk]) * SECTOR) + .sum::(); + let counted = vmm.bytes_read().unwrap() - ready; + if VMM_COUNTS_DISK_READS { + assert!( + counted >= asked, + "{upper:?}: the VMM counted {counted} bytes for the {asked} that the guest read" + ); + } else { + assert!( + counted * 8 < asked, + "{upper:?}: the VMM's read calls read {counted} bytes for the {asked} that the \ + guest read" + ); + } + drop(vmm); cloned.stop(); } assert_payload(&template, false); @@ -4792,7 +5361,7 @@ fn a_run_whose_clone_s_vm_exits_during_its_execution_ends_with_the_clone_s_failu // while the clone's VMM ends without a word of its guest, as a crash of the VMM would end it. let script = "printf running > /srv/data/running; exec sleep 600"; let execution = ["--identity", "0:0", "--", "/bin/sh", "-c", script]; - let (sandbox, mut running) = template.begin_in("cl-rl", "run", &start, &execution); + let (sandbox, _, mut running) = template.begin_in("cl-rl", "run", &start, &execution); let marker = export.path("running"); assert_eq!(sandbox.paused(&mut running, &marker), "running"); let vmm = sandbox.vmm().expect("the clone's VMM runs");