From d70acadfe1dc9cc0738960e2bb9f19057178d0dd Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 13:05:02 -0800 Subject: [PATCH 01/51] feat(mailbox): stage durable local admission and bounded guest submission --- .../src/bin/mvm-guest-agent/mailbox.rs | 147 +++- crates/mvm-agentd/src/flowmux_drive.rs | 161 +++- crates/mvm-agentd/src/mailbox_outbox.rs | 346 +++++++- crates/mvm-core/src/mailbox.rs | 95 +- crates/mvm-core/src/mailbox/frame.rs | 177 +++- crates/mvm-core/src/mailbox/journal.rs | 763 ++++++++++++++++ crates/mvm-core/src/mailbox/transport.rs | 16 +- .../mvm-core/src/mailbox/transport/tests.rs | 830 +++++++++++++++++- crates/mvm-hostd/src/mailbox_launch.rs | 113 ++- crates/mvm-hostd/src/mailbox_process.rs | 126 ++- crates/mvm-hostd/src/mailbox_service.rs | 237 ++++- .../mvm-hostd/src/mailbox_service/durable.rs | 308 +++++++ crates/mvm-hostd/tests/mission_demo.rs | 439 ++++++++- .../src/workload_runner/runner/spawner.rs | 1 + crates/mvm-vmm/src/host/flowmux_identity.rs | 127 ++- .../057-mailbox-semantics-and-threat-model.md | 31 +- 16 files changed, 3779 insertions(+), 138 deletions(-) create mode 100644 crates/mvm-core/src/mailbox/journal.rs create mode 100644 crates/mvm-hostd/src/mailbox_service/durable.rs diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs index f72fa0963b..4f619fa0bc 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs @@ -6,17 +6,116 @@ //! signed plan that carries a mailbox grant. Otherwise no thread starts and no //! port is held. Vsock-only; the container tier has none. -use std::os::fd::{FromRawFd, RawFd}; +use std::io::{self, Read, Write}; +use std::os::fd::{AsRawFd, FromRawFd, RawFd}; use std::path::Path; use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; -use mvm_agentd::mailbox_outbox::{MAILBOX_CMDLINE_TOKEN, OUTBOX_PATH, forward_outbox}; +use mvm_agentd::mailbox_outbox::{MAILBOX_CMDLINE_TOKEN, OUTBOX_PATH, wait_and_forward_outbox}; +use mvm_agentd::telemetry_capture::session::TimedStream; use mvm_contract::protocol::mailbox::MAILBOX_PORT; +use mvm_core::mailbox::frame::Refusal; use crate::globals::SHUTDOWN_REQUESTED; use crate::telemetry::cmdline_asserts; use crate::transport::{accept_vsock, bind_vsock_listener, unix_transport_selected}; +const MAILBOX_IO_TIMEOUT: Duration = Duration::from_secs(12); + +struct MailboxDeadline<'a, S> { + stream: &'a mut S, + until: Instant, +} + +impl<'a, S> MailboxDeadline<'a, S> { + fn new(stream: &'a mut S) -> Self { + Self { + stream, + until: Instant::now() + MAILBOX_IO_TIMEOUT, + } + } + + fn remaining(&self) -> io::Result { + let remaining = self.until.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + Err(io::Error::new( + io::ErrorKind::TimedOut, + "mailbox I/O deadline", + )) + } else { + Ok(remaining) + } + } +} + +impl Read for MailboxDeadline<'_, S> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + loop { + let millis = self.remaining()?.as_millis().max(1); + let millis = libc::c_int::try_from(millis).unwrap_or(libc::c_int::MAX); + let fd = self.stream.as_raw_fd(); + let mut ready = libc::pollfd { + fd, + events: libc::POLLIN, + revents: 0, + }; + // SAFETY: one valid pollfd is borrowed from the live socket. + let polled = unsafe { libc::poll(&raw mut ready, 1, millis) }; + if polled == 0 { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "mailbox read timeout", + )); + } + if polled < 0 { + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + // SAFETY: `fd` is live and `buf` is writable for its exact length. + let received = + unsafe { libc::recv(fd, buf.as_mut_ptr().cast(), buf.len(), libc::MSG_DONTWAIT) }; + if received >= 0 { + return Ok(received as usize); + } + let error = io::Error::last_os_error(); + if matches!( + error.kind(), + io::ErrorKind::Interrupted | io::ErrorKind::WouldBlock + ) { + continue; + } + return Err(error); + } + } +} + +impl Write for MailboxDeadline<'_, S> { + fn write(&mut self, buf: &[u8]) -> io::Result { + loop { + let remaining = self.remaining()?; + match self.stream.write_now(buf) { + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + self.stream.wait_writable(remaining)?; + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + result => return result, + } + } + } + + fn flush(&mut self) -> io::Result<()> { + self.remaining()?; + self.stream.flush() + } +} + /// Spawn the mailbox listener on a boot that asserts one. Called after PID-1 /// activation; a bind failure is logged and abandoned, never boot-fatal. pub(crate) fn init_mailbox() { @@ -45,8 +144,18 @@ fn accept_loop(listener_fd: RawFd) { // SAFETY: `cfd` is the just-accepted connection fd, owned here and // closed on drop. let mut stream = unsafe { std::fs::File::from_raw_fd(cfd) }; - if let Err(refusal) = forward_outbox(&mut stream, Path::new(OUTBOX_PATH)) { - eprintln!("mvm-guest-agent: mailbox frame not delivered: {refusal}"); + if let Err(error) = stream.set_timeouts(MAILBOX_IO_TIMEOUT, MAILBOX_IO_TIMEOUT) { + eprintln!("mvm-guest-agent: mailbox connection has no bounded I/O: {error}"); + continue; + } + match wait_and_forward_outbox( + &mut MailboxDeadline::new(&mut stream), + Path::new(OUTBOX_PATH), + ) { + Ok(()) | Err(Refusal::Unavailable) => {} + Err(refusal) => { + eprintln!("mvm-guest-agent: mailbox frame not delivered: {refusal}") + } } } } @@ -65,4 +174,34 @@ mod tests { assert!(!cmdline_asserts("xmvm.mailbox=1", MAILBOX_CMDLINE_TOKEN)); assert!(!cmdline_asserts("console=ttyS0", MAILBOX_CMDLINE_TOKEN)); } + + #[test] + fn a_mailbox_deadline_refuses_more_io_after_it_expires() { + let (mut stream, _peer) = std::os::unix::net::UnixStream::pair().expect("socket"); + let mut expired = MailboxDeadline { + stream: &mut stream, + until: Instant::now() - Duration::from_millis(1), + }; + assert_eq!( + expired.read(&mut [0u8; 1]).unwrap_err().kind(), + io::ErrorKind::TimedOut + ); + assert_eq!( + expired.write(b"pending").unwrap_err().kind(), + io::ErrorKind::TimedOut + ); + } + + #[test] + fn a_silent_host_cannot_keep_the_guest_reader_blocked() { + let (mut stream, _silent_peer) = std::os::unix::net::UnixStream::pair().expect("socket"); + let mut bounded = MailboxDeadline { + stream: &mut stream, + until: Instant::now() + Duration::from_millis(25), + }; + assert_eq!( + bounded.read(&mut [0u8; 1]).unwrap_err().kind(), + io::ErrorKind::TimedOut + ); + } } diff --git a/crates/mvm-agentd/src/flowmux_drive.rs b/crates/mvm-agentd/src/flowmux_drive.rs index 4b4c594a1a..a7c6cfe8cc 100644 --- a/crates/mvm-agentd/src/flowmux_drive.rs +++ b/crates/mvm-agentd/src/flowmux_drive.rs @@ -29,6 +29,10 @@ pub const IDENTITY_DRIVE_LABEL: &str = "mvm-identity"; /// Basename of the guest's 32-byte signing key on the drive. pub const GUEST_SIGNING_KEY_FILE: &str = "flowmux-guest-signing-key"; +/// Optional per-boot mailbox ACK signing seed, independently enrolled before use. +/// The root-owned guest agent alone may read this; the egress client cannot. +pub const MAILBOX_ACK_SIGNING_KEY_FILE: &str = "mailbox-ack-signing-key"; + /// Basename of the host-signer trust anchor on the drive. pub const HOST_SIGNER_PUB_FILE: &str = "host-signer.pub"; @@ -122,6 +126,9 @@ pub fn install_egress_ca_trust(run_dir: &Path, baked_roots: &[&Path]) -> std::io /// Mode the signing key is written with inside the guest. pub const GUEST_SIGNING_KEY_MODE: u32 = 0o400; +/// The mailbox ACK key stays root-owned, never assigned to the egress service. +pub const MAILBOX_ACK_SIGNING_KEY_MODE: u32 = 0o400; + /// Mode the public anchor is written with inside the guest. pub const HOST_SIGNER_PUB_MODE: u32 = 0o444; @@ -292,7 +299,8 @@ mod linux { use super::{ EGRESS_CA_CERT_FILE, EGRESS_CA_CERT_MODE, GUEST_SIGNING_KEY_FILE, GUEST_SIGNING_KEY_MODE, HOST_SIGNER_PUB_FILE, HOST_SIGNER_PUB_MODE, IDENTITY_DRIVE_LABEL, INGRESS_TARGETS_FILE, - INGRESS_TARGETS_MODE, IdentityDriveError, RUN_MVM_DIR, find_labeled_ext4_disk_among, + INGRESS_TARGETS_MODE, IdentityDriveError, MAILBOX_ACK_SIGNING_KEY_FILE, + MAILBOX_ACK_SIGNING_KEY_MODE, RUN_MVM_DIR, find_labeled_ext4_disk_among, virtio_block_devices, }; use std::path::Path; @@ -318,8 +326,12 @@ mod linux { pub fn provision_identity_from_drive_for_uid( signing_key_uid: u32, ) -> Result<(), IdentityDriveError> { - let device = find_labeled_ext4_disk_among(virtio_block_devices(), IDENTITY_DRIVE_LABEL) - .ok_or(IdentityDriveError::NotAttached)?; + let Some(device) = + find_labeled_ext4_disk_among(virtio_block_devices(), IDENTITY_DRIVE_LABEL) + else { + clear_mailbox_ack_key(&Path::new(RUN_MVM_DIR).join(MAILBOX_ACK_SIGNING_KEY_FILE))?; + return Err(IdentityDriveError::NotAttached); + }; std::fs::create_dir_all(MOUNTPOINT).map_err(io_err("create the identity mountpoint"))?; mount_ro(&device, MOUNTPOINT)?; @@ -357,7 +369,80 @@ mod linux { EGRESS_CA_CERT_FILE, EGRESS_CA_CERT_MODE, "the per-VM egress CA certificate", - ) + )?; + install_mailbox_ack_key(Path::new(MOUNTPOINT), Path::new(RUN_MVM_DIR)) + } + + pub(super) fn install_mailbox_ack_key( + drive_dir: &Path, + run_dir: &Path, + ) -> Result<(), IdentityDriveError> { + let to = run_dir.join(MAILBOX_ACK_SIGNING_KEY_FILE); + // A restored guest must never keep a previous boot's key, and a + // symlink at the destination must not redirect secret bytes. + clear_mailbox_ack_key(&to)?; + let seed = read_optional_mailbox_ack_key(&drive_dir.join(MAILBOX_ACK_SIGNING_KEY_FILE))?; + if let Some(seed) = seed { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(MAILBOX_ACK_SIGNING_KEY_MODE) + .custom_flags(libc::O_NOFOLLOW) + .open(&to) + .map_err(io_err("create the root-only mailbox ACK key"))?; + file.write_all(&seed) + .map_err(io_err("write the mailbox ACK key to /run/mvm"))?; + } + Ok(()) + } + + fn clear_mailbox_ack_key(path: &Path) -> Result<(), IdentityDriveError> { + match std::fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(IdentityDriveError::Unreadable(format!( + "replace stale mailbox ACK key: {error}" + ))); + } + } + Ok(()) + } + + pub(super) fn read_optional_mailbox_ack_key( + path: &Path, + ) -> Result, IdentityDriveError> { + use std::io::Read as _; + use std::os::unix::fs::OpenOptionsExt as _; + let file = match std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(IdentityDriveError::Unreadable(format!( + "open mailbox ACK key: {error}" + ))); + } + }; + let metadata = file.metadata().map_err(io_err("inspect mailbox ACK key"))?; + if !metadata.file_type().is_file() || metadata.len() != 32 { + return Err(IdentityDriveError::Unreadable( + "mailbox ACK key must be a regular 32-byte file".into(), + )); + } + let mut bytes = Vec::new(); + file.take(33) + .read_to_end(&mut bytes) + .map_err(io_err("read mailbox ACK key from the drive"))?; + let seed = bytes.try_into().map_err(|_: Vec| { + IdentityDriveError::Unreadable("mailbox ACK key must be exactly 32 bytes".into()) + })?; + Ok(Some(seed)) } /// Copy a drive file that a boot may legitimately not carry. @@ -450,6 +535,74 @@ mod linux { mod tests { use super::*; + #[cfg(target_os = "linux")] + #[test] + fn mailbox_ack_key_is_optional_but_present_material_must_be_exactly_32_bytes() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join(MAILBOX_ACK_SIGNING_KEY_FILE); + assert!( + linux::read_optional_mailbox_ack_key(&path) + .unwrap() + .is_none() + ); + for size in [0, 31, 33, 1024 * 1024] { + std::fs::write(&path, vec![7u8; size]).expect("write bad key"); + assert!( + linux::read_optional_mailbox_ack_key(&path).is_err(), + "{size}" + ); + } + std::fs::write(&path, [7u8; 32]).expect("write key"); + assert_eq!( + linux::read_optional_mailbox_ack_key(&path).expect("read key"), + Some([7u8; 32]) + ); + std::fs::remove_file(&path).expect("remove key"); + std::fs::create_dir(&path).expect("unreadable key"); + assert!(linux::read_optional_mailbox_ack_key(&path).is_err()); + std::fs::remove_dir(&path).expect("remove directory"); + std::os::unix::fs::symlink(dir.path().join("missing"), &path).expect("symlink"); + assert!( + linux::read_optional_mailbox_ack_key(&path).is_err(), + "a dangling symlink is not an absent key" + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn mailbox_ack_key_is_root_only_and_absence_clears_a_stale_boot() { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + let drive = tempfile::tempdir().expect("drive"); + let run = tempfile::tempdir().expect("run"); + let output = run.path().join(MAILBOX_ACK_SIGNING_KEY_FILE); + let supplied = drive.path().join(MAILBOX_ACK_SIGNING_KEY_FILE); + let outside = run.path().join("outside"); + std::fs::write(&outside, b"untouched").expect("sentinel"); + std::os::unix::fs::symlink(&outside, &output).expect("stale symlink"); + std::fs::write(&supplied, [42; 32]).expect("drive key"); + linux::install_mailbox_ack_key(drive.path(), run.path()).expect("copy"); + let meta = std::fs::symlink_metadata(&output).expect("installed key"); + assert!(meta.file_type().is_file()); + assert_eq!( + meta.permissions().mode() & 0o777, + MAILBOX_ACK_SIGNING_KEY_MODE + ); + assert_eq!( + meta.uid(), + std::fs::metadata(run.path()).unwrap().uid(), + "the key belongs to the provisioning principal, root in the guest" + ); + assert_eq!(std::fs::read(&output).unwrap(), [42; 32]); + assert_eq!(std::fs::read(&outside).unwrap(), b"untouched"); + std::fs::remove_file(&supplied).expect("no key on next boot"); + linux::install_mailbox_ack_key(drive.path(), run.path()).expect("remove stale key"); + assert!(!output.exists()); + std::fs::write(&output, [9; 32]).expect("plant previous boot key"); + std::fs::write(&supplied, [7; 31]).expect("malformed next boot key"); + assert!(linux::install_mailbox_ack_key(drive.path(), run.path()).is_err()); + assert!(!output.exists(), "malformed new key must clear the old one"); + } + #[test] fn the_trust_bundle_is_the_baked_roots_plus_the_delivered_certificate() { let run = tempfile::tempdir().expect("tempdir"); diff --git a/crates/mvm-agentd/src/mailbox_outbox.rs b/crates/mvm-agentd/src/mailbox_outbox.rs index d8907e5569..833eeab53d 100644 --- a/crates/mvm-agentd/src/mailbox_outbox.rs +++ b/crates/mvm-agentd/src/mailbox_outbox.rs @@ -2,8 +2,8 @@ //! plaintext message the workload left in the outbox and return the host's //! verdict. //! -//! The guest holds no key and seals nothing. The outbox carries a recipient -//! and a body; the host seals and signs it under the capability the signed +//! The guest holds no key and seals nothing. The outbox carries a stable +//! message ID, recipient and body; the host seals and signs under the capability the signed //! plan granted, then admits it. Encryption is host to host. //! //! [`write_outbox`] is the workload side: it leaves that one message where @@ -11,11 +11,13 @@ //! and size limits the host decodes with. use std::io::{Read, Write}; -use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; use std::path::Path; +use std::time::Duration; use mvm_contract::protocol::mailbox::MAX_MAILBOX_PAYLOAD_BYTES; use mvm_core::mailbox::frame::{OutboxMessage, Refusal, send_outbox_message}; +use mvm_core::util::atomic_io::{atomic_write_new, sync_dir}; /// Where the workload leaves its message and the agent reads it. pub const OUTBOX_PATH: &str = "/run/mvm/mailbox/outbox"; @@ -31,20 +33,117 @@ pub const MAILBOX_CMDLINE_TOKEN: &str = "mvm.mailbox=1"; /// can list, read or replace anything in it. pub const OUTBOX_DIR_MODE: u32 = 0o700; -/// Owner read/write, group read. Nobody else may read the plaintext. -pub const OUTBOX_FILE_MODE: u32 = 0o640; +/// Owner-only read/write. Nobody else may read the plaintext. +pub const OUTBOX_FILE_MODE: u32 = 0o600; -/// The recipient bytes beside the largest body the host will seal. -const MAX_OUTBOX_BYTES: u64 = MAX_MAILBOX_PAYLOAD_BYTES as u64 + 32; +/// The recipient and stable message ID beside the largest body the host seals. +const MAX_OUTBOX_BYTES: u64 = MAX_MAILBOX_PAYLOAD_BYTES as u64 + 64; +const MAX_ARCHIVED_OUTBOX: usize = 64; +const IDLE_CHECKS: usize = 16; +const IDLE_INTERVAL: Duration = Duration::from_millis(250); -/// Forward the outbox message over `stream`. An absent, oversized or -/// malformed outbox is refused here, before a byte is sent to the host. +/// Wait at most four seconds for a later workload write on an already +/// connected mailbox port. The host's read deadline is longer than this +/// window; an idle window closes cleanly so teardown is always bounded. +pub fn wait_and_forward_outbox( + stream: &mut S, + outbox: &Path, +) -> Result<(), Refusal> { + for _ in 0..IDLE_CHECKS { + if outbox.exists() { + return forward_outbox(stream, outbox); + } + std::thread::sleep(IDLE_INTERVAL); + } + Err(Refusal::Unavailable) +} + +/// Forward one immutable outbox operation over `stream`. A refused or uncertain +/// send leaves it in place for a same-ID retry; only an accepted verdict removes +/// it under the writer lock. pub fn forward_outbox(stream: &mut S, outbox: &Path) -> Result<(), Refusal> { + let dir = outbox.parent().ok_or(Refusal::Malformed)?; + let _lock = lock_outbox_dir(dir)?; let mut bytes = Vec::new(); - std::fs::File::open(outbox) + open_outbox(outbox) .and_then(|file| file.take(MAX_OUTBOX_BYTES + 1).read_to_end(&mut bytes)) .map_err(|_| Refusal::Malformed)?; - send_outbox_message(stream, &OutboxMessage::decode(&bytes)?) + let message = OutboxMessage::decode(&bytes)?; + match send_outbox_message(stream, &message) { + Ok(()) => {} + Err(refusal @ (Refusal::RetriesExhausted | Refusal::Oversized(_))) => { + let prefix = if matches!(refusal, Refusal::Oversized(_)) { + "rejected" + } else { + "uncertain" + }; + archive_outbox(outbox, &message, prefix)?; + return Err(refusal); + } + Err(error) => return Err(error), + } + std::fs::remove_file(outbox).map_err(|_| Refusal::Unavailable)?; + sync_dir(dir).map_err(|_| Refusal::Unavailable) +} + +fn archive_outbox(outbox: &Path, message: &OutboxMessage, prefix: &str) -> Result<(), Refusal> { + let dir = outbox.parent().ok_or(Refusal::Malformed)?; + let id = message + .message_id + .as_bytes() + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + let archived = dir.join(format!("{prefix}-{id}")); + let bytes = message.encode(); + let mut previous = Vec::new(); + match open_outbox(&archived) + .and_then(|file| file.take(MAX_OUTBOX_BYTES + 1).read_to_end(&mut previous)) + { + Ok(_) if previous != bytes => return Err(Refusal::Malformed), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let mut count = 0; + for entry in std::fs::read_dir(dir).map_err(|_| Refusal::Unavailable)? { + let entry = entry.map_err(|_| Refusal::Unavailable)?; + if entry.file_name() != "outbox" { + count += 1; + if count >= MAX_ARCHIVED_OUTBOX { + return Err(Refusal::QueueFull(MAX_ARCHIVED_OUTBOX)); + } + } + } + atomic_write_new(&archived, &bytes).map_err(|_| Refusal::Unavailable)?; + } + Err(_) => return Err(Refusal::Unavailable), + } + std::fs::remove_file(outbox).map_err(|_| Refusal::Unavailable)?; + sync_dir(dir).map_err(|_| Refusal::Unavailable) +} + +fn lock_outbox_dir(dir: &Path) -> Result { + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(dir) + .map_err(|_| Refusal::Unavailable)?; + file.try_lock().map_err(|_| Refusal::Unavailable)?; + Ok(file) +} + +fn open_outbox(outbox: &Path) -> std::io::Result { + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(outbox)?; + let metadata = file.metadata()?; + if !metadata.is_file() || metadata.nlink() != 1 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "outbox must be a single-link regular file", + )); + } + Ok(file) } /// Create the outbox directory, owned by `uid`:`gid` at [`OUTBOX_DIR_MODE`]. @@ -78,11 +177,16 @@ pub fn provision_outbox_dir_if_asserted( provision_outbox_dir(dir, uid, gid).map(|()| true) } -/// Workload side: atomically leave `message` at `outbox`. The directory must -/// already exist, since it is how the launch grants a mailbox; without it this -/// refuses and creates nothing. Written to a sibling temp file, then renamed, -/// so the agent never reads a partial frame. +/// Workload side: atomically publish one immutable message. A second writer +/// cannot replace a pending operation; retrying identical bytes is idempotent. +/// The directory must already exist as the launch grant. pub fn write_outbox(outbox: &Path, message: &OutboxMessage) -> Result<(), Refusal> { + if message.body.is_empty() { + return Err(Refusal::Malformed); + } + if message.body.len() > MAX_MAILBOX_PAYLOAD_BYTES { + return Err(Refusal::Oversized(MAX_MAILBOX_PAYLOAD_BYTES)); + } let bytes = message.encode(); // Same validation the host applies on receipt (empty, oversized). OutboxMessage::decode(&bytes)?; @@ -90,29 +194,30 @@ pub fn write_outbox(outbox: &Path, message: &OutboxMessage) -> Result<(), Refusa .parent() .filter(|d| d.is_dir()) .ok_or(Refusal::Malformed)?; - let tmp = dir.join(format!(".outbox.{}.tmp", std::process::id())); - let written = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(OUTBOX_FILE_MODE) - .open(&tmp) - .and_then(|mut f| f.write_all(&bytes)) - .and_then(|()| std::fs::rename(&tmp, outbox)); - if written.is_err() { - let _ = std::fs::remove_file(&tmp); - return Err(Refusal::Malformed); + let _lock = lock_outbox_dir(dir)?; + let mut existing = Vec::new(); + match open_outbox(outbox) + .and_then(|file| file.take(MAX_OUTBOX_BYTES + 1).read_to_end(&mut existing)) + { + Ok(_) if existing.len() as u64 > MAX_OUTBOX_BYTES => return Err(Refusal::Malformed), + Ok(_) if existing == bytes => return Ok(()), + Ok(_) => return Err(Refusal::QueueFull(1)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err(Refusal::Unavailable), } - Ok(()) + atomic_write_new(outbox, &bytes).map_err(|_| Refusal::Unavailable)?; + sync_dir(dir).map_err(|_| Refusal::Unavailable) } #[cfg(test)] mod tests { use super::*; - use mvm_contract::protocol::mailbox::PrincipalId; + use mvm_contract::protocol::mailbox::{MessageId, PrincipalId}; fn message(body: &[u8]) -> OutboxMessage { OutboxMessage { recipient: PrincipalId::from_bytes([7; 32]), + message_id: MessageId::from_bytes([8; 32]), body: body.to_vec(), } } @@ -137,10 +242,39 @@ mod tests { OutboxMessage::decode(&host.join().unwrap()), Ok(message(b"hello")) ); + assert!( + !outbox.exists(), + "accepted submission frees the single slot" + ); + } + + #[test] + fn a_connected_guest_delivers_a_message_written_after_the_host_dials() { + let dir = tempfile::tempdir().unwrap(); + let outbox = dir.path().join("outbox"); + let (mut ours, mut theirs) = std::os::unix::net::UnixStream::pair().unwrap(); + let host = std::thread::spawn(move || { + let mut len = [0u8; 4]; + theirs.read_exact(&mut len).unwrap(); + let mut body = vec![0u8; u32::from_be_bytes(len) as usize]; + theirs.read_exact(&mut body).unwrap(); + theirs.write_all(b"{\"Ok\":null}").unwrap(); + OutboxMessage::decode(&body).unwrap() + }); + let written = message(b"later"); + let path = outbox.clone(); + let workload = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(100)); + write_outbox(&path, &written).unwrap(); + }); + wait_and_forward_outbox(&mut ours, &outbox).expect("late write was forwarded"); + workload.join().unwrap(); + assert_eq!(host.join().unwrap(), message(b"later")); + assert!(!outbox.exists()); } #[test] - fn the_outbox_file_is_group_readable_never_world_readable_and_leaves_no_temp() { + fn the_outbox_file_is_owner_only_and_leaves_no_temp() { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); let outbox = dir.path().join("outbox"); @@ -196,13 +330,17 @@ mod tests { } #[test] - fn a_second_write_replaces_the_first_atomically() { + fn a_pending_outbox_cannot_be_overwritten_but_an_identical_retry_is_safe() { let dir = tempfile::tempdir().unwrap(); let outbox = dir.path().join("outbox"); write_outbox(&outbox, &message(b"one")).unwrap(); - write_outbox(&outbox, &message(b"two")).unwrap(); + write_outbox(&outbox, &message(b"one")).unwrap(); + assert_eq!( + write_outbox(&outbox, &message(b"two")), + Err(Refusal::QueueFull(1)) + ); let on_disk = std::fs::read(&outbox).unwrap(); - assert_eq!(OutboxMessage::decode(&on_disk), Ok(message(b"two"))); + assert_eq!(OutboxMessage::decode(&on_disk), Ok(message(b"one"))); } #[test] @@ -252,6 +390,148 @@ mod tests { assert!(seen.is_empty(), "nothing reached the host"); } + #[test] + fn an_unknown_verdict_keeps_the_same_operation_for_a_later_retry() { + let dir = tempfile::tempdir().unwrap(); + let outbox = dir.path().join("outbox"); + write_outbox(&outbox, &message(b"once")).unwrap(); + let original = std::fs::read(&outbox).unwrap(); + let (mut ours, mut theirs) = std::os::unix::net::UnixStream::pair().unwrap(); + let host = std::thread::spawn(move || { + let mut len = [0u8; 4]; + theirs.read_exact(&mut len).unwrap(); + let mut body = vec![0; u32::from_be_bytes(len) as usize]; + theirs.read_exact(&mut body).unwrap(); + body + }); + assert_eq!(forward_outbox(&mut ours, &outbox), Err(Refusal::Malformed)); + assert_eq!(host.join().unwrap(), original); + assert_eq!(std::fs::read(&outbox).unwrap(), original); + write_outbox(&outbox, &message(b"once")).expect("same-ID retry stays valid"); + } + + #[test] + fn exhausted_retries_preserve_an_uncertain_operation_without_blocking_new_sends() { + let dir = tempfile::tempdir().unwrap(); + let outbox = dir.path().join("outbox"); + let pending = message(b"once"); + write_outbox(&outbox, &pending).unwrap(); + let (mut ours, mut theirs) = std::os::unix::net::UnixStream::pair().unwrap(); + let host = std::thread::spawn(move || { + let mut len = [0u8; 4]; + theirs.read_exact(&mut len).unwrap(); + let mut body = vec![0; u32::from_be_bytes(len) as usize]; + theirs.read_exact(&mut body).unwrap(); + theirs.write_all(b"{\"Err\":\"RetriesExhausted\"}").unwrap(); + }); + assert_eq!( + forward_outbox(&mut ours, &outbox), + Err(Refusal::RetriesExhausted) + ); + host.join().unwrap(); + assert!( + !outbox.exists(), + "the next operation can use the single slot" + ); + let uncertain = std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().path()) + .next() + .expect("an uncertain operation is retained"); + assert_eq!( + OutboxMessage::decode(&std::fs::read(uncertain).unwrap()), + Ok(pending) + ); + write_outbox(&outbox, &message(b"next")).unwrap(); + } + + #[test] + fn a_signed_capability_payload_refusal_archives_the_exact_rejected_operation() { + let dir = tempfile::tempdir().unwrap(); + let outbox = dir.path().join("outbox"); + let pending = message(b"too large for this grant"); + write_outbox(&outbox, &pending).unwrap(); + let (mut ours, mut theirs) = std::os::unix::net::UnixStream::pair().unwrap(); + let host = std::thread::spawn(move || { + let mut len = [0u8; 4]; + theirs.read_exact(&mut len).unwrap(); + let mut body = vec![0; u32::from_be_bytes(len) as usize]; + theirs.read_exact(&mut body).unwrap(); + theirs.write_all(b"{\"Err\":{\"Oversized\":16}}").unwrap(); + }); + assert_eq!( + forward_outbox(&mut ours, &outbox), + Err(Refusal::Oversized(16)) + ); + host.join().unwrap(); + assert!(!outbox.exists()); + let rejected = std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().path()) + .next() + .expect("retain terminal refusal"); + assert!( + rejected + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("rejected-"), + "a known terminal refusal is not an uncertain delivery" + ); + assert_eq!( + OutboxMessage::decode(&std::fs::read(rejected).unwrap()), + Ok(pending) + ); + } + + #[test] + fn archive_exhaustion_retains_pending_bytes_and_refuses_more_disk_use() { + let dir = tempfile::tempdir().unwrap(); + let outbox = dir.path().join("outbox"); + let pending = message(b"once"); + write_outbox(&outbox, &pending).unwrap(); + for i in 0..MAX_ARCHIVED_OUTBOX { + std::fs::write(dir.path().join(format!("uncertain-{i}")), b"old").unwrap(); + } + let (mut ours, mut theirs) = std::os::unix::net::UnixStream::pair().unwrap(); + let host = std::thread::spawn(move || { + let mut len = [0u8; 4]; + theirs.read_exact(&mut len).unwrap(); + let mut body = vec![0; u32::from_be_bytes(len) as usize]; + theirs.read_exact(&mut body).unwrap(); + theirs.write_all(b"{\"Err\":\"RetriesExhausted\"}").unwrap(); + }); + assert_eq!( + forward_outbox(&mut ours, &outbox), + Err(Refusal::QueueFull(MAX_ARCHIVED_OUTBOX)) + ); + host.join().unwrap(); + assert_eq!( + OutboxMessage::decode(&std::fs::read(&outbox).unwrap()), + Ok(pending) + ); + } + + #[test] + fn an_outbox_symlink_is_never_followed_to_a_different_file() { + let dir = tempfile::tempdir().unwrap(); + let secret = dir.path().join("other"); + let outbox = dir.path().join("outbox"); + std::fs::write(&secret, message(b"secret").encode()).unwrap(); + std::os::unix::fs::symlink(&secret, &outbox).unwrap(); + let (mut ours, mut theirs) = std::os::unix::net::UnixStream::pair().unwrap(); + assert_eq!(forward_outbox(&mut ours, &outbox), Err(Refusal::Malformed)); + assert_eq!( + write_outbox(&outbox, &message(b"overwrite")), + Err(Refusal::Unavailable) + ); + drop(ours); + let mut seen = Vec::new(); + theirs.read_to_end(&mut seen).unwrap(); + assert!(seen.is_empty()); + assert_eq!(std::fs::read(secret).unwrap(), message(b"secret").encode()); + } + /// The wire carries the plaintext message and nothing else: no /// capability, envelope, signature or key, and this path names no key /// type to build one from. @@ -259,7 +539,7 @@ mod tests { fn the_frame_on_the_wire_is_exactly_the_plaintext_message() { let dir = tempfile::tempdir().unwrap(); let outbox = dir.path().join("outbox"); - let message = [&[9u8; 32][..], b"hello"].concat(); + let message = [&[9u8; 32][..], &[10u8; 32][..], b"hello"].concat(); std::fs::write(&outbox, &message).unwrap(); let (mut ours, mut theirs) = std::os::unix::net::UnixStream::pair().unwrap(); let host = std::thread::spawn(move || { diff --git a/crates/mvm-core/src/mailbox.rs b/crates/mvm-core/src/mailbox.rs index 34b126da03..f2d12b55f8 100644 --- a/crates/mvm-core/src/mailbox.rs +++ b/crates/mvm-core/src/mailbox.rs @@ -15,7 +15,7 @@ use mvm_contract::protocol::mailbox::{ MailboxOperation, MailboxRoutingFrame, MessageId, PrincipalId, ProtocolVersion, RevocationId, SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxMessage, TenantId, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::crypto::aead::{self, Key}; @@ -28,6 +28,9 @@ const ENVELOPE_AEAD_DOMAIN: &[u8] = b"mvm.mailbox.payload.v1\0"; /// The one-frame exchange the guest and the host mailbox service share. pub mod frame; +/// Bounded destination-side ciphertext journal and durable acceptance. +pub mod journal; + /// Overlay resolution and authenticated, bounded byte-transport seams. pub mod transport; @@ -58,7 +61,7 @@ pub struct OpenContext { /// Replay identity is scoped to the authenticated sender boot and execution /// epoch. The durable implementation may retain the same tuple in a journal. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub struct ReplayIdentity { sender: PrincipalId, sender_boot: BootId, @@ -402,12 +405,13 @@ fn verify_capability_scope( header: &MailboxEnvelopeHeader, capability: &SignedMailboxCapability, context: &OpenContext, + check_send_reference: bool, ) -> Result<(), MailboxError> { let grant = &capability.grant; if context.revoked_capabilities.contains(&grant.revocation_id) { return Err(MailboxError::CapabilityRevoked); } - if grant.capability_id != header.capability_id { + if check_send_reference && grant.capability_id != header.capability_id { return Err(MailboxError::WrongCapability); } if grant.issuer != context.expected_capability_issuer { @@ -457,6 +461,43 @@ fn verify_capability_scope( Ok(()) } +/// A message whose signed inner header, capability and replay identity passed +/// the destination's authority checks. The header can be journaled with the +/// unchanged ciphertext without parsing untrusted routing hints as identity. +#[derive(PartialEq, Eq)] +pub struct OpenedMailboxMessage { + header: MailboxEnvelopeHeader, + payload: Vec, +} + +impl std::fmt::Debug for OpenedMailboxMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpenedMailboxMessage") + .field("payload_len", &self.payload.len()) + .finish_non_exhaustive() + } +} + +impl OpenedMailboxMessage { + /// The authenticated inner header used to bind journal and dedup records. + #[must_use] + pub fn header(&self) -> &MailboxEnvelopeHeader { + &self.header + } + + /// Plaintext for the authorized recipient, not for a router or store. + #[must_use] + pub fn payload(&self) -> &[u8] { + &self.payload + } + + /// Consume the opened message to hand plaintext to its authorized reader. + #[must_use] + pub fn into_payload(self) -> Vec { + self.payload + } +} + /// Verify identities, signature, capability, current state, ciphertext, and /// replay identity before returning plaintext. pub fn open_envelope( @@ -468,6 +509,30 @@ pub fn open_envelope( authority_key: &VerifyingKey, replay: &mut impl ReplayProtector, ) -> Result, MailboxError> { + open_authenticated_envelope( + envelope, + capability, + context, + encryption_key, + sender_key, + authority_key, + replay, + ) + .map(OpenedMailboxMessage::into_payload) +} + +/// Open the envelope and retain the verified inner header for destination +/// admission. `replay` must stage its identity until any durable commit is +/// complete; an in-memory protector is appropriate only for ephemeral delivery. +pub fn open_authenticated_envelope( + envelope: &MailboxEnvelope, + capability: &SignedMailboxCapability, + context: &OpenContext, + encryption_key: &Key, + sender_key: &VerifyingKey, + authority_key: &VerifyingKey, + replay: &mut impl ReplayProtector, +) -> Result { envelope.validate_bounds()?; validate_outer_versions(envelope)?; let actual_commitment: [u8; 32] = Sha256::digest(&envelope.ciphertext).into(); @@ -504,12 +569,32 @@ pub fn open_envelope( verify_identity_and_time(&message.header, context)?; verify_capability_signature(capability, authority_key)?; - verify_capability_scope(&message.header, capability, context)?; + verify_capability_scope(&message.header, capability, context, true)?; if !replay.check_and_record(ReplayIdentity::from_header(&message.header)) { return Err(MailboxError::Replay); } - Ok(message.payload) + Ok(OpenedMailboxMessage { + header: message.header, + payload: message.payload, + }) +} + +/// Authorize a separate Receive capability for a verified signed message. +/// The header's capability ID names its original Send grant, not the reader's +/// independently issued Receive grant. +pub fn authorize_receive( + header: &MailboxEnvelopeHeader, + capability: &SignedMailboxCapability, + context: &OpenContext, + authority_key: &VerifyingKey, +) -> Result<(), MailboxError> { + if context.required_operation != MailboxOperation::Receive { + return Err(MailboxError::OperationNotAuthorized); + } + verify_identity_and_time(header, context)?; + verify_capability_signature(capability, authority_key)?; + verify_capability_scope(header, capability, context, false) } /// Sign a destination receipt in a domain distinct from envelopes and grants. diff --git a/crates/mvm-core/src/mailbox/frame.rs b/crates/mvm-core/src/mailbox/frame.rs index f28fd7cef4..f22a6f05d5 100644 --- a/crates/mvm-core/src/mailbox/frame.rs +++ b/crates/mvm-core/src/mailbox/frame.rs @@ -10,12 +10,19 @@ use std::io::{Read, Write}; +use ed25519_dalek::VerifyingKey; use mvm_contract::protocol::mailbox::{ - MAX_MAILBOX_PAYLOAD_BYTES, MailboxEnvelope, PrincipalId, SignedMailboxCapability, + DeliveryOutcome, MAX_MAILBOX_PAYLOAD_BYTES, MailboxAddress, MailboxEnvelope, + MailboxEnvelopeHeader, MessageId, PrincipalId, SignedDestinationReceipt, + SignedMailboxCapability, }; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use super::{decode_envelope, encode_envelope}; +use super::{decode_envelope, encode_envelope, verify_receipt}; + +// A verdict contains no payload. Bound the response even when a peer never +// closes or sends arbitrarily large error text. +const MAX_VERDICT_BYTES: u64 = 4 * 1024; /// Why a frame was not queued. Sent back to the peer verbatim. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)] @@ -24,8 +31,14 @@ pub enum Refusal { Oversized(usize), #[error("mailbox queue is full ({0} messages)")] QueueFull(usize), + #[error("durable mailbox storage quota is full")] + StorageFull, + #[error("mailbox retry budget exhausted; reconcile the stable message ID")] + RetriesExhausted, #[error("frame is malformed")] Malformed, + #[error("destination is unavailable")] + Unavailable, /// The mailbox gate refused it; carries the gate's own reason text. #[error("{0}")] Mailbox(String), @@ -55,23 +68,50 @@ pub fn decode_frame(body: &[u8]) -> Result<(SignedMailboxCapability, MailboxEnve )) } -/// What a guest asks its host to send: a recipient and a plaintext body. +/// What a guest asks its host to send: an immutable operation identity, +/// recipient, and plaintext body. /// Nothing here is sealed or signed; the host does that under keys the guest -/// never holds. Wire form: the 32 recipient bytes, then the body. -#[derive(Debug, Clone, PartialEq, Eq)] +/// never holds. Wire form: 32 recipient bytes, 32 message-ID bytes, then body. +#[derive(Clone, PartialEq, Eq)] pub struct OutboxMessage { pub recipient: PrincipalId, + pub message_id: MessageId, pub body: Vec, } +impl std::fmt::Debug for OutboxMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OutboxMessage") + .field("body_len", &self.body.len()) + .finish_non_exhaustive() + } +} + impl OutboxMessage { + /// Mint a fresh operation identity once, before the first submission; + /// retry by retaining this value rather than constructing another one. + #[must_use] + pub fn new(recipient: PrincipalId, body: Vec) -> Self { + Self { + recipient, + message_id: MessageId::from_bytes(rand::random()), + body, + } + } + pub fn encode(&self) -> Vec { - [self.recipient.as_bytes().as_slice(), &self.body].concat() + [ + self.recipient.as_bytes().as_slice(), + self.message_id.as_bytes().as_slice(), + &self.body, + ] + .concat() } /// Refuses a short, oversized or empty-bodied message. pub fn decode(bytes: &[u8]) -> Result { - let (recipient, body) = bytes.split_at_checked(32).ok_or(Refusal::Malformed)?; + let (recipient, rest) = bytes.split_at_checked(32).ok_or(Refusal::Malformed)?; + let (message_id, body) = rest.split_at_checked(32).ok_or(Refusal::Malformed)?; if body.is_empty() { return Err(Refusal::Malformed); } @@ -79,8 +119,10 @@ impl OutboxMessage { return Err(Refusal::Oversized(MAX_MAILBOX_PAYLOAD_BYTES)); } let recipient = <[u8; 32]>::try_from(recipient).map_err(|_| Refusal::Malformed)?; + let message_id = <[u8; 32]>::try_from(message_id).map_err(|_| Refusal::Malformed)?; Ok(Self { recipient: PrincipalId::from_bytes(recipient), + message_id: MessageId::from_bytes(message_id), body: body.to_vec(), }) } @@ -96,15 +138,56 @@ pub fn send_frame( exchange(stream, &encode_frame(capability, envelope)?) } +/// A durable destination verdict is useful only after verifying its signer +/// and binding to the exact canonical envelope sent on this connection. +pub fn send_receipted_frame( + stream: &mut S, + capability: &SignedMailboxCapability, + envelope: &MailboxEnvelope, + expected_header: &MailboxEnvelopeHeader, + destination_key: &VerifyingKey, +) -> Result { + let receipt: SignedDestinationReceipt = exchange(stream, &encode_frame(capability, envelope)?)?; + verify_receipt(&receipt, destination_key).map_err(|_| Refusal::Malformed)?; + let address = MailboxAddress { + tenant: expected_header.tenant, + authorization_domain: expected_header.authorization_domain, + mailbox: expected_header.mailbox, + recipient: expected_header.recipient, + }; + if receipt.receipt.delivery_id != envelope.routing.delivery_id + || receipt.receipt.ciphertext_commitment != envelope.ciphertext_commitment + || receipt.receipt.message_id != expected_header.message_id + || receipt.receipt.address != address + || receipt.receipt.execution_epoch != expected_header.execution_epoch + || !matches!( + receipt.receipt.outcome, + DeliveryOutcome::AcceptedDurable | DeliveryOutcome::Duplicate + ) + { + return Err(Refusal::Malformed); + } + Ok(receipt) +} + /// Guest side: write one plaintext message and read the host's verdict. pub fn send_outbox_message( stream: &mut S, message: &OutboxMessage, ) -> Result<(), Refusal> { + if message.body.is_empty() { + return Err(Refusal::Malformed); + } + if message.body.len() > MAX_MAILBOX_PAYLOAD_BYTES { + return Err(Refusal::Oversized(MAX_MAILBOX_PAYLOAD_BYTES)); + } exchange(stream, &message.encode()) } -fn exchange(stream: &mut S, body: &[u8]) -> Result<(), Refusal> { +fn exchange( + stream: &mut S, + body: &[u8], +) -> Result { // A service that refuses on length alone closes early; the verdict is // still readable, so a failed write is not itself the answer. let _ = stream @@ -112,9 +195,13 @@ fn exchange(stream: &mut S, body: &[u8]) -> Result<(), Refusal> .and_then(|()| stream.write_all(body)); let mut reply = Vec::new(); stream + .take(MAX_VERDICT_BYTES + 1) .read_to_end(&mut reply) .map_err(|_| Refusal::Malformed)?; - serde_json::from_slice::>(&reply).map_err(|_| Refusal::Malformed)? + if reply.len() as u64 > MAX_VERDICT_BYTES { + return Err(Refusal::Oversized(MAX_VERDICT_BYTES as usize)); + } + serde_json::from_slice::>(&reply).map_err(|_| Refusal::Malformed)? } #[cfg(test)] @@ -125,15 +212,79 @@ mod tests { fn an_outbox_message_round_trips_and_malformed_input_is_refused() { let message = OutboxMessage { recipient: PrincipalId::from_bytes([3; 32]), + message_id: MessageId::from_bytes([4; 32]), body: b"hello".to_vec(), }; assert_eq!(OutboxMessage::decode(&message.encode()), Ok(message)); - assert_eq!(OutboxMessage::decode(&[0; 31]), Err(Refusal::Malformed)); - assert_eq!(OutboxMessage::decode(&[0; 32]), Err(Refusal::Malformed)); - let big = vec![1; 32 + MAX_MAILBOX_PAYLOAD_BYTES + 1]; + assert_eq!(OutboxMessage::decode(&[0; 63]), Err(Refusal::Malformed)); + assert_eq!(OutboxMessage::decode(&[0; 64]), Err(Refusal::Malformed)); + let big = vec![1; 64 + MAX_MAILBOX_PAYLOAD_BYTES + 1]; assert_eq!( OutboxMessage::decode(&big), Err(Refusal::Oversized(MAX_MAILBOX_PAYLOAD_BYTES)) ); + let first = OutboxMessage::new(PrincipalId::from_bytes([3; 32]), b"hello".to_vec()); + let second = OutboxMessage::new(PrincipalId::from_bytes([3; 32]), b"hello".to_vec()); + assert_ne!( + first.message_id, second.message_id, + "each new send has a new identity" + ); + assert_eq!( + first.message_id, + first.clone().message_id, + "retry retains its identity" + ); + } + + #[test] + fn a_peer_cannot_make_the_verdict_reader_grow_without_bound() { + struct Peer(std::io::Cursor>); + impl Read for Peer { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.read(buf) + } + } + impl Write for Peer { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let mut peer = Peer(std::io::Cursor::new(vec![ + b'x'; + MAX_VERDICT_BYTES as usize + 2 + ])); + assert_eq!( + send_outbox_message( + &mut peer, + &OutboxMessage { + recipient: PrincipalId::from_bytes([3; 32]), + message_id: MessageId::from_bytes([4; 32]), + body: b"hello".to_vec(), + } + ), + Err(Refusal::Oversized(MAX_VERDICT_BYTES as usize)) + ); + assert_eq!(peer.0.position(), MAX_VERDICT_BYTES + 1); + } + + #[test] + fn an_oversized_outbox_is_refused_before_a_peer_sees_a_frame() { + let mut peer = std::io::Cursor::new(Vec::new()); + assert_eq!( + send_outbox_message( + &mut peer, + &OutboxMessage { + recipient: PrincipalId::from_bytes([3; 32]), + message_id: MessageId::from_bytes([4; 32]), + body: vec![0; MAX_MAILBOX_PAYLOAD_BYTES + 1], + } + ), + Err(Refusal::Oversized(MAX_MAILBOX_PAYLOAD_BYTES)) + ); + assert!(peer.into_inner().is_empty()); } } diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs new file mode 100644 index 0000000000..c82cda71ae --- /dev/null +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -0,0 +1,763 @@ +//! Destination-owned, bounded ciphertext admission journal. +//! +//! Each immutable entry is published with a no-clobber rename and a directory +//! sync. The signed acceptance receipt and dedup identity live in the same +//! authenticated record as the canonical envelope. A caller must retain the +//! returned tip in an independent trusted anchor to detect whole-journal +//! rollback; a hash chain alone cannot detect deletion of its suffix. + +use std::collections::BTreeMap; +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; +use ed25519_dalek::{SigningKey, VerifyingKey}; +use hmac::{Hmac, Mac, digest::KeyInit}; +use mvm_contract::protocol::mailbox::{ + DeliveryOutcome, DestinationReceipt, MailboxAddress, MailboxEnvelope, MailboxOperation, + ProtocolVersion, SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, +}; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use zeroize::Zeroizing; + +use super::{ + MailboxError, OpenContext, ReplayIdentity, ReplayProtector, authorize_receive, frame, + open_authenticated_envelope, sign_receipt, verify_receipt, +}; +use crate::crypto::aead::{self, Key}; +use crate::private_fs; +use crate::util::atomic_io::{FileLock, atomic_write_durable, atomic_write_new, sync_dir}; + +const DOMAIN: &[u8] = b"mvm.mailbox.journal.v1\0"; +const ENCRYPTION_DOMAIN: &[u8] = b"mvm.mailbox.journal.encryption.v1\0"; +const ANCHOR_DOMAIN: &[u8] = b"mvm.mailbox.anchor.v1\0"; +const MAX_ANCHOR_BYTES: u64 = 1024; +const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; +const MAX_RECORDS: usize = 100_000; +const MAX_JOURNAL_BYTES: u64 = 1024 * 1024 * 1024; + +/// Limits are checked both before admission and while reconstructing state. +#[derive(Debug, Clone, Copy)] +pub struct JournalLimits { + pub max_records: usize, + pub max_bytes: u64, +} + +/// Trusted external high-water mark. Persist this outside the journal's +/// rollback domain before exposing the receipt to the sender. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct JournalTip { + pub cursor: u64, + pub head: [u8; 32], +} + +impl JournalTip { + #[must_use] + pub const fn genesis() -> Self { + Self { + cursor: 0, + head: [0; 32], + } + } +} + +/// Fail-closed admission and recovery failures; no plaintext is included. +#[derive(Debug, thiserror::Error)] +pub enum JournalError { + #[error("invalid journal limits")] + InvalidLimits, + #[error("journal writer is already active")] + Busy, + #[error("journal quota exhausted")] + Full, + #[error("journal record or chain is invalid")] + Corrupt, + #[error("journal high-water mark does not match its trusted anchor")] + Rollback, + #[error("journal address or destination authority does not match")] + WrongAuthority, + #[error("a message identity was reused with different bytes")] + ConflictingRetry, + #[error("mailbox message does not exist")] + UnknownMessage, + #[error("journal cannot be used after an uncertain write")] + Uncertain, + #[error("mailbox authorization refused: {0}")] + Mailbox(#[from] MailboxError), + #[error("journal storage is unavailable: {0}")] + Storage(#[from] std::io::Error), +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Record { + version: u8, + cursor: u64, + previous: [u8; 32], + address: MailboxAddress, + event: JournalEvent, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum JournalEvent { + Accepted { + identity: ReplayIdentity, + frame: String, + receipt: Box, + }, + Acknowledged { + identity: ReplayIdentity, + acknowledged_at_unix_ms: i64, + }, +} + +struct Seen { + envelope: MailboxEnvelope, + capability: SignedMailboxCapability, + acknowledged: bool, +} + +#[derive(Default)] +struct Probe(Option); + +impl ReplayProtector for Probe { + fn check_and_record(&mut self, identity: ReplayIdentity) -> bool { + self.0 = Some(identity); + true // Only the durable journal records it after the fsync. + } +} + +/// An exclusive writer. Records contain only canonical ciphertext, capability +/// and signed metadata; plaintext never enters the journal. +pub struct MailboxJournal { + root: PathBuf, + address: MailboxAddress, + authority: VerifyingKey, + key: Zeroizing<[u8; 32]>, + limits: JournalLimits, + tip: JournalTip, + bytes: u64, + seen: BTreeMap, + order: Vec, + poisoned: bool, + _lock: FileLock, +} + +/// Inputs verified at the destination before ciphertext is committed. +pub struct JournalAdmission<'a> { + pub envelope: &'a MailboxEnvelope, + pub capability: &'a SignedMailboxCapability, + pub context: &'a OpenContext, + pub encryption_key: &'a Key, + pub sender_key: &'a VerifyingKey, + pub signer: &'a SigningKey, +} + +/// Independent reader authority and host-held decryption material for the +/// trusted-host profile. A sender's grant cannot substitute for Receive. +pub struct JournalReceive<'a> { + pub capability: &'a SignedMailboxCapability, + pub context: &'a OpenContext, + pub encryption_key: &'a Key, + pub sender_key: &'a VerifyingKey, +} + +/// A pending, independently authorized message. Debug omits its plaintext. +pub struct ReceivedMailboxMessage { + identity: ReplayIdentity, + opened: super::OpenedMailboxMessage, +} + +impl std::fmt::Debug for ReceivedMailboxMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReceivedMailboxMessage") + .field("identity", &self.identity) + .field("payload_len", &self.opened.payload().len()) + .finish_non_exhaustive() + } +} + +impl ReceivedMailboxMessage { + #[must_use] + pub const fn identity(&self) -> ReplayIdentity { + self.identity + } + + #[must_use] + pub fn payload(&self) -> &[u8] { + self.opened.payload() + } +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AnchorBody { + version: u8, + address: MailboxAddress, + tip: JournalTip, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AnchorRecord { + body: AnchorBody, + tag: [u8; 32], +} + +/// A software-first, locally fsynced anchor. Keep its file outside the +/// journal directory. It detects journal-only rollback, not simultaneous +/// rollback of the entire trusted host and its anchor storage. +pub struct FileJournalAnchor { + path: PathBuf, + address: MailboxAddress, + key: Zeroizing<[u8; 32]>, + tip: Mutex, + _lock: FileLock, +} + +impl FileJournalAnchor { + /// Open a private anchor file, or genesis if none exists. + pub fn open(path: &Path, address: MailboxAddress, key: [u8; 32]) -> Result { + let parent = path.parent().ok_or(JournalError::Corrupt)?; + private_fs::ensure_private_dir(parent)?; + let lock = FileLock::try_acquire(path) + .map_err(|e| std::io::Error::other(e.to_string()))? + .ok_or(JournalError::Busy)?; + let tip = match fs::symlink_metadata(path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => JournalTip::genesis(), + Err(e) => return Err(e.into()), + Ok(meta) if !meta.file_type().is_file() || meta.len() > MAX_ANCHOR_BYTES => { + return Err(JournalError::Corrupt); + } + Ok(_) => { + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(MAX_ANCHOR_BYTES + 1) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_ANCHOR_BYTES { + return Err(JournalError::Corrupt); + } + let record: AnchorRecord = + serde_json::from_slice(&bytes).map_err(|_| JournalError::Corrupt)?; + let body = serde_json::to_vec(&record.body).map_err(|_| JournalError::Corrupt)?; + if record.body.version != 1 + || record.body.address != address + || anchor_mac(&key, &body) != record.tag + { + return Err(JournalError::Corrupt); + } + record.body.tip + } + }; + Ok(Self { + path: path.to_owned(), + address, + key: Zeroizing::new(key), + tip: Mutex::new(tip), + _lock: lock, + }) + } + + #[must_use] + pub fn tip(&self) -> JournalTip { + *self.tip.lock().expect("mailbox anchor lock poisoned") + } + + /// Advance by one committed journal entry, fsyncing the file and its + /// containing directory before returning. + pub fn advance(&self, tip: JournalTip) -> Result<(), JournalError> { + let mut current = self.tip.lock().expect("mailbox anchor lock poisoned"); + if current.cursor.checked_add(1) != Some(tip.cursor) { + return Err(JournalError::Rollback); + } + let body = AnchorBody { + version: 1, + address: self.address, + tip, + }; + let tag = anchor_mac( + &self.key[..], + &serde_json::to_vec(&body).map_err(|_| JournalError::Corrupt)?, + ); + let bytes = + serde_json::to_vec(&AnchorRecord { body, tag }).map_err(|_| JournalError::Corrupt)?; + atomic_write_durable(&self.path, &bytes) + .map_err(|e| JournalError::Storage(std::io::Error::other(e.to_string())))?; + *current = tip; + Ok(()) + } +} + +fn anchor_mac(key: &[u8], body: &[u8]) -> [u8; 32] { + let mut mac = + as KeyInit>::new_from_slice(key).expect("a 32-byte HMAC key is valid"); + mac.update(ANCHOR_DOMAIN); + mac.update(body); + mac.finalize().into_bytes().into() +} + +impl MailboxJournal { + /// Reconstruct under an exclusive lock and require the separately trusted + /// high-water mark. Genesis is valid only for an empty journal. + pub fn open( + root: &Path, + address: MailboxAddress, + authority: VerifyingKey, + key: [u8; 32], + limits: JournalLimits, + expected: JournalTip, + ) -> Result { + Self::open_with_recovery(root, address, authority, key, limits, expected, |_| { + Err(JournalError::Rollback) + }) + } + + /// Recover a journal fsynced before its anchor during an interrupted + /// admission. Only a verified extension of the anchored prefix may advance + /// the independent anchor; rollback or divergence still refuses startup. + pub fn open_reconciling( + root: &Path, + address: MailboxAddress, + authority: VerifyingKey, + key: [u8; 32], + limits: JournalLimits, + anchor: &FileJournalAnchor, + ) -> Result { + if anchor.address != address { + return Err(JournalError::WrongAuthority); + } + Self::open_with_recovery(root, address, authority, key, limits, anchor.tip(), |tip| { + anchor.advance(tip) + }) + } + + fn open_with_recovery( + root: &Path, + address: MailboxAddress, + authority: VerifyingKey, + key: [u8; 32], + limits: JournalLimits, + expected: JournalTip, + mut reconcile: impl FnMut(JournalTip) -> Result<(), JournalError>, + ) -> Result { + if limits.max_records == 0 + || limits.max_records > MAX_RECORDS + || limits.max_bytes == 0 + || limits.max_bytes > MAX_JOURNAL_BYTES + { + return Err(JournalError::InvalidLimits); + } + if expected.cursor == 0 && expected != JournalTip::genesis() { + return Err(JournalError::Rollback); + } + private_fs::ensure_private_dir(root)?; + sync_dir(root.parent().ok_or(JournalError::Corrupt)?) + .map_err(|e| JournalError::Storage(std::io::Error::other(e.to_string())))?; + let lock = FileLock::try_acquire(&root.join("journal")) + .map_err(|e| std::io::Error::other(e.to_string()))? + .ok_or(JournalError::Busy)?; + let mut journal = Self { + root: root.to_owned(), + address, + authority, + key: Zeroizing::new(key), + limits, + tip: JournalTip::genesis(), + bytes: 0, + seen: BTreeMap::new(), + order: Vec::new(), + poisoned: false, + _lock: lock, + }; + let mut paths = Vec::new(); + for entry in fs::read_dir(root)? { + let entry = entry?; + if entry.file_name() != "journal.lock" { + paths.push(entry.path()); + if paths.len() > limits.max_records { + return Err(JournalError::Full); + } + } + } + paths.sort(); + for path in paths { + let cursor = journal + .tip + .cursor + .checked_add(1) + .ok_or(JournalError::Full)?; + if path != record_path(root, cursor) { + return Err(JournalError::Corrupt); + } + let bytes = read_record(&path)?; + journal.bytes = journal + .bytes + .checked_add(bytes.len() as u64) + .filter(|n| *n <= limits.max_bytes) + .ok_or(JournalError::Full)?; + let (tag, ciphertext) = bytes.split_at_checked(32).ok_or(JournalError::Corrupt)?; + let expected_tag = journal.mac(ciphertext); + if tag != expected_tag { + return Err(JournalError::Corrupt); + } + let plaintext = Zeroizing::new( + aead::open( + &journal.record_key(), + ciphertext, + &[DOMAIN, &journal.tip.head].concat(), + ) + .map_err(|_| JournalError::Corrupt)?, + ); + let record: Record = + serde_json::from_slice(&plaintext).map_err(|_| JournalError::Corrupt)?; + if record.version != 1 + || record.cursor != cursor + || record.previous != journal.tip.head + || record.address != address + { + return Err(JournalError::Corrupt); + } + match record.event { + JournalEvent::Accepted { + identity, + frame: encoded_frame, + receipt, + } => { + if receipt.receipt.address != address + || receipt.receipt.outcome != DeliveryOutcome::AcceptedDurable + || receipt.receipt.message_id != identity.message_id + || receipt.receipt.execution_epoch != identity.execution_epoch + { + return Err(JournalError::Corrupt); + } + verify_receipt(&receipt, &authority).map_err(|_| JournalError::Corrupt)?; + let raw = STANDARD_NO_PAD + .decode(&encoded_frame) + .map_err(|_| JournalError::Corrupt)?; + let (capability, envelope) = + frame::decode_frame(&raw).map_err(|_| JournalError::Corrupt)?; + if receipt.receipt.delivery_id != envelope.routing.delivery_id + || receipt.receipt.ciphertext_commitment != envelope.ciphertext_commitment + || journal + .seen + .insert( + identity, + Seen { + envelope, + capability, + acknowledged: false, + }, + ) + .is_some() + { + return Err(JournalError::Corrupt); + } + journal.order.push(identity); + } + JournalEvent::Acknowledged { + identity, + acknowledged_at_unix_ms, + } => { + let Some(entry) = journal.seen.get_mut(&identity) else { + return Err(JournalError::Corrupt); + }; + if entry.acknowledged || acknowledged_at_unix_ms < 0 { + return Err(JournalError::Corrupt); + } + entry.acknowledged = true; + } + } + let next = JournalTip { + cursor, + head: expected_tag, + }; + if cursor == expected.cursor && next != expected { + return Err(JournalError::Rollback); + } + if cursor > expected.cursor { + reconcile(next)?; + } + journal.tip = next; + } + if journal.tip.cursor < expected.cursor { + return Err(JournalError::Rollback); + } + Ok(journal) + } + + /// Verify the canonical envelope and capability, commit ciphertext and + /// dedup identity, then sign a destination verdict. `anchor` must commit the + /// new tip in a separate rollback domain before this returns success. + pub fn admit( + &mut self, + request: JournalAdmission<'_>, + anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, + ) -> Result { + let JournalAdmission { + envelope, + capability, + context, + encryption_key, + sender_key, + signer, + } = request; + if self.poisoned { + return Err(JournalError::Uncertain); + } + if self.address.tenant != context.expected_tenant + || self.address.authorization_domain != context.expected_authorization_domain + || self.address.mailbox != context.expected_mailbox + || self.address.recipient != context.expected_recipient + || signer.verifying_key() != self.authority + { + return Err(JournalError::WrongAuthority); + } + let mut probe = Probe::default(); + let opened = open_authenticated_envelope( + envelope, + capability, + context, + encryption_key, + sender_key, + &self.authority, + &mut probe, + )?; + let identity = probe.0.ok_or(JournalError::Corrupt)?; + let receipt = |outcome| DestinationReceipt { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: envelope.routing.delivery_id, + message_id: opened.header().message_id, + address: self.address, + execution_epoch: opened.header().execution_epoch, + outcome, + observed_at_unix_ms: context.now_unix_ms, + ciphertext_commitment: envelope.ciphertext_commitment, + }; + if let Some(previous) = self.seen.get(&identity) { + if previous.envelope != *envelope || previous.capability != *capability { + return Err(JournalError::ConflictingRetry); + } + return Ok(sign_receipt(receipt(DeliveryOutcome::Duplicate), signer)?); + } + if self.tip.cursor >= self.limits.max_records as u64 { + return Err(JournalError::Full); + } + let signed = sign_receipt(receipt(DeliveryOutcome::AcceptedDurable), signer)?; + let encoded_frame = + frame::encode_frame(capability, envelope).map_err(|_| JournalError::Corrupt)?; + self.append( + JournalEvent::Accepted { + identity, + frame: STANDARD_NO_PAD.encode(encoded_frame), + receipt: Box::new(signed.clone()), + }, + anchor, + )?; + self.seen.insert( + identity, + Seen { + envelope: envelope.clone(), + capability: capability.clone(), + acknowledged: false, + }, + ); + self.order.push(identity); + Ok(signed) + } + + fn append( + &mut self, + event: JournalEvent, + anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, + ) -> Result { + if self.poisoned { + return Err(JournalError::Uncertain); + } + let cursor = self.tip.cursor.checked_add(1).ok_or(JournalError::Full)?; + if cursor > self.limits.max_records as u64 { + return Err(JournalError::Full); + } + let record = Record { + version: 1, + cursor, + previous: self.tip.head, + address: self.address, + event, + }; + let body = Zeroizing::new(serde_json::to_vec(&record).map_err(|_| JournalError::Corrupt)?); + if body.len() + 32 + aead::NONCE_SIZE + aead::TAG_SIZE > MAX_RECORD_BYTES { + return Err(JournalError::Full); + } + let ciphertext = aead::seal( + &self.record_key(), + &body, + &[DOMAIN, &self.tip.head].concat(), + ); + if self + .bytes + .checked_add((ciphertext.len() + 32) as u64) + .is_none_or(|n| n > self.limits.max_bytes) + { + return Err(JournalError::Full); + } + let tag = self.mac(&ciphertext); + let mut bytes = tag.to_vec(); + bytes.extend_from_slice(&ciphertext); + let tip = JournalTip { cursor, head: tag }; + let result = atomic_write_new(&record_path(&self.root, cursor), &bytes) + .and_then(|()| sync_dir(&self.root)) + .map_err(|e| JournalError::Storage(std::io::Error::other(e.to_string()))); + if let Err(error) = result { + self.poisoned = true; + return Err(error); + } + self.tip = tip; + self.bytes += bytes.len() as u64; + if let Err(error) = anchor(tip) { + self.poisoned = true; + return Err(error); + } + Ok(tip) + } + + /// Inspect the oldest unacknowledged ciphertext under an independently + /// signed Receive grant. No journal state changes until `acknowledge`. + pub fn receive_next( + &self, + request: JournalReceive<'_>, + ) -> Result, JournalError> { + if self.poisoned { + return Err(JournalError::Uncertain); + } + for identity in &self.order { + let entry = self.seen.get(identity).ok_or(JournalError::Corrupt)?; + if entry.acknowledged { + continue; + } + let opened = self.open_for_reader(*identity, entry, &request)?; + return Ok(Some(ReceivedMailboxMessage { + identity: *identity, + opened, + })); + } + Ok(None) + } + + /// Fsync an append-only ACK and advance the trusted tip. An uncertain + /// commit poisons this writer; recovery must reconcile it before retry. + pub fn acknowledge( + &mut self, + identity: ReplayIdentity, + request: JournalReceive<'_>, + anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, + ) -> Result { + if self.poisoned { + return Err(JournalError::Uncertain); + } + let entry = self + .seen + .get(&identity) + .ok_or(JournalError::UnknownMessage)?; + let _opened = self.open_for_reader(identity, entry, &request)?; + if entry.acknowledged { + return Ok(self.tip); + } + let tip = self.append( + JournalEvent::Acknowledged { + identity, + acknowledged_at_unix_ms: request.context.now_unix_ms, + }, + anchor, + )?; + self.seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)? + .acknowledged = true; + Ok(tip) + } + + fn open_for_reader( + &self, + identity: ReplayIdentity, + entry: &Seen, + request: &JournalReceive<'_>, + ) -> Result { + if request.context.required_operation != MailboxOperation::Receive + || request.context.expected_tenant != self.address.tenant + || request.context.expected_authorization_domain != self.address.authorization_domain + || request.context.expected_mailbox != self.address.mailbox + || request.context.expected_recipient != self.address.recipient + { + return Err(JournalError::WrongAuthority); + } + let mut send_context = request.context.clone(); + send_context.acting_principal = entry.capability.grant.subject; + send_context.expected_capability_issuer = entry.capability.grant.issuer; + send_context.required_operation = MailboxOperation::Send; + let opened = open_authenticated_envelope( + &entry.envelope, + &entry.capability, + &send_context, + request.encryption_key, + request.sender_key, + &self.authority, + &mut Probe::default(), + )?; + if ReplayIdentity::from_header(opened.header()) != identity { + return Err(JournalError::Corrupt); + } + authorize_receive( + opened.header(), + request.capability, + request.context, + &self.authority, + )?; + Ok(opened) + } + + /// The last committed high-water mark for an independent trusted anchor. + #[must_use] + pub const fn tip(&self) -> JournalTip { + self.tip + } + + fn mac(&self, bytes: &[u8]) -> [u8; 32] { + let mut mac = as KeyInit>::new_from_slice(&self.key[..]) + .expect("a 32-byte HMAC key is valid"); + mac.update(DOMAIN); + mac.update(&self.tip.head); + mac.update(bytes); + mac.finalize().into_bytes().into() + } + + fn record_key(&self) -> Key { + let mut mac = as KeyInit>::new_from_slice(&self.key[..]) + .expect("a 32-byte HMAC key is valid"); + mac.update(ENCRYPTION_DOMAIN); + Key::from_bytes(mac.finalize().into_bytes().into()) + } +} + +fn record_path(root: &Path, cursor: u64) -> PathBuf { + root.join(format!("entry-{cursor:020}.bin")) +} + +fn read_record(path: &Path) -> Result, JournalError> { + let meta = fs::symlink_metadata(path)?; + if !meta.file_type().is_file() || meta.len() > MAX_RECORD_BYTES as u64 { + return Err(JournalError::Corrupt); + } + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(MAX_RECORD_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > MAX_RECORD_BYTES { + return Err(JournalError::Corrupt); + } + Ok(bytes) +} diff --git a/crates/mvm-core/src/mailbox/transport.rs b/crates/mvm-core/src/mailbox/transport.rs index b25512cafb..482bb5b3f2 100644 --- a/crates/mvm-core/src/mailbox/transport.rs +++ b/crates/mvm-core/src/mailbox/transport.rs @@ -546,14 +546,17 @@ pub fn prepare_mailbox_envelope( Ok(PreparedMailboxEnvelope { envelope, header }) } -/// Canonically encoded ciphertext admitted by mailbox authority. Its bytes are -/// private so a transport cannot be used as an arbitrary packet-forwarding API. +/// Canonically encoded ciphertext and the exact signed capability admitted by +/// mailbox authority. The bytes are private so a transport cannot be used as +/// an arbitrary packet-forwarding API. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AuthorizedMailboxFrame { bytes: Vec, + capability: SignedMailboxCapability, } impl AuthorizedMailboxFrame { + /// Canonical envelope length, excluding the separately carried capability. #[must_use] pub fn encoded_len(&self) -> usize { self.bytes.len() @@ -564,6 +567,12 @@ impl AuthorizedMailboxFrame { pub fn as_bytes(&self) -> &[u8] { &self.bytes } + + /// The signed send capability verified with this exact envelope. + #[must_use] + pub fn capability(&self) -> &SignedMailboxCapability { + &self.capability + } } /// Verify capability, lifecycle, and route binding before exposing a frame to @@ -590,7 +599,7 @@ pub fn authorize_outbound( } verify_identity_and_time(header, request.context)?; verify_capability_signature(request.capability, request.capability_authority_key)?; - verify_capability_scope(header, request.capability, request.context)?; + verify_capability_scope(header, request.capability, request.context, true)?; let claims = &request.route.claims; if claims.address.tenant != header.tenant || claims.address.authorization_domain != header.authorization_domain @@ -606,6 +615,7 @@ pub fn authorize_outbound( } Ok(AuthorizedMailboxFrame { bytes: encode_envelope(envelope)?, + capability: request.capability.clone(), }) } diff --git a/crates/mvm-core/src/mailbox/transport/tests.rs b/crates/mvm-core/src/mailbox/transport/tests.rs index b2767de338..01b192c044 100644 --- a/crates/mvm-core/src/mailbox/transport/tests.rs +++ b/crates/mvm-core/src/mailbox/transport/tests.rs @@ -1,13 +1,21 @@ use super::*; use mvm_contract::protocol::mailbox::{ AuthorizationDomainId, BootId, CapabilityId, CapabilityOperations, CausationId, - CiphertextPadding, ContentType, CorrelationId, DeliveryId, EncryptionAlgorithm, - EncryptionContext, MailboxCapabilityGrant, MailboxId, MailboxMessageKind, MailboxRoutingFrame, - MessageId, OrderingScope, ReplayNonce, RevocationId, SequenceNumber, TenantId, + CiphertextPadding, ContentType, CorrelationId, DeliveryId, DeliveryOutcome, + EncryptionAlgorithm, EncryptionContext, MailboxCapabilityGrant, MailboxId, MailboxMessageKind, + MailboxRoutingFrame, MessageId, OrderingScope, ReplayNonce, RevocationId, SequenceNumber, + TenantId, }; use crate::crypto::aead::Key; -use crate::mailbox::{SealRequest, sign_capability}; +use crate::mailbox::journal::{ + FileJournalAnchor, JournalAdmission, JournalError, JournalLimits, JournalReceive, JournalTip, + MailboxJournal, +}; +use crate::mailbox::{ + InMemoryReplayProtector, SealRequest, decode_envelope, open_authenticated_envelope, + sign_capability, +}; const NOW: i64 = 1_800_000_000_000; @@ -189,6 +197,780 @@ fn mailbox_fixture() -> MailboxFixture { } } +#[test] +fn only_an_authenticated_inner_header_can_supply_a_durable_replay_identity() { + let fixture = mailbox_fixture(); + let mut replay = InMemoryReplayProtector::default(); + let sender_key = SigningKey::from_bytes(&id(22)).verifying_key(); + let opened = open_authenticated_envelope( + fixture.prepared.envelope(), + &fixture.capability, + &fixture.context, + &Key::from_bytes(id(21)), + &sender_key, + &fixture.capability_authority.verifying_key(), + &mut replay, + ) + .expect("signed envelope and capability"); + assert_eq!(opened.header().message_id, MessageId::from_bytes(id(14))); + assert_eq!(opened.payload(), b"hello"); + assert!( + !format!("{opened:?}").contains("hello"), + "debug output must not disclose mailbox plaintext" + ); + assert_eq!( + open_authenticated_envelope( + fixture.prepared.envelope(), + &fixture.capability, + &fixture.context, + &Key::from_bytes(id(21)), + &sender_key, + &fixture.capability_authority.verifying_key(), + &mut replay, + ), + Err(MailboxError::Replay), + "the same verified header cannot be admitted twice" + ); + let mut tampered = fixture.capability.clone(); + tampered.signature[0] ^= 1; + assert_eq!( + open_authenticated_envelope( + fixture.prepared.envelope(), + &tampered, + &fixture.context, + &Key::from_bytes(id(21)), + &sender_key, + &fixture.capability_authority.verifying_key(), + &mut InMemoryReplayProtector::default(), + ), + Err(MailboxError::CapabilitySignatureInvalid), + "an unverified capability cannot yield a journal identity" + ); +} + +#[test] +fn receive_requires_its_own_signed_recipient_capability_not_the_send_grant() { + let fixture = mailbox_fixture(); + let header = mailbox_header_for_retry(); + let mut context = fixture.context.clone(); + context.acting_principal = address().recipient; + context.required_operation = MailboxOperation::Receive; + assert_eq!( + crate::mailbox::authorize_receive( + &header, + &fixture.capability, + &context, + &fixture.capability_authority.verifying_key() + ), + Err(MailboxError::WrongCapabilitySubject), + "the signed Send grant cannot double as Receive authority" + ); + let mut grant = fixture.capability.grant.clone(); + grant.capability_id = CapabilityId::from_bytes(id(99)); + grant.subject = address().recipient; + grant.operations = CapabilityOperations::from_operations(&[MailboxOperation::Receive]); + let receive = sign_capability(grant, &fixture.capability_authority).unwrap(); + crate::mailbox::authorize_receive( + &header, + &receive, + &context, + &fixture.capability_authority.verifying_key(), + ) + .expect("independent Receive capability is authorized"); + let mut stale = context.clone(); + stale.current_execution_epoch = ExecutionEpoch::new(8).unwrap(); + assert_eq!( + crate::mailbox::authorize_receive( + &header, + &receive, + &stale, + &fixture.capability_authority.verifying_key() + ), + Err(MailboxError::StaleExecutionEpoch) + ); +} + +#[test] +fn receive_and_ack_require_independent_authority_and_survive_recovery() { + let dir = tempfile::tempdir().expect("journal directory"); + let fixture = mailbox_fixture(); + let signer = &fixture.capability_authority; + let sender = SigningKey::from_bytes(&id(22)).verifying_key(); + let encryption_key = Key::from_bytes(id(21)); + let limits = JournalLimits { + max_records: 3, + max_bytes: 64 * 1024, + }; + let mut journal = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + JournalTip::genesis(), + ) + .expect("empty journal"); + let mut tip = JournalTip::genesis(); + journal + .admit( + JournalAdmission { + envelope: fixture.prepared.envelope(), + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &encryption_key, + sender_key: &sender, + signer, + }, + |committed| { + tip = committed; + Ok(()) + }, + ) + .expect("durable Send"); + let mut grant = fixture.capability.grant.clone(); + grant.capability_id = CapabilityId::from_bytes(id(99)); + grant.subject = address().recipient; + grant.operations = CapabilityOperations::from_operations(&[MailboxOperation::Receive]); + let receive_capability = sign_capability(grant, signer).expect("Receive capability"); + let mut reader = fixture.context.clone(); + reader.acting_principal = address().recipient; + reader.required_operation = MailboxOperation::Receive; + let request = |capability| JournalReceive { + capability, + context: &reader, + encryption_key: &encryption_key, + sender_key: &sender, + }; + assert!(matches!( + journal.receive_next(request(&fixture.capability)), + Err(JournalError::Mailbox(MailboxError::WrongCapabilitySubject)) + )); + let pending = journal + .receive_next(request(&receive_capability)) + .expect("authorized Receive") + .expect("one pending message"); + assert_eq!(pending.payload(), b"hello"); + let identity = pending.identity(); + assert_eq!( + journal + .receive_next(request(&receive_capability)) + .expect("at-least-once redelivery before ACK") + .expect("still pending") + .identity(), + identity + ); + journal + .acknowledge(identity, request(&receive_capability), |committed| { + tip = committed; + Ok(()) + }) + .expect("durable ACK"); + assert_eq!( + journal + .acknowledge(identity, request(&receive_capability), |_| { + panic!("an idempotent ACK cannot append a second event") + }) + .expect("duplicate ACK"), + tip + ); + let mut stale_reader = reader.clone(); + stale_reader.current_execution_epoch = ExecutionEpoch::new(8).unwrap(); + assert!(matches!( + journal.acknowledge( + identity, + JournalReceive { + capability: &receive_capability, + context: &stale_reader, + encryption_key: &encryption_key, + sender_key: &sender, + }, + |_| Ok(()), + ), + Err(JournalError::Mailbox(MailboxError::StaleExecutionEpoch)) + )); + assert!( + journal + .receive_next(request(&receive_capability)) + .expect("queue after ACK") + .is_none() + ); + drop(journal); + let recovered = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + tip, + ) + .expect("journal and ACK recover"); + assert!( + recovered + .receive_next(request(&receive_capability)) + .expect("recovered queue") + .is_none() + ); + drop(recovered); + let ack_path = dir.path().join("entry-00000000000000000002.bin"); + let ack = std::fs::read(&ack_path).expect("ACK record"); + let mut tampered = ack.clone(); + tampered[40] ^= 1; + std::fs::write(&ack_path, tampered).expect("tamper ACK fixture"); + assert!(matches!( + MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + tip + ), + Err(JournalError::Corrupt) + )); + std::fs::remove_file(&ack_path).expect("roll back ACK fixture"); + assert!(matches!( + MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + tip + ), + Err(JournalError::Rollback) + )); + std::fs::write(&ack_path, ack).expect("restore ACK fixture"); +} + +#[test] +fn durable_admission_recovers_ciphertext_dedup_and_a_signed_destination_verdict() { + use mvm_contract::protocol::mailbox::DeliveryOutcome; + + let dir = tempfile::tempdir().expect("journal directory"); + let fixture = mailbox_fixture(); + let signer = &fixture.capability_authority; + let sender = SigningKey::from_bytes(&id(22)).verifying_key(); + let limits = JournalLimits { + max_records: 2, + max_bytes: 64 * 1024, + }; + let mut journal = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + JournalTip::genesis(), + ) + .expect("empty journal"); + let mut anchored = None; + let receipt = journal + .admit( + JournalAdmission { + envelope: fixture.prepared.envelope(), + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &Key::from_bytes(id(21)), + sender_key: &sender, + signer, + }, + |tip| { + anchored = Some(tip); + Ok(()) + }, + ) + .expect("durable acceptance"); + assert_eq!(receipt.receipt.outcome, DeliveryOutcome::AcceptedDurable); + crate::mailbox::verify_receipt(&receipt, &signer.verifying_key()) + .expect("destination authority signature"); + let tip = anchored.expect("external anchor is published before acceptance"); + assert_eq!(journal.tip(), tip); + let bytes = + std::fs::read(dir.path().join("entry-00000000000000000001.bin")).expect("durable record"); + assert!( + !bytes.windows(5).any(|w| w == b"hello"), + "journal contains no plaintext" + ); + for metadata in [b"recipient".as_slice(), b"capability_id", b"message_id"] { + assert!( + !bytes.windows(metadata.len()).any(|w| w == metadata), + "journal record must conceal authenticated inner metadata" + ); + } + drop(journal); + + let mut recovered = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + tip, + ) + .expect("recover accepted entry"); + let duplicate = recovered + .admit( + JournalAdmission { + envelope: fixture.prepared.envelope(), + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &Key::from_bytes(id(21)), + sender_key: &sender, + signer, + }, + |_| panic!("a duplicate cannot advance the external anchor"), + ) + .expect("duplicate receipt"); + assert_eq!(duplicate.receipt.outcome, DeliveryOutcome::Duplicate); + assert_eq!( + recovered.tip(), + tip, + "duplicate does not append a second entry" + ); + let original = fixture.prepared.envelope(); + let conflicting = crate::mailbox::seal_envelope( + SealRequest { + routing: original.routing.clone(), + header: mailbox_header_for_retry(), + plaintext: b"hello", + }, + &Key::from_bytes(id(21)), + &SigningKey::from_bytes(&id(22)), + ) + .expect("fresh ciphertext with the same signed message identity"); + assert!(matches!( + recovered.admit( + JournalAdmission { + envelope: &conflicting, + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &Key::from_bytes(id(21)), + sender_key: &sender, + signer, + }, + |_| Ok(()), + ), + Err(JournalError::ConflictingRetry) + )); +} + +#[test] +fn a_transport_verdict_cannot_substitute_for_a_matching_signed_destination_receipt() { + use mvm_contract::protocol::mailbox::{ + DeliveryOutcome, DestinationReceipt, SignedDestinationReceipt, + }; + use std::io::{Read, Write}; + + struct Reply(std::io::Cursor>); + impl Read for Reply { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.read(buf) + } + } + impl Write for Reply { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let fixture = mailbox_fixture(); + let envelope = fixture.prepared.envelope(); + let header = &fixture.prepared.header; + let key = fixture.capability_authority.verifying_key(); + let receipt = crate::mailbox::sign_receipt( + DestinationReceipt { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: envelope.routing.delivery_id, + message_id: header.message_id, + address: address(), + execution_epoch: header.execution_epoch, + outcome: DeliveryOutcome::AcceptedDurable, + observed_at_unix_ms: NOW, + ciphertext_commitment: envelope.ciphertext_commitment, + }, + &fixture.capability_authority, + ) + .expect("signed receipt"); + let exchange = |signed: &SignedDestinationReceipt| { + let reply = serde_json::to_vec(&Ok::<_, crate::mailbox::frame::Refusal>(signed)) + .expect("reply serialization"); + crate::mailbox::frame::send_receipted_frame( + &mut Reply(std::io::Cursor::new(reply)), + &fixture.capability, + envelope, + header, + &key, + ) + }; + assert_eq!(exchange(&receipt), Ok(receipt.clone())); + + let mut wrong_message = receipt.clone(); + wrong_message.receipt.message_id = MessageId::from_bytes(id(77)); + wrong_message = + crate::mailbox::sign_receipt(wrong_message.receipt, &fixture.capability_authority) + .expect("different signed message ID"); + assert_eq!( + exchange(&wrong_message), + Err(crate::mailbox::frame::Refusal::Malformed) + ); + let mut forged = receipt.clone(); + forged.signature[0] ^= 1; + assert_eq!( + exchange(&forged), + Err(crate::mailbox::frame::Refusal::Malformed) + ); + let refused = crate::mailbox::sign_receipt( + DestinationReceipt { + outcome: DeliveryOutcome::Refused, + ..receipt.receipt + }, + &fixture.capability_authority, + ) + .expect("signed refusal"); + assert_eq!( + exchange(&refused), + Err(crate::mailbox::frame::Refusal::Malformed) + ); +} + +fn mailbox_header_for_retry() -> MailboxEnvelopeHeader { + // The fixture seals a stable message ID for a retry. Recovering the inner + // header through the authority gate avoids constructing a second identity. + let fixture = mailbox_fixture(); + let mut replay = InMemoryReplayProtector::default(); + open_authenticated_envelope( + fixture.prepared.envelope(), + &fixture.capability, + &fixture.context, + &Key::from_bytes(id(21)), + &SigningKey::from_bytes(&id(22)).verifying_key(), + &fixture.capability_authority.verifying_key(), + &mut replay, + ) + .expect("fixture opens") + .header() + .clone() +} + +#[test] +fn journal_recovery_refuses_wrong_key_tamper_truncation_and_rollback() { + let dir = tempfile::tempdir().expect("journal directory"); + let fixture = mailbox_fixture(); + let signer = &fixture.capability_authority; + let sender = SigningKey::from_bytes(&id(22)).verifying_key(); + let limits = JournalLimits { + max_records: 1, + max_bytes: 64 * 1024, + }; + let mut journal = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + JournalTip::genesis(), + ) + .expect("empty journal"); + journal + .admit( + JournalAdmission { + envelope: fixture.prepared.envelope(), + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &Key::from_bytes(id(21)), + sender_key: &sender, + signer, + }, + |_| Ok(()), + ) + .expect("accepted"); + let tip = journal.tip(); + drop(journal); + assert!(matches!( + MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(91), + limits, + tip + ), + Err(JournalError::Corrupt) + )); + let mut wrong_address = address(); + wrong_address.tenant = TenantId::from_bytes(id(99)); + assert!(matches!( + MailboxJournal::open( + dir.path(), + wrong_address, + signer.verifying_key(), + id(90), + limits, + tip + ), + Err(JournalError::Corrupt) + )); + assert!(matches!( + MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + JournalTip::genesis() + ), + Err(JournalError::Rollback) + )); + let path = dir.path().join("entry-00000000000000000001.bin"); + let original = std::fs::read(&path).expect("record"); + let mut tampered = original.clone(); + tampered[40] ^= 1; + std::fs::write(&path, tampered).expect("tamper fixture"); + assert!(matches!( + MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + tip + ), + Err(JournalError::Corrupt) + )); + std::fs::write(&path, &original[..original.len() - 1]).expect("truncate fixture"); + assert!(matches!( + MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + tip + ), + Err(JournalError::Corrupt) + )); +} + +#[test] +fn a_full_journal_never_publishes_a_durable_acceptance_or_advances_its_tip() { + let dir = tempfile::tempdir().expect("journal directory"); + let fixture = mailbox_fixture(); + let signer = &fixture.capability_authority; + let sender = SigningKey::from_bytes(&id(22)).verifying_key(); + let mut journal = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + JournalLimits { + max_records: 1, + max_bytes: 1, + }, + JournalTip::genesis(), + ) + .expect("empty journal"); + assert!(matches!( + journal.admit( + JournalAdmission { + envelope: fixture.prepared.envelope(), + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &Key::from_bytes(id(21)), + sender_key: &sender, + signer, + }, + |_| panic!("a quota refusal must not update the external anchor"), + ), + Err(JournalError::Full) + )); + assert_eq!(journal.tip(), JournalTip::genesis()); + assert_eq!( + std::fs::read_dir(dir.path()) + .expect("journal directory") + .count(), + 1, + "only the writer lock exists; no accepted record was published" + ); +} + +#[test] +fn anchor_failure_after_fsync_returns_uncertain_and_blocks_further_admission() { + let dir = tempfile::tempdir().expect("journal directory"); + let fixture = mailbox_fixture(); + let signer = &fixture.capability_authority; + let sender = SigningKey::from_bytes(&id(22)).verifying_key(); + let limits = JournalLimits { + max_records: 2, + max_bytes: 64 * 1024, + }; + let mut journal = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + JournalTip::genesis(), + ) + .expect("empty journal"); + let encryption_key = Key::from_bytes(id(21)); + let request = || JournalAdmission { + envelope: fixture.prepared.envelope(), + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &encryption_key, + sender_key: &sender, + signer, + }; + assert!(matches!( + journal.admit(request(), |_| Err(JournalError::Uncertain)), + Err(JournalError::Uncertain) + )); + assert_eq!( + journal.tip().cursor, + 1, + "ciphertext was committed before anchoring" + ); + assert!( + matches!( + journal.admit(request(), |_| Ok(())), + Err(JournalError::Uncertain) + ), + "a failed anchor cannot be mistaken for a fresh acceptance" + ); + drop(journal); + assert!( + matches!( + MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + JournalTip::genesis() + ), + Err(JournalError::Rollback) + ), + "recovery with the old anchor fails closed until reconciliation" + ); +} + +#[test] +fn startup_reconciles_only_a_verified_fsynced_suffix_before_accepting_retries() { + let dir = tempfile::tempdir().expect("destination"); + let root = dir.path().join("journal"); + let anchor_path = dir.path().join("anchor"); + let fixture = mailbox_fixture(); + let signer = &fixture.capability_authority; + let sender = SigningKey::from_bytes(&id(22)).verifying_key(); + let encryption_key = Key::from_bytes(id(21)); + let limits = JournalLimits { + max_records: 2, + max_bytes: 64 * 1024, + }; + let anchor = FileJournalAnchor::open(&anchor_path, address(), id(91)).expect("anchor"); + let mut journal = MailboxJournal::open_reconciling( + &root, + address(), + signer.verifying_key(), + id(90), + limits, + &anchor, + ) + .expect("empty journal"); + let request = || JournalAdmission { + envelope: fixture.prepared.envelope(), + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &encryption_key, + sender_key: &sender, + signer, + }; + assert!(matches!( + journal.admit(request(), |_| Err(JournalError::Uncertain)), + Err(JournalError::Uncertain) + )); + assert_eq!(anchor.tip(), JournalTip::genesis()); + let fsynced = journal.tip(); + drop(journal); + drop(anchor); + + let anchor = FileJournalAnchor::open(&anchor_path, address(), id(91)).expect("reopen anchor"); + let mut recovered = MailboxJournal::open_reconciling( + &root, + address(), + signer.verifying_key(), + id(90), + limits, + &anchor, + ) + .expect("reconcile verified suffix"); + assert_eq!(anchor.tip(), fsynced); + let duplicate = recovered + .admit(request(), |_| panic!("a duplicate must not append")) + .expect("retry after reconciliation"); + assert_eq!(duplicate.receipt.outcome, DeliveryOutcome::Duplicate); + assert_eq!(recovered.tip(), fsynced); + drop(recovered); + drop(anchor); + + let bad_anchor_path = dir.path().join("wrong-anchor"); + let bad_anchor = FileJournalAnchor::open(&bad_anchor_path, address(), id(91)) + .expect("independent wrong anchor"); + bad_anchor + .advance(JournalTip { + cursor: 1, + head: id(99), + }) + .expect("valid MAC, wrong chain head"); + assert!(matches!( + MailboxJournal::open_reconciling( + &root, + address(), + signer.verifying_key(), + id(90), + limits, + &bad_anchor, + ), + Err(JournalError::Rollback) + )); +} + +#[test] +fn private_anchor_survives_reopen_and_refuses_wrong_key_tamper_and_backward_move() { + let dir = tempfile::tempdir().expect("anchor directory"); + let path = dir.path().join("mailbox-anchor"); + let anchor = FileJournalAnchor::open(&path, address(), id(91)).expect("genesis anchor"); + assert_eq!(anchor.tip(), JournalTip::genesis()); + let tip = JournalTip { + cursor: 1, + head: id(92), + }; + anchor.advance(tip).expect("durable advance"); + assert!(matches!(anchor.advance(tip), Err(JournalError::Rollback))); + drop(anchor); + assert_eq!( + FileJournalAnchor::open(&path, address(), id(91)) + .expect("reopen anchor") + .tip(), + tip + ); + assert!(matches!( + FileJournalAnchor::open(&path, address(), id(90)), + Err(JournalError::Corrupt) + )); + let mut tampered = std::fs::read(&path).expect("anchor bytes"); + tampered[18] ^= 1; + std::fs::write(&path, tampered).expect("tamper fixture"); + assert!(matches!( + FileJournalAnchor::open(&path, address(), id(91)), + Err(JournalError::Corrupt) + )); +} + #[test] fn binding_verification_rejects_tamper_identity_epoch_policy_time_and_assurance() { let authority = SigningKey::from_bytes(&id(30)); @@ -396,6 +1178,46 @@ fn authorized_frame(route: &AuthorizedRoute) -> AuthorizedMailboxFrame { .expect("outbound authority") } +#[test] +fn authorized_frame_retains_the_verified_capability_without_reencoding_the_envelope() { + let authority = SigningKey::from_bytes(&id(53)); + let (_, route) = signed_route(TransportPath::Direct, 54, &authority); + let mut fixture = mailbox_fixture(); + let frame = authorize_outbound(OutboundAuthorization { + prepared: &fixture.prepared, + capability: &fixture.capability, + context: &fixture.context, + capability_authority_key: &fixture.capability_authority.verifying_key(), + route: &route, + }) + .expect("outbound authority"); + let original = fixture.capability.clone(); + fixture.capability.signature[0] ^= 1; + assert_eq!( + frame.capability(), + &original, + "the verified grant is retained" + ); + assert_eq!( + decode_envelope(frame.as_bytes()).expect("canonical envelope"), + *fixture.prepared.envelope(), + "the envelope remains unchanged and separate from its capability" + ); + assert_eq!( + authorize_outbound(OutboundAuthorization { + prepared: &fixture.prepared, + capability: &fixture.capability, + context: &fixture.context, + capability_authority_key: &fixture.capability_authority.verifying_key(), + route: &route, + }), + Err(OutboundAuthorizationError::Mailbox( + MailboxError::CapabilitySignatureInvalid + )), + "tampered capability must never reach a transport" + ); +} + #[test] fn authenticated_connection_is_not_mailbox_authority() { let authority = SigningKey::from_bytes(&id(44)); diff --git a/crates/mvm-hostd/src/mailbox_launch.rs b/crates/mvm-hostd/src/mailbox_launch.rs index bc45b8fe8e..cd303c5823 100644 --- a/crates/mvm-hostd/src/mailbox_launch.rs +++ b/crates/mvm-hostd/src/mailbox_launch.rs @@ -29,9 +29,7 @@ use mvm_core::mailbox::OpenContext; use sha2::Sha256; use zeroize::Zeroizing; -use crate::mailbox_service::{ - GuestSealer, MailboxLimits, MailboxPolicy, MailboxService, serve_grant, -}; +use crate::mailbox_service::{GuestSealer, MailboxLimits, MailboxPolicy, MailboxService}; const SIGNER_SECRET: &str = "host-signer.ed25519"; const SIGNER_PUBLIC: &str = "host-signer.pub"; @@ -105,6 +103,8 @@ pub struct DerivedMailboxKeys { sender: [u8; 32], authority: [u8; 32], encryption: [u8; 32], + journal_mac: [u8; 32], + anchor_mac: [u8; 32], template: MailboxTemplate, } @@ -122,10 +122,21 @@ impl Drop for DerivedMailboxKeys { self.sender.zeroize(); self.authority.zeroize(); self.encryption.zeroize(); + self.journal_mac.zeroize(); + self.anchor_mac.zeroize(); } } impl DerivedMailboxKeys { + /// Mailbox-scoped destination signer and separate journal/anchor MAC keys. + pub(crate) fn durable_material(&self) -> (SigningKey, [u8; 32], [u8; 32]) { + ( + SigningKey::from_bytes(&self.authority), + self.journal_mac, + self.anchor_mac, + ) + } + /// The host trust and sealer these keys stand for. Pure: no file is read, /// so it can be rebuilt on every dial attempt. pub fn trust(&self) -> (MailboxPolicy, GuestSealer) { @@ -161,6 +172,7 @@ impl DerivedMailboxKeys { header: template.header, routing: template.routing, sent: 0, + intent_dir: None, }; (policy, sealer) } @@ -184,6 +196,8 @@ pub fn derive_launch_keys(keys_dir: &Path, grant: &MailboxGrant) -> Result, } -/// Dial the guest's mailbox port from its own thread, retrying for up to 30 s -/// while the guest comes up, and serve it under `keys`. Returns at once. +/// Bind the local service and dial the guest for the VM's lifetime. The first +/// guest connection has a bounded 30-second readiness window; after readiness +/// an idle outbox is normal and does not stop later submissions. pub fn serve_derived( grant: MailboxGrant, keys: DerivedMailboxKeys, @@ -347,19 +363,40 @@ pub fn serve_derived( let service = Arc::new(OnceLock::new()); let (thread_stop, thread_service) = (stop.clone(), service.clone()); std::thread::spawn(move || { - for _ in 0..DIAL_ATTEMPTS { - if thread_stop.load(Ordering::SeqCst) { - return; - } - let (trust, sealer) = keys.trust(); - let guest = (paths.guest_port.as_path(), paths.firecracker_mux.as_path()); - match serve_grant(&grant, trust, sealer, limits, &paths.socket, guest) { - Ok(served) => { - let _ = thread_service.set(served); - return; + let (trust, mut sealer) = keys.trust(); + sealer.intent_dir = paths.intent_dir; + let (Ok(trust), Ok(mut sealer)) = (trust.with_grant(&grant), sealer.with_grant(&grant)) + else { + tracing::warn!("mailbox launch refused mismatched host-held grant"); + return; + }; + let Ok(served) = MailboxService::bind_live(&paths.socket, limits, trust) else { + tracing::warn!("mailbox service could not bind"); + return; + }; + let _ = thread_service.set(served); + let Some(served) = thread_service.get() else { + return; + }; + let guest = (paths.guest_port.as_path(), paths.firecracker_mux.as_path()); + let mut ready = false; + let mut attempts = 0; + while !thread_stop.load(Ordering::SeqCst) { + match served.serve_guest(guest.0, guest.1, &mut sealer) { + Ok(_) => { + ready = true; + attempts = 0; + } + Err(_) if !ready => { + attempts += 1; + if attempts >= DIAL_ATTEMPTS { + tracing::warn!("mailbox guest port did not become ready"); + return; + } } - Err(_) => std::thread::sleep(DIAL_INTERVAL), + Err(_) => {} } + std::thread::sleep(DIAL_INTERVAL); } }); MailboxLaunch { stop, service } @@ -374,7 +411,8 @@ pub const LAUNCH_LIMITS: MailboxLimits = MailboxLimits { #[cfg(test)] mod tests { use super::*; - use mvm_contract::protocol::mailbox::ExecutionEpoch; + use mvm_contract::protocol::mailbox::{ExecutionEpoch, MessageId}; + use mvm_core::mailbox::frame::OutboxMessage; fn keys_dir() -> tempfile::TempDir { let dir = tempfile::tempdir().unwrap(); @@ -391,6 +429,47 @@ mod tests { } } + #[test] + fn a_retry_reuses_exact_sealed_intent_after_helper_restart() { + let dir = keys_dir(); + let grant = grant(dir.path(), [4; 32]); + let keys = derive_launch_keys(dir.path(), &grant).unwrap(); + let intents = dir.path().join("intents"); + let sealer = || { + let (_, mut sealer) = keys.trust(); + sealer.intent_dir = Some(intents.clone()); + sealer.with_grant(&grant).unwrap() + }; + let message = OutboxMessage { + recipient: sealer().header.recipient, + message_id: MessageId::from_bytes([45; 32]), + body: b"first".to_vec(), + }; + let first = sealer().seal(message.clone()).expect("first seal"); + let retry = sealer().seal(message.clone()).expect("retry after restart"); + assert_eq!( + first, retry, + "retry must reuse canonical ciphertext and delivery ID" + ); + for _ in 2..8 { + assert_eq!( + sealer().seal(message.clone()).expect("bounded retry"), + first + ); + } + assert_eq!( + sealer().seal(message.clone()), + Err(crate::mailbox_service::Refusal::RetriesExhausted), + "the attempt count survives a fresh helper using the same source intent" + ); + let mut conflicting = message; + conflicting.body = b"different".to_vec(); + assert!(matches!( + sealer().seal(conflicting), + Err(crate::mailbox_service::Refusal::Mailbox(_)) + )); + } + #[test] fn without_a_template_file_the_derived_defaults_serve_a_matching_grant() { let dir = keys_dir(); diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index a5704f4120..8c67764d23 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -20,17 +20,18 @@ use std::io::{Error, ErrorKind, Result, Write as _}; use std::os::unix::process::CommandExt as _; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use std::sync::mpsc::sync_channel; use std::time::Duration; use mvm_contract::protocol::mailbox::{MAILBOX_PORT, MailboxGrant}; +use mvm_core::mailbox::journal::{FileJournalAnchor, JournalLimits, MailboxJournal}; use mvm_core::plan::ExecutionPlan; use mvm_core::vm_backend::{BackendKind, VmId, VmStartConfig}; use mvm_runtime::backend::AnyBackend; use mvm_vmm::host::aux_bin::AuxBin; -use crate::mailbox_launch::{ - DerivedMailboxKeys, LAUNCH_LIMITS, ServePaths, derive_launch_keys, serve_derived, -}; +use crate::mailbox_launch::{DerivedMailboxKeys, LAUNCH_LIMITS, derive_launch_keys}; +use crate::mailbox_service::durable::DurableMailboxService; use crate::vm_lifetime::{endpoint_needed, launcher_is_alive}; /// The helper binary this module spawns. @@ -40,6 +41,8 @@ pub const MAILBOX_BIN_ENV: &str = "MVM_MAILBOX_PATH"; /// Where the process's pid is recorded, beside the VM. pub const MAILBOX_PID_FILE: &str = "mailbox.pid"; const POLL_INTERVAL: Duration = Duration::from_millis(500); +const READY_TIMEOUT: Duration = Duration::from_secs(10); +const READY_MARKER: &[u8; 6] = b"READY\n"; /// What the process reads on stdin. `deny_unknown_fields`, so a field a newer /// launcher adds fails closed here. @@ -62,6 +65,16 @@ pub fn parse(raw: &[u8]) -> serde_json::Result { /// Serve until neither the launcher nor the VM is left; the exit code. pub fn serve(config: MailboxProcessConfig) -> i32 { + match serve_durable(config) { + Ok(()) => 0, + Err(error) => { + tracing::error!(%error, "mailbox service failed closed"); + 1 + } + } +} + +fn serve_durable(config: MailboxProcessConfig) -> Result<()> { // SAFETY: `getppid` is async-signal-safe and infallible. let launcher = unsafe { libc::getppid() }; let MailboxProcessConfig { @@ -72,20 +85,44 @@ pub fn serve(config: MailboxProcessConfig) -> i32 { firecracker_mux, vm_state_dir, } = config; - let _serving = serve_derived( - grant, - keys, - ServePaths { - socket, - guest_port, - firecracker_mux, + let (policy, mut sealer) = keys.trust(); + let (signer, journal_key, anchor_key) = keys.durable_material(); + sealer.intent_dir = Some(vm_state_dir.join("mailbox-intents")); + let sealer = sealer.with_grant(&grant)?; + let address = sealer.capability.address; + let anchor = FileJournalAnchor::open(&vm_state_dir.join("mailbox-anchor"), address, anchor_key) + .map_err(Error::other)?; + let journal = MailboxJournal::open_reconciling( + &vm_state_dir.join("mailbox-journal"), + address, + signer.verifying_key(), + journal_key, + JournalLimits { + max_records: 64, + max_bytes: 4 * 1024 * 1024, }, + &anchor, + ) + .map_err(Error::other)?; + let service = DurableMailboxService::bind( + &socket, LAUNCH_LIMITS, - ); + policy.with_grant(&grant)?, + signer, + journal, + anchor, + )?; + std::io::stdout().write_all(READY_MARKER)?; + let mut sealer = sealer; while endpoint_needed(launcher_is_alive(launcher), vm_is_alive(&vm_state_dir)) { - std::thread::sleep(POLL_INTERVAL); + if service + .serve_guest(&guest_port, &firecracker_mux, &mut sealer) + .is_err() + { + std::thread::sleep(POLL_INTERVAL); + } } - 0 + Ok(()) } fn vm_is_alive(state_dir: &Path) -> bool { @@ -121,12 +158,31 @@ pub fn build_config( fn command(bin: &Path) -> Command { let mut cmd = mvm_core::env_hygiene::helper_command(bin); cmd.stdin(Stdio::piped()) - .stdout(Stdio::null()) + .stdout(Stdio::piped()) .stderr(Stdio::null()) .process_group(0); cmd } +fn wait_ready(mut stdout: impl std::io::Read + Send + 'static, timeout: Duration) -> Result<()> { + let (tx, rx) = sync_channel(1); + std::thread::spawn(move || { + let mut marker = [0u8; READY_MARKER.len()]; + let outcome = stdout.read_exact(&mut marker).map(|()| marker); + let _ = tx.send(outcome); + }); + let marker = rx + .recv_timeout(timeout) + .map_err(|_| Error::new(ErrorKind::TimedOut, "mailbox helper readiness timed out"))??; + if &marker != READY_MARKER { + return Err(Error::new( + ErrorKind::InvalidData, + "mailbox helper sent an invalid readiness marker", + )); + } + Ok(()) +} + /// Refuse a launch whose plan grants a mailbox on a backend that cannot reach /// the guest's mailbox port. The guest would be told to serve a port nobody /// dials, so the launch fails instead. @@ -219,7 +275,21 @@ pub fn spawn_for_launch( let _ = child.wait(); return Err(error); } - std::fs::write(state_dir.join(MAILBOX_PID_FILE), child.id().to_string())?; + let ready = child + .stdout + .take() + .ok_or_else(|| Error::other("mvm-mailbox stdout was not piped")) + .and_then(|stdout| wait_ready(stdout, READY_TIMEOUT)); + if let Err(error) = ready { + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + if let Err(error) = std::fs::write(state_dir.join(MAILBOX_PID_FILE), child.id().to_string()) { + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } Ok(Some(child.id())) } @@ -228,6 +298,32 @@ mod tests { use super::*; use mvm_contract::protocol::mailbox::{ExecutionEpoch, MailboxId}; + #[test] + fn helper_readiness_requires_the_exact_marker_before_launch_succeeds() { + let timeout = Duration::from_millis(100); + assert!(wait_ready(std::io::Cursor::new(*READY_MARKER), timeout).is_ok()); + assert_eq!( + wait_ready(std::io::Cursor::new(*b"WRONG\n"), timeout) + .expect_err("wrong marker") + .kind(), + ErrorKind::InvalidData + ); + assert_eq!( + wait_ready(std::io::Cursor::new(Vec::::new()), timeout) + .expect_err("helper exited") + .kind(), + ErrorKind::UnexpectedEof + ); + let (blocked, peer) = std::os::unix::net::UnixStream::pair().unwrap(); + assert_eq!( + wait_ready(blocked, Duration::from_millis(1)) + .expect_err("silent helper cannot hang the launch") + .kind(), + ErrorKind::TimedOut + ); + drop(peer); + } + fn keys_dir() -> tempfile::TempDir { let dir = tempfile::tempdir().unwrap(); crate::audit::host_keypair::load_or_init_at(dir.path()).unwrap(); diff --git a/crates/mvm-hostd/src/mailbox_service.rs b/crates/mvm-hostd/src/mailbox_service.rs index 2ff5d89943..66675d46b8 100644 --- a/crates/mvm-hostd/src/mailbox_service.rs +++ b/crates/mvm-hostd/src/mailbox_service.rs @@ -12,27 +12,37 @@ use std::collections::VecDeque; use std::io::{Read, Write}; +use std::os::unix::fs::MetadataExt; use std::os::unix::net::{UnixListener, UnixStream}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread::JoinHandle; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use ed25519_dalek::{SigningKey, VerifyingKey}; use mvm_contract::protocol::mailbox::{ - ExecutionEpoch, MailboxCapabilityGrant, MailboxEnvelope, MailboxEnvelopeHeader, MailboxGrant, - MailboxRoutingFrame, MessageId, ReplayNonce, SequenceNumber, SignedMailboxCapability, + BootId, CorrelationId, DeliveryId, ExecutionEpoch, MailboxCapabilityGrant, MailboxEnvelope, + MailboxEnvelopeHeader, MailboxGrant, MailboxRoutingFrame, ReplayNonce, SequenceNumber, + SignedMailboxCapability, }; use mvm_core::crypto::aead::Key; pub use mvm_core::mailbox::frame::Refusal; -use mvm_core::mailbox::frame::{OutboxMessage, decode_frame, send_frame}; +use mvm_core::mailbox::frame::{OutboxMessage, decode_frame, encode_frame, send_frame}; use mvm_core::mailbox::{ - InMemoryReplayProtector, OpenContext, SealRequest, open_envelope, seal_envelope, - sign_capability, + InMemoryReplayProtector, OpenContext, SealRequest, open_authenticated_envelope, open_envelope, + seal_envelope, sign_capability, }; +use mvm_core::private_fs; +use mvm_core::util::atomic_io::{FileLock, atomic_write_durable, atomic_write_new, sync_dir}; + +/// Separate destination path that returns a signed, journal-backed verdict. +pub mod durable; const IO_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_SOURCE_INTENTS: usize = 64; +const MAX_SOURCE_INTENT_BYTES: usize = 64 * 1024; +const MAX_SOURCE_ATTEMPTS: u8 = 8; /// Bounds on what one node will hold or accept. #[derive(Debug, Clone, Copy)] @@ -77,6 +87,8 @@ pub struct GuestSealer { pub routing: MailboxRoutingFrame, /// Messages sealed so far; start at 0. pub sent: u64, + /// Private, bounded source intents retain exact sealed bytes for retries. + pub intent_dir: Option, } impl GuestSealer { @@ -92,16 +104,126 @@ impl GuestSealer { self.capability.address.mailbox = grant.mailbox; self.header.mailbox = grant.mailbox; self.header.execution_epoch = grant.epoch; + // The template is stable across restarts; its counter starts over in each + // process. A fresh boot identity keeps those message/replay IDs distinct. + self.header.sender_boot = BootId::from_bytes(rand::random()); Ok(self) } - fn seal( + pub(crate) fn seal( &mut self, message: OutboxMessage, ) -> Result<(SignedMailboxCapability, MailboxEnvelope), Refusal> { + let Some(dir) = self.intent_dir.clone() else { + let (capability, envelope, n) = self.seal_new(&message)?; + self.sent = n; + return Ok((capability, envelope)); + }; + private_fs::ensure_private_dir(&dir).map_err(|_| Refusal::Unavailable)?; + sync_dir(dir.parent().ok_or(Refusal::Malformed)?).map_err(|_| Refusal::Unavailable)?; + let _lock = FileLock::try_acquire(&dir.join("intent")) + .map_err(|_| Refusal::Unavailable)? + .ok_or(Refusal::Unavailable)?; + let path = dir.join(hex::encode(message.message_id.as_bytes())); + if path.exists() { + let reused = self.read_intent(&path, &message)?; + charge_source_attempt(&dir, &message)?; + return Ok(reused); + } + let mut count = 0; + for entry in std::fs::read_dir(&dir).map_err(|_| Refusal::Unavailable)? { + let entry = entry.map_err(|_| Refusal::Unavailable)?; + let name = entry.file_name(); + if name == "intent.lock" || name.to_string_lossy().ends_with(".attempts") { + continue; + } + count += 1; + if count >= MAX_SOURCE_INTENTS { + return Err(Refusal::QueueFull(MAX_SOURCE_INTENTS)); + } + } + let (capability, envelope, n) = self.seal_new(&message)?; + let bytes = encode_frame(&capability, &envelope)?; + if bytes.len() > MAX_SOURCE_INTENT_BYTES { + return Err(Refusal::Oversized(MAX_SOURCE_INTENT_BYTES)); + } + atomic_write_new(&path, &bytes).map_err(|_| Refusal::Unavailable)?; + sync_dir(&dir).map_err(|_| Refusal::Unavailable)?; + charge_source_attempt(&dir, &message)?; + self.sent = n; + Ok((capability, envelope)) + } + + fn read_intent( + &self, + path: &Path, + message: &OutboxMessage, + ) -> Result<(SignedMailboxCapability, MailboxEnvelope), Refusal> { + let meta = std::fs::symlink_metadata(path).map_err(|_| Refusal::Unavailable)?; + if !meta.file_type().is_file() + || meta.nlink() != 1 + || meta.len() > MAX_SOURCE_INTENT_BYTES as u64 + { + return Err(Refusal::Malformed); + } + let mut bytes = Vec::new(); + std::fs::File::open(path) + .map_err(|_| Refusal::Unavailable)? + .take(MAX_SOURCE_INTENT_BYTES as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|_| Refusal::Unavailable)?; + if bytes.len() > MAX_SOURCE_INTENT_BYTES { + return Err(Refusal::Oversized(MAX_SOURCE_INTENT_BYTES)); + } + let (capability, envelope) = decode_frame(&bytes)?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| Refusal::Unavailable)? + .as_millis() + .try_into() + .map_err(|_| Refusal::Unavailable)?; + let context = OpenContext { + expected_tenant: self.header.tenant, + expected_authorization_domain: self.header.authorization_domain, + expected_mailbox: self.header.mailbox, + expected_sender: self.header.sender, + expected_recipient: message.recipient, + acting_principal: self.capability.subject, + expected_capability_issuer: self.capability.issuer, + current_execution_epoch: self.header.execution_epoch, + current_policy_epoch: self.header.policy_epoch, + required_operation: mvm_contract::protocol::mailbox::MailboxOperation::Send, + now_unix_ms: now, + accepted_content_types: vec![self.header.content_type.clone()], + revoked_capabilities: Vec::new(), + }; + let opened = open_authenticated_envelope( + &envelope, + &capability, + &context, + &self.encryption_key, + &self.sender_key.verifying_key(), + &self.authority_key.verifying_key(), + &mut InMemoryReplayProtector::default(), + ) + .map_err(|e| Refusal::Mailbox(e.to_string()))?; + if opened.header().message_id != message.message_id + || opened.header().kind != self.header.kind + || opened.payload() != message.body + { + return Err(Refusal::Mailbox( + "message identity was reused with different bytes".into(), + )); + } + Ok((capability, envelope)) + } + + fn seal_new( + &self, + message: &OutboxMessage, + ) -> Result<(SignedMailboxCapability, MailboxEnvelope, u64), Refusal> { let gate = |e: mvm_core::mailbox::MailboxError| Refusal::Mailbox(e.to_string()); - self.sent += 1; - let n = self.sent; + let n = self.sent.checked_add(1).ok_or(Refusal::Malformed)?; let counter = |width: usize| { let mut bytes = vec![0; width]; bytes[width - 8..].copy_from_slice(&n.to_be_bytes()); @@ -109,15 +231,22 @@ impl GuestSealer { }; let mut header = self.header.clone(); header.recipient = message.recipient; - header.message_id = MessageId::from_bytes(counter(32).try_into().unwrap()); + header.message_id = message.message_id; + header.correlation_id = CorrelationId::from_bytes(*message.message_id.as_bytes()); header.nonce = ReplayNonce::from_bytes(counter(16).try_into().unwrap()); header.sequence = SequenceNumber::new(n).map_err(|_| Refusal::Malformed)?; - header.plaintext_len = message.body.len() as u32; + header.plaintext_len = message + .body + .len() + .try_into() + .map_err(|_| Refusal::Malformed)?; let capability = sign_capability(self.capability.clone(), &self.authority_key).map_err(gate)?; + let mut routing = self.routing.clone(); + routing.delivery_id = DeliveryId::from_bytes(rand::random()); let envelope = seal_envelope( SealRequest { - routing: self.routing.clone(), + routing, header, plaintext: &message.body, }, @@ -125,10 +254,34 @@ impl GuestSealer { &self.sender_key, ) .map_err(gate)?; - Ok((capability, envelope)) + Ok((capability, envelope, n)) } } +fn charge_source_attempt(dir: &Path, message: &OutboxMessage) -> Result<(), Refusal> { + let path = dir.join(format!( + "{}.attempts", + hex::encode(message.message_id.as_bytes()) + )); + let count = match std::fs::symlink_metadata(&path) { + Ok(meta) if meta.file_type().is_file() && meta.len() <= 3 => { + let raw = std::fs::read(&path).map_err(|_| Refusal::Unavailable)?; + std::str::from_utf8(&raw) + .ok() + .and_then(|n| n.parse::().ok()) + .ok_or(Refusal::Malformed)? + } + Ok(_) => return Err(Refusal::Malformed), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => 0, + Err(_) => return Err(Refusal::Unavailable), + }; + if count >= MAX_SOURCE_ATTEMPTS { + return Err(Refusal::RetriesExhausted); + } + atomic_write_durable(&path, (count + 1).to_string().as_bytes()) + .map_err(|_| Refusal::Unavailable) +} + /// Launch-path entry: serve the guest's mailbox port only when the signed /// plan's `grant` is present. No grant means no socket bound, no dial, and /// `None`; a grant yields the service holding whatever the guest sent, sealed @@ -159,11 +312,17 @@ pub fn serve_grant( guest: (&Path, &Path), ) -> std::io::Result { let mut sealer = sealer.with_grant(grant)?; - let service = MailboxService::bind(socket, limits, trust.with_grant(grant)?)?; - service.serve_guest(guest.0, guest.1, &mut sealer)?; + let service = MailboxService::bind_live(socket, limits, trust.with_grant(grant)?)?; + let _ = service.serve_guest(guest.0, guest.1, &mut sealer)?; Ok(service) } +#[derive(Clone, Copy)] +enum Clock { + Context, + Wall, +} + struct State { queue: VecDeque>, replay: InMemoryReplayProtector, @@ -172,6 +331,7 @@ struct State { struct Shared { limits: MailboxLimits, + clock: Clock, state: Mutex, } @@ -183,16 +343,36 @@ pub struct MailboxService { } impl MailboxService { - /// Bind `path` (replacing a stale socket file) and serve until dropped. + /// Bind `path` (replacing a stale socket file) with the caller-supplied + /// context clock; useful for deterministic callers. The production launch + /// uses `bind_live` to advance the clock at every admission. pub fn bind( path: &Path, limits: MailboxLimits, policy: MailboxPolicy, + ) -> std::io::Result { + Self::bind_with_clock(path, limits, policy, Clock::Context) + } + + pub(crate) fn bind_live( + path: &Path, + limits: MailboxLimits, + policy: MailboxPolicy, + ) -> std::io::Result { + Self::bind_with_clock(path, limits, policy, Clock::Wall) + } + + fn bind_with_clock( + path: &Path, + limits: MailboxLimits, + policy: MailboxPolicy, + clock: Clock, ) -> std::io::Result { let _ = std::fs::remove_file(path); let listener = UnixListener::bind(path)?; let shared = Arc::new(Shared { limits, + clock, state: Mutex::new(State { queue: VecDeque::new(), replay: InMemoryReplayProtector::default(), @@ -247,17 +427,16 @@ impl MailboxService { per_port: &Path, firecracker_mux: &Path, sealer: &mut GuestSealer, - ) -> std::io::Result<()> { + ) -> std::io::Result { let stream = crate::telemetry_collector::dial_guest_port( per_port, firecracker_mux, mvm_contract::protocol::mailbox::MAILBOX_PORT, )?; - serve(&self.shared, stream, |body| { + stream.set_read_timeout(Some(Duration::from_secs(7)))?; + Ok(serve(&self.shared, stream, |body| { sealer.seal(OutboxMessage::decode(body)?) - }) - .then_some(()) - .ok_or_else(|| std::io::Error::other("the guest sent no mailbox frame")) + })) } /// Take the oldest accepted plaintext, if any. @@ -309,9 +488,11 @@ fn serve( mut stream: UnixStream, prepare: impl FnOnce(&[u8]) -> Result<(SignedMailboxCapability, MailboxEnvelope), Refusal>, ) -> bool { - let _ = stream.set_read_timeout(Some(IO_TIMEOUT)); + if stream.read_timeout().ok().flatten().is_none() { + let _ = stream.set_read_timeout(Some(IO_TIMEOUT)); + } let _ = stream.set_write_timeout(Some(IO_TIMEOUT)); - let frame = read_frame(shared, &mut stream); + let frame = read_frame(shared.limits.max_frame_bytes, &mut stream); let received = frame.is_ok(); let verdict = frame .and_then(|body| prepare(&body)) @@ -322,8 +503,7 @@ fn serve( received } -fn read_frame(shared: &Shared, stream: &mut UnixStream) -> Result, Refusal> { - let max = shared.limits.max_frame_bytes; +fn read_frame(max: usize, stream: &mut UnixStream) -> Result, Refusal> { let mut len = [0u8; 4]; stream .read_exact(&mut len) @@ -349,6 +529,13 @@ fn admit( if state.queue.len() >= shared.limits.max_depth { return Err(Refusal::QueueFull(shared.limits.max_depth)); } + if let Clock::Wall = shared.clock { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| Refusal::Malformed)? + .as_millis(); + state.policy.context.now_unix_ms = now.try_into().map_err(|_| Refusal::Malformed)?; + } let p = &state.policy; let plaintext = open_envelope( envelope, diff --git a/crates/mvm-hostd/src/mailbox_service/durable.rs b/crates/mvm-hostd/src/mailbox_service/durable.rs new file mode 100644 index 0000000000..50fc3a7dd2 --- /dev/null +++ b/crates/mvm-hostd/src/mailbox_service/durable.rs @@ -0,0 +1,308 @@ +//! Destination-owned local endpoint for receipt-bearing durable admission. +//! +//! This listener is separate from the ephemeral mailbox queue. It never +//! interprets a successful connection as authority or a signed verdict: +//! authorization, journal fsync, and external anchor publication precede the +//! destination signature being returned to the caller. + +use std::io::Write; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::{SystemTime, UNIX_EPOCH}; + +use ed25519_dalek::{SigningKey, VerifyingKey}; +use mvm_contract::protocol::mailbox::{ + MailboxEnvelope, MailboxEnvelopeHeader, SignedDestinationReceipt, SignedMailboxCapability, +}; +use mvm_core::mailbox::frame::{OutboxMessage, Refusal, decode_frame, send_receipted_frame}; +use mvm_core::mailbox::journal::{ + FileJournalAnchor, JournalAdmission, JournalError, JournalReceive, JournalTip, MailboxJournal, + ReceivedMailboxMessage, +}; +use mvm_core::mailbox::{MailboxError, OpenContext, ReplayIdentity}; + +use super::{GuestSealer, IO_TIMEOUT, MailboxLimits, MailboxPolicy, read_frame}; + +/// A trusted anchor advances outside the journal's rollback domain. +pub trait JournalAnchor: Send + Sync { + fn commit(&self, tip: JournalTip) -> Result<(), JournalError>; +} + +impl JournalAnchor for F +where + F: Fn(JournalTip) -> Result<(), JournalError> + Send + Sync, +{ + fn commit(&self, tip: JournalTip) -> Result<(), JournalError> { + self(tip) + } +} + +impl JournalAnchor for FileJournalAnchor { + fn commit(&self, tip: JournalTip) -> Result<(), JournalError> { + self.advance(tip) + } +} + +struct State { + policy: MailboxPolicy, + journal: MailboxJournal, +} + +struct Shared { + limits: MailboxLimits, + state: Mutex, + signer: SigningKey, + anchor: Box, +} + +/// One destination authority's bounded, journal-backed local admission socket. +pub struct DurableMailboxService { + shared: Arc, + path: PathBuf, + stop: Arc, + worker: Option>, +} + +impl DurableMailboxService { + /// Bind a live destination. Caller provides an already-recovered journal + /// and an anchor backed by a different rollback domain. + pub fn bind( + path: &Path, + limits: MailboxLimits, + policy: MailboxPolicy, + signer: SigningKey, + journal: MailboxJournal, + anchor: impl JournalAnchor + 'static, + ) -> std::io::Result { + if signer.verifying_key() != policy.authority_key { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "destination signer does not match the trusted authority", + )); + } + if limits.max_frame_bytes == 0 || limits.max_depth == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "mailbox limits must be nonzero", + )); + } + let _ = std::fs::remove_file(path); + let listener = UnixListener::bind(path)?; + let shared = Arc::new(Shared { + limits, + state: Mutex::new(State { policy, journal }), + signer, + anchor: Box::new(anchor), + }); + let stop = Arc::new(AtomicBool::new(false)); + let worker = { + let (shared, stop) = (shared.clone(), stop.clone()); + std::thread::spawn(move || { + for stream in listener.incoming() { + if stop.load(Ordering::SeqCst) { + break; + } + if let Ok(stream) = stream { + serve(&shared, stream); + } + } + }) + }; + Ok(Self { + shared, + path: path.to_owned(), + stop, + worker: Some(worker), + }) + } + + /// Read one guest outbox operation over its dedicated mailbox port. The + /// guest sees success only after the canonical ciphertext, dedup identity, + /// and anchor have committed; an idle connection returns `false`. + pub fn serve_guest( + &self, + per_port: &Path, + firecracker_mux: &Path, + sealer: &mut GuestSealer, + ) -> std::io::Result { + let mut stream = crate::telemetry_collector::dial_guest_port( + per_port, + firecracker_mux, + mvm_contract::protocol::mailbox::MAILBOX_PORT, + )?; + stream.set_read_timeout(Some(std::time::Duration::from_secs(7)))?; + stream.set_write_timeout(Some(IO_TIMEOUT))?; + let body = read_frame(self.shared.limits.max_frame_bytes, &mut stream); + let received = body.is_ok(); + let verdict: Result<(), Refusal> = body + .and_then(|body| OutboxMessage::decode(&body)) + .and_then(|message| sealer.seal(message)) + .and_then(|(capability, envelope)| { + admit(&self.shared, &capability, &envelope).map(|_| ()) + }); + if let Ok(reply) = serde_json::to_vec(&verdict) { + let _ = stream.write_all(&reply); + } + Ok(received) + } + + /// A trusted host caller may receive only with a separate signed Receive + /// capability. This is not a transport-facing unauthenticated read. + pub fn receive_next( + &self, + capability: &SignedMailboxCapability, + reader: &OpenContext, + ) -> Result, JournalError> { + let state = self + .shared + .state + .lock() + .expect("mailbox journal lock poisoned"); + let context = reader_context(&state.policy, reader)?; + state.journal.receive_next(JournalReceive { + capability, + context: &context, + encryption_key: &state.policy.encryption_key, + sender_key: &state.policy.sender_key, + }) + } + + /// Persist an ACK from a previously authenticated reader; a lost reply + /// may be retried with the same identity without a second journal event. + pub fn acknowledge( + &self, + identity: ReplayIdentity, + capability: &SignedMailboxCapability, + reader: &OpenContext, + ) -> Result { + let mut state = self + .shared + .state + .lock() + .expect("mailbox journal lock poisoned"); + let context = reader_context(&state.policy, reader)?; + let State { policy, journal } = &mut *state; + journal.acknowledge( + identity, + JournalReceive { + capability, + context: &context, + encryption_key: &policy.encryption_key, + sender_key: &policy.sender_key, + }, + |tip| self.shared.anchor.commit(tip), + ) + } +} + +fn reader_context( + policy: &MailboxPolicy, + reader: &OpenContext, +) -> Result { + if reader.expected_tenant != policy.context.expected_tenant + || reader.expected_authorization_domain != policy.context.expected_authorization_domain + || reader.expected_mailbox != policy.context.expected_mailbox + || reader.expected_sender != policy.context.expected_sender + || reader.expected_recipient != policy.context.expected_recipient + || reader.current_execution_epoch != policy.context.current_execution_epoch + || reader.current_policy_epoch != policy.context.current_policy_epoch + { + return Err(JournalError::WrongAuthority); + } + let mut context = reader.clone(); + context.required_operation = mvm_contract::protocol::mailbox::MailboxOperation::Receive; + context.now_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| JournalError::Uncertain)? + .as_millis() + .try_into() + .map_err(|_| JournalError::Uncertain)?; + Ok(context) +} + +impl Drop for DurableMailboxService { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + let _ = UnixStream::connect(&self.path); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + let _ = std::fs::remove_file(&self.path); + } +} + +fn serve(shared: &Shared, mut stream: UnixStream) { + let _ = stream.set_read_timeout(Some(IO_TIMEOUT)); + let _ = stream.set_write_timeout(Some(IO_TIMEOUT)); + let result = read_frame(shared.limits.max_frame_bytes, &mut stream) + .and_then(|body| decode_frame(&body)) + .and_then(|(capability, envelope)| admit(shared, &capability, &envelope)); + if let Ok(reply) = serde_json::to_vec(&result) { + let _ = stream.write_all(&reply); + } +} + +fn admit( + shared: &Shared, + capability: &SignedMailboxCapability, + envelope: &MailboxEnvelope, +) -> Result { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| Refusal::Unavailable)? + .as_millis(); + let now: i64 = now.try_into().map_err(|_| Refusal::Unavailable)?; + let mut state = shared.state.lock().expect("mailbox journal lock poisoned"); + state.policy.context.now_unix_ms = now; + let State { policy, journal } = &mut *state; + journal + .admit( + JournalAdmission { + envelope, + capability, + context: &policy.context, + encryption_key: &policy.encryption_key, + sender_key: &policy.sender_key, + signer: &shared.signer, + }, + |tip| shared.anchor.commit(tip), + ) + .map_err(|error| match error { + JournalError::Mailbox(MailboxError::CapabilityPayloadExceeded) => { + Refusal::Oversized(capability.grant.max_payload_bytes as usize) + } + JournalError::Mailbox(error) => Refusal::Mailbox(error.to_string()), + JournalError::Full => Refusal::StorageFull, + JournalError::ConflictingRetry => Refusal::Mailbox(error.to_string()), + _ => Refusal::Unavailable, + }) +} + +/// Send an immutable, already-sealed delivery to the local destination. +/// A transport connection alone is never enough: the signed receipt must +/// identify the exact expected header and ciphertext commitment. +pub fn send( + path: &Path, + capability: &SignedMailboxCapability, + envelope: &MailboxEnvelope, + expected_header: &MailboxEnvelopeHeader, + destination_key: &VerifyingKey, +) -> Result { + let mut stream = UnixStream::connect(path).map_err(|_| Refusal::Unavailable)?; + stream + .set_read_timeout(Some(IO_TIMEOUT)) + .map_err(|_| Refusal::Unavailable)?; + stream + .set_write_timeout(Some(IO_TIMEOUT)) + .map_err(|_| Refusal::Unavailable)?; + send_receipted_frame( + &mut stream, + capability, + envelope, + expected_header, + destination_key, + ) +} diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index b807434e88..e6ad837517 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -320,6 +320,7 @@ fn sealer(channel: &Channel, operations: &[MailboxOperation]) -> GuestSealer { header: header(1, 1), routing: routing(), sent: 0, + intent_dir: None, } } @@ -327,6 +328,7 @@ fn sealer(channel: &Channel, operations: &[MailboxOperation]) -> GuestSealer { fn outbox_message() -> OutboxMessage { OutboxMessage { recipient: aircraft_principal(), + message_id: MessageId::from_bytes([25; 32]), body: b"hello".to_vec(), } } @@ -611,7 +613,7 @@ fn a_guest_flooding_the_mailbox_port_is_refused_at_its_own_queue() { // A Firecracker-style mux: the guest end answers CONNECT, then sends. let guest = std::thread::spawn(move || { let mut verdicts = Vec::new(); - for _ in 0..=LIMITS.max_depth { + for n in 0..=LIMITS.max_depth { let (stream, _) = listener.accept().unwrap(); let mut line = String::new(); BufReader::new(stream.try_clone().unwrap()) @@ -620,7 +622,9 @@ fn a_guest_flooding_the_mailbox_port_is_refused_at_its_own_queue() { assert_eq!(line.trim(), "CONNECT 5257"); (&stream).write_all(b"OK 1\n").unwrap(); let mut stream = stream; - verdicts.push(send_outbox_message(&mut stream, &outbox_message())); + let mut message = outbox_message(); + message.message_id = MessageId::from_bytes([n as u8 + 1; 32]); + verdicts.push(send_outbox_message(&mut stream, &message)); } verdicts }); @@ -735,7 +739,14 @@ fn a_signed_mailbox_grant_alone_makes_the_host_serve_the_guest_port() { let listener = UnixListener::bind(&per_port).unwrap(); listener.set_nonblocking(true).unwrap(); let (plan, trust) = plan_and_trust(&channel, false); - let sealer_for = || sealer(&channel, &ops); + let sealer_for = || { + let mut sealer = sealer(&channel, &ops); + let future = 4_000_000_000_000; + sealer.capability.expires_at_unix_ms = future; + sealer.header.expires_at_unix_ms = future; + sealer.routing.expires_at_unix_ms = future; + sealer + }; let none = serve_granted_guest( plan.mailbox.as_ref(), trust, @@ -754,7 +765,7 @@ fn a_signed_mailbox_grant_alone_makes_the_host_serve_the_guest_port() { // A grant naming an authority the host does not hold is refused. let (mut plan, trust) = plan_and_trust(&channel, true); plan.mailbox.as_mut().unwrap().authority_key = [1; 32]; - let refused = serve_granted_guest( + let foreign = serve_granted_guest( plan.mailbox.as_ref(), trust, sealer_for(), @@ -762,7 +773,7 @@ fn a_signed_mailbox_grant_alone_makes_the_host_serve_the_guest_port() { &socket, (&per_port, &absent_mux), ); - assert!(refused.is_err(), "a foreign authority key is not honored"); + assert!(foreign.is_err(), "a foreign authority key is not honored"); // Grant present: the guest forwards plaintext, the host seals and queues. listener.set_nonblocking(false).unwrap(); @@ -783,6 +794,23 @@ fn a_signed_mailbox_grant_alone_makes_the_host_serve_the_guest_port() { .expect("a granted plan is served"); assert_eq!(guest.join().unwrap(), Ok(())); assert_eq!(service.receive().unwrap(), b"hello"); + + let expired = seal_envelope( + SealRequest { + routing: routing(), + header: header(1, 2), + plaintext: b"hello", + }, + &channel.encryption_key, + &channel.sender_key, + ) + .unwrap(); + let capability = sign_capability(grant(&ops), &channel.authority_key).unwrap(); + assert_eq!( + send(&socket, &capability, &expired), + Err(refused(MailboxError::Expired)), + "live admission must not reuse the clock captured when the service started" + ); } /// A host key directory in a tempdir: the signing key, its public half and the @@ -824,6 +852,26 @@ fn granted_plan(authority_key: [u8; 32]) -> ExecutionPlan { plan } +#[test] +fn a_restarted_mailbox_sealer_does_not_reuse_its_sender_boot() { + use mvm_hostd::mailbox_launch::{authority_key_for, derive_launch_keys}; + + let dir = tempfile::tempdir().unwrap(); + host_keys(dir.path(), 5); + let authority = authority_key_for(dir.path(), &address().mailbox).unwrap(); + let grant = granted_plan(authority.to_bytes()).mailbox.unwrap(); + let keys = derive_launch_keys(dir.path(), &grant).unwrap(); + let (_, first) = keys.trust(); + let (_, restarted) = keys.trust(); + let first = first.with_grant(&grant).unwrap(); + let restarted = restarted.with_grant(&grant).unwrap(); + + assert_ne!( + first.header.sender_boot, restarted.header.sender_boot, + "a fresh process starts its sequence at one but must not replay the prior boot's identity" + ); +} + /// The sealing keys are derived from the host key per mailbox: they differ /// between mailboxes and hosts, and none is the host signing key. #[test] @@ -886,6 +934,7 @@ fn derived_keys_serve_the_guest_port_and_queue_what_the_guest_sent() { socket: dir.path().join("mailbox.sock"), guest_port: dir.path().join("port.sock"), firecracker_mux: dir.path().join("absent-mux"), + intent_dir: Some(dir.path().join("intents")), }, LAUNCH_LIMITS, ); @@ -899,6 +948,297 @@ fn derived_keys_serve_the_guest_port_and_queue_what_the_guest_sent() { assert_eq!(received.unwrap(), b"hello"); } +#[test] +fn a_running_guest_submits_again_after_its_first_message_was_consumed() { + use mvm_hostd::mailbox_launch::{ + LAUNCH_LIMITS, ServePaths, authority_key_for, derive_launch_keys, serve_derived, + }; + use std::os::unix::net::UnixListener; + + let dir = tempfile::tempdir().unwrap(); + host_keys(dir.path(), 5); + let listener = UnixListener::bind(dir.path().join("port.sock")).unwrap(); + let authority = authority_key_for(dir.path(), &address().mailbox).unwrap(); + let grant = granted_plan(authority.to_bytes()).mailbox.unwrap(); + let keys = derive_launch_keys(dir.path(), &grant).unwrap(); + let outbox = dir.path().join("outbox"); + mvm_agentd::mailbox_outbox::write_outbox(&outbox, &outbox_message()).unwrap(); + let guest_outbox = outbox.clone(); + let guest = std::thread::spawn(move || { + for _ in 0..2 { + let (mut stream, _) = listener.accept().unwrap(); + mvm_agentd::mailbox_outbox::wait_and_forward_outbox(&mut stream, &guest_outbox) + .expect("guest submits pending operation"); + } + }); + let launch = serve_derived( + grant, + keys, + ServePaths { + socket: dir.path().join("mailbox.sock"), + guest_port: dir.path().join("port.sock"), + firecracker_mux: dir.path().join("absent-mux"), + intent_dir: Some(dir.path().join("intents")), + }, + LAUNCH_LIMITS, + ); + let receive = || { + (0..100) + .find_map(|_| { + launch.receive().or_else(|| { + std::thread::sleep(std::time::Duration::from_millis(50)); + None + }) + }) + .expect("the live guest message arrives") + }; + assert_eq!(receive(), b"hello"); + let next = OutboxMessage::new(aircraft_principal(), b"after-start".to_vec()); + mvm_agentd::mailbox_outbox::write_outbox(&outbox, &next).unwrap(); + assert_eq!(receive(), b"after-start"); + guest.join().unwrap(); +} + +#[test] +fn durable_local_destination_signs_only_after_journal_and_anchor_commit() { + use mvm_contract::protocol::mailbox::DeliveryOutcome; + use mvm_core::mailbox::journal::{FileJournalAnchor, JournalLimits, MailboxJournal}; + use mvm_hostd::mailbox_service::durable::{DurableMailboxService, send as send_durable}; + + let (node, channel) = channel_with_one_node(); + let journal_dir = node.home.path().join("journal"); + let anchor_path = node.home.path().join("mailbox-anchor"); + let socket = node.home.path().join("durable.sock"); + let signer = channel.authority_key.clone(); + let authority = signer.verifying_key(); + let limits = JournalLimits { + max_records: 2, + max_bytes: 64 * 1024, + }; + let mut header = header(1, 3); + header.expires_at_unix_ms = 4_000_000_000_000; + let mut routing = routing(); + routing.expires_at_unix_ms = header.expires_at_unix_ms; + let mut capability_grant = grant(&[MailboxOperation::Send]); + capability_grant.expires_at_unix_ms = header.expires_at_unix_ms; + let mut undersized_grant = capability_grant.clone(); + undersized_grant.max_payload_bytes = 4; + let undersized = sign_capability(undersized_grant, &signer).unwrap(); + let capability = sign_capability(capability_grant, &signer).unwrap(); + let envelope = seal_envelope( + SealRequest { + routing, + header: header.clone(), + plaintext: b"hello", + }, + &channel.encryption_key, + &channel.sender_key, + ) + .unwrap(); + + let start = || { + let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]) + .expect("private local anchor"); + let journal = MailboxJournal::open( + &journal_dir, + address(), + authority, + [90; 32], + limits, + anchor.tip(), + ) + .expect("journal recovery"); + DurableMailboxService::bind( + &socket, + LIMITS, + MailboxPolicy { + context: context(1), + encryption_key: Key::from_bytes([23; 32]), + sender_key: channel.sender_key.verifying_key(), + authority_key: authority, + }, + signer.clone(), + journal, + anchor, + ) + .expect("destination listener") + }; + + let service = start(); + assert_eq!( + send_durable(&socket, &undersized, &envelope, &header, &authority), + Err(Refusal::Oversized(4)), + "the signed payload ceiling is a terminal refusal, not a durable receipt" + ); + let accepted = send_durable(&socket, &capability, &envelope, &header, &authority) + .expect("destination signed durable verdict"); + assert_eq!(accepted.receipt.outcome, DeliveryOutcome::AcceptedDurable); + let mut receive_grant = grant(&[MailboxOperation::Receive]); + receive_grant.subject = aircraft_principal(); + receive_grant.capability_id = CapabilityId::from_bytes([88; 32]); + receive_grant.expires_at_unix_ms = header.expires_at_unix_ms; + let receive_capability = sign_capability(receive_grant, &signer).unwrap(); + let mut reader = context(1); + reader.acting_principal = aircraft_principal(); + reader.required_operation = MailboxOperation::Receive; + assert!(matches!( + service.receive_next(&capability, &reader), + Err(mvm_core::mailbox::journal::JournalError::Mailbox( + MailboxError::WrongCapabilitySubject + )) + )); + let pending = service + .receive_next(&receive_capability, &reader) + .expect("independent Receive authority") + .expect("pending message"); + assert_eq!(pending.payload(), b"hello"); + let identity = pending.identity(); + service + .acknowledge(identity, &receive_capability, &reader) + .expect("durable ACK"); + assert!( + service + .receive_next(&receive_capability, &reader) + .expect("after ACK") + .is_none() + ); + drop(service); + let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]) + .expect("anchor survived destination restart"); + let tip = anchor.tip(); + assert_eq!(tip.cursor, 2, "acceptance and ACK each advanced the anchor"); + drop(anchor); + let service = start(); + assert!( + service + .receive_next(&receive_capability, &reader) + .expect("ACK survived restart") + .is_none() + ); + let duplicate = send_durable(&socket, &capability, &envelope, &header, &authority) + .expect("retry after service restart"); + assert_eq!(duplicate.receipt.outcome, DeliveryOutcome::Duplicate); + drop(service); + let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]) + .expect("retry did not corrupt the anchor"); + assert_eq!(anchor.tip(), tip, "retry must not append a record"); +} + +#[test] +fn durable_mailbox_helper_process() { + let Ok(root) = std::env::var("MVM_MAILBOX_TEST_ROOT") else { + return; + }; + use mvm_core::mailbox::journal::{FileJournalAnchor, JournalLimits, MailboxJournal}; + use mvm_hostd::mailbox_service::durable::DurableMailboxService; + + let root = std::path::Path::new(&root); + let signer = SigningKey::from_bytes(&[22; 32]); + let authority = signer.verifying_key(); + let anchor = FileJournalAnchor::open(&root.join("anchor"), address(), [91; 32]) + .expect("recover software anchor"); + let journal = MailboxJournal::open_reconciling( + &root.join("journal"), + address(), + authority, + [90; 32], + JournalLimits { + max_records: 2, + max_bytes: 64 * 1024, + }, + &anchor, + ) + .expect("recover journal and dedup"); + let _service = DurableMailboxService::bind( + &root.join("durable.sock"), + LIMITS, + MailboxPolicy { + context: context(1), + encryption_key: Key::from_bytes([23; 32]), + sender_key: SigningKey::from_bytes(&[21; 32]).verifying_key(), + authority_key: authority, + }, + signer, + journal, + anchor, + ) + .expect("bind destination"); + loop { + std::thread::sleep(std::time::Duration::from_secs(60)); + } +} + +#[test] +fn a_killed_destination_reopens_its_anchor_and_deduplicates_the_retry() { + use mvm_contract::protocol::mailbox::DeliveryOutcome; + use mvm_core::mailbox::journal::FileJournalAnchor; + use mvm_hostd::mailbox_service::durable::send as send_durable; + + let dir = tempfile::tempdir_in("/tmp").expect("short Unix socket path"); + let socket = dir.path().join("durable.sock"); + let signer = SigningKey::from_bytes(&[22; 32]); + let authority = signer.verifying_key(); + let mut header = header(1, 3); + header.expires_at_unix_ms = 4_000_000_000_000; + let mut routing = routing(); + routing.expires_at_unix_ms = header.expires_at_unix_ms; + let mut capability_grant = grant(&[MailboxOperation::Send]); + capability_grant.expires_at_unix_ms = header.expires_at_unix_ms; + let capability = sign_capability(capability_grant, &signer).unwrap(); + let envelope = seal_envelope( + SealRequest { + routing, + header: header.clone(), + plaintext: b"hello", + }, + &Key::from_bytes([23; 32]), + &SigningKey::from_bytes(&[21; 32]), + ) + .unwrap(); + let launch = || { + KillOnDrop( + std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "durable_mailbox_helper_process"]) + .env("MVM_MAILBOX_TEST_ROOT", dir.path()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("start destination process"), + dir.path().join("child.log"), + ) + }; + let submit = || { + (0..100) + .find_map(|_| { + match send_durable(&socket, &capability, &envelope, &header, &authority) { + Ok(receipt) => Some(receipt), + Err(Refusal::Unavailable) => { + std::thread::sleep(std::time::Duration::from_millis(20)); + None + } + Err(other) => panic!("unexpected destination refusal: {other}"), + } + }) + .expect("destination was ready within two seconds") + }; + + let mut process = launch(); + assert_eq!(submit().receipt.outcome, DeliveryOutcome::AcceptedDurable); + process.0.kill().expect("crash destination"); + process.0.wait().expect("reap crashed process"); + drop(process); + let anchor = FileJournalAnchor::open(&dir.path().join("anchor"), address(), [91; 32]) + .expect("anchor survives process crash"); + assert_eq!( + anchor.tip().cursor, + 1, + "the signed verdict followed durable commit" + ); + drop(anchor); + let _restarted = launch(); + assert_eq!(submit().receipt.outcome, DeliveryOutcome::Duplicate); +} + struct KillOnDrop(std::process::Child, std::path::PathBuf); impl KillOnDrop { @@ -986,6 +1326,95 @@ fn the_mailbox_process_serves_a_granted_mailbox_from_derived_keys_alone() { "the host service bound its socket; stderr:\n{}", process.stderr() ); + assert!( + !outbox.exists(), + "guest received a durable admission verdict" + ); + assert!(state_dir.join("mailbox-anchor").exists()); + let record = std::fs::read( + state_dir + .join("mailbox-journal") + .join("entry-00000000000000000001.bin"), + ) + .expect("accepted canonical ciphertext is durable"); + assert!( + !record.windows(5).any(|w| w == b"hello"), + "the durable record never stores guest plaintext" + ); + let later = OutboxMessage::new(aircraft_principal(), b"after-start".to_vec()); + mvm_agentd::mailbox_outbox::write_outbox(&outbox, &later).unwrap(); + let mut stream = process.accept(&listener); + assert_eq!( + mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &outbox), + Ok(()), + "the running process keeps serving the live guest" + ); + assert!( + state_dir + .join("mailbox-journal") + .join("entry-00000000000000000002.bin") + .exists(), + "a post-start operation committed durably" + ); + process.0.kill().expect("crash the mailbox helper"); + process.0.wait().expect("reap the crashed helper"); + drop(process); + mvm_agentd::mailbox_outbox::write_outbox(&outbox, &outbox_message()).unwrap(); + let mut restarted = spawn_mailbox_process(&wire); + let forwarded = (0..8).any(|_| { + let mut stream = restarted.accept(&listener); + matches!( + mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &outbox), + Ok(()) + ) + }); + assert!( + forwarded, + "the same immutable guest operation must be reconciled after restart" + ); + assert!( + !state_dir + .join("mailbox-journal") + .join("entry-00000000000000000003.bin") + .exists(), + "a recovered duplicate does not create a second durable acceptance" + ); +} + +#[test] +fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { + use mvm_hostd::mailbox_process::build_config; + + let dir = tempfile::tempdir().unwrap(); + let keys = dir.path().join("keys"); + std::fs::create_dir_all(&keys).unwrap(); + host_keys(&keys, 5); + let state_dir = dir.path().join("vm"); + std::fs::create_dir_all(&state_dir).unwrap(); + let authority = mvm_hostd::mailbox_launch::authority_key_for(&keys, &address().mailbox) + .unwrap() + .to_bytes(); + let grant = granted_plan(authority).mailbox.unwrap(); + let config = build_config(&keys, &grant, &state_dir).unwrap(); + std::fs::write(state_dir.join("mailbox-anchor"), b"{").unwrap(); + let mut process = spawn_mailbox_process(&serde_json::to_vec(&config).unwrap()); + let status = (0..100).find_map(|_| { + let exited = process.0.try_wait().unwrap(); + if exited.is_none() { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + exited + }); + assert_eq!( + status.and_then(|status| status.code()), + Some(1), + "a corrupt anchor must fail closed; stderr:\n{}", + process.stderr() + ); + assert!( + !config.socket.exists(), + "no mailbox endpoint may be published from a corrupt anchor" + ); } // ---- two nodes over a loopback TCP PeerTransport ---------------------------- diff --git a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs index 68efcc9318..1db908768f 100644 --- a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs +++ b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs @@ -294,6 +294,7 @@ impl NetworkEndpointSpawner for RealNetworkEndpointSpawner { &IdentityDriveContents { ingress: req.ingress, egress_ca_cert_pem: egress_ca.guest_cert_pem(), + mailbox_ack_key: None, }, )?; // Persisted so a warm child claimed from this VM pins the key diff --git a/crates/mvm-vmm/src/host/flowmux_identity.rs b/crates/mvm-vmm/src/host/flowmux_identity.rs index 4761ada64f..daaf12830d 100644 --- a/crates/mvm-vmm/src/host/flowmux_identity.rs +++ b/crates/mvm-vmm/src/host/flowmux_identity.rs @@ -40,7 +40,7 @@ use mvm_core::atomic_io::write_private_ephemeral; // cannot leave the writer and the reader describing different drives. pub use mvm_agentd::flowmux_drive::{ EGRESS_CA_CERT_FILE, GUEST_SIGNING_KEY_FILE, GuestIngressTarget, HOST_SIGNER_PUB_FILE, - IDENTITY_DRIVE_LABEL, INGRESS_TARGETS_FILE, + IDENTITY_DRIVE_LABEL, INGRESS_TARGETS_FILE, MAILBOX_ACK_SIGNING_KEY_FILE, }; /// Mode the guest signing key is stored with. The drive is mounted read-only, @@ -60,7 +60,7 @@ const EGRESS_CA_CERT_MODE: u16 = 0o444; /// optional-shaped, both are "material the host projects into the guest", and a /// third one is likelier than not. `Default` is the empty projection — the /// builder VM's own identity drive carries neither. -#[derive(Debug, Default, Clone, Copy)] +#[derive(Default, Clone, Copy)] pub struct IdentityDriveContents<'a> { /// The signed plan's guest-loopback ingress targets. pub ingress: &'a [mvm_core::plan::IngressMapping], @@ -68,6 +68,52 @@ pub struct IdentityDriveContents<'a> { /// so an unmodified TLS client accepts a host-terminated bound-host flow. /// `None` when the plan binds no destination, and never a key. pub egress_ca_cert_pem: Option<&'a str>, + /// Only project a distinct guest-held ACK key when a signed Receive + /// capability and current-boot enrollment will authorize its use. + pub mailbox_ack_key: Option<&'a MailboxAckKeyMaterial>, +} + +impl std::fmt::Debug for IdentityDriveContents<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IdentityDriveContents") + .field("ingress", &self.ingress) + .field("egress_ca_cert_pem", &self.egress_ca_cert_pem) + .field( + "mailbox_ack_key", + &self.mailbox_ack_key.map(|_| ""), + ) + .finish() + } +} + +/// Independently minted guest mailbox ACK identity, never a FlowMux or host sealing key. +pub struct MailboxAckKeyMaterial { + seed: zeroize::Zeroizing<[u8; 32]>, +} + +impl std::fmt::Debug for MailboxAckKeyMaterial { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MailboxAckKeyMaterial") + .field("seed", &"") + .finish() + } +} + +impl MailboxAckKeyMaterial { + /// Mint a new identity for one guest boot; enrollment and Receive authority + /// must be checked separately before accepting an ACK signed by this key. + pub fn mint() -> Result { + let mut seed = zeroize::Zeroizing::new([0u8; 32]); + rand::rngs::SysRng + .try_fill_bytes(seed.as_mut()) + .context("drawing system entropy for the mailbox ACK key")?; + Ok(Self { seed }) + } + + /// Public key for binding the guest-held identity to an authorized boot. + pub fn verifying_key(&self) -> VerifyingKey { + SigningKey::from_bytes(&self.seed).verifying_key() + } } /// A minted per-boot identity: what the endpoint is told, and what the guest is @@ -294,6 +340,15 @@ impl GuestIdentityMaterial { owner: Owner::ROOT, }); } + if let Some(ack_key) = contents.mailbox_ack_key { + nodes.push(Node::File { + path: format!("/{MAILBOX_ACK_SIGNING_KEY_FILE}"), + mode: GUEST_SIGNING_KEY_MODE, + data: ack_key.seed.to_vec(), + xattrs: Vec::new(), + owner: Owner::ROOT, + }); + } let image = build_image( nodes, &BuildOptions::default().with_volume_name(IDENTITY_DRIVE_LABEL.as_bytes()), @@ -430,6 +485,51 @@ mod tests { ); } + #[test] + fn mailbox_ack_key_is_distinct_optional_and_redacted() { + let anchor = host_key().verifying_key(); + let guest = GuestIdentityMaterial::mint(anchor).expect("guest key"); + let ack = MailboxAckKeyMaterial::mint().expect("mailbox key"); + let other = MailboxAckKeyMaterial::mint().expect("next boot"); + assert_ne!(ack.verifying_key(), guest.verifying_key()); + assert_ne!(ack.verifying_key(), other.verifying_key()); + assert_ne!(ack.verifying_key(), host_key().verifying_key()); + + let dir = tempfile::tempdir().expect("tempdir"); + let drive = dir.path().join(IDENTITY_DRIVE_FILE); + guest + .write_drive_with(&drive, &IdentityDriveContents::default()) + .expect("ordinary identity drive"); + let ordinary = std::fs::read(&drive).expect("ordinary drive"); + assert!( + !ordinary + .windows(MAILBOX_ACK_SIGNING_KEY_FILE.len()) + .any(|part| part == MAILBOX_ACK_SIGNING_KEY_FILE.as_bytes()) + ); + + let contents = IdentityDriveContents { + mailbox_ack_key: Some(&ack), + ..Default::default() + }; + assert!(!format!("{ack:?}").contains(&hex::encode(ack.seed.as_slice()))); + assert!(!format!("{contents:?}").contains(&hex::encode(ack.seed.as_slice()))); + guest + .write_drive_with(&drive, &contents) + .expect("mailbox identity drive"); + let enrolled = std::fs::read(&drive).expect("mailbox drive"); + assert!( + enrolled + .windows(MAILBOX_ACK_SIGNING_KEY_FILE.len()) + .any(|part| part == MAILBOX_ACK_SIGNING_KEY_FILE.as_bytes()) + ); + assert!(enrolled.windows(32).any(|part| part == ack.seed.as_slice())); + assert!( + !enrolled + .windows(32) + .any(|part| part == host_key().to_bytes()) + ); + } + #[test] fn the_published_guest_key_is_the_one_on_the_drive() { let material = FlowMuxIdentityMaterial::mint("s1", &host_key()).expect("mint"); @@ -449,6 +549,27 @@ mod tests { #[test] fn writing_an_identity_drive_is_not_artifact_hashing() { + // Acquisition is a process-global counter. Other tests hash artifacts + // concurrently, so measure this operation in its own test process. + if std::env::var_os("MVM_DRIVE_HASH_TEST_CHILD").is_none() { + let test_name = format!( + "{}::writing_an_identity_drive_is_not_artifact_hashing", + module_path!().trim_start_matches(concat!(env!("CARGO_CRATE_NAME"), "::")) + ); + let result = std::process::Command::new(std::env::current_exe().expect("test binary")) + .arg("--exact") + .arg(test_name) + .env("MVM_DRIVE_HASH_TEST_CHILD", "1") + .output() + .expect("run isolated drive test"); + assert!( + result.status.success() + && String::from_utf8_lossy(&result.stdout).contains("1 passed"), + "isolated identity-drive measurement failed: {}", + String::from_utf8_lossy(&result.stdout) + ); + return; + } let dir = tempfile::tempdir().unwrap(); let path = dir.path().join(IDENTITY_DRIVE_FILE); let material = FlowMuxIdentityMaterial::mint("cold", &host_key()).unwrap(); @@ -569,6 +690,7 @@ mod tests { &IdentityDriveContents { ingress: &[], egress_ca_cert_pem: Some(ca.cert_pem()), + mailbox_ack_key: None, }, ) .expect("write drive"); @@ -604,6 +726,7 @@ mod tests { &IdentityDriveContents { ingress: &[], egress_ca_cert_pem: Some(poisoned), + mailbox_ack_key: None, }, ); diff --git a/specs/adrs/057-mailbox-semantics-and-threat-model.md b/specs/adrs/057-mailbox-semantics-and-threat-model.md index 9595dc8c42..afcabb5440 100644 --- a/specs/adrs/057-mailbox-semantics-and-threat-model.md +++ b/specs/adrs/057-mailbox-semantics-and-threat-model.md @@ -193,6 +193,18 @@ revocation state before payload disclosure or queue allocation. A sender field inside the envelope never authorizes itself. Delegation may only remove authority or tighten bounds. +Guest Receive and ACK use a separate, signed Receive capability and a distinct +per-boot guest-held mailbox ACK signing key, not the FlowMux egress identity or +the host's mailbox sealing key. The destination authority enrolls its public +key against the signed plan ID/nonce, recipient, current authenticated guest +boot and reader session, execution/policy epoch, capability ID, and validity +window. The guest proves possession over a fresh challenge before delivery; +the destination signs a lease binding that session, delivery, ciphertext +commitment, and deadline. ACKs are signed by the enrolled guest key over the +exact lease and are replay-checked and journaled before a terminal disposition. +A transport connection, inherited warm-parent key, or structurally valid +enrollment without signature and current-boot verification authorizes nothing. + ### 5. `call`, `cast`, reply, cancellation, and admission Version 1 has three application message kinds: @@ -376,11 +388,13 @@ Overlay resolution conveys no send authority. Transport authentication conveys no mailbox authority. The encoded canonical envelope crosses the byte transport unchanged. -Iroh is the first planned cross-host transport behind `PeerTransport`. Direct -and customer-configured relay paths are the default; public relay use requires -explicit configuration and cannot be a silent fallback. HeatNet/UOR and a -future virtual NIC are evaluated separately and cannot smuggle packet -forwarding into mailbox authority. +`mvm` defines the mailbox contract, authorization gates, and transport-neutral +interfaces; it does not implement a cross-host network. `mvmd` owns concrete +cross-host transport and orchestration, including the planned Iroh adapter +behind `PeerTransport`. Direct and customer-configured relay paths are the +default; public relay use requires explicit configuration and cannot be a +silent fallback. HeatNet/UOR and a future virtual NIC are evaluated separately +and cannot smuggle packet forwarding into mailbox authority. `mvmd` may enroll principals, place workloads, distribute signed route hints, and store or forward opaque ciphertext. It cannot define a second envelope, @@ -389,9 +403,10 @@ an ordinary coordinator; or become required for local delivery. The MVM interfaces reuse operational lessons from the earlier `mvmd` `AgentTransport` work without importing an `mvmd` crate or runtime dependency. -Optional coordinator metadata enters through a narrow route-hint adapter that -selects an already authorized path; it cannot edit, decrypt, or re-sign the -canonical envelope. +`mvmd` implements cross-host adapters for the interfaces and may adapt its +coordinator metadata into authorized route hints; it cannot edit, decrypt, or +re-sign the canonical envelope. Local mailbox delivery remains usable without +`mvmd`. ### 14. Prototype `mvmd` type migration From 8448257796d1a831a4ae25cf41e8c50a210bced7 Mon Sep 17 00:00:00 2001 From: Ari Date: Mon, 9 Nov 2026 18:38:03 -0800 Subject: [PATCH 02/51] feat(mailbox): define typed reports and reader delivery contract --- crates/mvm-build/src/guest_bins/mod.rs | 5 +- crates/mvm-contract/src/protocol/mailbox.rs | 474 ++++++++++++++++++ crates/mvm-contract/src/protocol/mod.rs | 2 + .../src/protocol/workload_report.rs | 235 +++++++++ crates/mvm-contract/tests/mailbox_delivery.rs | 428 ++++++++++++++++ crates/mvm-contract/tests/workload_report.rs | 124 +++++ crates/mvm-core/tests/mailbox_crypto.rs | 86 ++++ 7 files changed, 1352 insertions(+), 2 deletions(-) create mode 100644 crates/mvm-contract/src/protocol/workload_report.rs create mode 100644 crates/mvm-contract/tests/mailbox_delivery.rs create mode 100644 crates/mvm-contract/tests/workload_report.rs diff --git a/crates/mvm-build/src/guest_bins/mod.rs b/crates/mvm-build/src/guest_bins/mod.rs index 6703407aba..b49e1febd0 100644 --- a/crates/mvm-build/src/guest_bins/mod.rs +++ b/crates/mvm-build/src/guest_bins/mod.rs @@ -1058,8 +1058,9 @@ mod tests { #[test] fn the_producer_records_the_producing_tree_and_every_member() { - let workspace = guest_agent_build::detect_source_workspace() - .expect("the test runs inside the mvm workspace"); + let workspace = + guest_agent_build::source_workspace_from(Path::new(env!("CARGO_MANIFEST_DIR"))) + .expect("the test crate lives inside the mvm workspace"); let tmp = tempfile::tempdir().unwrap(); let cache_root = tmp.path().join("cache"); for arch in [GuestArch::Aarch64, GuestArch::X86_64] { diff --git a/crates/mvm-contract/src/protocol/mailbox.rs b/crates/mvm-contract/src/protocol/mailbox.rs index e354f67f11..16ad25f1d5 100644 --- a/crates/mvm-contract/src/protocol/mailbox.rs +++ b/crates/mvm-contract/src/protocol/mailbox.rs @@ -9,6 +9,8 @@ use alloc::vec::Vec; use serde::{Deserialize, Serialize}; +use crate::plan::{Nonce, PlanId}; + /// Dedicated mailbox vsock service. Its own port, so mailbox traffic never /// shares a queue or credit with machine control (5252), `NetworkFlow` (5253), /// telemetry (5254), display (5255) or GPU (5256). @@ -23,6 +25,8 @@ pub const AES_256_GCM_ALGORITHM: u16 = 1; pub const MAX_MAILBOX_PAYLOAD_BYTES: usize = 1024 * 1024; /// Maximum encoded envelope accepted before deserialization. pub const MAX_ENCODED_ENVELOPE_BYTES: usize = 8 * 1024 * 1024; +/// Maximum encoded delivery lease or ACK before parsing untrusted JSON. +pub const MAX_MAILBOX_DELIVERY_CONTROL_BYTES: usize = 16 * 1024; /// Maximum number of operations, kinds, or content types in one capability. pub const MAX_CAPABILITY_LIST_ITEMS: usize = 32; /// Ed25519 signature length. @@ -106,6 +110,17 @@ fixed_id!( 16, "Fresh sender nonce authenticated with the message; stable IDs, not this nonce, key replay detection." ); +fixed_id!( + ConsumerSessionId, + 32, + "Fresh MVM-bound authenticated reader session; a caller-supplied transport ID is not authority." +); +fixed_id!( + DeliveryLeaseNonce, + 16, + "Fresh nonce for one delivery lease." +); +fixed_id!(DeliveryAckNonce, 16, "Fresh nonce for one application ACK."); /// Protocol version carried on every signed object. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -640,6 +655,465 @@ pub struct SignedDestinationReceipt { pub signature: Vec, } +/// Versioned signature domains for guest reader enrollment, lease, and ACK. +pub const READER_ENROLLMENT_SIGNATURE_DOMAIN: &[u8] = b"mvm.mailbox.reader-enrollment.v1\0"; +pub const DELIVERY_LEASE_SIGNATURE_DOMAIN: &[u8] = b"mvm.mailbox.delivery-lease.v1\0"; +pub const DELIVERY_ACK_SIGNATURE_DOMAIN: &[u8] = b"mvm.mailbox.delivery-ack.v1\0"; + +/// Guest-held mailbox ACK key enrolled for one signed plan and fresh boot. +/// +/// Host-prepared seeds are permitted only in the trusted-host profile. A +/// host-blind profile must instead release keys inside a measured guest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MailboxGuestReaderEnrollment { + pub protocol_version: ProtocolVersion, + pub signature_algorithm: SignatureAlgorithm, + pub plan_id: PlanId, + pub plan_nonce: Nonce, + pub address: MailboxAddress, + pub consumer: PrincipalId, + pub consumer_boot: BootId, + pub consumer_session: ConsumerSessionId, + pub receive_capability_id: CapabilityId, + pub ack_public_key: [u8; 32], + pub execution_epoch: ExecutionEpoch, + pub policy_epoch: u64, + pub issued_at_unix_ms: i64, + pub expires_at_unix_ms: i64, +} + +/// Plan- and boot-derived expectations supplied by the verifying runtime. +/// +/// Construct these only from an authenticated current execution; transport +/// claims and unverified plan JSON are not trusted sources of this context. +#[derive(Clone, Copy)] +pub struct MailboxGuestReaderScope<'a> { + pub plan: &'a MailboxGrant, + pub plan_id: &'a PlanId, + pub plan_nonce: &'a Nonce, + pub receive: &'a MailboxCapabilityGrant, + pub current_boot: BootId, + pub current_session: ConsumerSessionId, + pub now_unix_ms: i64, +} + +/// Enrollment signed by the authority pinned in the admitted mailbox grant. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SignedMailboxGuestReaderEnrollment { + pub enrollment: MailboxGuestReaderEnrollment, + #[serde(with = "base64_bytes")] + pub signature: Vec, +} + +impl SignedMailboxGuestReaderEnrollment { + fn validate_shape(&self) -> Result<(), MailboxDeliveryError> { + let enrollment = &self.enrollment; + if enrollment.protocol_version != ProtocolVersion::V1 { + return Err(MailboxDeliveryError::UnsupportedVersion); + } + if enrollment.signature_algorithm != SignatureAlgorithm::ED25519 { + return Err(MailboxDeliveryError::UnsupportedAlgorithm); + } + if self.signature.len() != ED25519_SIGNATURE_BYTES { + return Err(MailboxDeliveryError::InvalidSignatureLength); + } + if enrollment.plan_id.0.is_empty() || enrollment.plan_id.0.len() > 128 { + return Err(MailboxDeliveryError::WrongPlanIdentity); + } + if enrollment.ack_public_key == [0; 32] { + return Err(MailboxDeliveryError::InvalidReaderKey); + } + if enrollment.expires_at_unix_ms <= enrollment.issued_at_unix_ms { + return Err(MailboxDeliveryError::InvalidTimeWindow); + } + Ok(()) + } + + /// Encodes a bounded enrollment; runtime still verifies its signature. + pub fn encode(&self) -> Result, MailboxDeliveryError> { + self.validate_shape()?; + let bytes = serde_json::to_vec(self).map_err(|_| MailboxDeliveryError::Malformed)?; + if bytes.len() > MAX_MAILBOX_DELIVERY_CONTROL_BYTES { + return Err(MailboxDeliveryError::Oversized); + } + Ok(bytes) + } + + /// Bounds an untrusted enrollment before parsing its JSON and signature. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_MAILBOX_DELIVERY_CONTROL_BYTES { + return Err(MailboxDeliveryError::Oversized); + } + let enrolled: Self = + serde_json::from_slice(bytes).map_err(|_| MailboxDeliveryError::Malformed)?; + enrolled.validate_shape()?; + Ok(enrolled) + } + + /// Checks structure against an authenticated current plan, boot, and reader. + /// + /// The runtime must verify this enrollment's signature under the plan's + /// mailbox authority key, the Receive capability signature/revocation, + /// and guest possession of the bound per-boot key before accepting an ACK. + pub fn check_scope( + &self, + scope: &MailboxGuestReaderScope<'_>, + ) -> Result<(), MailboxDeliveryError> { + self.validate_shape()?; + let enrolled = &self.enrollment; + if enrolled.address.mailbox != scope.plan.mailbox { + return Err(MailboxDeliveryError::WrongPlanMailbox); + } + if &enrolled.plan_id != scope.plan_id || &enrolled.plan_nonce != scope.plan_nonce { + return Err(MailboxDeliveryError::WrongPlanIdentity); + } + if enrolled.execution_epoch != scope.plan.epoch + || enrolled.execution_epoch < scope.receive.minimum_execution_epoch + || enrolled.execution_epoch > scope.receive.maximum_execution_epoch + { + return Err(MailboxDeliveryError::StaleExecutionEpoch); + } + if enrolled.address != scope.receive.address { + return Err(MailboxDeliveryError::WrongAddress); + } + if enrolled.consumer != scope.receive.subject { + return Err(MailboxDeliveryError::WrongConsumer); + } + if enrolled.consumer_boot != scope.current_boot { + return Err(MailboxDeliveryError::StaleConsumerBoot); + } + if enrolled.consumer_session != scope.current_session { + return Err(MailboxDeliveryError::StaleConsumerSession); + } + if enrolled.receive_capability_id != scope.receive.capability_id { + return Err(MailboxDeliveryError::WrongCapability); + } + if !scope.receive.operations.allows(MailboxOperation::Receive) { + return Err(MailboxDeliveryError::ReceiveNotAuthorized); + } + if enrolled.policy_epoch != scope.receive.policy_epoch { + return Err(MailboxDeliveryError::StalePolicyEpoch); + } + if scope.now_unix_ms < enrolled.issued_at_unix_ms + || enrolled.issued_at_unix_ms < scope.receive.not_before_unix_ms + { + return Err(MailboxDeliveryError::NotYetValid); + } + if scope.now_unix_ms >= enrolled.expires_at_unix_ms + || scope.now_unix_ms >= scope.receive.expires_at_unix_ms + || enrolled.expires_at_unix_ms > scope.receive.expires_at_unix_ms + { + return Err(MailboxDeliveryError::Expired); + } + Ok(()) + } +} + +/// One destination-issued lease to an authorized consumer of a specific message. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MailboxDeliveryLease { + pub protocol_version: ProtocolVersion, + pub signature_algorithm: SignatureAlgorithm, + pub address: MailboxAddress, + pub sender: PrincipalId, + pub sender_boot: BootId, + pub message_id: MessageId, + pub delivery_id: DeliveryId, + pub execution_epoch: ExecutionEpoch, + pub consumer: PrincipalId, + pub consumer_session: ConsumerSessionId, + pub receive_capability_id: CapabilityId, + pub policy_epoch: u64, + pub ciphertext_commitment: [u8; 32], + pub issued_at_unix_ms: i64, + pub expires_at_unix_ms: i64, + pub lease_nonce: DeliveryLeaseNonce, +} + +/// Lease signed by the destination mailbox authority, not by a transport. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SignedMailboxDeliveryLease { + pub lease: MailboxDeliveryLease, + #[serde(with = "base64_bytes")] + pub signature: Vec, +} + +impl MailboxDeliveryLease { + fn validate_shape(&self) -> Result<(), MailboxDeliveryError> { + if self.protocol_version != ProtocolVersion::V1 { + return Err(MailboxDeliveryError::UnsupportedVersion); + } + if self.signature_algorithm != SignatureAlgorithm::ED25519 { + return Err(MailboxDeliveryError::UnsupportedAlgorithm); + } + if self.expires_at_unix_ms <= self.issued_at_unix_ms { + return Err(MailboxDeliveryError::InvalidTimeWindow); + } + Ok(()) + } + + /// Checks data consistency with a verified plan and Receive capability. + /// + /// This does not verify signatures, authenticate the supplied current session, + /// or check replay. The destination must do so before payload disclosure. + pub fn check_scope( + &self, + plan: &MailboxGrant, + receive: &MailboxCapabilityGrant, + consumer_session: ConsumerSessionId, + now_unix_ms: i64, + ) -> Result<(), MailboxDeliveryError> { + let lease = self; + lease.validate_shape()?; + if lease.address.mailbox != plan.mailbox { + return Err(MailboxDeliveryError::WrongPlanMailbox); + } + if lease.execution_epoch != plan.epoch + || lease.execution_epoch < receive.minimum_execution_epoch + || lease.execution_epoch > receive.maximum_execution_epoch + { + return Err(MailboxDeliveryError::StaleExecutionEpoch); + } + if lease.address != receive.address { + return Err(MailboxDeliveryError::WrongAddress); + } + if lease.consumer != receive.subject { + return Err(MailboxDeliveryError::WrongConsumer); + } + if lease.consumer_session != consumer_session { + return Err(MailboxDeliveryError::StaleConsumerSession); + } + if lease.receive_capability_id != receive.capability_id { + return Err(MailboxDeliveryError::WrongCapability); + } + if !receive.operations.allows(MailboxOperation::Receive) { + return Err(MailboxDeliveryError::ReceiveNotAuthorized); + } + if lease.policy_epoch != receive.policy_epoch { + return Err(MailboxDeliveryError::StalePolicyEpoch); + } + if now_unix_ms < lease.issued_at_unix_ms + || lease.issued_at_unix_ms < receive.not_before_unix_ms + || now_unix_ms < receive.not_before_unix_ms + { + return Err(MailboxDeliveryError::NotYetValid); + } + if now_unix_ms >= lease.expires_at_unix_ms + || now_unix_ms >= receive.expires_at_unix_ms + || lease.expires_at_unix_ms > receive.expires_at_unix_ms + { + return Err(MailboxDeliveryError::Expired); + } + Ok(()) + } +} + +impl SignedMailboxDeliveryLease { + fn validate_shape(&self) -> Result<(), MailboxDeliveryError> { + self.lease.validate_shape()?; + if self.signature.len() != ED25519_SIGNATURE_BYTES { + return Err(MailboxDeliveryError::InvalidSignatureLength); + } + Ok(()) + } + + /// Encodes a bounded lease after checking its version and signature shape. + pub fn encode(&self) -> Result, MailboxDeliveryError> { + self.validate_shape()?; + let bytes = serde_json::to_vec(self).map_err(|_| MailboxDeliveryError::Malformed)?; + if bytes.len() > MAX_MAILBOX_DELIVERY_CONTROL_BYTES { + return Err(MailboxDeliveryError::Oversized); + } + Ok(bytes) + } + + /// Bounds untrusted input before parsing a destination-signed lease. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_MAILBOX_DELIVERY_CONTROL_BYTES { + return Err(MailboxDeliveryError::Oversized); + } + let lease: Self = + serde_json::from_slice(bytes).map_err(|_| MailboxDeliveryError::Malformed)?; + lease.validate_shape()?; + Ok(lease) + } + + /// Checks the signature length and structural scope, not signature validity. + pub fn check_scope( + &self, + plan: &MailboxGrant, + receive: &MailboxCapabilityGrant, + consumer_session: ConsumerSessionId, + now_unix_ms: i64, + ) -> Result<(), MailboxDeliveryError> { + self.validate_shape()?; + self.lease + .check_scope(plan, receive, consumer_session, now_unix_ms) + } +} + +/// Consumer-authored acknowledgement of the exact destination-signed lease. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MailboxDeliveryAck { + pub protocol_version: ProtocolVersion, + pub signature_algorithm: SignatureAlgorithm, + pub lease: SignedMailboxDeliveryLease, + pub ack_nonce: DeliveryAckNonce, + pub acknowledged_at_unix_ms: i64, +} + +/// ACK signed by the Receive capability's consumer after application acceptance. +/// +/// The consumer holds an independently scoped mailbox signing key: a local host +/// reader may hold one in host custody, a user holds their own, and a guest must +/// enroll a boot-bound key before guest ACK can be enabled. The FlowMux key is +/// not mailbox authority. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SignedMailboxDeliveryAck { + pub ack: MailboxDeliveryAck, + #[serde(with = "base64_bytes")] + pub signature: Vec, +} + +/// One ACK operation per lease; changed signed ACK bytes conflict on retry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MailboxAckReplayIdentity { + address: MailboxAddress, + consumer: PrincipalId, + consumer_session: ConsumerSessionId, + delivery_id: DeliveryId, + lease_nonce: DeliveryLeaseNonce, +} + +impl SignedMailboxDeliveryAck { + fn validate_shape(&self) -> Result<(), MailboxDeliveryError> { + self.ack.lease.validate_shape()?; + if self.ack.protocol_version != ProtocolVersion::V1 { + return Err(MailboxDeliveryError::UnsupportedVersion); + } + if self.ack.signature_algorithm != SignatureAlgorithm::ED25519 { + return Err(MailboxDeliveryError::UnsupportedAlgorithm); + } + if self.signature.len() != ED25519_SIGNATURE_BYTES { + return Err(MailboxDeliveryError::InvalidSignatureLength); + } + if self.ack.acknowledged_at_unix_ms < self.ack.lease.lease.issued_at_unix_ms + || self.ack.acknowledged_at_unix_ms >= self.ack.lease.lease.expires_at_unix_ms + { + return Err(MailboxDeliveryError::AckOutsideLease); + } + Ok(()) + } + + /// Encodes a bounded ACK; runtime signature verification is still required. + pub fn encode(&self) -> Result, MailboxDeliveryError> { + self.validate_shape()?; + let bytes = serde_json::to_vec(self).map_err(|_| MailboxDeliveryError::Malformed)?; + if bytes.len() > MAX_MAILBOX_DELIVERY_CONTROL_BYTES { + return Err(MailboxDeliveryError::Oversized); + } + Ok(bytes) + } + + /// Bounds untrusted input before parsing a consumer-signed ACK. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_MAILBOX_DELIVERY_CONTROL_BYTES { + return Err(MailboxDeliveryError::Oversized); + } + let ack: Self = + serde_json::from_slice(bytes).map_err(|_| MailboxDeliveryError::Malformed)?; + ack.validate_shape()?; + Ok(ack) + } + + /// Stable key for the runtime's bounded, crash-recoverable ACK replay ledger. + /// + /// Use only after verifying both signatures and current reader authority. + /// Retain the signed ACK commitment with this key: changing even the ACK + /// nonce on retry is a conflict, not a second acknowledgement. + #[must_use] + pub fn replay_identity(&self) -> MailboxAckReplayIdentity { + let lease = &self.ack.lease.lease; + MailboxAckReplayIdentity { + address: lease.address, + consumer: lease.consumer, + consumer_session: lease.consumer_session, + delivery_id: lease.delivery_id, + lease_nonce: lease.lease_nonce, + } + } + + /// Checks the nested lease and ACK time, not signatures or replay state. + pub fn check_scope( + &self, + plan: &MailboxGrant, + receive: &MailboxCapabilityGrant, + consumer_session: ConsumerSessionId, + now_unix_ms: i64, + ) -> Result<(), MailboxDeliveryError> { + self.validate_shape()?; + self.ack + .lease + .check_scope(plan, receive, consumer_session, now_unix_ms)?; + if self.ack.acknowledged_at_unix_ms > now_unix_ms { + return Err(MailboxDeliveryError::AckOutsideLease); + } + Ok(()) + } +} + +/// Data-only lease/ACK refusals; cryptographic and replay errors are runtime-owned. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum MailboxDeliveryError { + #[error("delivery control frame exceeds the encoded bound")] + Oversized, + #[error("delivery control frame is malformed")] + Malformed, + #[error("delivery protocol version is unsupported")] + UnsupportedVersion, + #[error("delivery signature algorithm is unsupported")] + UnsupportedAlgorithm, + #[error("delivery signature length is invalid")] + InvalidSignatureLength, + #[error("delivery time window is invalid")] + InvalidTimeWindow, + #[error("delivery mailbox is absent from the signed plan")] + WrongPlanMailbox, + #[error("reader enrollment is bound to a different signed plan")] + WrongPlanIdentity, + #[error("delivery execution epoch is stale")] + StaleExecutionEpoch, + #[error("delivery address does not match the receive capability")] + WrongAddress, + #[error("delivery consumer does not match the receive capability")] + WrongConsumer, + #[error("reader enrollment is bound to a stale boot")] + StaleConsumerBoot, + #[error("delivery consumer session is stale")] + StaleConsumerSession, + #[error("reader enrollment signing key is invalid")] + InvalidReaderKey, + #[error("delivery receive capability does not match")] + WrongCapability, + #[error("receive operation is not authorized")] + ReceiveNotAuthorized, + #[error("delivery policy epoch is stale")] + StalePolicyEpoch, + #[error("delivery lease is not yet valid")] + NotYetValid, + #[error("delivery lease or capability has expired")] + Expired, + #[error("acknowledgement is outside its delivery lease")] + AckOutsideLease, +} + /// Data-only contract validation failures. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum MailboxContractError { diff --git a/crates/mvm-contract/src/protocol/mod.rs b/crates/mvm-contract/src/protocol/mod.rs index 82a880d08e..4559f68b98 100644 --- a/crates/mvm-contract/src/protocol/mod.rs +++ b/crates/mvm-contract/src/protocol/mod.rs @@ -35,3 +35,5 @@ pub mod signing; pub mod upstream_tools; pub mod vcpu_quota; pub mod vm_backend; +/// Typed application results and errors inside canonical mailbox casts. +pub mod workload_report; diff --git a/crates/mvm-contract/src/protocol/workload_report.rs b/crates/mvm-contract/src/protocol/workload_report.rs new file mode 100644 index 0000000000..b0bd189b08 --- /dev/null +++ b/crates/mvm-contract/src/protocol/workload_report.rs @@ -0,0 +1,235 @@ +//! Versioned application results and errors carried as canonical mailbox payloads. +//! +//! This is content inside a `MailboxMessageKind::Cast`, not another envelope or +//! admission path. The receiver must verify the signed mailbox envelope and +//! match this identity to the admitted plan before trusting the report. + +use alloc::string::{String, ToString}; +use alloc::vec::Vec; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use serde::{Deserialize, Serialize}; + +use crate::plan::{Nonce, PlanId}; +use crate::protocol::mailbox::{ContentType, MAX_MAILBOX_PAYLOAD_BYTES, MessageId}; + +/// Content type used in the enclosing signed mailbox header. +pub const WORKLOAD_REPORT_CONTENT_TYPE: &str = "application/vnd.mvm.workload-report"; +/// Version of the application report body, independent of the mailbox protocol. +pub const WORKLOAD_REPORT_VERSION: u16 = 1; + +/// Identifies one report within an admitted execution and its mailbox message. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReportIdentity { + pub plan_id: PlanId, + pub plan_nonce: Nonce, + pub message_id: MessageId, +} + +/// Stable, payload-free application failure code. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "String", into = "String")] +pub struct ReportErrorCode(String); + +impl core::fmt::Debug for ReportErrorCode { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("ReportErrorCode([redacted])") + } +} + +impl ReportErrorCode { + /// Rejects free-form exception text; details belong in the encrypted payload. + pub fn parse(value: &str) -> Result { + if value.is_empty() + || value.len() > 64 + || !value.bytes().all(|b| { + b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'_' | b'-' | b'.') + }) + { + return Err(ReportError::InvalidErrorCode); + } + Ok(Self(value.to_string())) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom for ReportErrorCode { + type Error = ReportError; + + fn try_from(value: String) -> Result { + Self::parse(&value) + } +} + +impl From for String { + fn from(value: ReportErrorCode) -> Self { + value.0 + } +} + +/// Application-authored outcome; host-observed exit and publication are separate. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReportKind { + Result, + Error { code: ReportErrorCode }, +} + +/// Bounded, immutable application content inside an authenticated mailbox cast. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "ReportWire", into = "ReportWire")] +pub struct WorkloadReport { + identity: ReportIdentity, + kind: ReportKind, + content_type: ContentType, + payload: Vec, +} + +impl core::fmt::Debug for WorkloadReport { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("WorkloadReport") + .field("kind", &self.kind) + .field("payload_len", &self.payload.len()) + .finish_non_exhaustive() + } +} + +impl WorkloadReport { + /// Builds a report that fits the canonical mailbox plaintext bound. + /// + /// The transport frame and signed capability may impose a smaller limit; + /// those are checked by destination admission, never assumed here. + pub fn new( + identity: ReportIdentity, + kind: ReportKind, + content_type: ContentType, + payload: Vec, + ) -> Result { + let report = Self { + identity, + kind, + content_type, + payload, + }; + report.validate()?; + Ok(report) + } + + #[must_use] + pub fn identity(&self) -> &ReportIdentity { + &self.identity + } + + /// Compares the untrusted report claim with identity from the verified plan and envelope. + pub fn verify_identity(&self, expected: &ReportIdentity) -> Result<(), ReportError> { + if &self.identity != expected { + return Err(ReportError::WrongIdentity); + } + Ok(()) + } + + #[must_use] + pub fn kind(&self) -> &ReportKind { + &self.kind + } + + #[must_use] + pub fn content_type(&self) -> &ContentType { + &self.content_type + } + + #[must_use] + pub fn payload(&self) -> &[u8] { + &self.payload + } + + /// Encodes one complete report body; the caller must publish it as a Cast. + pub fn encode(&self) -> Result, ReportError> { + let bytes = serde_json::to_vec(self).map_err(|_| ReportError::Malformed)?; + if bytes.len() > MAX_MAILBOX_PAYLOAD_BYTES { + return Err(ReportError::Oversized); + } + Ok(bytes) + } + + /// Bounds input before parsing an untrusted report body. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_MAILBOX_PAYLOAD_BYTES { + return Err(ReportError::Oversized); + } + let wire: ReportWire = serde_json::from_slice(bytes).map_err(|_| ReportError::Malformed)?; + Self::try_from(wire) + } + + fn validate(&self) -> Result<(), ReportError> { + if self.identity.plan_id.0.is_empty() || self.identity.plan_id.0.len() > 128 { + return Err(ReportError::InvalidPlanId); + } + if self.payload.len() > MAX_MAILBOX_PAYLOAD_BYTES { + return Err(ReportError::Oversized); + } + self.encode().map(|_| ()) + } +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ReportWire { + schema_version: u16, + identity: ReportIdentity, + kind: ReportKind, + content_type: ContentType, + payload_b64: String, +} + +impl From for ReportWire { + fn from(report: WorkloadReport) -> Self { + Self { + schema_version: WORKLOAD_REPORT_VERSION, + identity: report.identity, + kind: report.kind, + content_type: report.content_type, + payload_b64: BASE64.encode(report.payload), + } + } +} + +impl TryFrom for WorkloadReport { + type Error = ReportError; + + fn try_from(wire: ReportWire) -> Result { + if wire.schema_version != WORKLOAD_REPORT_VERSION { + return Err(ReportError::UnsupportedVersion); + } + if wire.payload_b64.len() > MAX_MAILBOX_PAYLOAD_BYTES { + return Err(ReportError::Oversized); + } + let payload = BASE64 + .decode(wire.payload_b64.as_bytes()) + .map_err(|_| ReportError::Malformed)?; + Self::new(wire.identity, wire.kind, wire.content_type, payload) + } +} + +/// Payload-free refusal when a report cannot be represented or admitted. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum ReportError { + #[error("workload report is malformed")] + Malformed, + #[error("workload report version is unsupported")] + UnsupportedVersion, + #[error("workload report error code is invalid")] + InvalidErrorCode, + #[error("workload report plan identity is invalid")] + InvalidPlanId, + #[error("workload report identity does not match admission")] + WrongIdentity, + #[error("workload report exceeds the mailbox plaintext limit")] + Oversized, +} diff --git a/crates/mvm-contract/tests/mailbox_delivery.rs b/crates/mvm-contract/tests/mailbox_delivery.rs new file mode 100644 index 0000000000..e1769a7ec2 --- /dev/null +++ b/crates/mvm-contract/tests/mailbox_delivery.rs @@ -0,0 +1,428 @@ +use mvm_contract::plan::{Nonce, PlanId}; +use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, BootId, CapabilityId, CapabilityOperations, ConsumerSessionId, + ContentType, DeliveryAckNonce, DeliveryId, DeliveryLeaseNonce, ExecutionEpoch, MailboxAddress, + MailboxCapabilityGrant, MailboxDeliveryAck, MailboxDeliveryError, MailboxDeliveryLease, + MailboxGrant, MailboxGuestReaderEnrollment, MailboxGuestReaderScope, MailboxId, + MailboxMessageKind, MailboxOperation, MessageId, PrincipalId, ProtocolVersion, RevocationId, + SignatureAlgorithm, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, + SignedMailboxGuestReaderEnrollment, TenantId, +}; + +fn address() -> MailboxAddress { + MailboxAddress { + tenant: TenantId::from_bytes([1; 32]), + authorization_domain: AuthorizationDomainId::from_bytes([2; 32]), + mailbox: MailboxId::from_bytes([3; 32]), + recipient: PrincipalId::from_bytes([4; 32]), + } +} + +fn plan_grant() -> MailboxGrant { + MailboxGrant { + mailbox: address().mailbox, + authority_key: [5; 32], + epoch: ExecutionEpoch::new(7).expect("epoch"), + } +} + +fn receive_capability() -> MailboxCapabilityGrant { + MailboxCapabilityGrant { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + capability_id: CapabilityId::from_bytes([6; 32]), + issuer: PrincipalId::from_bytes([5; 32]), + subject: PrincipalId::from_bytes([8; 32]), + address: address(), + operations: CapabilityOperations::from_operations(&[MailboxOperation::Receive]), + message_kinds: vec![MailboxMessageKind::Cast], + content_types: vec![ + ContentType::parse("application/vnd.mvm.workload-report", 1) + .expect("static content type"), + ], + max_payload_bytes: 1024, + max_messages_per_minute: 60, + max_outstanding_calls: 1, + max_queue_contribution_bytes: 4096, + minimum_execution_epoch: plan_grant().epoch, + maximum_execution_epoch: plan_grant().epoch, + policy_epoch: 2, + not_before_unix_ms: 50, + expires_at_unix_ms: 300, + revocation_id: RevocationId::from_bytes([9; 32]), + delegation_depth: 0, + } +} + +fn current_session() -> ConsumerSessionId { + ConsumerSessionId::from_bytes([14; 32]) +} + +fn lease() -> MailboxDeliveryLease { + MailboxDeliveryLease { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + address: address(), + sender: PrincipalId::from_bytes([10; 32]), + sender_boot: BootId::from_bytes([11; 32]), + message_id: MessageId::from_bytes([12; 32]), + delivery_id: DeliveryId::from_bytes([13; 32]), + execution_epoch: plan_grant().epoch, + consumer: receive_capability().subject, + consumer_session: current_session(), + receive_capability_id: receive_capability().capability_id, + policy_epoch: 2, + ciphertext_commitment: [15; 32], + issued_at_unix_ms: 100, + expires_at_unix_ms: 200, + lease_nonce: DeliveryLeaseNonce::from_bytes([16; 16]), + } +} + +fn reader_plan_id() -> PlanId { + PlanId("sha256:".to_owned() + &"a".repeat(64)) +} + +fn reader_plan_nonce() -> Nonce { + Nonce::from_bytes([26; 16]) +} + +fn reader_boot() -> BootId { + BootId::from_bytes([27; 32]) +} + +fn reader_enrollment() -> SignedMailboxGuestReaderEnrollment { + SignedMailboxGuestReaderEnrollment { + enrollment: MailboxGuestReaderEnrollment { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + plan_id: reader_plan_id(), + plan_nonce: reader_plan_nonce(), + address: address(), + consumer: receive_capability().subject, + consumer_boot: reader_boot(), + consumer_session: current_session(), + receive_capability_id: receive_capability().capability_id, + ack_public_key: [28; 32], + execution_epoch: plan_grant().epoch, + policy_epoch: 2, + issued_at_unix_ms: 100, + expires_at_unix_ms: 200, + }, + signature: vec![29; 64], + } +} + +#[test] +fn enrolled_guest_reader_requires_its_current_plan_boot_and_receive_grant() { + let plan = plan_grant(); + let cap = receive_capability(); + let plan_id = reader_plan_id(); + let plan_nonce = reader_plan_nonce(); + let scope = MailboxGuestReaderScope { + plan: &plan, + plan_id: &plan_id, + plan_nonce: &plan_nonce, + receive: &cap, + current_boot: reader_boot(), + current_session: current_session(), + now_unix_ms: 150, + }; + let enrolled = reader_enrollment(); + assert_eq!(enrolled.check_scope(&scope), Ok(())); + assert_eq!( + SignedMailboxGuestReaderEnrollment::decode(&enrolled.encode().expect("bounded enrollment")) + .expect("decode enrollment"), + enrolled + ); + + let mut stale_parent = scope; + stale_parent.current_boot = BootId::from_bytes([30; 32]); + assert_eq!( + enrolled.check_scope(&stale_parent), + Err(MailboxDeliveryError::StaleConsumerBoot) + ); + let stale_session = MailboxGuestReaderScope { + current_session: ConsumerSessionId::from_bytes([33; 32]), + ..scope + }; + assert_eq!( + enrolled.check_scope(&stale_session), + Err(MailboxDeliveryError::StaleConsumerSession) + ); + let wrong_nonce = Nonce::from_bytes([31; 16]); + let wrong_plan = MailboxGuestReaderScope { + plan_nonce: &wrong_nonce, + ..scope + }; + assert_eq!( + enrolled.check_scope(&wrong_plan), + Err(MailboxDeliveryError::WrongPlanIdentity) + ); + let mut wrong_recipient = cap.clone(); + wrong_recipient.address.recipient = PrincipalId::from_bytes([32; 32]); + let wrong_scope = MailboxGuestReaderScope { + receive: &wrong_recipient, + ..scope + }; + assert_eq!( + enrolled.check_scope(&wrong_scope), + Err(MailboxDeliveryError::WrongAddress) + ); + let expired = MailboxGuestReaderScope { + now_unix_ms: 200, + ..scope + }; + assert_eq!( + enrolled.check_scope(&expired), + Err(MailboxDeliveryError::Expired) + ); +} + +#[test] +fn forged_or_stale_reader_enrollment_refuses_without_exposing_an_ack_key() { + let plan = plan_grant(); + let cap = receive_capability(); + let plan_id = reader_plan_id(); + let plan_nonce = reader_plan_nonce(); + let scope = MailboxGuestReaderScope { + plan: &plan, + plan_id: &plan_id, + plan_nonce: &plan_nonce, + receive: &cap, + current_boot: reader_boot(), + current_session: current_session(), + now_unix_ms: 150, + }; + let mut enrolled = reader_enrollment(); + enrolled.enrollment.ack_public_key = [0; 32]; + assert_eq!( + enrolled.check_scope(&scope), + Err(MailboxDeliveryError::InvalidReaderKey) + ); + enrolled = reader_enrollment(); + enrolled.signature.clear(); + assert_eq!( + enrolled.check_scope(&scope), + Err(MailboxDeliveryError::InvalidSignatureLength) + ); + let mut stale = reader_enrollment(); + stale.enrollment.execution_epoch = ExecutionEpoch::new(8).expect("epoch"); + assert_eq!( + stale.check_scope(&scope), + Err(MailboxDeliveryError::StaleExecutionEpoch) + ); + stale = reader_enrollment(); + stale.enrollment.policy_epoch = 3; + assert_eq!( + stale.check_scope(&scope), + Err(MailboxDeliveryError::StalePolicyEpoch) + ); + let mut send_only = cap.clone(); + send_only.operations = CapabilityOperations::from_operations(&[MailboxOperation::Send]); + let wrong_scope = MailboxGuestReaderScope { + receive: &send_only, + ..scope + }; + assert_eq!( + reader_enrollment().check_scope(&wrong_scope), + Err(MailboxDeliveryError::ReceiveNotAuthorized) + ); +} + +#[test] +fn signed_lease_and_ack_round_trip_with_exact_reader_scope() { + let signed_lease = SignedMailboxDeliveryLease { + lease: lease(), + signature: vec![17; 64], + }; + let ack = SignedMailboxDeliveryAck { + ack: MailboxDeliveryAck { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + lease: signed_lease.clone(), + ack_nonce: DeliveryAckNonce::from_bytes([18; 16]), + acknowledged_at_unix_ms: 150, + }, + signature: vec![19; 64], + }; + assert_eq!( + signed_lease.check_scope(&plan_grant(), &receive_capability(), current_session(), 150), + Ok(()) + ); + assert_eq!( + ack.check_scope(&plan_grant(), &receive_capability(), current_session(), 150), + Ok(()) + ); + let wire = serde_json::to_vec(&ack).expect("encode"); + assert_eq!( + serde_json::from_slice::(&wire).expect("decode"), + ack + ); + assert_eq!(ack.ack.lease.lease.delivery_id, lease().delivery_id); + assert_eq!( + ack.ack.lease.lease.consumer_session, + lease().consumer_session + ); + let lease_wire = signed_lease.encode().expect("bounded lease"); + assert_eq!( + SignedMailboxDeliveryLease::decode(&lease_wire).expect("decode lease"), + signed_lease + ); + let ack_wire = ack.encode().expect("bounded acknowledgement"); + let retried = SignedMailboxDeliveryAck::decode(&ack_wire).expect("decode ACK"); + assert_eq!(retried, ack); + assert_eq!(retried.replay_identity(), ack.replay_identity()); + let mut conflicting = retried.clone(); + conflicting.ack.lease.lease.message_id = MessageId::from_bytes([42; 32]); + assert_eq!( + conflicting.replay_identity(), + ack.replay_identity(), + "same ACK operation with changed content must be a conflict, not a fresh identity" + ); + conflicting.ack.ack_nonce = DeliveryAckNonce::from_bytes([43; 16]); + assert_eq!( + conflicting.replay_identity(), + ack.replay_identity(), + "a second nonce cannot create a second ACK for one lease" + ); +} + +#[test] +fn control_frames_are_bounded_before_parsing_and_reject_unknown_fields() { + use mvm_contract::protocol::mailbox::MAX_MAILBOX_DELIVERY_CONTROL_BYTES; + + let oversized = vec![b'x'; MAX_MAILBOX_DELIVERY_CONTROL_BYTES + 1]; + assert_eq!( + SignedMailboxDeliveryLease::decode(&oversized), + Err(MailboxDeliveryError::Oversized) + ); + assert_eq!( + SignedMailboxDeliveryAck::decode(&oversized), + Err(MailboxDeliveryError::Oversized) + ); + assert_eq!( + SignedMailboxGuestReaderEnrollment::decode(&oversized), + Err(MailboxDeliveryError::Oversized) + ); + let mut wire = serde_json::to_value(SignedMailboxDeliveryLease { + lease: lease(), + signature: vec![17; 64], + }) + .expect("encode lease"); + wire["lease"]["unknown"] = true.into(); + assert_eq!( + SignedMailboxDeliveryLease::decode(&serde_json::to_vec(&wire).expect("JSON")), + Err(MailboxDeliveryError::Malformed) + ); + let mut lease = SignedMailboxDeliveryLease { + lease: lease(), + signature: vec![17; 64], + }; + lease.lease.protocol_version = ProtocolVersion(2); + assert_eq!( + lease.encode(), + Err(MailboxDeliveryError::UnsupportedVersion) + ); + lease.lease.protocol_version = ProtocolVersion::V1; + lease.signature.clear(); + assert_eq!( + lease.encode(), + Err(MailboxDeliveryError::InvalidSignatureLength) + ); +} + +#[test] +fn receive_is_never_authorized_by_address_or_transport_alone() { + let lease = lease(); + let mut capability = receive_capability(); + capability.operations = CapabilityOperations::from_operations(&[MailboxOperation::Send]); + assert_eq!( + lease.check_scope(&plan_grant(), &capability, current_session(), 150), + Err(MailboxDeliveryError::ReceiveNotAuthorized) + ); + + let mut capability = receive_capability(); + capability.address.tenant = TenantId::from_bytes([20; 32]); + assert_eq!( + lease.check_scope(&plan_grant(), &capability, current_session(), 150), + Err(MailboxDeliveryError::WrongAddress) + ); + let mut capability = receive_capability(); + capability.address.recipient = PrincipalId::from_bytes([21; 32]); + assert_eq!( + lease.check_scope(&plan_grant(), &capability, current_session(), 150), + Err(MailboxDeliveryError::WrongAddress) + ); + let mut capability = receive_capability(); + capability.subject = PrincipalId::from_bytes([22; 32]); + assert_eq!( + lease.check_scope(&plan_grant(), &capability, current_session(), 150), + Err(MailboxDeliveryError::WrongConsumer) + ); + assert_eq!( + lease.check_scope( + &plan_grant(), + &receive_capability(), + ConsumerSessionId::from_bytes([99; 32]), + 150 + ), + Err(MailboxDeliveryError::StaleConsumerSession) + ); + let mut capability = receive_capability(); + capability.capability_id = CapabilityId::from_bytes([23; 32]); + assert_eq!( + lease.check_scope(&plan_grant(), &capability, current_session(), 150), + Err(MailboxDeliveryError::WrongCapability) + ); +} + +#[test] +fn stale_epoch_policy_and_expired_lease_cannot_acknowledge() { + let mut plan = plan_grant(); + plan.epoch = ExecutionEpoch::new(8).expect("new epoch"); + assert_eq!( + lease().check_scope(&plan, &receive_capability(), current_session(), 150), + Err(MailboxDeliveryError::StaleExecutionEpoch) + ); + plan = plan_grant(); + plan.mailbox = MailboxId::from_bytes([24; 32]); + assert_eq!( + lease().check_scope(&plan, &receive_capability(), current_session(), 150), + Err(MailboxDeliveryError::WrongPlanMailbox) + ); + let mut capability = receive_capability(); + capability.policy_epoch = 3; + assert_eq!( + lease().check_scope(&plan_grant(), &capability, current_session(), 150), + Err(MailboxDeliveryError::StalePolicyEpoch) + ); + assert_eq!( + lease().check_scope(&plan_grant(), &receive_capability(), current_session(), 200), + Err(MailboxDeliveryError::Expired) + ); + + let mut ack = SignedMailboxDeliveryAck { + ack: MailboxDeliveryAck { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + lease: SignedMailboxDeliveryLease { + lease: lease(), + signature: vec![17; 64], + }, + ack_nonce: DeliveryAckNonce::from_bytes([18; 16]), + acknowledged_at_unix_ms: 201, + }, + signature: vec![19; 64], + }; + assert_eq!( + ack.check_scope(&plan_grant(), &receive_capability(), current_session(), 150), + Err(MailboxDeliveryError::AckOutsideLease) + ); + ack.ack.acknowledged_at_unix_ms = 150; + ack.signature.clear(); + assert_eq!( + ack.check_scope(&plan_grant(), &receive_capability(), current_session(), 150), + Err(MailboxDeliveryError::InvalidSignatureLength) + ); +} diff --git a/crates/mvm-contract/tests/workload_report.rs b/crates/mvm-contract/tests/workload_report.rs new file mode 100644 index 0000000000..6eca2e9224 --- /dev/null +++ b/crates/mvm-contract/tests/workload_report.rs @@ -0,0 +1,124 @@ +use mvm_contract::plan::{Nonce, PlanId}; +use mvm_contract::protocol::mailbox::{ContentType, MessageId}; +use mvm_contract::protocol::workload_report::{ + ReportError, ReportErrorCode, ReportIdentity, ReportKind, WORKLOAD_REPORT_CONTENT_TYPE, + WORKLOAD_REPORT_VERSION, WorkloadReport, +}; + +fn identity() -> ReportIdentity { + ReportIdentity { + plan_id: PlanId("sha256:".to_owned() + &"a".repeat(64)), + plan_nonce: Nonce::from_bytes([7; 16]), + message_id: MessageId::from_bytes([8; 32]), + } +} + +fn content_type() -> ContentType { + ContentType::parse("application/json", 1).expect("static content type") +} + +#[test] +fn result_and_error_are_distinct_versioned_mailbox_values() { + let result = WorkloadReport::new( + identity(), + ReportKind::Result, + content_type(), + b"\0binary result".to_vec(), + ) + .expect("bounded result"); + assert_eq!(result.payload(), b"\0binary result"); + assert_eq!(result.identity(), &identity()); + assert_eq!(result.verify_identity(&identity()), Ok(())); + let mut different_message = identity(); + different_message.message_id = MessageId::from_bytes([9; 32]); + assert_eq!( + result.verify_identity(&different_message), + Err(ReportError::WrongIdentity) + ); + let mut different_plan = identity(); + different_plan.plan_id = PlanId("another-plan".into()); + assert_eq!( + result.verify_identity(&different_plan), + Err(ReportError::WrongIdentity) + ); + assert_eq!(result.kind(), &ReportKind::Result); + let wire = result.encode().expect("encode result"); + assert_eq!( + WorkloadReport::decode(&wire).expect("decode result"), + result + ); + assert_eq!( + serde_json::from_slice::(&wire).expect("serde decode"), + result + ); + assert!( + !format!("{result:?}").contains("binary result"), + "debug output must not disclose application payload" + ); + + let error = WorkloadReport::new( + identity(), + ReportKind::Error { + code: ReportErrorCode::parse("dependency_unavailable").expect("static code"), + }, + content_type(), + b"{\"reason\":\"try later\"}".to_vec(), + ) + .expect("bounded error"); + assert_eq!( + WorkloadReport::decode(&error.encode().expect("encode error")).expect("decode error"), + error + ); + assert!(!format!("{error:?}").contains("dependency_unavailable")); + assert!(!format!("{error:?}").contains("try later")); + assert_ne!(result, error); + assert_eq!(WORKLOAD_REPORT_VERSION, 1); + assert_eq!( + WORKLOAD_REPORT_CONTENT_TYPE, + "application/vnd.mvm.workload-report" + ); +} + +#[test] +fn invalid_version_code_payload_and_unknown_fields_fail_closed() { + let report = WorkloadReport::new(identity(), ReportKind::Result, content_type(), vec![9]) + .expect("bounded result"); + let mut wire: serde_json::Value = + serde_json::from_slice(&report.encode().expect("encode")).expect("JSON"); + wire["schema_version"] = 2.into(); + assert_eq!( + WorkloadReport::decode(&serde_json::to_vec(&wire).expect("serialize")).unwrap_err(), + ReportError::UnsupportedVersion + ); + wire["schema_version"] = 1.into(); + wire["unexpected"] = true.into(); + assert_eq!( + WorkloadReport::decode(&serde_json::to_vec(&wire).expect("serialize")).unwrap_err(), + ReportError::Malformed + ); + wire.as_object_mut().expect("object").remove("unexpected"); + wire["payload_b64"] = "***".into(); + assert_eq!( + WorkloadReport::decode(&serde_json::to_vec(&wire).expect("serialize")).unwrap_err(), + ReportError::Malformed + ); + assert_eq!( + ReportErrorCode::parse("contains a secret: value").unwrap_err(), + ReportError::InvalidErrorCode + ); +} + +#[test] +fn encoded_report_is_bounded_before_parsing_or_publication() { + let too_large = WorkloadReport::new( + identity(), + ReportKind::Result, + content_type(), + vec![0; 1024 * 1024], + ); + assert_eq!(too_large.unwrap_err(), ReportError::Oversized); + assert_eq!( + WorkloadReport::decode(&vec![b'x'; 1024 * 1024 + 1]).unwrap_err(), + ReportError::Oversized + ); +} diff --git a/crates/mvm-core/tests/mailbox_crypto.rs b/crates/mvm-core/tests/mailbox_crypto.rs index f555d16db8..448442b5b4 100644 --- a/crates/mvm-core/tests/mailbox_crypto.rs +++ b/crates/mvm-core/tests/mailbox_crypto.rs @@ -1,4 +1,5 @@ use ed25519_dalek::SigningKey; +use mvm_contract::plan::{Nonce, PlanId}; use mvm_contract::protocol::mailbox::{ AuthorizationDomainId, AuthorizationDomainRouteLabel, BootId, CapabilityId, CapabilityOperations, CausationId, CiphertextPadding, ContentType, CorrelationId, DeliveryId, @@ -8,6 +9,10 @@ use mvm_contract::protocol::mailbox::{ PrincipalId, ProtocolVersion, ReplayNonce, ReplyRoute, RevocationId, SequenceNumber, SignatureAlgorithm, TenantId, TransportBindingId, }; +use mvm_contract::protocol::workload_report::{ + ReportError, ReportIdentity, ReportKind, WORKLOAD_REPORT_CONTENT_TYPE, WORKLOAD_REPORT_VERSION, + WorkloadReport, +}; use mvm_core::crypto::aead::Key; use mvm_core::mailbox::{ InMemoryReplayProtector, MailboxError, OpenContext, SealRequest, decode_envelope, @@ -969,6 +974,87 @@ fn cast_is_admitted_only_when_the_capability_names_it_and_bounds_it() { ); } +#[test] +fn typed_workload_report_uses_the_verified_canonical_cast() { + let report_type = + ContentType::parse(WORKLOAD_REPORT_CONTENT_TYPE, WORKLOAD_REPORT_VERSION).expect("type"); + let mut cast = header(); + cast.kind = MailboxMessageKind::Cast; + cast.reply_to = None; + cast.content_type = report_type.clone(); + let identity = ReportIdentity { + plan_id: PlanId("sha256:".to_owned() + &"a".repeat(64)), + plan_nonce: Nonce::from_bytes([24; 16]), + message_id: cast.message_id, + }; + let report = WorkloadReport::new( + identity.clone(), + ReportKind::Result, + ContentType::parse("application/json", 1).expect("application content type"), + b"{\"private\":\"result\"}".to_vec(), + ) + .expect("report"); + let bytes = report.encode().expect("bounded report"); + cast.plaintext_len = u32::try_from(bytes.len()).expect("mailbox bound fits u32"); + let envelope = seal_with(cast, &bytes); + let encoded = encode_envelope(&envelope).expect("canonical envelope"); + assert!( + !String::from_utf8_lossy(&encoded).contains("private"), + "routing frame must not disclose report content" + ); + + let mut permitted = grant(); + permitted.message_kinds.push(MailboxMessageKind::Cast); + let mut receiving = context(); + receiving.accepted_content_types.push(report_type.clone()); + assert_eq!( + open_under( + &envelope, + permitted.clone(), + &receiving, + &mut InMemoryReplayProtector::default() + ) + .expect_err("the signed capability did not grant this report content type"), + MailboxError::ContentTypeNotAuthorized + ); + permitted.content_types.push(report_type.clone()); + assert_eq!( + report_type.name(), + WORKLOAD_REPORT_CONTENT_TYPE, + "the signed envelope must declare the typed report content" + ); + let opened = open_under( + &envelope, + permitted.clone(), + &receiving, + &mut InMemoryReplayProtector::default(), + ) + .expect("capability-authorized result"); + let decoded = WorkloadReport::decode(&opened).expect("typed report"); + assert_eq!(decoded, report); + decoded + .verify_identity(&identity) + .expect("report matches admitted run and signed message"); + let mut wrong_run = identity; + wrong_run.plan_nonce = Nonce::from_bytes([25; 16]); + assert_eq!( + decoded.verify_identity(&wrong_run), + Err(ReportError::WrongIdentity) + ); + + permitted.max_payload_bytes = u32::try_from(bytes.len() - 1).expect("bounded report"); + assert_eq!( + open_under( + &envelope, + permitted, + &receiving, + &mut InMemoryReplayProtector::default() + ) + .expect_err("the signed grant must bound the whole encoded report"), + MailboxError::CapabilityPayloadExceeded + ); +} + #[test] fn the_delivery_and_message_vocabularies_are_closed() { // The exhaustive matches stop compiling when a variant is added, so a new From 2cd2e341a93dcaeb6b604d93afc71325872000f2 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 14:04:16 -0800 Subject: [PATCH 03/51] feat(mailbox): verify guest reader enrollment and signed delivery controls --- crates/mvm-core/src/mailbox.rs | 30 +- crates/mvm-core/src/mailbox/reader.rs | 285 +++++++++++++++ crates/mvm-core/tests/mailbox_reader.rs | 440 ++++++++++++++++++++++++ 3 files changed, 752 insertions(+), 3 deletions(-) create mode 100644 crates/mvm-core/src/mailbox/reader.rs create mode 100644 crates/mvm-core/tests/mailbox_reader.rs diff --git a/crates/mvm-core/src/mailbox.rs b/crates/mvm-core/src/mailbox.rs index f2d12b55f8..470cff7434 100644 --- a/crates/mvm-core/src/mailbox.rs +++ b/crates/mvm-core/src/mailbox.rs @@ -11,9 +11,10 @@ use mvm_contract::protocol::mailbox::{ AES_256_GCM_ALGORITHM, AuthorizationDomainId, BootId, ContentType, DestinationReceipt, ED25519_ALGORITHM, ED25519_SIGNATURE_BYTES, ExecutionEpoch, MAILBOX_PROTOCOL_VERSION, MAX_ENCODED_ENVELOPE_BYTES, MAX_MAILBOX_PAYLOAD_BYTES, MailboxCapabilityGrant, - MailboxContractError, MailboxEnvelope, MailboxEnvelopeHeader, MailboxId, MailboxMessage, - MailboxOperation, MailboxRoutingFrame, MessageId, PrincipalId, ProtocolVersion, RevocationId, - SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxMessage, TenantId, + MailboxContractError, MailboxDeliveryError, MailboxEnvelope, MailboxEnvelopeHeader, MailboxId, + MailboxMessage, MailboxOperation, MailboxRoutingFrame, MessageId, PrincipalId, ProtocolVersion, + RevocationId, SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxMessage, + TenantId, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -31,6 +32,9 @@ pub mod frame; /// Bounded destination-side ciphertext journal and durable acceptance. pub mod journal; +/// Cryptographic guest reader enrollment, separate from transport admission. +pub mod reader; + /// Overlay resolution and authenticated, bounded byte-transport seams. pub mod transport; @@ -122,12 +126,26 @@ pub enum MailboxError { UnsupportedContentType, #[error("mailbox contract bounds are invalid: {0}")] Contract(MailboxContractError), + #[error("mailbox delivery contract is invalid: {0}")] + DeliveryContract(MailboxDeliveryError), #[error("mailbox sender signature is invalid")] SignatureInvalid, #[error("mailbox capability signature is invalid")] CapabilitySignatureInvalid, #[error("destination receipt signature is invalid")] ReceiptSignatureInvalid, + #[error("guest reader enrollment signature is invalid")] + ReaderEnrollmentSignatureInvalid, + #[error("guest reader key is invalid")] + ReaderKeyInvalid, + #[error("guest reader possession signature is invalid")] + ReaderPossessionInvalid, + #[error("destination delivery lease signature is invalid")] + DeliveryLeaseSignatureInvalid, + #[error("guest delivery ACK signature is invalid")] + DeliveryAckSignatureInvalid, + #[error("delivery lease does not identify the committed message")] + DeliveryLeaseMismatch, #[error("mailbox ciphertext commitment is invalid")] CiphertextCommitmentInvalid, #[error("mailbox payload decryption failed")] @@ -182,6 +200,12 @@ impl From for MailboxError { } } +impl From for MailboxError { + fn from(value: MailboxDeliveryError) -> Self { + Self::DeliveryContract(value) + } +} + fn canonical_bytes(domain: &[u8], value: &T) -> Result, MailboxError> { let encoded = serde_jcs::to_vec(value).map_err(|_| MailboxError::Canonicalization)?; let mut bytes = Vec::with_capacity(domain.len().saturating_add(encoded.len())); diff --git a/crates/mvm-core/src/mailbox/reader.rs b/crates/mvm-core/src/mailbox/reader.rs new file mode 100644 index 0000000000..4cfdbb9a23 --- /dev/null +++ b/crates/mvm-core/src/mailbox/reader.rs @@ -0,0 +1,285 @@ +//! Cryptographic guest reader enrollment; the runtime owns boot/session provenance. +//! +//! A valid signature or a connected vsock peer alone is not Receive authority. +//! Callers must obtain the scope from an authenticated current guest boot and +//! persist one-time challenge use before publishing a lease. + +use ed25519_dalek::{Signature, Signer as _, SigningKey, Verifier as _, VerifyingKey}; +use mvm_contract::protocol::mailbox::{ + ConsumerSessionId, DELIVERY_ACK_SIGNATURE_DOMAIN, DELIVERY_LEASE_SIGNATURE_DOMAIN, + MailboxAddress, MailboxDeliveryAck, MailboxDeliveryLease, MailboxEnvelope, + MailboxEnvelopeHeader, MailboxGrant, MailboxGuestReaderEnrollment, MailboxGuestReaderScope, + READER_ENROLLMENT_SIGNATURE_DOMAIN, RevocationId, SignedMailboxCapability, + SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, +}; + +use super::{MailboxError, canonical_bytes, verify_capability_signature}; + +const READER_POSSESSION_SIGNATURE_DOMAIN: &[u8] = b"mvm.mailbox.reader-possession.v1\0"; + +/// Sign an enrollment with the destination authority. The caller must bind +/// its body to the verified plan and current authenticated guest boot first. +pub fn sign_guest_reader_enrollment( + enrollment: MailboxGuestReaderEnrollment, + authority: &SigningKey, +) -> Result { + let signature = authority + .sign(&canonical_bytes( + READER_ENROLLMENT_SIGNATURE_DOMAIN, + &enrollment, + )?) + .to_bytes() + .to_vec(); + let signed = SignedMailboxGuestReaderEnrollment { + enrollment, + signature, + }; + signed.encode()?; + Ok(signed) +} + +/// Verify the destination signature and independently signed Receive grant. +/// +/// `scope` must come from an already authenticated current plan/boot/session. +/// The returned key is *not* reader authority until fresh possession is +/// proven and that session/challenge binding is persisted and replay-fenced. +pub fn verify_guest_reader_enrollment( + signed: &SignedMailboxGuestReaderEnrollment, + capability: &SignedMailboxCapability, + scope: &MailboxGuestReaderScope<'_>, + authority: &VerifyingKey, + revoked: &[RevocationId], +) -> Result { + if scope.plan.authority_key != authority.to_bytes() { + return Err(MailboxError::WrongCapabilityIssuer); + } + if scope.receive != &capability.grant { + return Err(MailboxError::WrongCapability); + } + if revoked.contains(&capability.grant.revocation_id) { + return Err(MailboxError::CapabilityRevoked); + } + verify_capability_signature(capability, authority)?; + signed.check_scope(scope)?; + signed.encode()?; + let signature: [u8; 64] = signed + .signature + .as_slice() + .try_into() + .map_err(|_| MailboxError::ReaderEnrollmentSignatureInvalid)?; + authority + .verify( + &canonical_bytes(READER_ENROLLMENT_SIGNATURE_DOMAIN, &signed.enrollment)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| MailboxError::ReaderEnrollmentSignatureInvalid)?; + VerifyingKey::from_bytes(&signed.enrollment.ack_public_key) + .map_err(|_| MailboxError::ReaderKeyInvalid) +} + +fn possession_bytes( + signed: &SignedMailboxGuestReaderEnrollment, + challenge: &[u8; 32], +) -> Result, MailboxError> { + if *challenge == [0; 32] { + return Err(MailboxError::ReaderPossessionInvalid); + } + signed.encode()?; + canonical_bytes(READER_POSSESSION_SIGNATURE_DOMAIN, &(signed, challenge)) +} + +/// Respond to a destination-minted one-time challenge with the guest-held +/// mailbox key, never the FlowMux key or host sealing key. +pub fn sign_reader_possession( + signed: &SignedMailboxGuestReaderEnrollment, + challenge: &[u8; 32], + key: &SigningKey, +) -> Result<[u8; 64], MailboxError> { + if key.verifying_key().to_bytes() != signed.enrollment.ack_public_key { + return Err(MailboxError::ReaderKeyInvalid); + } + Ok(key.sign(&possession_bytes(signed, challenge)?).to_bytes()) +} + +/// Verify possession only; the destination must also authenticate the live +/// boot/session, mint the nonce, and remember its single use durably. +pub fn verify_reader_possession( + signed: &SignedMailboxGuestReaderEnrollment, + challenge: &[u8; 32], + signature: &[u8], + key: &VerifyingKey, +) -> Result<(), MailboxError> { + if key.to_bytes() != signed.enrollment.ack_public_key { + return Err(MailboxError::ReaderKeyInvalid); + } + let signature: [u8; 64] = signature + .try_into() + .map_err(|_| MailboxError::ReaderPossessionInvalid)?; + key.verify( + &possession_bytes(signed, challenge)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| MailboxError::ReaderPossessionInvalid) +} + +/// Sign a destination lease only after the referenced ciphertext and dedup +/// identity have committed. Signing alone does not publish the lease. +pub fn sign_delivery_lease( + lease: MailboxDeliveryLease, + authority: &SigningKey, +) -> Result { + let signature = authority + .sign(&canonical_bytes(DELIVERY_LEASE_SIGNATURE_DOMAIN, &lease)?) + .to_bytes() + .to_vec(); + let signed = SignedMailboxDeliveryLease { lease, signature }; + signed.encode()?; + Ok(signed) +} + +/// Already-authenticated destination facts for one leased ciphertext. +/// The runtime must derive these from its verified plan, active reader +/// session, current revocation state, and committed journal entry. +#[derive(Clone, Copy)] +pub struct ReaderDeliveryContext<'a> { + pub plan: &'a MailboxGrant, + pub capability: &'a SignedMailboxCapability, + pub reader_scope: MailboxGuestReaderScope<'a>, + pub consumer_session: ConsumerSessionId, + pub now_unix_ms: i64, + pub envelope: &'a MailboxEnvelope, + pub header: &'a MailboxEnvelopeHeader, + pub authority: &'a VerifyingKey, + pub revoked: &'a [RevocationId], +} + +/// Verify a lease against the exact committed ciphertext, a separate signed +/// Receive capability and an authenticated current consumer session. +pub fn verify_delivery_lease( + signed: &SignedMailboxDeliveryLease, + context: &ReaderDeliveryContext<'_>, +) -> Result<(), MailboxError> { + let ReaderDeliveryContext { + plan, + capability, + consumer_session, + now_unix_ms, + envelope, + header, + authority, + revoked, + .. + } = context; + if plan.authority_key != authority.to_bytes() { + return Err(MailboxError::WrongCapabilityIssuer); + } + if revoked.contains(&capability.grant.revocation_id) { + return Err(MailboxError::CapabilityRevoked); + } + verify_capability_signature(capability, authority)?; + signed.check_scope(plan, &capability.grant, *consumer_session, *now_unix_ms)?; + signed.encode()?; + let lease = &signed.lease; + let address = MailboxAddress { + tenant: header.tenant, + authorization_domain: header.authorization_domain, + mailbox: header.mailbox, + recipient: header.recipient, + }; + if lease.address != address + || lease.sender != header.sender + || lease.sender_boot != header.sender_boot + || lease.message_id != header.message_id + || lease.delivery_id != envelope.routing.delivery_id + || lease.execution_epoch != header.execution_epoch + || lease.ciphertext_commitment != envelope.ciphertext_commitment + { + return Err(MailboxError::DeliveryLeaseMismatch); + } + let signature: [u8; 64] = signed + .signature + .as_slice() + .try_into() + .map_err(|_| MailboxError::DeliveryLeaseSignatureInvalid)?; + authority + .verify( + &canonical_bytes(DELIVERY_LEASE_SIGNATURE_DOMAIN, lease)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| MailboxError::DeliveryLeaseSignatureInvalid) +} + +/// Sign one acknowledgement with the guest's independently enrolled mailbox +/// key; this does not assert that the destination committed the ACK. +pub fn sign_delivery_ack( + ack: MailboxDeliveryAck, + guest: &SigningKey, +) -> Result { + let signature = guest + .sign(&canonical_bytes(DELIVERY_ACK_SIGNATURE_DOMAIN, &ack)?) + .to_bytes() + .to_vec(); + let signed = SignedMailboxDeliveryAck { ack, signature }; + signed.encode()?; + Ok(signed) +} + +/// Verify the current signed enrollment, exact active lease and guest ACK. +/// The runtime must authenticate the supplied boot/session, prove possession +/// before disclosure, and durably fence ACK replay; this does not mutate state. +pub fn verify_delivery_ack( + signed: &SignedMailboxDeliveryAck, + active: &SignedMailboxDeliveryLease, + context: &ReaderDeliveryContext<'_>, + enrollment: &SignedMailboxGuestReaderEnrollment, + guest: &VerifyingKey, +) -> Result<(), MailboxError> { + if context.reader_scope.plan != context.plan + || context.reader_scope.current_session != context.consumer_session + || context.reader_scope.now_unix_ms != context.now_unix_ms + { + return Err(MailboxError::ReaderKeyInvalid); + } + let enrolled_key = verify_guest_reader_enrollment( + enrollment, + context.capability, + &context.reader_scope, + context.authority, + context.revoked, + )?; + if enrolled_key != *guest { + return Err(MailboxError::ReaderKeyInvalid); + } + if enrollment.enrollment.ack_public_key != guest.to_bytes() + || enrollment.enrollment.consumer_session != context.consumer_session + || enrollment.enrollment.consumer != context.capability.grant.subject + || enrollment.enrollment.address != context.capability.grant.address + || enrollment.enrollment.receive_capability_id != context.capability.grant.capability_id + || enrollment.enrollment.execution_epoch != context.plan.epoch + || enrollment.enrollment.policy_epoch != context.capability.grant.policy_epoch + { + return Err(MailboxError::ReaderKeyInvalid); + } + signed.check_scope( + context.plan, + &context.capability.grant, + context.consumer_session, + context.now_unix_ms, + )?; + signed.encode()?; + if &signed.ack.lease != active { + return Err(MailboxError::DeliveryLeaseMismatch); + } + verify_delivery_lease(active, context)?; + let signature: [u8; 64] = signed + .signature + .as_slice() + .try_into() + .map_err(|_| MailboxError::DeliveryAckSignatureInvalid)?; + guest + .verify( + &canonical_bytes(DELIVERY_ACK_SIGNATURE_DOMAIN, &signed.ack)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| MailboxError::DeliveryAckSignatureInvalid) +} diff --git a/crates/mvm-core/tests/mailbox_reader.rs b/crates/mvm-core/tests/mailbox_reader.rs new file mode 100644 index 0000000000..4652167c06 --- /dev/null +++ b/crates/mvm-core/tests/mailbox_reader.rs @@ -0,0 +1,440 @@ +use ed25519_dalek::SigningKey; +use mvm_contract::plan::{Nonce, PlanId}; +use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, AuthorizationDomainRouteLabel, BootId, CapabilityId, + CapabilityOperations, ConsumerSessionId, ContentType, CorrelationId, DeliveryAckNonce, + DeliveryId, DeliveryLeaseNonce, DestinationRouteToken, EncryptionAlgorithm, EncryptionContext, + ExecutionEpoch, MailboxAddress, MailboxCapabilityGrant, MailboxDeliveryAck, + MailboxDeliveryLease, MailboxEnvelopeHeader, MailboxGrant, MailboxGuestReaderEnrollment, + MailboxGuestReaderScope, MailboxId, MailboxMessageKind, MailboxOperation, MailboxRoutingFrame, + MessageId, OrderingScope, PrincipalId, ProtocolVersion, ReplayNonce, RevocationId, + SequenceNumber, SignatureAlgorithm, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, + TenantId, TransportBindingId, +}; +use mvm_core::crypto::aead::Key; +use mvm_core::mailbox::reader::{ + ReaderDeliveryContext, sign_delivery_ack, sign_delivery_lease, sign_guest_reader_enrollment, + sign_reader_possession, verify_delivery_ack, verify_delivery_lease, + verify_guest_reader_enrollment, verify_reader_possession, +}; +use mvm_core::mailbox::{MailboxError, SealRequest, seal_envelope, sign_capability}; + +fn fixture() -> ( + SigningKey, + SigningKey, + MailboxGrant, + PlanId, + Nonce, + MailboxCapabilityGrant, + MailboxGuestReaderEnrollment, +) { + let authority = SigningKey::from_bytes(&[5; 32]); + let guest = SigningKey::from_bytes(&[6; 32]); + let address = MailboxAddress { + tenant: TenantId::from_bytes([1; 32]), + authorization_domain: AuthorizationDomainId::from_bytes([2; 32]), + mailbox: MailboxId::from_bytes([3; 32]), + recipient: PrincipalId::from_bytes([4; 32]), + }; + let epoch = ExecutionEpoch::new(7).unwrap(); + let plan = MailboxGrant { + mailbox: address.mailbox, + authority_key: authority.verifying_key().to_bytes(), + epoch, + }; + let plan_id = PlanId(format!("sha256:{}", "a".repeat(64))); + let plan_nonce = Nonce::from_bytes([7; 16]); + let receive = MailboxCapabilityGrant { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + capability_id: CapabilityId::from_bytes([8; 32]), + issuer: PrincipalId::from_bytes([9; 32]), + subject: PrincipalId::from_bytes([10; 32]), + address, + operations: CapabilityOperations::from_operations(&[MailboxOperation::Receive]), + message_kinds: vec![MailboxMessageKind::Cast], + content_types: vec![ContentType::parse("application/vnd.mvm.workload-report", 1).unwrap()], + max_payload_bytes: 1024, + max_messages_per_minute: 60, + max_outstanding_calls: 1, + max_queue_contribution_bytes: 4096, + minimum_execution_epoch: epoch, + maximum_execution_epoch: epoch, + policy_epoch: 12, + not_before_unix_ms: 100, + expires_at_unix_ms: 300, + revocation_id: RevocationId::from_bytes([11; 32]), + delegation_depth: 0, + }; + let enrollment = MailboxGuestReaderEnrollment { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + plan_id: plan_id.clone(), + plan_nonce: plan_nonce.clone(), + address, + consumer: receive.subject, + consumer_boot: BootId::from_bytes([12; 32]), + consumer_session: ConsumerSessionId::from_bytes([13; 32]), + receive_capability_id: receive.capability_id, + ack_public_key: guest.verifying_key().to_bytes(), + execution_epoch: epoch, + policy_epoch: receive.policy_epoch, + issued_at_unix_ms: 110, + expires_at_unix_ms: 200, + }; + ( + authority, guest, plan, plan_id, plan_nonce, receive, enrollment, + ) +} + +#[test] +fn enrolled_reader_requires_two_authority_signatures_and_fresh_guest_possession() { + let (authority, guest, plan, plan_id, plan_nonce, receive, enrollment) = fixture(); + let cap = sign_capability(receive.clone(), &authority).unwrap(); + let signed = sign_guest_reader_enrollment(enrollment, &authority).unwrap(); + let scope = MailboxGuestReaderScope { + plan: &plan, + plan_id: &plan_id, + plan_nonce: &plan_nonce, + receive: &receive, + current_boot: signed.enrollment.consumer_boot, + current_session: signed.enrollment.consumer_session, + now_unix_ms: 150, + }; + let key = + verify_guest_reader_enrollment(&signed, &cap, &scope, &authority.verifying_key(), &[]) + .expect("current signed plan and Receive grant"); + let challenge = [20; 32]; + let proof = sign_reader_possession(&signed, &challenge, &guest).unwrap(); + verify_reader_possession(&signed, &challenge, &proof, &key).unwrap(); + assert_eq!( + verify_reader_possession(&signed, &[21; 32], &proof, &key), + Err(MailboxError::ReaderPossessionInvalid), + "a proof for one fresh host challenge cannot answer another" + ); + assert_eq!( + verify_reader_possession(&signed, &[0; 32], &proof, &key), + Err(MailboxError::ReaderPossessionInvalid) + ); + assert_eq!( + sign_reader_possession(&signed, &challenge, &authority), + Err(MailboxError::ReaderKeyInvalid), + "the host authority must not substitute for the guest-held key" + ); + + let mut forged_enrollment = signed.clone(); + forged_enrollment.enrollment.ack_public_key = + SigningKey::from_bytes(&[44; 32]).verifying_key().to_bytes(); + assert_eq!( + verify_guest_reader_enrollment( + &forged_enrollment, + &cap, + &scope, + &authority.verifying_key(), + &[], + ), + Err(MailboxError::ReaderEnrollmentSignatureInvalid) + ); + let mut forged_capability = cap.clone(); + forged_capability.signature[0] ^= 1; + assert_eq!( + verify_guest_reader_enrollment( + &signed, + &forged_capability, + &scope, + &authority.verifying_key(), + &[], + ), + Err(MailboxError::CapabilitySignatureInvalid) + ); + assert_eq!( + verify_guest_reader_enrollment( + &signed, + &cap, + &scope, + &authority.verifying_key(), + &[receive.revocation_id], + ), + Err(MailboxError::CapabilityRevoked) + ); +} + +#[test] +fn enrollment_refuses_stale_boot_plan_recipient_and_policy_before_key_use() { + let (authority, _, plan, plan_id, plan_nonce, receive, enrollment) = fixture(); + let signed = sign_guest_reader_enrollment(enrollment, &authority).unwrap(); + let cap = sign_capability(receive.clone(), &authority).unwrap(); + let scope = MailboxGuestReaderScope { + plan: &plan, + plan_id: &plan_id, + plan_nonce: &plan_nonce, + receive: &receive, + current_boot: signed.enrollment.consumer_boot, + current_session: signed.enrollment.consumer_session, + now_unix_ms: 150, + }; + let mut previous_boot = scope; + previous_boot.current_boot = BootId::from_bytes([99; 32]); + assert!( + verify_guest_reader_enrollment( + &signed, + &cap, + &previous_boot, + &authority.verifying_key(), + &[], + ) + .is_err() + ); + let wrong_nonce = Nonce::from_bytes([98; 16]); + let wrong_plan = MailboxGuestReaderScope { + plan_nonce: &wrong_nonce, + ..scope + }; + assert!(verify_guest_reader_enrollment( + &signed, + &cap, + &wrong_plan, + &authority.verifying_key(), + &[], + ) + .is_err()); + let mut wrong_recipient = receive.clone(); + wrong_recipient.address.recipient = PrincipalId::from_bytes([97; 32]); + let wrong_scope = MailboxGuestReaderScope { + receive: &wrong_recipient, + ..scope + }; + assert_eq!( + verify_guest_reader_enrollment( + &signed, + &cap, + &wrong_scope, + &authority.verifying_key(), + &[], + ), + Err(MailboxError::WrongCapability), + "the verified capability must be exactly the scoped grant" + ); + let mut stale_policy = receive.clone(); + stale_policy.policy_epoch += 1; + let stale_cap = sign_capability(stale_policy.clone(), &authority).unwrap(); + let stale_scope = MailboxGuestReaderScope { + receive: &stale_policy, + ..scope + }; + assert!( + verify_guest_reader_enrollment( + &signed, + &stale_cap, + &stale_scope, + &authority.verifying_key(), + &[], + ) + .is_err() + ); +} + +#[test] +fn lease_and_ack_signatures_bind_committed_ciphertext_and_consumer() { + let (authority, guest, plan, plan_id, plan_nonce, receive, enrollment) = fixture(); + let signed_enrollment = sign_guest_reader_enrollment(enrollment.clone(), &authority).unwrap(); + let capability = sign_capability(receive.clone(), &authority).unwrap(); + let header = MailboxEnvelopeHeader { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + tenant: receive.address.tenant, + authorization_domain: receive.address.authorization_domain, + sender: PrincipalId::from_bytes([30; 32]), + recipient: receive.address.recipient, + mailbox: plan.mailbox, + sender_boot: BootId::from_bytes([31; 32]), + execution_epoch: plan.epoch, + message_id: MessageId::from_bytes([32; 32]), + correlation_id: CorrelationId::from_bytes([33; 32]), + causation_id: mvm_contract::protocol::mailbox::CausationId::from_bytes([34; 32]), + kind: MailboxMessageKind::Cast, + reply_to: None, + ordering_scope: OrderingScope::parse("report").unwrap(), + sequence: SequenceNumber::new(1).unwrap(), + issued_at_unix_ms: 110, + expires_at_unix_ms: 200, + nonce: ReplayNonce::from_bytes([35; 16]), + content_type: receive.content_types[0].clone(), + capability_id: CapabilityId::from_bytes([36; 32]), + policy_epoch: receive.policy_epoch, + encryption: EncryptionContext { + algorithm: EncryptionAlgorithm::AES_256_GCM, + key_id: [37; 32], + }, + plaintext_len: 5, + }; + let envelope = seal_envelope( + SealRequest { + routing: MailboxRoutingFrame { + protocol_version: ProtocolVersion::V1, + authorization_domain_route_label: AuthorizationDomainRouteLabel::from_bytes( + [38; 32], + ), + destination_route_token: DestinationRouteToken::from_bytes([39; 32]), + delivery_id: DeliveryId::from_bytes([40; 32]), + ciphertext_len: 0, + padding: mvm_contract::protocol::mailbox::CiphertextPadding::Exact, + expires_at_unix_ms: header.expires_at_unix_ms, + hop_limit: 1, + transport_binding_id: TransportBindingId::from_bytes([41; 32]), + }, + header: header.clone(), + plaintext: b"hello", + }, + &Key::from_bytes([42; 32]), + &SigningKey::from_bytes(&[43; 32]), + ) + .unwrap(); + let lease = sign_delivery_lease( + MailboxDeliveryLease { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + address: receive.address, + sender: header.sender, + sender_boot: header.sender_boot, + message_id: header.message_id, + delivery_id: envelope.routing.delivery_id, + execution_epoch: plan.epoch, + consumer: receive.subject, + consumer_session: enrollment.consumer_session, + receive_capability_id: receive.capability_id, + policy_epoch: receive.policy_epoch, + ciphertext_commitment: envelope.ciphertext_commitment, + issued_at_unix_ms: 120, + expires_at_unix_ms: 190, + lease_nonce: DeliveryLeaseNonce::from_bytes([44; 16]), + }, + &authority, + ) + .unwrap(); + let authority_vk = authority.verifying_key(); + let context = ReaderDeliveryContext { + plan: &plan, + capability: &capability, + reader_scope: MailboxGuestReaderScope { + plan: &plan, + plan_id: &plan_id, + plan_nonce: &plan_nonce, + receive: &receive, + current_boot: enrollment.consumer_boot, + current_session: enrollment.consumer_session, + now_unix_ms: 150, + }, + consumer_session: enrollment.consumer_session, + now_unix_ms: 150, + envelope: &envelope, + header: &header, + authority: &authority_vk, + revoked: &[], + }; + let verify_lease = |value: &SignedMailboxDeliveryLease| verify_delivery_lease(value, &context); + verify_lease(&lease).unwrap(); + let ack = sign_delivery_ack( + MailboxDeliveryAck { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + lease: lease.clone(), + ack_nonce: DeliveryAckNonce::from_bytes([45; 16]), + acknowledged_at_unix_ms: 150, + }, + &guest, + ) + .unwrap(); + let verify_ack = |value: &SignedMailboxDeliveryAck, session| { + verify_delivery_ack( + value, + &lease, + &ReaderDeliveryContext { + consumer_session: session, + ..context + }, + &signed_enrollment, + &guest.verifying_key(), + ) + }; + verify_ack(&ack, enrollment.consumer_session).unwrap(); + let wrong_guest = SigningKey::from_bytes(&[99; 32]).verifying_key(); + assert_eq!( + verify_delivery_ack(&ack, &lease, &context, &signed_enrollment, &wrong_guest), + Err(MailboxError::ReaderKeyInvalid) + ); + let mut forged_enrollment = signed_enrollment.clone(); + forged_enrollment.signature[0] ^= 1; + assert_eq!( + verify_delivery_ack( + &ack, + &lease, + &context, + &forged_enrollment, + &guest.verifying_key(), + ), + Err(MailboxError::ReaderEnrollmentSignatureInvalid) + ); + let other_guest = SigningKey::from_bytes(&[98; 32]); + let forged_ack = sign_delivery_ack(ack.ack.clone(), &other_guest).unwrap(); + assert_eq!( + verify_delivery_ack( + &forged_ack, + &lease, + &context, + &signed_enrollment, + &guest.verifying_key(), + ), + Err(MailboxError::DeliveryAckSignatureInvalid) + ); + assert_eq!( + verify_delivery_lease( + &lease, + &ReaderDeliveryContext { + revoked: &[receive.revocation_id], + ..context + }, + ), + Err(MailboxError::CapabilityRevoked) + ); + let mut different_ciphertext = envelope.clone(); + different_ciphertext.ciphertext_commitment[0] ^= 1; + assert_eq!( + verify_delivery_lease( + &lease, + &ReaderDeliveryContext { + envelope: &different_ciphertext, + ..context + }, + ), + Err(MailboxError::DeliveryLeaseMismatch) + ); + let mut bad_lease = lease.clone(); + bad_lease.signature[0] ^= 1; + assert_eq!( + verify_lease(&bad_lease), + Err(MailboxError::DeliveryLeaseSignatureInvalid) + ); + bad_lease = lease.clone(); + bad_lease.lease.ciphertext_commitment[0] ^= 1; + assert_eq!( + verify_lease(&bad_lease), + Err(MailboxError::DeliveryLeaseMismatch) + ); + let mut bad_ack = ack.clone(); + bad_ack.signature[0] ^= 1; + assert_eq!( + verify_ack(&bad_ack, enrollment.consumer_session), + Err(MailboxError::DeliveryAckSignatureInvalid) + ); + bad_ack = ack.clone(); + bad_ack.ack.lease.lease.lease_nonce = DeliveryLeaseNonce::from_bytes([46; 16]); + assert_eq!( + verify_ack(&bad_ack, enrollment.consumer_session), + Err(MailboxError::DeliveryLeaseMismatch) + ); + assert!( + verify_ack(&ack, ConsumerSessionId::from_bytes([47; 32])).is_err(), + "an inherited or stale reader session must not ACK this lease" + ); +} From dfc16918c1db2a08eb364aff9d3a62fb8a25eb57 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 14:50:23 -0800 Subject: [PATCH 04/51] fix(mailbox): reconcile runtime refusal and process teardown --- crates/mvm-core/tests/mailbox_reader.rs | 35 +++++++++++++++++++ crates/mvm-hostd/src/mailbox_launch.rs | 17 +++++++-- crates/mvm-hostd/src/mailbox_process.rs | 8 ++++- crates/mvm-hostd/src/mailbox_service.rs | 3 +- .../mvm-hostd/src/mailbox_service/durable.rs | 26 ++++++++++---- crates/mvm-hostd/tests/mission_demo.rs | 12 +++---- 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/crates/mvm-core/tests/mailbox_reader.rs b/crates/mvm-core/tests/mailbox_reader.rs index 4652167c06..d4ec8bd35c 100644 --- a/crates/mvm-core/tests/mailbox_reader.rs +++ b/crates/mvm-core/tests/mailbox_reader.rs @@ -375,6 +375,41 @@ fn lease_and_ack_signatures_bind_committed_ciphertext_and_consumer() { ), Err(MailboxError::ReaderEnrollmentSignatureInvalid) ); + let previous_boot = ReaderDeliveryContext { + reader_scope: MailboxGuestReaderScope { + current_boot: BootId::from_bytes([92; 32]), + ..context.reader_scope + }, + ..context + }; + assert!( + verify_delivery_ack( + &ack, + &lease, + &previous_boot, + &signed_enrollment, + &guest.verifying_key(), + ) + .is_err() + ); + let stale_nonce = Nonce::from_bytes([93; 16]); + let wrong_plan = ReaderDeliveryContext { + reader_scope: MailboxGuestReaderScope { + plan_nonce: &stale_nonce, + ..context.reader_scope + }, + ..context + }; + assert!( + verify_delivery_ack( + &ack, + &lease, + &wrong_plan, + &signed_enrollment, + &guest.verifying_key(), + ) + .is_err() + ); let other_guest = SigningKey::from_bytes(&[98; 32]); let forged_ack = sign_delivery_ack(ack.ack.clone(), &other_guest).unwrap(); assert_eq!( diff --git a/crates/mvm-hostd/src/mailbox_launch.rs b/crates/mvm-hostd/src/mailbox_launch.rs index caeb2d3fa7..f0dc1237aa 100644 --- a/crates/mvm-hostd/src/mailbox_launch.rs +++ b/crates/mvm-hostd/src/mailbox_launch.rs @@ -442,7 +442,7 @@ mod tests { use mvm_core::mailbox::frame::OutboxMessage; fn keys_dir() -> tempfile::TempDir { - let dir = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir_in("/tmp").unwrap(); crate::audit::host_keypair::load_or_init_at(dir.path()).unwrap(); dir } @@ -576,7 +576,11 @@ mod tests { std::thread::spawn(move || { for mut stream in listener.incoming().flatten() { let v = mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &path); - if v != Err(mvm_core::mailbox::frame::Refusal::Malformed) { + if !matches!( + v, + Err(mvm_core::mailbox::frame::Refusal::Malformed + | mvm_core::mailbox::frame::Refusal::Unavailable) + ) { let _ = tx.send(v); } } @@ -617,7 +621,14 @@ mod tests { ..LAUNCH_LIMITS }; let launch = launch_against(dir.path(), limits); - assert!(send_one(&outbox, &rx, 9, b"wrong recipient").is_err()); + assert_eq!( + send_one(&outbox, &rx, 9, b"wrong recipient"), + Err(mvm_core::mailbox::frame::Refusal::NotAuthorized) + ); + assert!( + !outbox.exists(), + "a terminal refusal must free the live outbox slot" + ); assert_eq!(send_one(&outbox, &rx, 7, b"first"), Ok(())); assert_eq!(send_one(&outbox, &rx, 7, b"second"), Ok(())); assert!(send_one(&outbox, &rx, 7, b"over the bound").is_err()); diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index 1da45c9150..7741058f84 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -335,7 +335,7 @@ mod tests { } fn keys_dir() -> tempfile::TempDir { - let dir = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir_in("/tmp").unwrap(); crate::audit::host_keypair::load_or_init_at(dir.path()).unwrap(); dir } @@ -521,7 +521,13 @@ mod tests { guest_route: HostDialRoute::PortSocket(dir.path().join("absent-port")), vm_state_dir: state_path, }; + let socket = config.socket.clone(); let serving = std::thread::spawn(move || serve(config)); + let started = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !socket.exists() { + assert!(std::time::Instant::now() < started, "mailbox never bound"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } drop(state); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); while !serving.is_finished() { diff --git a/crates/mvm-hostd/src/mailbox_service.rs b/crates/mvm-hostd/src/mailbox_service.rs index 6487643eb7..c625d3ed38 100644 --- a/crates/mvm-hostd/src/mailbox_service.rs +++ b/crates/mvm-hostd/src/mailbox_service.rs @@ -331,7 +331,8 @@ pub fn serve_grant( socket: &Path, guest: (&Path, &Path), ) -> std::io::Result { - let (service, mut sealer) = bind_grant(grant, trust, sealer, limits, socket)?; + let mut sealer = sealer.with_grant(grant)?; + let service = MailboxService::bind_live(socket, limits, trust.with_grant(grant)?)?; let _ = service.serve_guest(guest.0, guest.1, &mut sealer)?; Ok(service) } diff --git a/crates/mvm-hostd/src/mailbox_service/durable.rs b/crates/mvm-hostd/src/mailbox_service/durable.rs index ff11801528..59c707a34b 100644 --- a/crates/mvm-hostd/src/mailbox_service/durable.rs +++ b/crates/mvm-hostd/src/mailbox_service/durable.rs @@ -27,6 +27,8 @@ use mvm_core::vm_backend::HostDialRoute; use super::{GuestSealer, IO_TIMEOUT, MailboxLimits, MailboxPolicy, read_frame}; +const ACCEPT_POLL: std::time::Duration = std::time::Duration::from_millis(20); + /// A trusted anchor advances outside the journal's rollback domain. pub trait JournalAnchor: Send + Sync { fn commit(&self, tip: JournalTip) -> Result<(), JournalError>; @@ -92,6 +94,7 @@ impl DurableMailboxService { } let _ = std::fs::remove_file(path); let listener = UnixListener::bind(path)?; + listener.set_nonblocking(true)?; let shared = Arc::new(Shared { limits, state: Mutex::new(State { policy, journal }), @@ -102,12 +105,22 @@ impl DurableMailboxService { let worker = { let (shared, stop) = (shared.clone(), stop.clone()); std::thread::spawn(move || { - for stream in listener.incoming() { - if stop.load(Ordering::SeqCst) { - break; - } - if let Ok(stream) = stream { - serve(&shared, stream); + while !stop.load(Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + // Some platforms inherit nonblocking on accepted sockets. + let _ = stream.set_nonblocking(false); + serve(&shared, stream); + } + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted + ) => + { + std::thread::sleep(ACCEPT_POLL); + } + Err(_) => break, } } }) @@ -225,7 +238,6 @@ fn reader_context( impl Drop for DurableMailboxService { fn drop(&mut self) { self.stop.store(true, Ordering::SeqCst); - let _ = UnixStream::connect(&self.path); if let Some(worker) = self.worker.take() { let _ = worker.join(); } diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index f34069a433..3cad888445 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -94,7 +94,7 @@ impl Node { let emitter = AuditEmitter::with_dir(audit_key.clone(), audit.path()).unwrap(); Self { name, - home: tempfile::tempdir().unwrap(), + home: tempfile::tempdir_in("/tmp").unwrap(), audit, audit_key, identity: SigningKey::from_bytes(&[seed.wrapping_add(100); 32]), @@ -750,7 +750,7 @@ fn a_guest_addresses_its_own_mailbox_without_the_derived_recipient() { let Err(refusal) = verdict else { panic!("a recipient the policy does not name was sealed"); }; - assert!(matches!(refusal, Refusal::Mailbox(_))); + assert_eq!(refusal, Refusal::NotAuthorized); assert!(queued.is_none()); // The only thing the guest reads back is a verdict, and it names no // recipient: the derived value stays with the host. @@ -979,7 +979,7 @@ fn derived_keys_serve_the_guest_port_and_queue_what_the_guest_sent() { }; use std::os::unix::net::UnixListener; - let dir = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir_in("/tmp").unwrap(); host_keys(dir.path(), 5); let listener = UnixListener::bind(dir.path().join("port.sock")).unwrap(); let authority = authority_key_for(dir.path(), &address().mailbox).unwrap(); @@ -1019,7 +1019,7 @@ fn a_running_guest_submits_again_after_its_first_message_was_consumed() { }; use std::os::unix::net::UnixListener; - let dir = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir_in("/tmp").unwrap(); host_keys(dir.path(), 5); let listener = UnixListener::bind(dir.path().join("port.sock")).unwrap(); let authority = authority_key_for(dir.path(), &address().mailbox).unwrap(); @@ -1572,7 +1572,7 @@ fn served_mailbox_process(dir: &std::path::Path) -> (KillOnDrop, std::path::Path fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { use mvm_hostd::mailbox_process::build_config; - let dir = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir_in("/tmp").unwrap(); let keys = dir.path().join("keys"); std::fs::create_dir_all(&keys).unwrap(); host_keys(&keys, 5); @@ -1609,7 +1609,7 @@ fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { /// ends a serving process: stopping its service must not need that path. #[test] fn the_mailbox_process_exits_when_its_vm_state_dir_is_removed() { - let dir = tempfile::tempdir().unwrap(); + let dir = tempfile::tempdir_in("/tmp").unwrap(); let (mut process, state_dir) = served_mailbox_process(dir.path()); std::fs::remove_dir_all(&state_dir).unwrap(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); From 454738f66b16cab830b756e05640c8b270e907cd Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 16:20:02 -0800 Subject: [PATCH 05/51] feat(mailbox): journal signed guest leases and bind cold boot ACK keys --- crates/mvm-core/src/mailbox/journal.rs | 338 +++++++++++++++++- .../mvm-core/src/mailbox/transport/tests.rs | 284 +++++++++++++++ crates/mvm-hostd/src/mailbox_process.rs | 104 +++++- .../mvm-runtime/src/workload_runner/claim.rs | 4 + .../mvm-runtime/src/workload_runner/runner.rs | 11 + .../src/workload_runner/runner/spawner.rs | 127 ++++++- crates/mvm-vmm/src/host/flowmux_identity.rs | 153 ++++++++ 7 files changed, 1015 insertions(+), 6 deletions(-) diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index c82cda71ae..19c1d3dc31 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -6,7 +6,7 @@ //! returned tip in an independent trusted anchor to detect whole-journal //! rollback; a hash chain alone cannot detect deletion of its suffix. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::io::Read; use std::path::{Path, PathBuf}; @@ -16,13 +16,19 @@ use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; use ed25519_dalek::{SigningKey, VerifyingKey}; use hmac::{Hmac, Mac, digest::KeyInit}; use mvm_contract::protocol::mailbox::{ - DeliveryOutcome, DestinationReceipt, MailboxAddress, MailboxEnvelope, MailboxOperation, - ProtocolVersion, SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, + DeliveryLeaseNonce, DeliveryOutcome, DestinationReceipt, MailboxAddress, MailboxDeliveryLease, + MailboxEnvelope, MailboxGrant, MailboxGuestReaderScope, MailboxOperation, ProtocolVersion, + SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, + SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, }; use serde::{Deserialize, Serialize}; use sha2::Sha256; use zeroize::Zeroizing; +use super::reader::{ + ReaderDeliveryContext, sign_delivery_lease, verify_delivery_ack, verify_delivery_lease, + verify_guest_reader_enrollment, verify_reader_possession, +}; use super::{ MailboxError, OpenContext, ReplayIdentity, ReplayProtector, authorize_receive, frame, open_authenticated_envelope, sign_receipt, verify_receipt, @@ -83,6 +89,8 @@ pub enum JournalError { ConflictingRetry, #[error("mailbox message does not exist")] UnknownMessage, + #[error("mailbox message has no active delivery lease")] + NoActiveLease, #[error("journal cannot be used after an uncertain write")] Uncertain, #[error("mailbox authorization refused: {0}")] @@ -113,12 +121,32 @@ enum JournalEvent { identity: ReplayIdentity, acknowledged_at_unix_ms: i64, }, + Leased { + identity: ReplayIdentity, + challenge: [u8; 32], + lease: Box, + enrollment: Box, + receive_capability: Box, + }, + AcknowledgedSigned { + identity: ReplayIdentity, + ack: Box, + }, +} + +struct LeaseState { + challenge: [u8; 32], + signed: SignedMailboxDeliveryLease, + enrollment: SignedMailboxGuestReaderEnrollment, + receive_capability: SignedMailboxCapability, } struct Seen { envelope: MailboxEnvelope, capability: SignedMailboxCapability, acknowledged: bool, + lease: Option, + signed_ack: Option, } #[derive(Default)] @@ -143,6 +171,7 @@ pub struct MailboxJournal { bytes: u64, seen: BTreeMap, order: Vec, + used_challenges: BTreeSet<[u8; 32]>, poisoned: bool, _lock: FileLock, } @@ -159,6 +188,7 @@ pub struct JournalAdmission<'a> { /// Independent reader authority and host-held decryption material for the /// trusted-host profile. A sender's grant cannot substitute for Receive. +#[derive(Clone, Copy)] pub struct JournalReceive<'a> { pub capability: &'a SignedMailboxCapability, pub context: &'a OpenContext, @@ -166,6 +196,46 @@ pub struct JournalReceive<'a> { pub sender_key: &'a VerifyingKey, } +/// Verified current guest facts and proof for issuing one durable lease. +/// The caller must obtain the boot/session and challenge from its authenticated +/// guest channel, not from untrusted request fields. +pub struct JournalGuestLease<'a> { + pub receive: JournalReceive<'a>, + pub plan: &'a MailboxGrant, + pub scope: MailboxGuestReaderScope<'a>, + pub enrollment: &'a SignedMailboxGuestReaderEnrollment, + pub challenge: &'a [u8; 32], + pub possession_signature: &'a [u8], + pub signer: &'a SigningKey, +} + +/// One guest-authored ACK of the previously persisted active lease. +pub struct JournalGuestAck<'a> { + pub receive: JournalReceive<'a>, + pub plan: &'a MailboxGrant, + pub scope: MailboxGuestReaderScope<'a>, + pub enrollment: &'a SignedMailboxGuestReaderEnrollment, + pub signed: &'a SignedMailboxDeliveryAck, +} + +/// A message whose signed lease was durably committed before payload release. +pub struct LeasedMailboxMessage { + message: ReceivedMailboxMessage, + lease: SignedMailboxDeliveryLease, +} + +impl LeasedMailboxMessage { + #[must_use] + pub fn payload(&self) -> &[u8] { + self.message.payload() + } + + #[must_use] + pub fn lease(&self) -> &SignedMailboxDeliveryLease { + &self.lease + } +} + /// A pending, independently authorized message. Debug omits its plaintext. pub struct ReceivedMailboxMessage { identity: ReplayIdentity, @@ -370,6 +440,7 @@ impl MailboxJournal { bytes: 0, seen: BTreeMap::new(), order: Vec::new(), + used_challenges: BTreeSet::new(), poisoned: false, _lock: lock, }; @@ -450,6 +521,8 @@ impl MailboxJournal { envelope, capability, acknowledged: false, + lease: None, + signed_ack: None, }, ) .is_some() @@ -470,6 +543,55 @@ impl MailboxJournal { } entry.acknowledged = true; } + JournalEvent::Leased { + identity, + challenge, + lease, + enrollment, + receive_capability, + } => { + let entry = journal + .seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)?; + if entry.acknowledged + || lease.encode().is_err() + || enrollment.encode().is_err() + || lease.lease.address != address + || lease.lease.delivery_id != entry.envelope.routing.delivery_id + || lease.lease.message_id != identity.message_id + || lease.lease.execution_epoch != identity.execution_epoch + || lease.lease.ciphertext_commitment != entry.envelope.ciphertext_commitment + || lease.lease.consumer_session != enrollment.enrollment.consumer_session + || lease.lease.receive_capability_id + != receive_capability.grant.capability_id + || challenge == [0; 32] + { + return Err(JournalError::Corrupt); + } + entry.lease = Some(LeaseState { + challenge, + signed: *lease, + enrollment: *enrollment, + receive_capability: *receive_capability, + }); + if !journal.used_challenges.insert(challenge) { + return Err(JournalError::Corrupt); + } + } + JournalEvent::AcknowledgedSigned { identity, ack } => { + let entry = journal + .seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)?; + let active = entry.lease.as_ref().ok_or(JournalError::Corrupt)?; + if entry.acknowledged || ack.encode().is_err() || ack.ack.lease != active.signed + { + return Err(JournalError::Corrupt); + } + entry.acknowledged = true; + entry.signed_ack = Some(*ack); + } } let next = JournalTip { cursor, @@ -564,6 +686,8 @@ impl MailboxJournal { envelope: envelope.clone(), capability: capability.clone(), acknowledged: false, + lease: None, + signed_ack: None, }, ); self.order.push(identity); @@ -648,6 +772,132 @@ impl MailboxJournal { Ok(None) } + /// Commit one signed lease and its one-time possession challenge before + /// releasing plaintext to the authenticated guest reader. + pub fn lease_next( + &mut self, + request: JournalGuestLease<'_>, + anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, + ) -> Result, JournalError> { + if self.poisoned { + return Err(JournalError::Uncertain); + } + if request.signer.verifying_key() != self.authority + || request.scope.plan != request.plan + || request.scope.receive != &request.receive.capability.grant + || request.scope.now_unix_ms != request.receive.context.now_unix_ms + || request.scope.current_session.as_bytes() == &[0; 32] + || request.scope.current_boot.as_bytes() == &[0; 32] + || request.receive.context.acting_principal != request.enrollment.enrollment.consumer + { + return Err(JournalError::WrongAuthority); + } + let key = verify_guest_reader_enrollment( + request.enrollment, + request.receive.capability, + &request.scope, + &self.authority, + &request.receive.context.revoked_capabilities, + )?; + verify_reader_possession( + request.enrollment, + request.challenge, + request.possession_signature, + &key, + )?; + let Some(message) = self.receive_next(request.receive)? else { + return Ok(None); + }; + let identity = message.identity; + let entry = self.seen.get(&identity).ok_or(JournalError::Corrupt)?; + let now = request.scope.now_unix_ms; + if let Some(active) = &entry.lease + && active.signed.lease.expires_at_unix_ms > now + { + if active.challenge != *request.challenge + || active.enrollment != *request.enrollment + || active.receive_capability != *request.receive.capability + { + return Err(JournalError::Busy); + } + return Ok(Some(LeasedMailboxMessage { + message, + lease: active.signed.clone(), + })); + } + if self.used_challenges.contains(request.challenge) { + return Err(JournalError::ConflictingRetry); + } + let header = message.opened.header(); + let expires_at_unix_ms = now + .saturating_add(30_000) + .min(header.expires_at_unix_ms) + .min(request.receive.capability.grant.expires_at_unix_ms) + .min(request.enrollment.enrollment.expires_at_unix_ms); + if expires_at_unix_ms <= now { + return Err(JournalError::Mailbox(MailboxError::Expired)); + } + let signed = sign_delivery_lease( + MailboxDeliveryLease { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + address: self.address, + sender: header.sender, + sender_boot: header.sender_boot, + message_id: header.message_id, + delivery_id: entry.envelope.routing.delivery_id, + execution_epoch: header.execution_epoch, + consumer: request.enrollment.enrollment.consumer, + consumer_session: request.scope.current_session, + receive_capability_id: request.receive.capability.grant.capability_id, + policy_epoch: request.receive.capability.grant.policy_epoch, + ciphertext_commitment: entry.envelope.ciphertext_commitment, + issued_at_unix_ms: now, + expires_at_unix_ms, + lease_nonce: DeliveryLeaseNonce::from_bytes(rand::random()), + }, + request.signer, + )?; + verify_delivery_lease( + &signed, + &ReaderDeliveryContext { + plan: request.plan, + capability: request.receive.capability, + reader_scope: request.scope, + consumer_session: request.scope.current_session, + now_unix_ms: now, + envelope: &entry.envelope, + header, + authority: &self.authority, + revoked: &request.receive.context.revoked_capabilities, + }, + )?; + self.append( + JournalEvent::Leased { + identity, + challenge: *request.challenge, + lease: Box::new(signed.clone()), + enrollment: Box::new(request.enrollment.clone()), + receive_capability: Box::new(request.receive.capability.clone()), + }, + anchor, + )?; + self.used_challenges.insert(*request.challenge); + self.seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)? + .lease = Some(LeaseState { + challenge: *request.challenge, + signed: signed.clone(), + enrollment: request.enrollment.clone(), + receive_capability: request.receive.capability.clone(), + }); + Ok(Some(LeasedMailboxMessage { + message, + lease: signed, + })) + } + /// Fsync an append-only ACK and advance the trusted tip. An uncertain /// commit poisons this writer; recovery must reconcile it before retry. pub fn acknowledge( @@ -681,6 +931,88 @@ impl MailboxJournal { Ok(tip) } + /// Verify the enrolled guest signature over the exact active lease, + /// commit its replay identity and signed bytes, then advance the anchor. + pub fn acknowledge_signed( + &mut self, + request: JournalGuestAck<'_>, + anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, + ) -> Result { + if self.poisoned { + return Err(JournalError::Uncertain); + } + if request.scope.plan != request.plan + || request.scope.receive != &request.receive.capability.grant + || request.scope.now_unix_ms != request.receive.context.now_unix_ms + { + return Err(JournalError::WrongAuthority); + } + let lease = &request.signed.ack.lease.lease; + let identity = ReplayIdentity { + sender: lease.sender, + sender_boot: lease.sender_boot, + execution_epoch: lease.execution_epoch, + message_id: lease.message_id, + }; + let entry = self + .seen + .get(&identity) + .ok_or(JournalError::UnknownMessage)?; + let active = entry.lease.as_ref().ok_or(JournalError::NoActiveLease)?; + if active.signed != request.signed.ack.lease + || active.enrollment != *request.enrollment + || active.receive_capability != *request.receive.capability + { + return Err(JournalError::ConflictingRetry); + } + if request + .receive + .context + .revoked_capabilities + .contains(&request.receive.capability.grant.revocation_id) + { + return Err(JournalError::Mailbox(MailboxError::CapabilityRevoked)); + } + if entry.acknowledged { + return if entry.signed_ack.as_ref() == Some(request.signed) { + Ok(self.tip) + } else { + Err(JournalError::ConflictingRetry) + }; + } + let opened = self.open_for_reader(identity, entry, &request.receive)?; + let guest_key = VerifyingKey::from_bytes(&request.enrollment.enrollment.ack_public_key) + .map_err(|_| JournalError::Mailbox(MailboxError::ReaderKeyInvalid))?; + verify_delivery_ack( + request.signed, + &active.signed, + &ReaderDeliveryContext { + plan: request.plan, + capability: request.receive.capability, + reader_scope: request.scope, + consumer_session: request.scope.current_session, + now_unix_ms: request.scope.now_unix_ms, + envelope: &entry.envelope, + header: opened.header(), + authority: &self.authority, + revoked: &request.receive.context.revoked_capabilities, + }, + request.enrollment, + &guest_key, + )?; + let tip = self.append( + JournalEvent::AcknowledgedSigned { + identity, + ack: Box::new(request.signed.clone()), + }, + anchor, + )?; + let entry = self.seen.get_mut(&identity).ok_or(JournalError::Corrupt)?; + entry.acknowledged = true; + entry.signed_ack = Some(request.signed.clone()); + Ok(tip) + } + fn open_for_reader( &self, identity: ReplayIdentity, diff --git a/crates/mvm-core/src/mailbox/transport/tests.rs b/crates/mvm-core/src/mailbox/transport/tests.rs index 01b192c044..a2e4974aa2 100644 --- a/crates/mvm-core/src/mailbox/transport/tests.rs +++ b/crates/mvm-core/src/mailbox/transport/tests.rs @@ -442,6 +442,290 @@ fn receive_and_ack_require_independent_authority_and_survive_recovery() { std::fs::write(&ack_path, ack).expect("restore ACK fixture"); } +#[test] +fn guest_lease_and_signed_ack_are_durable_replay_fenced_and_boot_bound() { + use crate::mailbox::journal::{JournalGuestAck, JournalGuestLease}; + use crate::mailbox::reader::{ + sign_delivery_ack, sign_guest_reader_enrollment, sign_reader_possession, + }; + use mvm_contract::plan::{Nonce, PlanId}; + use mvm_contract::protocol::mailbox::{ + ConsumerSessionId, DeliveryAckNonce, MailboxDeliveryAck, MailboxGrant, + MailboxGuestReaderEnrollment, MailboxGuestReaderScope, + }; + + let dir = tempfile::tempdir().expect("journal directory"); + let fixture = mailbox_fixture(); + let signer = &fixture.capability_authority; + let sender = SigningKey::from_bytes(&id(22)).verifying_key(); + let encryption_key = Key::from_bytes(id(21)); + let limits = JournalLimits { + max_records: 5, + max_bytes: 128 * 1024, + }; + let mut journal = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + JournalTip::genesis(), + ) + .unwrap(); + let mut tip = JournalTip::genesis(); + journal + .admit( + JournalAdmission { + envelope: fixture.prepared.envelope(), + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &encryption_key, + sender_key: &sender, + signer, + }, + |next| { + tip = next; + Ok(()) + }, + ) + .expect("durable send before guest reader"); + let mut receive_grant = fixture.capability.grant.clone(); + receive_grant.capability_id = CapabilityId::from_bytes(id(51)); + receive_grant.subject = address().recipient; + receive_grant.operations = CapabilityOperations::from_operations(&[MailboxOperation::Receive]); + let receive_cap = sign_capability(receive_grant, signer).unwrap(); + let mut reader = fixture.context.clone(); + reader.acting_principal = address().recipient; + reader.required_operation = MailboxOperation::Receive; + let receive = || JournalReceive { + capability: &receive_cap, + context: &reader, + encryption_key: &encryption_key, + sender_key: &sender, + }; + let plan = MailboxGrant { + mailbox: address().mailbox, + authority_key: signer.verifying_key().to_bytes(), + epoch: epoch(), + }; + let plan_id = PlanId(format!("sha256:{}", "a".repeat(64))); + let plan_nonce = Nonce::from_bytes(id(52)); + let boot = BootId::from_bytes(id(53)); + let session = ConsumerSessionId::from_bytes(id(54)); + let scope = MailboxGuestReaderScope { + plan: &plan, + plan_id: &plan_id, + plan_nonce: &plan_nonce, + receive: &receive_cap.grant, + current_boot: boot, + current_session: session, + now_unix_ms: NOW, + }; + let guest = SigningKey::from_bytes(&id(55)); + let enrollment = sign_guest_reader_enrollment( + MailboxGuestReaderEnrollment { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + plan_id: plan_id.clone(), + plan_nonce: plan_nonce.clone(), + address: address(), + consumer: address().recipient, + consumer_boot: boot, + consumer_session: session, + receive_capability_id: receive_cap.grant.capability_id, + ack_public_key: guest.verifying_key().to_bytes(), + execution_epoch: epoch(), + policy_epoch: receive_cap.grant.policy_epoch, + issued_at_unix_ms: NOW - 1, + expires_at_unix_ms: NOW + 5_000, + }, + signer, + ) + .unwrap(); + let challenge = id(56); + let proof = sign_reader_possession(&enrollment, &challenge, &guest).unwrap(); + let lease_req = |scope| JournalGuestLease { + receive: receive(), + plan: &plan, + scope, + enrollment: &enrollment, + challenge: &challenge, + possession_signature: &proof, + signer, + }; + assert!(matches!( + journal.lease_next( + JournalGuestLease { + possession_signature: &[0; 64], + ..lease_req(scope) + }, + |_| panic!("forgery cannot commit"), + ), + Err(JournalError::Mailbox(MailboxError::ReaderPossessionInvalid)) + )); + assert!(matches!( + journal.lease_next( + lease_req(MailboxGuestReaderScope { + current_boot: BootId::from_bytes(id(57)), + ..scope + }), + |_| panic!("stale boot cannot commit"), + ), + Err(JournalError::Mailbox(MailboxError::DeliveryContract(_))) + )); + assert_eq!(journal.tip(), tip); + let leased = journal + .lease_next(lease_req(scope), |next| { + tip = next; + Ok(()) + }) + .expect("verified guest obtains a lease") + .expect("pending ciphertext"); + assert_eq!(leased.payload(), b"hello"); + let signed_lease = leased.lease().clone(); + assert_eq!(journal.tip(), tip); + assert_eq!( + journal + .lease_next(lease_req(scope), |_| panic!("same lease does not append")) + .unwrap() + .unwrap() + .lease(), + &signed_lease + ); + let ack = sign_delivery_ack( + MailboxDeliveryAck { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + lease: signed_lease.clone(), + ack_nonce: DeliveryAckNonce::from_bytes(id(58)), + acknowledged_at_unix_ms: NOW, + }, + &guest, + ) + .unwrap(); + let ack_req = |signed| JournalGuestAck { + receive: receive(), + plan: &plan, + scope, + enrollment: &enrollment, + signed, + }; + let forged = sign_delivery_ack(ack.ack.clone(), signer).unwrap(); + assert!(matches!( + journal.acknowledge_signed(ack_req(&forged), |_| panic!("wrong signer")), + Err(JournalError::Mailbox( + MailboxError::DeliveryAckSignatureInvalid + )) + )); + assert_eq!(journal.tip(), tip); + journal + .acknowledge_signed(ack_req(&ack), |next| { + tip = next; + Ok(()) + }) + .expect("signed ACK commits"); + assert_eq!( + journal + .acknowledge_signed(ack_req(&ack), |_| panic!("exact ACK is idempotent")) + .unwrap(), + tip + ); + let mut changed_ack = ack.ack.clone(); + changed_ack.ack_nonce = DeliveryAckNonce::from_bytes(id(59)); + let conflict = sign_delivery_ack(changed_ack, &guest).unwrap(); + assert!(matches!( + journal.acknowledge_signed(ack_req(&conflict), |_| panic!("conflicting ACK")), + Err(JournalError::ConflictingRetry) + )); + assert!(journal.receive_next(receive()).unwrap().is_none()); + drop(journal); + let mut recovered = MailboxJournal::open( + dir.path(), + address(), + signer.verifying_key(), + id(90), + limits, + tip, + ) + .expect("lease and ACK survive restart"); + assert!(recovered.receive_next(receive()).unwrap().is_none()); + assert_eq!( + recovered + .acknowledge_signed(ack_req(&ack), |_| panic!("recovered ACK is idempotent")) + .unwrap(), + tip + ); + assert!(matches!( + recovered.acknowledge_signed(ack_req(&conflict), |_| panic!("replayed ACK conflict")), + Err(JournalError::ConflictingRetry) + )); + assert!( + matches!( + recovered.lease_next(lease_req(scope), |_| panic!("challenge replay")), + Ok(None), + ), + "a consumed message cannot be leased a second time" + ); + let mut next_header = fixture.prepared.header.clone(); + next_header.message_id = MessageId::from_bytes(id(61)); + next_header.nonce = ReplayNonce::from_bytes(id(62)); + next_header.sequence = SequenceNumber::new(2).unwrap(); + let mut next_routing = fixture.prepared.envelope().routing.clone(); + next_routing.delivery_id = DeliveryId::from_bytes(id(63)); + let next = crate::mailbox::seal_envelope( + SealRequest { + routing: next_routing, + header: next_header, + plaintext: b"hello", + }, + &encryption_key, + &SigningKey::from_bytes(&id(22)), + ) + .unwrap(); + recovered + .admit( + JournalAdmission { + envelope: &next, + capability: &fixture.capability, + context: &fixture.context, + encryption_key: &encryption_key, + sender_key: &sender, + signer, + }, + |committed| { + tip = committed; + Ok(()) + }, + ) + .unwrap(); + assert!(matches!( + recovered.lease_next(lease_req(scope), |_| panic!("reused challenge")), + Err(JournalError::ConflictingRetry) + )); + assert_eq!(recovered.tip(), tip); + let fresh_challenge = id(64); + let fresh_proof = sign_reader_possession(&enrollment, &fresh_challenge, &guest).unwrap(); + let next_lease = recovered + .lease_next( + JournalGuestLease { + challenge: &fresh_challenge, + possession_signature: &fresh_proof, + ..lease_req(scope) + }, + |committed| { + tip = committed; + Ok(()) + }, + ) + .unwrap() + .unwrap(); + assert_eq!( + next_lease.lease().lease.message_id, + MessageId::from_bytes(id(61)) + ); + assert_eq!(recovered.tip(), tip); +} + #[test] fn durable_admission_recovers_ciphertext_dedup_and_a_signed_destination_verdict() { use mvm_contract::protocol::mailbox::DeliveryOutcome; diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index 7741058f84..2c7d378bf1 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -32,6 +32,7 @@ use mvm_core::plan::ExecutionPlan; use mvm_core::vm_backend::{HostDialRoute, VmId, VmStartConfig}; use mvm_runtime::backend::AnyBackend; use mvm_vmm::host::aux_bin::AuxBin; +use mvm_vmm::host::flowmux_identity::{MAILBOX_READER_BOOT_FILE, MailboxReaderBootBinding}; use crate::mailbox_launch::{DerivedMailboxKeys, LAUNCH_LIMITS, derive_launch_keys}; use crate::mailbox_service::durable::DurableMailboxService; @@ -56,6 +57,9 @@ pub struct MailboxProcessConfig { pub keys: DerivedMailboxKeys, pub socket: PathBuf, pub guest_route: HostDialRoute, + /// Public ACK key and boot identity pinned to the admitted cold plan. + /// Absent on a warm child until it rekeys and enrolls. + pub reader_boot: Option, /// The VM whose lifetime bounds this process. pub vm_state_dir: PathBuf, } @@ -84,8 +88,17 @@ fn serve_durable(config: MailboxProcessConfig) -> Result<()> { keys, socket, guest_route, + reader_boot, vm_state_dir, } = config; + if let Some(binding) = &reader_boot + && (binding.mailbox != grant.mailbox || binding.execution_epoch != grant.epoch) + { + return Err(Error::new( + ErrorKind::PermissionDenied, + "mailbox reader boot does not match the helper's signed grant", + )); + } let (policy, mut sealer) = keys.trust(); let (signer, journal_key, anchor_key) = keys.durable_material(); sealer.intent_dir = Some(vm_state_dir.join("mailbox-intents")); @@ -149,10 +162,50 @@ pub fn build_config( "backend cannot dial the guest mailbox port", ) })?, + reader_boot: None, vm_state_dir: state_dir.to_path_buf(), }) } +fn load_reader_boot_for_plan( + state_dir: &Path, + plan: &ExecutionPlan, +) -> Result> { + use base64::Engine as _; + let path = state_dir.join(MAILBOX_READER_BOOT_FILE); + match std::fs::symlink_metadata(&path) { + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + Ok(metadata) if !metadata.file_type().is_file() => { + return Err(Error::new( + ErrorKind::PermissionDenied, + "mailbox reader boot binding is not a regular file", + )); + } + Ok(_) => {} + } + let binding = MailboxReaderBootBinding::load(state_dir).map_err(Error::other)?; + binding.check_plan(plan).map_err(|_| { + Error::new( + ErrorKind::PermissionDenied, + "mailbox reader boot binding does not match the admitted plan", + ) + })?; + let identity = mvm_vmm::host::flowmux_identity::load_inheritable_identity(state_dir) + .map_err(Error::other)? + .ok_or_else(|| Error::new(ErrorKind::PermissionDenied, "guest identity is absent"))?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(identity.guest_verifying_key_base64) + .map_err(|_| Error::new(ErrorKind::InvalidData, "guest identity encoding is invalid"))?; + if bytes != binding.guest_identity_key { + return Err(Error::new( + ErrorKind::PermissionDenied, + "mailbox ACK key belongs to a different guest boot identity", + )); + } + Ok(Some(binding)) +} + /// The command that starts the process: no arguments, an empty environment, /// its own process group, config over a pipe. The process parses guest bytes /// and holds derived keys, and reads no variable, so it is given none. @@ -262,12 +315,13 @@ pub fn spawn_for_launch( }; require_wired(plan, backend)?; let state_dir = mvm_core::config::vm_state_dir(vm_name); - let config = build_config( + let mut config = build_config( &mvm_core::config::mvm_keys_dir(), grant, &state_dir, backend, )?; + config.reader_boot = load_reader_boot_for_plan(&state_dir, plan)?; let bin = mvm_vmm::host::broker_services_spawn::resolve_subprocess_bin_to_spawn(&AuxBin::new( MAILBOX_BIN, MAILBOX_BIN_ENV, @@ -414,6 +468,53 @@ mod tests { ); } + #[test] + fn guest_reader_boot_must_match_the_admitted_plan_and_pinned_guest_identity() { + use base64::Engine as _; + use mvm_vmm::host::flowmux_identity::{ + InheritableIdentity, MAILBOX_READER_BOOT_FILE, MailboxAckKeyMaterial, + }; + let keys = keys_dir(); + let state = tempfile::tempdir().unwrap(); + let guest = ed25519_dalek::SigningKey::from_bytes(&[66; 32]); + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(grant(keys.path())); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + let ack = MailboxAckKeyMaterial::mint().unwrap(); + let binding = MailboxReaderBootBinding::mint(&plan, guest.verifying_key(), &ack).unwrap(); + binding.persist(state.path()).unwrap(); + InheritableIdentity { + session_id: "guest-current".into(), + guest_verifying_key_base64: base64::engine::general_purpose::STANDARD + .encode(guest.verifying_key().as_bytes()), + } + .persist(state.path()) + .unwrap(); + assert_eq!( + load_reader_boot_for_plan(state.path(), &plan).unwrap(), + Some(binding.clone()) + ); + let mut stale = plan.clone(); + stale.nonce = mvm_core::plan::Nonce::from_bytes([99; 16]); + assert!(load_reader_boot_for_plan(state.path(), &stale).is_err()); + InheritableIdentity { + session_id: "guest-other".into(), + guest_verifying_key_base64: base64::engine::general_purpose::STANDARD.encode( + ed25519_dalek::SigningKey::from_bytes(&[67; 32]) + .verifying_key() + .as_bytes(), + ), + } + .persist(state.path()) + .unwrap(); + assert!(load_reader_boot_for_plan(state.path(), &plan).is_err()); + std::fs::remove_file(state.path().join(MAILBOX_READER_BOOT_FILE)).unwrap(); + let outside = state.path().join("outside"); + std::fs::write(&outside, serde_json::to_vec(&binding).unwrap()).unwrap(); + std::os::unix::fs::symlink(&outside, state.path().join(MAILBOX_READER_BOOT_FILE)).unwrap(); + assert!(load_reader_boot_for_plan(state.path(), &plan).is_err()); + } + #[test] fn the_config_carries_no_seed_and_the_command_carries_no_argv_or_env_secret() { let dir = keys_dir(); @@ -519,6 +620,7 @@ mod tests { grant, socket: dir.path().join("mailbox.sock"), guest_route: HostDialRoute::PortSocket(dir.path().join("absent-port")), + reader_boot: None, vm_state_dir: state_path, }; let socket = config.socket.clone(); diff --git a/crates/mvm-runtime/src/workload_runner/claim.rs b/crates/mvm-runtime/src/workload_runner/claim.rs index 690a5a1b63..470dfa9aea 100644 --- a/crates/mvm-runtime/src/workload_runner/claim.rs +++ b/crates/mvm-runtime/src/workload_runner/claim.rs @@ -204,6 +204,8 @@ pub struct EndpointSpawnInputs<'a> { pub network_limits: mvm_core::plan::NetworkLimits, /// Exact signed ingress mappings owned by this endpoint. pub ingress: &'a [mvm_core::plan::IngressMapping], + /// The admitted plan for a cold boot that grants a guest mailbox. + pub reader_plan: Option<&'a mvm_core::plan::ExecutionPlan>, /// Where this boot's FlowMux identity comes from. A cold boot mints one; a /// warm claim inherits its parent's, because the restored child already /// holds the parent's signing key in memory. @@ -311,6 +313,7 @@ impl<'a> ClaimGuards<'a> { network_policy: inputs.network_policy, network_limits: inputs.network_limits, ingress: inputs.ingress, + reader_plan: inputs.reader_plan, identity: inputs.identity, }; if inputs.secrets.is_empty() @@ -519,6 +522,7 @@ mod tests { network_policy: policy, network_limits: mvm_core::plan::NetworkLimits::default(), ingress: &[], + reader_plan: None, } } diff --git a/crates/mvm-runtime/src/workload_runner/runner.rs b/crates/mvm-runtime/src/workload_runner/runner.rs index 8f42f3f728..b47b73e552 100644 --- a/crates/mvm-runtime/src/workload_runner/runner.rs +++ b/crates/mvm-runtime/src/workload_runner/runner.rs @@ -441,6 +441,13 @@ impl WorkloadRunner )?; let network_limits = admitted_network_limits(inputs.config.plan_json.as_deref())?; let ingress = admitted_ingress(inputs.config.plan_json.as_deref())?; + let reader_plan = inputs + .config + .plan_json + .as_deref() + .map(mvm_core::plan::plan_from_admitted_json) + .transpose() + .context("parse admitted mailbox reader plan")?; // Spawn the per-child substitution endpoint through the shared // `ClaimGuards`, so a warm claim stands up the identical guarded endpoint @@ -457,6 +464,7 @@ impl WorkloadRunner network_policy: inputs.network_policy, network_limits, ingress: &ingress, + reader_plan: reader_plan.as_ref().filter(|plan| plan.mailbox.is_some()), // A cold boot mints this guest's identity and hands it a drive. identity: FlowMuxIdentitySource::Mint, }, @@ -918,6 +926,9 @@ impl WorkloadRunner network_policy: &claim.network_policy, network_limits, ingress: &ingress, + // A warm child must rekey and enroll independently; it + // cannot inherit a parent's mailbox ACK authority. + reader_plan: None, // A restored child already holds its parent's signing key // in the memory image it woke from, and there is no way to // put a different one there. Its endpoint pins what the diff --git a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs index 89307042c2..b1e30ea61e 100644 --- a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs +++ b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs @@ -39,6 +39,8 @@ pub struct NetworkEndpointSpawnRequest<'a> { pub network_limits: mvm_core::plan::NetworkLimits, /// Exact signed ingress mappings owned by this endpoint. pub ingress: &'a [mvm_core::plan::IngressMapping], + /// Verified cold plan for optional per-boot mailbox reader key delivery. + pub reader_plan: Option<&'a mvm_core::plan::ExecutionPlan>, /// Where the guest's half of the authenticated session comes from. pub identity: FlowMuxIdentitySource<'a>, } @@ -191,6 +193,7 @@ pub(super) fn prepare_standby_identity( network_policy: &NetworkPolicy::deny_all(), network_limits: mvm_core::plan::NetworkLimits::default(), ingress: &[], + reader_plan: None, identity: FlowMuxIdentitySource::Mint, }) .map_err(|error| { @@ -286,21 +289,28 @@ impl NetworkEndpointSpawner for RealNetworkEndpointSpawner { FlowMuxIdentitySource::Mint => { let material = FlowMuxIdentityMaterial::mint_from_host_signer(req.vm_name) .context("minting this boot's FlowMux identity")?; + let mailbox_ack_key = mint_mailbox_ack_key(req)?; let drive = req.state_dir.join(IDENTITY_DRIVE_FILE); material.write_drive_with( &drive, &IdentityDriveContents { ingress: req.ingress, egress_ca_cert_pem: egress_ca.guest_cert_pem(), - mailbox_ack_key: None, + mailbox_ack_key: mailbox_ack_key.as_ref(), }, )?; + persist_mailbox_reader_boot( + req, + material.guest_verifying_key(), + mailbox_ack_key.as_ref(), + )?; // Persisted so a warm child claimed from this VM pins the key // its restored guest actually holds. material.persist_inheritable(req.state_dir)?; (material.spawn_config().clone(), Some(drive)) } FlowMuxIdentitySource::InheritFrom(parent_state_dir) => { + clear_mailbox_reader_boot(req.state_dir)?; let inherited = load_inheritable_identity(parent_state_dir)?.with_context(|| { format!( @@ -366,6 +376,41 @@ impl NetworkEndpointSpawner for RealNetworkEndpointSpawner { } } +fn mint_mailbox_ack_key( + req: &NetworkEndpointSpawnRequest<'_>, +) -> Result> { + req.reader_plan + .filter(|plan| plan.mailbox.is_some()) + .map(|_| mvm_vmm::host::flowmux_identity::MailboxAckKeyMaterial::mint()) + .transpose() +} + +fn clear_mailbox_reader_boot(state_dir: &Path) -> Result<()> { + let path = state_dir.join(mvm_vmm::host::flowmux_identity::MAILBOX_READER_BOOT_FILE); + match std::fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| format!("retire {}", path.display())), + } +} + +fn persist_mailbox_reader_boot( + req: &NetworkEndpointSpawnRequest<'_>, + guest_identity: ed25519_dalek::VerifyingKey, + key: Option<&mvm_vmm::host::flowmux_identity::MailboxAckKeyMaterial>, +) -> Result<()> { + match (req.reader_plan, key) { + (Some(plan), Some(key)) => mvm_vmm::host::flowmux_identity::MailboxReaderBootBinding::mint( + plan, + guest_identity, + key, + )? + .persist(req.state_dir), + (None, None) => clear_mailbox_reader_boot(req.state_dir), + _ => anyhow::bail!("mailbox reader key projection disagrees with admitted plan"), + } +} + fn prepare_observation_identity( req: &NetworkEndpointSpawnRequest<'_>, anchor_path: &Path, @@ -381,8 +426,16 @@ fn prepare_observation_identity( let anchor = mvm_agentd::vsock::load_host_signer_verifying_key(anchor_path)? .context("guest observations require the host signer public anchor")?; let guest = GuestIdentityMaterial::mint(anchor)?; + let mailbox_ack_key = mint_mailbox_ack_key(req)?; let drive = req.state_dir.join(IDENTITY_DRIVE_FILE); - guest.write_drive_with(&drive, &IdentityDriveContents::default())?; + guest.write_drive_with( + &drive, + &IdentityDriveContents { + mailbox_ack_key: mailbox_ack_key.as_ref(), + ..IdentityDriveContents::default() + }, + )?; + persist_mailbox_reader_boot(req, guest.verifying_key(), mailbox_ack_key.as_ref())?; ( InheritableIdentity { session_id: req.vm_name.to_string(), @@ -393,6 +446,7 @@ fn prepare_observation_identity( ) } FlowMuxIdentitySource::InheritFrom(parent) => { + clear_mailbox_reader_boot(req.state_dir)?; let identity = load_inheritable_identity(parent)? .context("restored guest has no registered observation identity")?; let bytes = base64::engine::general_purpose::STANDARD @@ -625,6 +679,7 @@ mod tests { network_policy: &NetworkPolicy::deny_all(), network_limits: mvm_core::plan::NetworkLimits::default(), ingress: &[], + reader_plan: None, identity, }, anchor, @@ -690,6 +745,74 @@ mod tests { ); } + #[test] + fn cold_mailbox_boot_projects_a_distinct_ack_key_and_warm_child_refuses_inheritance() { + use mvm_contract::protocol::mailbox::{ExecutionEpoch, MailboxGrant, MailboxId}; + use mvm_vmm::host::flowmux_identity::{ + MAILBOX_ACK_SIGNING_KEY_FILE, MAILBOX_READER_BOOT_FILE, MailboxReaderBootBinding, + }; + let root = tempfile::tempdir().unwrap(); + let anchor = root.path().join("host-signer.pub"); + let host = ed25519_dalek::SigningKey::from_bytes(&[37; 32]); + std::fs::write(&anchor, host.verifying_key().as_bytes()).unwrap(); + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(MailboxGrant { + mailbox: MailboxId::from_bytes([9; 32]), + authority_key: host.verifying_key().to_bytes(), + epoch: ExecutionEpoch::new(3).unwrap(), + }); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + let redaction = RedactionPolicy::default(); + let policy = NetworkPolicy::deny_all(); + let parent = root.path().join("parent"); + let request = NetworkEndpointSpawnRequest { + vm_name: "mailbox-parent", + state_dir: &parent, + tenant: "tenant", + secrets: &[], + redaction: &redaction, + network_policy: &policy, + network_limits: mvm_core::plan::NetworkLimits::default(), + ingress: &[], + reader_plan: Some(&plan), + identity: FlowMuxIdentitySource::Mint, + }; + let drive = prepare_observation_identity(&request, &anchor) + .unwrap() + .unwrap(); + let binding = MailboxReaderBootBinding::load(&parent).unwrap(); + binding.check_plan(&plan).unwrap(); + let image = std::fs::read(&drive).unwrap(); + assert!( + image + .windows(MAILBOX_ACK_SIGNING_KEY_FILE.len()) + .any(|part| { part == MAILBOX_ACK_SIGNING_KEY_FILE.as_bytes() }) + ); + assert_ne!(binding.ack_public_key, binding.guest_identity_key); + + let child = root.path().join("child"); + std::fs::create_dir_all(&child).unwrap(); + std::fs::copy( + parent.join(MAILBOX_READER_BOOT_FILE), + child.join(MAILBOX_READER_BOOT_FILE), + ) + .unwrap(); + assert!( + prepare_observation_identity( + &NetworkEndpointSpawnRequest { + state_dir: &child, + identity: FlowMuxIdentitySource::InheritFrom(&parent), + reader_plan: None, + ..request + }, + &anchor, + ) + .unwrap() + .is_none() + ); + assert!(!child.join(MAILBOX_READER_BOOT_FILE).exists()); + } + #[test] fn restored_observation_identity_preserves_the_key_without_minting_a_drive() { use base64::Engine as _; diff --git a/crates/mvm-vmm/src/host/flowmux_identity.rs b/crates/mvm-vmm/src/host/flowmux_identity.rs index daaf12830d..fc9ef32c33 100644 --- a/crates/mvm-vmm/src/host/flowmux_identity.rs +++ b/crates/mvm-vmm/src/host/flowmux_identity.rs @@ -35,6 +35,114 @@ use rand::TryRng; use super::network_endpoint_spawn::FlowMuxIdentitySpawnConfig; use mvm_core::atomic_io::write_private_ephemeral; +/// Host-only public binding for the ACK key carried by one cold boot's drive. +/// A warm-restored guest must rekey before a new binding can be issued. +pub const MAILBOX_READER_BOOT_FILE: &str = "mailbox-reader-boot.json"; +const MAX_MAILBOX_READER_BOOT_BYTES: u64 = 2048; + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MailboxReaderBootBinding { + pub plan_id: mvm_core::plan::PlanId, + pub plan_nonce: mvm_core::plan::Nonce, + pub mailbox: mvm_contract::protocol::mailbox::MailboxId, + pub execution_epoch: mvm_contract::protocol::mailbox::ExecutionEpoch, + pub boot: mvm_contract::protocol::mailbox::BootId, + pub guest_identity_key: [u8; 32], + pub ack_public_key: [u8; 32], +} + +impl MailboxReaderBootBinding { + /// Bind a fresh key to an admitted plan and the separately pinned guest + /// identity; possession and Receive authority still require live checks. + pub fn mint( + plan: &mvm_core::plan::ExecutionPlan, + guest_identity: VerifyingKey, + ack_key: &MailboxAckKeyMaterial, + ) -> Result { + let grant = plan + .mailbox + .context("reader boot requires a mailbox grant")?; + anyhow::ensure!( + plan.plan_id == mvm_core::plan::compute_plan_id(plan), + "mailbox reader boot requires a content-addressed admitted plan" + ); + let mut boot = [0u8; 32]; + rand::rngs::SysRng + .try_fill_bytes(&mut boot) + .context("drawing system entropy for the mailbox reader boot")?; + Ok(Self { + plan_id: plan.plan_id.clone(), + plan_nonce: plan.nonce.clone(), + mailbox: grant.mailbox, + execution_epoch: grant.epoch, + boot: mvm_contract::protocol::mailbox::BootId::from_bytes(boot), + guest_identity_key: guest_identity.to_bytes(), + ack_public_key: ack_key.verifying_key().to_bytes(), + }) + } + + /// Persist the host-side public binding atomically before the guest boots. + pub fn persist(&self, state_dir: &Path) -> Result<()> { + mvm_core::private_fs::ensure_private_dir(state_dir)?; + let bytes = serde_json::to_vec(self).context("encode mailbox reader boot binding")?; + anyhow::ensure!( + bytes.len() <= MAX_MAILBOX_READER_BOOT_BYTES as usize, + "mailbox reader boot binding exceeds the encoded bound" + ); + mvm_core::atomic_io::write_private(&state_dir.join(MAILBOX_READER_BOOT_FILE), &bytes) + .context("persist mailbox reader boot binding") + } + + /// Reopen the bounded host record without following a symlink. + pub fn load(state_dir: &Path) -> Result { + use std::io::Read as _; + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + let path = state_dir.join(MAILBOX_READER_BOOT_FILE); + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(&path) + .with_context(|| format!("open {}", path.display()))?; + let metadata = file.metadata()?; + anyhow::ensure!( + metadata.is_file() + && metadata.nlink() == 1 + && metadata.len() <= MAX_MAILBOX_READER_BOOT_BYTES, + "mailbox reader boot binding is not a bounded regular file" + ); + let mut bytes = Vec::new(); + file.take(MAX_MAILBOX_READER_BOOT_BYTES + 1) + .read_to_end(&mut bytes)?; + anyhow::ensure!( + bytes.len() as u64 <= MAX_MAILBOX_READER_BOOT_BYTES, + "mailbox reader boot binding exceeds the encoded bound" + ); + serde_json::from_slice(&bytes).context("decode mailbox reader boot binding") + } + + /// Require the exact admitted plan, not a stale parent or moved mailbox. + pub fn check_plan(&self, plan: &mvm_core::plan::ExecutionPlan) -> Result<()> { + let grant = plan + .mailbox + .context("reader boot requires a mailbox grant")?; + anyhow::ensure!( + self.plan_id == plan.plan_id + && self.plan_nonce == plan.nonce + && self.mailbox == grant.mailbox + && self.execution_epoch == grant.epoch + && self.boot.as_bytes() != &[0; 32] + && self.ack_public_key != [0; 32] + && self.guest_identity_key != [0; 32], + "mailbox reader boot binding is stale or does not match the admitted plan" + ); + VerifyingKey::from_bytes(&self.ack_public_key).context("invalid mailbox ACK public key")?; + VerifyingKey::from_bytes(&self.guest_identity_key) + .context("invalid guest identity public key")?; + Ok(()) + } +} + // The drive's label and filenames are declared on the reading side, in // `mvm_agentd::flowmux_keys`, and used here. One declaration, so a rename // cannot leave the writer and the reader describing different drives. @@ -219,6 +327,11 @@ impl FlowMuxIdentityMaterial { &self.spawn } + /// Public guest identity pinned by this boot's host endpoint. + pub fn guest_verifying_key(&self) -> VerifyingKey { + self.guest.verifying_key() + } + /// The host-signer trust anchor the guest pins. pub fn host_signer_verifying_key(&self) -> Result { VerifyingKey::from_bytes(&self.guest.host_signer_pub) @@ -530,6 +643,46 @@ mod tests { ); } + #[test] + fn mailbox_reader_boot_binding_is_private_current_and_rejects_a_parent_or_symlink() { + use mvm_contract::protocol::mailbox::{ExecutionEpoch, MailboxGrant, MailboxId}; + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(MailboxGrant { + mailbox: MailboxId::from_bytes([8; 32]), + authority_key: host_key().verifying_key().to_bytes(), + epoch: ExecutionEpoch::new(3).unwrap(), + }); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + let dir = tempfile::tempdir().unwrap(); + let guest = GuestIdentityMaterial::mint(host_key().verifying_key()).unwrap(); + let ack = MailboxAckKeyMaterial::mint().unwrap(); + let binding = MailboxReaderBootBinding::mint(&plan, guest.verifying_key(), &ack).unwrap(); + binding.persist(dir.path()).unwrap(); + assert_eq!(MailboxReaderBootBinding::load(dir.path()).unwrap(), binding); + binding.check_plan(&plan).unwrap(); + assert_eq!(binding.ack_public_key, ack.verifying_key().to_bytes()); + assert_ne!(binding.guest_identity_key, binding.ack_public_key); + let mut stale = plan.clone(); + stale.nonce = mvm_core::plan::Nonce::from_bytes([9; 16]); + assert!(binding.check_plan(&stale).is_err()); + stale = plan.clone(); + stale.mailbox.as_mut().unwrap().epoch = ExecutionEpoch::new(4).unwrap(); + assert!(binding.check_plan(&stale).is_err()); + + let path = dir.path().join(MAILBOX_READER_BOOT_FILE); + std::fs::write( + &path, + vec![b'x'; MAX_MAILBOX_READER_BOOT_BYTES as usize + 1], + ) + .unwrap(); + assert!(MailboxReaderBootBinding::load(dir.path()).is_err()); + std::fs::remove_file(&path).unwrap(); + let outside = dir.path().join("outside"); + std::fs::write(&outside, serde_json::to_vec(&binding).unwrap()).unwrap(); + std::os::unix::fs::symlink(&outside, &path).unwrap(); + assert!(MailboxReaderBootBinding::load(dir.path()).is_err()); + } + #[test] fn the_published_guest_key_is_the_one_on_the_drive() { let material = FlowMuxIdentityMaterial::mint("s1", &host_key()).expect("mint"); From d7848dae86d81c35718e09db5b1d6f29ac4c6ba3 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 16:42:56 -0800 Subject: [PATCH 06/51] feat(mailbox): carry boot-bound ACK key into guest activation --- .../src/bin/mvm-guest-agent/init.rs | 1 + .../src/bin/mvm-guest-agent/mailbox.rs | 80 ++++++++- crates/mvm-agentd/src/flowmux_drive.rs | 110 +++++++++++- crates/mvm-core/src/mailbox/reader.rs | 125 ++++++++++++- crates/mvm-hostd/src/mailbox_process.rs | 7 +- .../src/workload_runner/runner/spawner.rs | 53 +++--- crates/mvm-vmm/src/host/flowmux_identity.rs | 167 +++++++----------- 7 files changed, 405 insertions(+), 138 deletions(-) diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/init.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/init.rs index fff76bcc79..37e79eb0fd 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/init.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/init.rs @@ -184,6 +184,7 @@ pub(crate) fn apply_activation( // The telemetry signing key is the egress client's, mode 0400; read it now // or the listener can never answer a session. crate::telemetry::capture_session_keys_before_privilege_drop(); + crate::mailbox::capture_ack_key_before_privilege_drop(); guest_mount::drop_guest_agent_privilege(guest_mount::WORKLOAD_UID, guest_mount::WORKLOAD_GID)?; boot_state.set_activation(ActivationState::Activated); diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs index 963c989616..e7239e5cb8 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs @@ -9,19 +9,78 @@ use std::io::{self, Read, Write}; use std::os::fd::{AsRawFd, FromRawFd, RawFd}; use std::path::Path; -use std::sync::atomic::Ordering; +use std::sync::{OnceLock, atomic::Ordering}; use std::time::{Duration, Instant}; +use ed25519_dalek::SigningKey; +use mvm_agentd::flowmux_drive::{MAILBOX_ACK_SIGNING_KEY_FILE, RUN_MVM_DIR}; use mvm_agentd::mailbox_outbox::{MAILBOX_CMDLINE_TOKEN, OUTBOX_PATH, wait_and_forward_outbox}; use mvm_agentd::telemetry_capture::session::TimedStream; use mvm_contract::protocol::mailbox::MAILBOX_PORT; use mvm_core::mailbox::frame::Refusal; +use mvm_core::mailbox::reader::MailboxReaderBootBinding; +use zeroize::Zeroizing; use crate::globals::SHUTDOWN_REQUESTED; use crate::telemetry::cmdline_asserts; use crate::transport::{accept_vsock, bind_vsock_listener, unix_transport_selected}; const MAILBOX_IO_TIMEOUT: Duration = Duration::from_secs(12); +static ACK_SEED: OnceLock> = OnceLock::new(); +static READER_BOOT: OnceLock = OnceLock::new(); + +/// Retain the root-only ACK key before PID 1 drops to the workload UID. +/// A missing or malformed key disables Receive; the existing Send path +/// remains available, and warm children never inherit reader authority. +pub(crate) fn capture_ack_key_before_privilege_drop() { + let dir = Path::new(RUN_MVM_DIR); + let binding = match MailboxReaderBootBinding::load(dir) { + Ok(binding) => binding, + Err(error) => { + if dir + .join(mvm_core::mailbox::reader::MAILBOX_READER_BOOT_FILE) + .exists() + { + eprintln!("mvm-guest-agent: mailbox reader boot unavailable: {error}"); + } + return; + } + }; + match load_ack_seed(&dir.join(MAILBOX_ACK_SIGNING_KEY_FILE), 0) { + Ok(seed) + if SigningKey::from_bytes(&seed).verifying_key().to_bytes() + == binding.ack_public_key => + { + if ACK_SEED.set(seed).is_err() || READER_BOOT.set(binding).is_err() { + eprintln!("mvm-guest-agent: mailbox ACK identity already initialized"); + } + } + Ok(_) | Err(_) => eprintln!("mvm-guest-agent: mailbox ACK key does not match this boot"), + } +} + +fn load_ack_seed(path: &Path, owner_uid: u32) -> io::Result> { + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _}; + let mut file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path)?; + let meta = file.metadata()?; + if !meta.is_file() + || meta.len() != 32 + || meta.nlink() != 1 + || meta.uid() != owner_uid + || meta.permissions().mode() & 0o777 != 0o400 + { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "mailbox ACK key is not root-only", + )); + } + let mut seed = Zeroizing::new([0u8; 32]); + file.read_exact(seed.as_mut())?; + Ok(seed) +} struct MailboxDeadline<'a, S> { stream: &'a mut S, @@ -208,4 +267,23 @@ mod tests { io::ErrorKind::TimedOut ); } + + #[test] + fn ack_seed_loader_requires_exact_private_regular_file() { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + let dir = tempfile::tempdir().unwrap(); + let uid = dir.path().metadata().unwrap().uid(); + let path = dir.path().join(MAILBOX_ACK_SIGNING_KEY_FILE); + std::fs::write(&path, [8; 32]).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o400)).unwrap(); + assert_eq!(*load_ack_seed(&path, uid).unwrap(), [8; 32]); + assert!(load_ack_seed(&path, uid.wrapping_add(1)).is_err()); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(load_ack_seed(&path, uid).is_err()); + std::fs::write(&path, [8; 31]).unwrap(); + assert!(load_ack_seed(&path, uid).is_err()); + std::fs::remove_file(&path).unwrap(); + std::os::unix::fs::symlink(dir.path().join("missing"), &path).unwrap(); + assert!(load_ack_seed(&path, uid).is_err()); + } } diff --git a/crates/mvm-agentd/src/flowmux_drive.rs b/crates/mvm-agentd/src/flowmux_drive.rs index a7c6cfe8cc..d68a76fd1a 100644 --- a/crates/mvm-agentd/src/flowmux_drive.rs +++ b/crates/mvm-agentd/src/flowmux_drive.rs @@ -33,6 +33,9 @@ pub const GUEST_SIGNING_KEY_FILE: &str = "flowmux-guest-signing-key"; /// The root-owned guest agent alone may read this; the egress client cannot. pub const MAILBOX_ACK_SIGNING_KEY_FILE: &str = "mailbox-ack-signing-key"; +/// Public boot/plan binding that accompanies the guest-held ACK key. +pub use mvm_core::mailbox::reader::{MAILBOX_READER_BOOT_FILE, MAX_MAILBOX_READER_BOOT_BYTES}; + /// Basename of the host-signer trust anchor on the drive. pub const HOST_SIGNER_PUB_FILE: &str = "host-signer.pub"; @@ -300,8 +303,8 @@ mod linux { EGRESS_CA_CERT_FILE, EGRESS_CA_CERT_MODE, GUEST_SIGNING_KEY_FILE, GUEST_SIGNING_KEY_MODE, HOST_SIGNER_PUB_FILE, HOST_SIGNER_PUB_MODE, IDENTITY_DRIVE_LABEL, INGRESS_TARGETS_FILE, INGRESS_TARGETS_MODE, IdentityDriveError, MAILBOX_ACK_SIGNING_KEY_FILE, - MAILBOX_ACK_SIGNING_KEY_MODE, RUN_MVM_DIR, find_labeled_ext4_disk_among, - virtio_block_devices, + MAILBOX_ACK_SIGNING_KEY_MODE, MAILBOX_READER_BOOT_FILE, MAX_MAILBOX_READER_BOOT_BYTES, + RUN_MVM_DIR, find_labeled_ext4_disk_among, virtio_block_devices, }; use std::path::Path; @@ -330,6 +333,7 @@ mod linux { find_labeled_ext4_disk_among(virtio_block_devices(), IDENTITY_DRIVE_LABEL) else { clear_mailbox_ack_key(&Path::new(RUN_MVM_DIR).join(MAILBOX_ACK_SIGNING_KEY_FILE))?; + clear_mailbox_reader_boot(&Path::new(RUN_MVM_DIR).join(MAILBOX_READER_BOOT_FILE))?; return Err(IdentityDriveError::NotAttached); }; @@ -370,7 +374,76 @@ mod linux { EGRESS_CA_CERT_MODE, "the per-VM egress CA certificate", )?; - install_mailbox_ack_key(Path::new(MOUNTPOINT), Path::new(RUN_MVM_DIR)) + let run_dir = Path::new(RUN_MVM_DIR); + clear_mailbox_reader_boot(&run_dir.join(MAILBOX_READER_BOOT_FILE))?; + install_mailbox_ack_key(Path::new(MOUNTPOINT), run_dir)?; + copy_mailbox_reader_boot(Path::new(MOUNTPOINT), run_dir)?; + if run_dir.join(MAILBOX_ACK_SIGNING_KEY_FILE).exists() + != run_dir.join(MAILBOX_READER_BOOT_FILE).exists() + { + clear_mailbox_ack_key(&run_dir.join(MAILBOX_ACK_SIGNING_KEY_FILE))?; + clear_mailbox_reader_boot(&run_dir.join(MAILBOX_READER_BOOT_FILE))?; + return Err(IdentityDriveError::Unreadable( + "mailbox ACK key and boot binding must be provisioned together".into(), + )); + } + Ok(()) + } + + fn clear_mailbox_reader_boot(path: &Path) -> Result<(), IdentityDriveError> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(IdentityDriveError::Unreadable(format!( + "retire stale mailbox reader boot: {error}" + ))), + } + } + + pub(super) fn copy_mailbox_reader_boot( + drive_dir: &Path, + run_dir: &Path, + ) -> Result<(), IdentityDriveError> { + use std::io::{Read as _, Write as _}; + use std::os::unix::fs::OpenOptionsExt as _; + let from = drive_dir.join(MAILBOX_READER_BOOT_FILE); + let file = match std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(&from) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(io_err("open mailbox reader boot")(error)), + }; + let metadata = file + .metadata() + .map_err(io_err("inspect mailbox reader boot"))?; + if !metadata.is_file() || metadata.len() > MAX_MAILBOX_READER_BOOT_BYTES { + return Err(IdentityDriveError::Unreadable( + "mailbox reader boot is not a bounded regular file".into(), + )); + } + let mut bytes = Vec::new(); + file.take(MAX_MAILBOX_READER_BOOT_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(io_err("read mailbox reader boot"))?; + if bytes.len() as u64 > MAX_MAILBOX_READER_BOOT_BYTES { + return Err(IdentityDriveError::Unreadable( + "mailbox reader boot exceeds the encoded bound".into(), + )); + } + let to = run_dir.join(MAILBOX_READER_BOOT_FILE); + let mut output = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o444) + .custom_flags(libc::O_NOFOLLOW) + .open(&to) + .map_err(io_err("create mailbox reader boot"))?; + output + .write_all(&bytes) + .map_err(io_err("copy mailbox reader boot")) } pub(super) fn install_mailbox_ack_key( @@ -603,6 +676,37 @@ mod tests { assert!(!output.exists(), "malformed new key must clear the old one"); } + #[cfg(target_os = "linux")] + #[test] + fn reader_boot_binding_copy_is_bounded_nofollow_and_clears_stale_boot() { + use std::os::unix::fs::PermissionsExt as _; + let drive = tempfile::tempdir().unwrap(); + let run = tempfile::tempdir().unwrap(); + let source = drive.path().join(MAILBOX_READER_BOOT_FILE); + let dest = run.path().join(MAILBOX_READER_BOOT_FILE); + std::fs::write(&source, b"{\"boot\":\"synthetic\"}").unwrap(); + linux::copy_mailbox_reader_boot(drive.path(), run.path()).unwrap(); + assert_eq!(std::fs::read(&dest).unwrap(), b"{\"boot\":\"synthetic\"}"); + assert_eq!( + std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777, + 0o444 + ); + std::fs::write( + &source, + vec![b'x'; MAX_MAILBOX_READER_BOOT_BYTES as usize + 1], + ) + .unwrap(); + std::fs::remove_file(&dest).unwrap(); + assert!(linux::copy_mailbox_reader_boot(drive.path(), run.path()).is_err()); + assert!(!dest.exists()); + std::fs::remove_file(&source).unwrap(); + std::os::unix::fs::symlink(drive.path().join("missing"), &source).unwrap(); + assert!(linux::copy_mailbox_reader_boot(drive.path(), run.path()).is_err()); + std::fs::remove_file(&source).unwrap(); + linux::copy_mailbox_reader_boot(drive.path(), run.path()).unwrap(); + assert!(!dest.exists()); + } + #[test] fn the_trust_bundle_is_the_baked_roots_plus_the_delivered_certificate() { let run = tempfile::tempdir().expect("tempdir"); diff --git a/crates/mvm-core/src/mailbox/reader.rs b/crates/mvm-core/src/mailbox/reader.rs index 4cfdbb9a23..4fcd71c4b7 100644 --- a/crates/mvm-core/src/mailbox/reader.rs +++ b/crates/mvm-core/src/mailbox/reader.rs @@ -4,17 +4,136 @@ //! Callers must obtain the scope from an authenticated current guest boot and //! persist one-time challenge use before publishing a lease. +use std::path::Path; + +use anyhow::{Context as _, Result as AnyResult}; use ed25519_dalek::{Signature, Signer as _, SigningKey, Verifier as _, VerifyingKey}; use mvm_contract::protocol::mailbox::{ - ConsumerSessionId, DELIVERY_ACK_SIGNATURE_DOMAIN, DELIVERY_LEASE_SIGNATURE_DOMAIN, - MailboxAddress, MailboxDeliveryAck, MailboxDeliveryLease, MailboxEnvelope, + BootId, ConsumerSessionId, DELIVERY_ACK_SIGNATURE_DOMAIN, DELIVERY_LEASE_SIGNATURE_DOMAIN, + ExecutionEpoch, MailboxAddress, MailboxDeliveryAck, MailboxDeliveryLease, MailboxEnvelope, MailboxEnvelopeHeader, MailboxGrant, MailboxGuestReaderEnrollment, MailboxGuestReaderScope, - READER_ENROLLMENT_SIGNATURE_DOMAIN, RevocationId, SignedMailboxCapability, + MailboxId, READER_ENROLLMENT_SIGNATURE_DOMAIN, RevocationId, SignedMailboxCapability, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, }; +use rand::TryRng as _; use super::{MailboxError, canonical_bytes, verify_capability_signature}; +/// Name shared by the host writer and guest reader of one public boot binding. +pub const MAILBOX_READER_BOOT_FILE: &str = "mailbox-reader-boot.json"; +/// Maximum encoded size before parsing a guest-visible boot binding. +pub const MAX_MAILBOX_READER_BOOT_BYTES: u64 = 2048; + +/// Public plan/boot/ACK-key binding projected into one cold guest identity +/// drive and retained privately by the host for helper restart. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MailboxReaderBootBinding { + pub plan_id: crate::plan::PlanId, + pub plan_nonce: crate::plan::Nonce, + pub mailbox: MailboxId, + pub authority_key: [u8; 32], + pub execution_epoch: ExecutionEpoch, + pub boot: BootId, + pub guest_identity_key: [u8; 32], + pub ack_public_key: [u8; 32], +} + +impl MailboxReaderBootBinding { + /// Mint a distinct boot ID for a content-addressed admitted plan. + pub fn mint( + plan: &crate::plan::ExecutionPlan, + guest_identity: VerifyingKey, + ack_public_key: [u8; 32], + ) -> AnyResult { + let grant = plan + .mailbox + .context("reader boot requires a mailbox grant")?; + anyhow::ensure!( + plan.plan_id == crate::plan::compute_plan_id(plan), + "mailbox reader boot requires a content-addressed admitted plan" + ); + VerifyingKey::from_bytes(&ack_public_key).context("invalid mailbox ACK public key")?; + let mut boot = [0u8; 32]; + rand::rngs::SysRng + .try_fill_bytes(&mut boot) + .context("drawing system entropy for the mailbox reader boot")?; + Ok(Self { + plan_id: plan.plan_id.clone(), + plan_nonce: plan.nonce.clone(), + mailbox: grant.mailbox, + authority_key: grant.authority_key, + execution_epoch: grant.epoch, + boot: BootId::from_bytes(boot), + guest_identity_key: guest_identity.to_bytes(), + ack_public_key, + }) + } + + /// Persist the host-side public binding atomically before the guest boots. + pub fn persist(&self, state_dir: &Path) -> AnyResult<()> { + crate::private_fs::ensure_private_dir(state_dir)?; + let bytes = serde_json::to_vec(self).context("encode mailbox reader boot binding")?; + anyhow::ensure!( + bytes.len() <= MAX_MAILBOX_READER_BOOT_BYTES as usize, + "mailbox reader boot binding exceeds the encoded bound" + ); + crate::atomic_io::write_private(&state_dir.join(MAILBOX_READER_BOOT_FILE), &bytes) + .context("persist mailbox reader boot binding") + } + + /// Reopen a bounded host record without following a symlink. + pub fn load(state_dir: &Path) -> AnyResult { + use std::io::Read as _; + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + let path = state_dir.join(MAILBOX_READER_BOOT_FILE); + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(&path) + .with_context(|| format!("open {}", path.display()))?; + let metadata = file.metadata()?; + anyhow::ensure!( + metadata.is_file() + && metadata.nlink() == 1 + && metadata.len() <= MAX_MAILBOX_READER_BOOT_BYTES, + "mailbox reader boot binding is not a bounded regular file" + ); + let mut bytes = Vec::new(); + file.take(MAX_MAILBOX_READER_BOOT_BYTES + 1) + .read_to_end(&mut bytes)?; + anyhow::ensure!( + bytes.len() as u64 <= MAX_MAILBOX_READER_BOOT_BYTES, + "mailbox reader boot binding exceeds the encoded bound" + ); + serde_json::from_slice(&bytes).context("decode mailbox reader boot binding") + } + + /// Refuse a stale parent or moved mailbox before this boot can enroll. + pub fn check_plan(&self, plan: &crate::plan::ExecutionPlan) -> AnyResult<()> { + let grant = plan + .mailbox + .context("reader boot requires a mailbox grant")?; + anyhow::ensure!( + self.plan_id == plan.plan_id + && self.plan_nonce == plan.nonce + && self.mailbox == grant.mailbox + && self.authority_key == grant.authority_key + && self.execution_epoch == grant.epoch + && self.boot.as_bytes() != &[0; 32] + && self.ack_public_key != [0; 32] + && self.guest_identity_key != [0; 32], + "mailbox reader boot binding is stale or does not match the admitted plan" + ); + VerifyingKey::from_bytes(&self.ack_public_key).context("invalid mailbox ACK public key")?; + VerifyingKey::from_bytes(&self.guest_identity_key) + .context("invalid guest identity public key")?; + VerifyingKey::from_bytes(&self.authority_key) + .context("invalid mailbox authority public key")?; + Ok(()) + } +} + const READER_POSSESSION_SIGNATURE_DOMAIN: &[u8] = b"mvm.mailbox.reader-possession.v1\0"; /// Sign an enrollment with the destination authority. The caller must bind diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index 2c7d378bf1..db9a16d9d3 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -481,7 +481,12 @@ mod tests { plan.mailbox = Some(grant(keys.path())); plan.plan_id = mvm_core::plan::compute_plan_id(&plan); let ack = MailboxAckKeyMaterial::mint().unwrap(); - let binding = MailboxReaderBootBinding::mint(&plan, guest.verifying_key(), &ack).unwrap(); + let binding = MailboxReaderBootBinding::mint( + &plan, + guest.verifying_key(), + ack.verifying_key().to_bytes(), + ) + .unwrap(); binding.persist(state.path()).unwrap(); InheritableIdentity { session_id: "guest-current".into(), diff --git a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs index b1e30ea61e..adab036ff2 100644 --- a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs +++ b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs @@ -289,21 +289,18 @@ impl NetworkEndpointSpawner for RealNetworkEndpointSpawner { FlowMuxIdentitySource::Mint => { let material = FlowMuxIdentityMaterial::mint_from_host_signer(req.vm_name) .context("minting this boot's FlowMux identity")?; - let mailbox_ack_key = mint_mailbox_ack_key(req)?; + let reader_boot = mint_mailbox_reader_boot(req, material.guest_verifying_key())?; let drive = req.state_dir.join(IDENTITY_DRIVE_FILE); material.write_drive_with( &drive, &IdentityDriveContents { ingress: req.ingress, egress_ca_cert_pem: egress_ca.guest_cert_pem(), - mailbox_ack_key: mailbox_ack_key.as_ref(), + mailbox_ack_key: reader_boot.as_ref().map(|(key, _)| key), + mailbox_reader_boot: reader_boot.as_ref().map(|(_, binding)| binding), }, )?; - persist_mailbox_reader_boot( - req, - material.guest_verifying_key(), - mailbox_ack_key.as_ref(), - )?; + persist_mailbox_reader_boot(req, reader_boot.as_ref().map(|(_, binding)| binding))?; // Persisted so a warm child claimed from this VM pins the key // its restored guest actually holds. material.persist_inheritable(req.state_dir)?; @@ -376,12 +373,26 @@ impl NetworkEndpointSpawner for RealNetworkEndpointSpawner { } } -fn mint_mailbox_ack_key( +fn mint_mailbox_reader_boot( req: &NetworkEndpointSpawnRequest<'_>, -) -> Result> { + guest_identity: ed25519_dalek::VerifyingKey, +) -> Result< + Option<( + mvm_vmm::host::flowmux_identity::MailboxAckKeyMaterial, + mvm_vmm::host::flowmux_identity::MailboxReaderBootBinding, + )>, +> { req.reader_plan .filter(|plan| plan.mailbox.is_some()) - .map(|_| mvm_vmm::host::flowmux_identity::MailboxAckKeyMaterial::mint()) + .map(|plan| { + let key = mvm_vmm::host::flowmux_identity::MailboxAckKeyMaterial::mint()?; + let binding = mvm_vmm::host::flowmux_identity::MailboxReaderBootBinding::mint( + plan, + guest_identity, + key.verifying_key().to_bytes(), + )?; + Ok((key, binding)) + }) .transpose() } @@ -396,16 +407,13 @@ fn clear_mailbox_reader_boot(state_dir: &Path) -> Result<()> { fn persist_mailbox_reader_boot( req: &NetworkEndpointSpawnRequest<'_>, - guest_identity: ed25519_dalek::VerifyingKey, - key: Option<&mvm_vmm::host::flowmux_identity::MailboxAckKeyMaterial>, + binding: Option<&mvm_vmm::host::flowmux_identity::MailboxReaderBootBinding>, ) -> Result<()> { - match (req.reader_plan, key) { - (Some(plan), Some(key)) => mvm_vmm::host::flowmux_identity::MailboxReaderBootBinding::mint( - plan, - guest_identity, - key, - )? - .persist(req.state_dir), + match (req.reader_plan, binding) { + (Some(plan), Some(binding)) => { + binding.check_plan(plan)?; + binding.persist(req.state_dir) + } (None, None) => clear_mailbox_reader_boot(req.state_dir), _ => anyhow::bail!("mailbox reader key projection disagrees with admitted plan"), } @@ -426,16 +434,17 @@ fn prepare_observation_identity( let anchor = mvm_agentd::vsock::load_host_signer_verifying_key(anchor_path)? .context("guest observations require the host signer public anchor")?; let guest = GuestIdentityMaterial::mint(anchor)?; - let mailbox_ack_key = mint_mailbox_ack_key(req)?; + let reader_boot = mint_mailbox_reader_boot(req, guest.verifying_key())?; let drive = req.state_dir.join(IDENTITY_DRIVE_FILE); guest.write_drive_with( &drive, &IdentityDriveContents { - mailbox_ack_key: mailbox_ack_key.as_ref(), + mailbox_ack_key: reader_boot.as_ref().map(|(key, _)| key), + mailbox_reader_boot: reader_boot.as_ref().map(|(_, binding)| binding), ..IdentityDriveContents::default() }, )?; - persist_mailbox_reader_boot(req, guest.verifying_key(), mailbox_ack_key.as_ref())?; + persist_mailbox_reader_boot(req, reader_boot.as_ref().map(|(_, binding)| binding))?; ( InheritableIdentity { session_id: req.vm_name.to_string(), diff --git a/crates/mvm-vmm/src/host/flowmux_identity.rs b/crates/mvm-vmm/src/host/flowmux_identity.rs index fc9ef32c33..5a8d590c0d 100644 --- a/crates/mvm-vmm/src/host/flowmux_identity.rs +++ b/crates/mvm-vmm/src/host/flowmux_identity.rs @@ -35,113 +35,10 @@ use rand::TryRng; use super::network_endpoint_spawn::FlowMuxIdentitySpawnConfig; use mvm_core::atomic_io::write_private_ephemeral; -/// Host-only public binding for the ACK key carried by one cold boot's drive. -/// A warm-restored guest must rekey before a new binding can be issued. -pub const MAILBOX_READER_BOOT_FILE: &str = "mailbox-reader-boot.json"; -const MAX_MAILBOX_READER_BOOT_BYTES: u64 = 2048; - -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MailboxReaderBootBinding { - pub plan_id: mvm_core::plan::PlanId, - pub plan_nonce: mvm_core::plan::Nonce, - pub mailbox: mvm_contract::protocol::mailbox::MailboxId, - pub execution_epoch: mvm_contract::protocol::mailbox::ExecutionEpoch, - pub boot: mvm_contract::protocol::mailbox::BootId, - pub guest_identity_key: [u8; 32], - pub ack_public_key: [u8; 32], -} - -impl MailboxReaderBootBinding { - /// Bind a fresh key to an admitted plan and the separately pinned guest - /// identity; possession and Receive authority still require live checks. - pub fn mint( - plan: &mvm_core::plan::ExecutionPlan, - guest_identity: VerifyingKey, - ack_key: &MailboxAckKeyMaterial, - ) -> Result { - let grant = plan - .mailbox - .context("reader boot requires a mailbox grant")?; - anyhow::ensure!( - plan.plan_id == mvm_core::plan::compute_plan_id(plan), - "mailbox reader boot requires a content-addressed admitted plan" - ); - let mut boot = [0u8; 32]; - rand::rngs::SysRng - .try_fill_bytes(&mut boot) - .context("drawing system entropy for the mailbox reader boot")?; - Ok(Self { - plan_id: plan.plan_id.clone(), - plan_nonce: plan.nonce.clone(), - mailbox: grant.mailbox, - execution_epoch: grant.epoch, - boot: mvm_contract::protocol::mailbox::BootId::from_bytes(boot), - guest_identity_key: guest_identity.to_bytes(), - ack_public_key: ack_key.verifying_key().to_bytes(), - }) - } - - /// Persist the host-side public binding atomically before the guest boots. - pub fn persist(&self, state_dir: &Path) -> Result<()> { - mvm_core::private_fs::ensure_private_dir(state_dir)?; - let bytes = serde_json::to_vec(self).context("encode mailbox reader boot binding")?; - anyhow::ensure!( - bytes.len() <= MAX_MAILBOX_READER_BOOT_BYTES as usize, - "mailbox reader boot binding exceeds the encoded bound" - ); - mvm_core::atomic_io::write_private(&state_dir.join(MAILBOX_READER_BOOT_FILE), &bytes) - .context("persist mailbox reader boot binding") - } - - /// Reopen the bounded host record without following a symlink. - pub fn load(state_dir: &Path) -> Result { - use std::io::Read as _; - use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; - let path = state_dir.join(MAILBOX_READER_BOOT_FILE); - let file = std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_NOFOLLOW) - .open(&path) - .with_context(|| format!("open {}", path.display()))?; - let metadata = file.metadata()?; - anyhow::ensure!( - metadata.is_file() - && metadata.nlink() == 1 - && metadata.len() <= MAX_MAILBOX_READER_BOOT_BYTES, - "mailbox reader boot binding is not a bounded regular file" - ); - let mut bytes = Vec::new(); - file.take(MAX_MAILBOX_READER_BOOT_BYTES + 1) - .read_to_end(&mut bytes)?; - anyhow::ensure!( - bytes.len() as u64 <= MAX_MAILBOX_READER_BOOT_BYTES, - "mailbox reader boot binding exceeds the encoded bound" - ); - serde_json::from_slice(&bytes).context("decode mailbox reader boot binding") - } - - /// Require the exact admitted plan, not a stale parent or moved mailbox. - pub fn check_plan(&self, plan: &mvm_core::plan::ExecutionPlan) -> Result<()> { - let grant = plan - .mailbox - .context("reader boot requires a mailbox grant")?; - anyhow::ensure!( - self.plan_id == plan.plan_id - && self.plan_nonce == plan.nonce - && self.mailbox == grant.mailbox - && self.execution_epoch == grant.epoch - && self.boot.as_bytes() != &[0; 32] - && self.ack_public_key != [0; 32] - && self.guest_identity_key != [0; 32], - "mailbox reader boot binding is stale or does not match the admitted plan" - ); - VerifyingKey::from_bytes(&self.ack_public_key).context("invalid mailbox ACK public key")?; - VerifyingKey::from_bytes(&self.guest_identity_key) - .context("invalid guest identity public key")?; - Ok(()) - } -} +/// A warm-restored guest must rekey before a new reader binding can be issued. +pub use mvm_core::mailbox::reader::{ + MAILBOX_READER_BOOT_FILE, MAX_MAILBOX_READER_BOOT_BYTES, MailboxReaderBootBinding, +}; // The drive's label and filenames are declared on the reading side, in // `mvm_agentd::flowmux_keys`, and used here. One declaration, so a rename @@ -179,6 +76,8 @@ pub struct IdentityDriveContents<'a> { /// Only project a distinct guest-held ACK key when a signed Receive /// capability and current-boot enrollment will authorize its use. pub mailbox_ack_key: Option<&'a MailboxAckKeyMaterial>, + /// Public plan/boot binding accompanying the root-only ACK key. + pub mailbox_reader_boot: Option<&'a MailboxReaderBootBinding>, } impl std::fmt::Debug for IdentityDriveContents<'_> { @@ -190,6 +89,7 @@ impl std::fmt::Debug for IdentityDriveContents<'_> { "mailbox_ack_key", &self.mailbox_ack_key.map(|_| ""), ) + .field("mailbox_reader_boot", &self.mailbox_reader_boot) .finish() } } @@ -453,7 +353,18 @@ impl GuestIdentityMaterial { owner: Owner::ROOT, }); } + if contents.mailbox_ack_key.is_some() != contents.mailbox_reader_boot.is_some() { + bail!("mailbox reader boot binding and ACK key must be projected together"); + } if let Some(ack_key) = contents.mailbox_ack_key { + let binding = contents + .mailbox_reader_boot + .context("mailbox reader binding")?; + if binding.ack_public_key != ack_key.verifying_key().to_bytes() + || binding.guest_identity_key != self.verifying_key().to_bytes() + { + bail!("mailbox ACK key and boot binding do not match this identity drive"); + } nodes.push(Node::File { path: format!("/{MAILBOX_ACK_SIGNING_KEY_FILE}"), mode: GUEST_SIGNING_KEY_MODE, @@ -461,6 +372,13 @@ impl GuestIdentityMaterial { xattrs: Vec::new(), owner: Owner::ROOT, }); + nodes.push(Node::File { + path: format!("/{MAILBOX_READER_BOOT_FILE}"), + mode: HOST_SIGNER_PUB_MODE, + data: serde_json::to_vec(binding).context("encode guest mailbox reader binding")?, + xattrs: Vec::new(), + owner: Owner::ROOT, + }); } let image = build_image( nodes, @@ -600,6 +518,7 @@ mod tests { #[test] fn mailbox_ack_key_is_distinct_optional_and_redacted() { + use mvm_contract::protocol::mailbox::{ExecutionEpoch, MailboxGrant, MailboxId}; let anchor = host_key().verifying_key(); let guest = GuestIdentityMaterial::mint(anchor).expect("guest key"); let ack = MailboxAckKeyMaterial::mint().expect("mailbox key"); @@ -620,8 +539,22 @@ mod tests { .any(|part| part == MAILBOX_ACK_SIGNING_KEY_FILE.as_bytes()) ); + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(MailboxGrant { + mailbox: MailboxId::from_bytes([8; 32]), + authority_key: anchor.to_bytes(), + epoch: ExecutionEpoch::new(3).unwrap(), + }); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + let binding = MailboxReaderBootBinding::mint( + &plan, + guest.verifying_key(), + ack.verifying_key().to_bytes(), + ) + .unwrap(); let contents = IdentityDriveContents { mailbox_ack_key: Some(&ack), + mailbox_reader_boot: Some(&binding), ..Default::default() }; assert!(!format!("{ack:?}").contains(&hex::encode(ack.seed.as_slice()))); @@ -636,11 +569,22 @@ mod tests { .any(|part| part == MAILBOX_ACK_SIGNING_KEY_FILE.as_bytes()) ); assert!(enrolled.windows(32).any(|part| part == ack.seed.as_slice())); + assert!( + enrolled + .windows(MAILBOX_READER_BOOT_FILE.len()) + .any(|part| part == MAILBOX_READER_BOOT_FILE.as_bytes()) + ); assert!( !enrolled .windows(32) .any(|part| part == host_key().to_bytes()) ); + let mismatched = IdentityDriveContents { + mailbox_ack_key: Some(&other), + mailbox_reader_boot: Some(&binding), + ..Default::default() + }; + assert!(guest.write_drive_with(&drive, &mismatched).is_err()); } #[test] @@ -656,7 +600,12 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let guest = GuestIdentityMaterial::mint(host_key().verifying_key()).unwrap(); let ack = MailboxAckKeyMaterial::mint().unwrap(); - let binding = MailboxReaderBootBinding::mint(&plan, guest.verifying_key(), &ack).unwrap(); + let binding = MailboxReaderBootBinding::mint( + &plan, + guest.verifying_key(), + ack.verifying_key().to_bytes(), + ) + .unwrap(); binding.persist(dir.path()).unwrap(); assert_eq!(MailboxReaderBootBinding::load(dir.path()).unwrap(), binding); binding.check_plan(&plan).unwrap(); @@ -844,6 +793,7 @@ mod tests { ingress: &[], egress_ca_cert_pem: Some(ca.cert_pem()), mailbox_ack_key: None, + mailbox_reader_boot: None, }, ) .expect("write drive"); @@ -880,6 +830,7 @@ mod tests { ingress: &[], egress_ca_cert_pem: Some(poisoned), mailbox_ack_key: None, + mailbox_reader_boot: None, }, ); From 4bc9e6ee946d9e3656784da7fb98575ead3cbb3a Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 18:10:28 -0800 Subject: [PATCH 07/51] wip(mailbox): hand off boot-bound reader foundation --- crates/mvm-agentd/src/guest_mount.rs | 25 ++- crates/mvm-contract/src/protocol/mailbox.rs | 4 + crates/mvm-core/src/mailbox.rs | 3 + crates/mvm-core/src/mailbox/reader.rs | 69 +++++++- crates/mvm-core/src/mailbox/reader_wire.rs | 168 +++++++++++++++++++ crates/mvm-core/tests/mailbox_reader.rs | 110 +++++++++++- crates/mvm-core/tests/mailbox_reader_wire.rs | 92 ++++++++++ 7 files changed, 466 insertions(+), 5 deletions(-) create mode 100644 crates/mvm-core/src/mailbox/reader_wire.rs create mode 100644 crates/mvm-core/tests/mailbox_reader_wire.rs diff --git a/crates/mvm-agentd/src/guest_mount.rs b/crates/mvm-agentd/src/guest_mount.rs index 8ca3f24aa6..36d9e0a850 100644 --- a/crates/mvm-agentd/src/guest_mount.rs +++ b/crates/mvm-agentd/src/guest_mount.rs @@ -660,7 +660,15 @@ pub fn drop_privilege_raw(uid: u32, gid: u32) -> std::io::Result<()> { /// #[cfg(target_os = "linux")] pub fn drop_guest_agent_privilege_raw(uid: u32, gid: u32) -> std::io::Result<()> { - assume_identity_retaining(uid, gid, RESTORE_AGENT_CAPABILITIES) + assume_identity_retaining(uid, gid, RESTORE_AGENT_CAPABILITIES)?; + // The agent retains per-boot mailbox and telemetry signing keys after + // changing to the workload UID. Do this *after* setuid, which can reset + // dumpability, so a same-UID workload cannot ptrace/read agent memory. + // SAFETY: prctl takes only integer arguments and touches no memory. + if unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) } /// Become `uid`/`gid` from root, keeping exactly `keep`. Async-signal-safe, so @@ -2394,5 +2402,20 @@ mod privilege_tests { )), "the agent must retain only the workload group; got:\n{status}" ); + // SAFETY: PR_GET_DUMPABLE has no pointer arguments and reads only + // the current process's dumpability. + assert_eq!( + unsafe { libc::prctl(libc::PR_GET_DUMPABLE, 0, 0, 0, 0) }, + 0, + "the workload UID must not be able to inspect the agent's keys" + ); + let output = std::process::Command::new("/bin/cat") + .arg(format!("/proc/{}/mem", std::process::id())) + .output() + .expect("spawn a same-UID process to probe the agent memory"); + assert!( + !output.status.success(), + "a same-UID workload read the agent's process memory" + ); } } diff --git a/crates/mvm-contract/src/protocol/mailbox.rs b/crates/mvm-contract/src/protocol/mailbox.rs index 16ad25f1d5..d6c8e3fdcf 100644 --- a/crates/mvm-contract/src/protocol/mailbox.rs +++ b/crates/mvm-contract/src/protocol/mailbox.rs @@ -184,6 +184,10 @@ impl From for u64 { /// What a signed `ExecutionPlan` admits for one workload's mailbox: which /// mailbox, the key whose capabilities are honored, and the execution epoch. +/// Presence grants that workload its own mailbox, including the ability for +/// the host to issue a separate boot-bound Receive capability. Absence grants +/// neither Send nor Receive. This grant is not itself a capability: the host +/// must still authenticate the current guest and sign each scoped operation. /// Everything else the receiving node trusts (encryption and sender keys, /// identities) is host-held and never rides in a plan. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/mvm-core/src/mailbox.rs b/crates/mvm-core/src/mailbox.rs index 470cff7434..f691339632 100644 --- a/crates/mvm-core/src/mailbox.rs +++ b/crates/mvm-core/src/mailbox.rs @@ -35,6 +35,9 @@ pub mod journal; /// Cryptographic guest reader enrollment, separate from transport admission. pub mod reader; +/// Bounded, versioned guest-reader frames on the existing mailbox port. +pub mod reader_wire; + /// Overlay resolution and authenticated, bounded byte-transport seams. pub mod transport; diff --git a/crates/mvm-core/src/mailbox/reader.rs b/crates/mvm-core/src/mailbox/reader.rs index 4fcd71c4b7..685d903d42 100644 --- a/crates/mvm-core/src/mailbox/reader.rs +++ b/crates/mvm-core/src/mailbox/reader.rs @@ -16,8 +16,9 @@ use mvm_contract::protocol::mailbox::{ SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, }; use rand::TryRng as _; +use sha2::{Digest as _, Sha256}; -use super::{MailboxError, canonical_bytes, verify_capability_signature}; +use super::{MailboxError, canonical_bytes, sign_capability, verify_capability_signature}; /// Name shared by the host writer and guest reader of one public boot binding. pub const MAILBOX_READER_BOOT_FILE: &str = "mailbox-reader-boot.json"; @@ -134,6 +135,72 @@ impl MailboxReaderBootBinding { } } +/// Derive a *separate* Receive capability for a guest already granted its +/// own mailbox by the signed plan. The plan's presence is the opt-in; a Send +/// capability or a boot from another plan cannot confer Receive authority. +/// +/// The authority must still verify a fresh possession proof from this boot +/// before releasing a message. The capability is limited by the host's +/// existing send-policy window, address, content types, and quotas. +/// `expected_address` must come from the admitted host mailbox policy, never +/// from the guest's request. +pub fn mint_guest_receive_capability( + grant: &MailboxGrant, + binding: &MailboxReaderBootBinding, + send: &SignedMailboxCapability, + expected_address: MailboxAddress, + authority: &SigningKey, +) -> Result { + if binding.mailbox != grant.mailbox + || binding.authority_key != grant.authority_key + || binding.execution_epoch != grant.epoch + || binding.boot.as_bytes() == &[0; 32] + || binding.ack_public_key == [0; 32] + || binding.guest_identity_key == [0; 32] + || grant.authority_key != authority.verifying_key().to_bytes() + || expected_address.mailbox != grant.mailbox + || send.grant.address != expected_address + || send.grant.minimum_execution_epoch != grant.epoch + || send.grant.maximum_execution_epoch != grant.epoch + || send.grant.issuer.as_bytes() != &grant.authority_key + || !send + .grant + .operations + .allows(mvm_contract::protocol::mailbox::MailboxOperation::Send) + || send + .grant + .operations + .allows(mvm_contract::protocol::mailbox::MailboxOperation::Receive) + { + return Err(MailboxError::WrongCapability); + } + VerifyingKey::from_bytes(&binding.ack_public_key) + .map_err(|_| MailboxError::ReaderKeyInvalid)?; + VerifyingKey::from_bytes(&binding.guest_identity_key) + .map_err(|_| MailboxError::ReaderKeyInvalid)?; + verify_capability_signature(send, &authority.verifying_key())?; + let id = |purpose: &[u8]| { + let mut digest = Sha256::new(); + digest.update(b"mvm.mailbox.own-reader.v1\0"); + digest.update(purpose); + digest.update(binding.plan_id.0.as_bytes()); + digest.update(binding.plan_nonce.as_hex().as_bytes()); + digest.update(binding.boot.as_bytes()); + digest.update(send.grant.capability_id.as_bytes()); + digest.finalize().into() + }; + let mut receive = send.grant.clone(); + receive.capability_id = + mvm_contract::protocol::mailbox::CapabilityId::from_bytes(id(b"capability")); + receive.revocation_id = + mvm_contract::protocol::mailbox::RevocationId::from_bytes(id(b"revocation")); + receive.subject = receive.address.recipient; + receive.operations = mvm_contract::protocol::mailbox::CapabilityOperations::from_operations(&[ + mvm_contract::protocol::mailbox::MailboxOperation::Receive, + ]); + sign_capability(receive, authority) +} + const READER_POSSESSION_SIGNATURE_DOMAIN: &[u8] = b"mvm.mailbox.reader-possession.v1\0"; /// Sign an enrollment with the destination authority. The caller must bind diff --git a/crates/mvm-core/src/mailbox/reader_wire.rs b/crates/mvm-core/src/mailbox/reader_wire.rs new file mode 100644 index 0000000000..a91114cc56 --- /dev/null +++ b/crates/mvm-core/src/mailbox/reader_wire.rs @@ -0,0 +1,168 @@ +//! Versioned control frames for the on-demand guest reader exchange. +//! +//! The `MVMR` discriminator must be dispatched before the outbox decoder; +//! it is not itself a claim of reader authority. +//! Decoding never supplies authority: the destination must verify the signed +//! Receive capability, current boot, possession proof, lease and ACK against +//! its journal before releasing plaintext or committing an acknowledgement. +//! The guest listener and destination service do not serve these frames yet. + +use std::io::{Read, Write}; + +use base64::Engine as _; +use mvm_contract::protocol::mailbox::{ + SignedMailboxCapability, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, + SignedMailboxGuestReaderEnrollment, +}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; + +use super::frame::Refusal; + +const READER_MAGIC: &[u8; 5] = b"MVMR\x01"; +const MAX_REQUEST_BYTES: usize = 16 * 1024; +const MAX_RESPONSE_BYTES: usize = 64 * 1024; + +/// One on-demand guest request. `Possession` answers the immediately preceding +/// destination challenge on the same connection, not an arbitrary old one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)] +pub enum ReaderRequest { + Receive, + Possession { + signature: Vec, + }, + Ack { + signed: Box, + }, +} + +/// Destination response; a delivery is never sent before its lease is durable. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] +pub enum ReaderResponse { + Challenge { + enrollment: Box, + capability: Box, + challenge: [u8; 32], + }, + Delivery { + lease: Box, + #[serde(with = "payload_base64")] + payload: Vec, + }, + Empty, + Acknowledged, + Refused { + reason: Refusal, + }, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Versioned { + version: u16, + value: T, +} + +/// Write one request, bounded before a byte is put on the stream. +pub fn write_request(stream: &mut impl Write, request: &ReaderRequest) -> Result<(), Refusal> { + write_frame(stream, request, MAX_REQUEST_BYTES) +} + +/// Read a request only after bounding its complete length. +pub fn read_request(stream: &mut impl Read) -> Result { + read_frame(stream, MAX_REQUEST_BYTES) +} + +/// Write one destination response, including at most a bounded payload. +pub fn write_response(stream: &mut impl Write, response: &ReaderResponse) -> Result<(), Refusal> { + write_frame(stream, response, MAX_RESPONSE_BYTES) +} + +/// Read a response only after bounding its complete length. +pub fn read_response(stream: &mut impl Read) -> Result { + read_frame(stream, MAX_RESPONSE_BYTES) +} + +fn write_frame( + stream: &mut impl Write, + value: &T, + limit: usize, +) -> Result<(), Refusal> { + let mut json = BoundedBytes { + bytes: Vec::new(), + limit: limit - READER_MAGIC.len(), + }; + serde_json::to_writer(&mut json, &Versioned { version: 1, value }) + .map_err(|_| Refusal::Oversized(limit))?; + let len = READER_MAGIC.len() + json.bytes.len(); + stream + .write_all(&(len as u32).to_be_bytes()) + .and_then(|()| stream.write_all(READER_MAGIC)) + .and_then(|()| stream.write_all(&json.bytes)) + .map_err(|_| Refusal::Unavailable) +} + +struct BoundedBytes { + bytes: Vec, + limit: usize, +} + +impl Write for BoundedBytes { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > self.limit - self.bytes.len() { + return Err(std::io::Error::other("mailbox reader frame exceeds limit")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +mod payload_base64 { + use super::*; + use serde::{Deserializer, Serializer, de::Error as _}; + + pub(super) fn serialize(bytes: &[u8], serializer: S) -> Result { + serializer.serialize_str(&base64::engine::general_purpose::STANDARD.encode(bytes)) + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + let encoded = String::deserialize(deserializer)?; + base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(D::Error::custom) + } +} + +fn read_frame(stream: &mut impl Read, limit: usize) -> Result { + let mut len = [0u8; 4]; + stream + .read_exact(&mut len) + .map_err(|_| Refusal::Malformed)?; + let len = u32::from_be_bytes(len) as usize; + if !(READER_MAGIC.len()..=limit).contains(&len) { + return Err(Refusal::Oversized(limit)); + } + let mut magic = [0u8; 5]; + stream + .read_exact(&mut magic) + .map_err(|_| Refusal::Malformed)?; + if &magic != READER_MAGIC { + return Err(Refusal::Malformed); + } + let mut json = vec![0; len - magic.len()]; + stream + .read_exact(&mut json) + .map_err(|_| Refusal::Malformed)?; + let frame: Versioned = serde_json::from_slice(&json).map_err(|_| Refusal::Malformed)?; + if frame.version != 1 { + return Err(Refusal::Malformed); + } + Ok(frame.value) +} diff --git a/crates/mvm-core/tests/mailbox_reader.rs b/crates/mvm-core/tests/mailbox_reader.rs index d4ec8bd35c..fca1890af4 100644 --- a/crates/mvm-core/tests/mailbox_reader.rs +++ b/crates/mvm-core/tests/mailbox_reader.rs @@ -13,9 +13,10 @@ use mvm_contract::protocol::mailbox::{ }; use mvm_core::crypto::aead::Key; use mvm_core::mailbox::reader::{ - ReaderDeliveryContext, sign_delivery_ack, sign_delivery_lease, sign_guest_reader_enrollment, - sign_reader_possession, verify_delivery_ack, verify_delivery_lease, - verify_guest_reader_enrollment, verify_reader_possession, + MailboxReaderBootBinding, ReaderDeliveryContext, mint_guest_receive_capability, + sign_delivery_ack, sign_delivery_lease, sign_guest_reader_enrollment, sign_reader_possession, + verify_delivery_ack, verify_delivery_lease, verify_guest_reader_enrollment, + verify_reader_possession, }; use mvm_core::mailbox::{MailboxError, SealRequest, seal_envelope, sign_capability}; @@ -87,6 +88,109 @@ fn fixture() -> ( ) } +#[test] +fn own_mailbox_grant_mints_a_distinct_boot_bound_receive_capability() { + let (authority, guest, grant, plan_id, plan_nonce, mut send, mut enrollment) = fixture(); + let binding = MailboxReaderBootBinding { + plan_id: plan_id.clone(), + plan_nonce: plan_nonce.clone(), + mailbox: grant.mailbox, + authority_key: grant.authority_key, + execution_epoch: grant.epoch, + boot: enrollment.consumer_boot, + guest_identity_key: SigningKey::from_bytes(&[17; 32]).verifying_key().to_bytes(), + ack_public_key: guest.verifying_key().to_bytes(), + }; + send.issuer = PrincipalId::from_bytes(authority.verifying_key().to_bytes()); + send.subject = PrincipalId::from_bytes([21; 32]); + send.operations = CapabilityOperations::from_operations(&[MailboxOperation::Send]); + let signed_send = sign_capability(send.clone(), &authority).unwrap(); + let receive = + mint_guest_receive_capability(&grant, &binding, &signed_send, send.address, &authority) + .unwrap(); + assert!(receive.grant.operations.allows(MailboxOperation::Receive)); + assert!(!receive.grant.operations.allows(MailboxOperation::Send)); + assert_eq!(receive.grant.subject, send.address.recipient); + assert_eq!(receive.grant.address, send.address); + assert_eq!(receive.grant.expires_at_unix_ms, send.expires_at_unix_ms); + assert_ne!(receive.grant.capability_id, send.capability_id); + assert_ne!(receive.grant.revocation_id, send.revocation_id); + let mut next_boot = binding.clone(); + next_boot.boot = BootId::from_bytes([42; 32]); + assert_ne!( + mint_guest_receive_capability(&grant, &next_boot, &signed_send, send.address, &authority,) + .unwrap() + .grant + .capability_id, + receive.grant.capability_id, + "a warm child must receive a fresh boot-scoped capability", + ); + + enrollment.consumer = receive.grant.subject; + enrollment.receive_capability_id = receive.grant.capability_id; + let signed_enrollment = sign_guest_reader_enrollment(enrollment, &authority).unwrap(); + let scope = MailboxGuestReaderScope { + plan: &grant, + plan_id: &plan_id, + plan_nonce: &plan_nonce, + receive: &receive.grant, + current_boot: binding.boot, + current_session: signed_enrollment.enrollment.consumer_session, + now_unix_ms: 150, + }; + verify_guest_reader_enrollment( + &signed_enrollment, + &receive, + &scope, + &authority.verifying_key(), + &[], + ) + .unwrap(); + + let mut stale = binding.clone(); + stale.boot = BootId::from_bytes([0; 32]); + assert!( + mint_guest_receive_capability(&grant, &stale, &signed_send, send.address, &authority) + .is_err() + ); + stale = binding.clone(); + stale.execution_epoch = ExecutionEpoch::new(8).unwrap(); + assert!( + mint_guest_receive_capability(&grant, &stale, &signed_send, send.address, &authority) + .is_err() + ); + stale = binding.clone(); + stale.mailbox = MailboxId::from_bytes([99; 32]); + assert!( + mint_guest_receive_capability(&grant, &stale, &signed_send, send.address, &authority) + .is_err() + ); + let mut wrong_address = send.clone(); + wrong_address.address.tenant = TenantId::from_bytes([99; 32]); + let signed_other = sign_capability(wrong_address, &authority).unwrap(); + assert!( + mint_guest_receive_capability(&grant, &binding, &signed_other, send.address, &authority) + .is_err(), + "a signed Send grant for another tenant must not authorize our inbox", + ); + let mut tampered = signed_send.clone(); + tampered.grant.address.recipient = PrincipalId::from_bytes([24; 32]); + assert!( + mint_guest_receive_capability(&grant, &binding, &tampered, send.address, &authority) + .is_err() + ); + assert!( + mint_guest_receive_capability( + &grant, + &binding, + &signed_send, + send.address, + &SigningKey::from_bytes(&[75; 32]), + ) + .is_err() + ); +} + #[test] fn enrolled_reader_requires_two_authority_signatures_and_fresh_guest_possession() { let (authority, guest, plan, plan_id, plan_nonce, receive, enrollment) = fixture(); diff --git a/crates/mvm-core/tests/mailbox_reader_wire.rs b/crates/mvm-core/tests/mailbox_reader_wire.rs new file mode 100644 index 0000000000..2128e3898a --- /dev/null +++ b/crates/mvm-core/tests/mailbox_reader_wire.rs @@ -0,0 +1,92 @@ +use mvm_core::mailbox::frame::Refusal; +use mvm_core::mailbox::reader_wire::{ + ReaderRequest, ReaderResponse, read_request, read_response, write_request, write_response, +}; + +const MAX_REQUEST_BYTES: usize = 16 * 1024; + +#[test] +fn reader_request_is_versioned_and_bounded_before_parsing() { + let mut frame = Vec::new(); + write_request(&mut frame, &ReaderRequest::Receive).unwrap(); + assert_eq!( + read_request(&mut frame.as_slice()), + Ok(ReaderRequest::Receive) + ); + + let mut bad_version = frame.clone(); + let one = bad_version + .windows(b"\"version\":1".len()) + .position(|w| w == b"\"version\":1") + .unwrap(); + bad_version[one + b"\"version\":".len()] = b'2'; + assert_eq!( + read_request(&mut bad_version.as_slice()), + Err(Refusal::Malformed) + ); + + let mut wrong_plane = frame.clone(); + wrong_plane[4] = b'X'; + assert_eq!( + read_request(&mut wrong_plane.as_slice()), + Err(Refusal::Malformed) + ); + assert_eq!( + read_request(&mut ((MAX_REQUEST_BYTES + 1) as u32).to_be_bytes().as_slice()), + Err(Refusal::Oversized(MAX_REQUEST_BYTES)) + ); +} + +#[test] +fn truncation_and_large_requests_refuse_without_writing_a_prefix() { + assert_eq!( + read_request(&mut [0, 0, 0, 1].as_slice()), + Err(Refusal::Oversized(MAX_REQUEST_BYTES)) + ); + let mut frame = Vec::new(); + write_request( + &mut frame, + &ReaderRequest::Possession { + signature: vec![4; 64], + }, + ) + .unwrap(); + frame.pop(); + assert_eq!(read_request(&mut frame.as_slice()), Err(Refusal::Malformed)); + + let mut huge = Vec::new(); + assert_eq!( + write_request( + &mut huge, + &ReaderRequest::Possession { + signature: vec![4; MAX_REQUEST_BYTES] + } + ), + Err(Refusal::Oversized(MAX_REQUEST_BYTES)), + ); + assert!(huge.is_empty(), "oversized request must not write a prefix"); +} + +#[test] +fn a_refusal_is_not_an_ack_or_an_empty_mailbox() { + let mut frame = Vec::new(); + write_response( + &mut frame, + &ReaderResponse::Refused { + reason: Refusal::NotAuthorized, + }, + ) + .unwrap(); + assert_eq!( + read_response(&mut frame.as_slice()), + Ok(ReaderResponse::Refused { + reason: Refusal::NotAuthorized + }) + ); + frame.clear(); + write_response(&mut frame, &ReaderResponse::Empty).unwrap(); + assert_eq!( + read_response(&mut frame.as_slice()), + Ok(ReaderResponse::Empty) + ); +} From 634d1cc687898f947129a0aec9dc89d7b117de65 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 22:05:16 -0800 Subject: [PATCH 08/51] wip(mailbox): wire boot-bound guest Receive and explicit ACK --- crates/mvm-agentd/README.md | 36 +- .../examples/mailbox_reader_client.rs | 70 ++++ .../examples/mailbox_writer_client.rs | 22 + crates/mvm-agentd/src/bin/mvm-guest-agent.rs | 2 + .../src/bin/mvm-guest-agent/init.rs | 1 + .../src/bin/mvm-guest-agent/mailbox.rs | 103 ++++- .../src/bin/mvm-guest-agent/mailbox_reader.rs | 382 ++++++++++++++++++ crates/mvm-agentd/src/bin/syscall-probe.rs | 19 +- crates/mvm-agentd/src/workload_seccomp.rs | 95 ++++- crates/mvm-agentd/tests/seccomp_apply.rs | 26 ++ crates/mvm-core/src/mailbox/journal.rs | 70 +++- crates/mvm-core/src/mailbox/reader.rs | 35 +- crates/mvm-core/src/mailbox/reader_wire.rs | 78 +++- crates/mvm-core/tests/mailbox_reader_wire.rs | 40 +- crates/mvm-hostd/src/mailbox_process.rs | 16 + .../mvm-hostd/src/mailbox_process/reader.rs | 289 +++++++++++++ crates/mvm-hostd/src/mailbox_service.rs | 2 +- .../mvm-hostd/src/mailbox_service/durable.rs | 305 +++++++++++++- crates/mvm-hostd/tests/mission_demo.rs | 340 ++++++++++++++++ 19 files changed, 1861 insertions(+), 70 deletions(-) create mode 100644 crates/mvm-agentd/examples/mailbox_reader_client.rs create mode 100644 crates/mvm-agentd/examples/mailbox_writer_client.rs create mode 100644 crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs create mode 100644 crates/mvm-hostd/src/mailbox_process/reader.rs diff --git a/crates/mvm-agentd/README.md b/crates/mvm-agentd/README.md index ac76b2bf23..d1cd4b1e13 100644 --- a/crates/mvm-agentd/README.md +++ b/crates/mvm-agentd/README.md @@ -34,9 +34,39 @@ Specialized binaries keep privilege and dependency boundaries small: `mvm-setpriv` crate, applies the final identity. The guest treats host replies as protocol input, while the host treats every -guest request as untrusted. Secrets and authorization decisions remain -host-side; the agent receives only the capabilities and values needed for the -admitted execution. +guest request as untrusted. Host sealing and destination signing material +remain host-side; an opt-in mailbox reader additionally holds its own per-boot +ACK signing key inside the guest agent, not in the workload. + +## Guest mailbox reader + +Only a signed plan granting this workload's own mailbox enables Receive. The +agent exposes `/run/mvm/mailbox/receive.sock` to that workload when its separate +boot ACK key was provisioned. The local protocol is versioned, length-prefixed, +and bounded; use `ApplicationReaderRequest` and `ReaderResponse` from +`mvm_core::mailbox::reader_wire`. `Receive` returns a destination-signed lease, +content type, and payload. Process the payload **before** sending +`Ack { lease }` on a new connection. An empty inbox and a refusal are not +successful delivery; reading never automatically ACKs. + +For a guest application example, build: + +```sh +cargo build -p mvm-agentd --example mailbox_reader_client --target aarch64-unknown-linux-musl +``` + +Run it without arguments to read without acknowledging; pass `--ack` only when +writing and flushing its stdout counts as completed processing. The returned +`Acknowledged { ack }` echoes the exact guest-signed operation after the local +journal and anchor commit. This is a trusted-local confirmation, **not** a +destination-signed, exportable ACK receipt. A lost delivery or ACK response is +**unknown**, not success. Retry with the same lease and signed ACK identity +while this boot's agent remains active, or reconcile against the durable +journal; a different ACK for the same lease conflicts. + +`mailbox_writer_client` is a separate guest example for staging a message in +the existing outbox. It reports **local staging only**, not signed destination +acceptance; use a host-side durable receipt or journal witness for that claim. ## Bounded entrypoint capture diff --git a/crates/mvm-agentd/examples/mailbox_reader_client.rs b/crates/mvm-agentd/examples/mailbox_reader_client.rs new file mode 100644 index 0000000000..96a2882457 --- /dev/null +++ b/crates/mvm-agentd/examples/mailbox_reader_client.rs @@ -0,0 +1,70 @@ +//! Guest application example: read one message from this workload's own inbox. +//! Pass `--ack` only after the application has processed and flushed its output. + +use std::io::{self, Write as _}; +use std::os::unix::net::UnixStream; +use std::time::Duration; + +use mvm_core::mailbox::reader_wire::{ + ApplicationReaderRequest, GUEST_READER_SOCKET_PATH, ReaderResponse, read_response, + write_application_request, +}; + +fn connect() -> io::Result { + let stream = UnixStream::connect(GUEST_READER_SOCKET_PATH)?; + stream.set_read_timeout(Some(Duration::from_secs(12)))?; + stream.set_write_timeout(Some(Duration::from_secs(12)))?; + Ok(stream) +} + +fn main() -> Result<(), Box> { + let ack = match std::env::args().skip(1).collect::>().as_slice() { + [] => false, + [flag] if flag == "--ack" => true, + _ => return Err("usage: mailbox_reader_client [--ack]".into()), + }; + let mut stream = connect()?; + write_application_request(&mut stream, &ApplicationReaderRequest::Receive)?; + let response = read_response(&mut stream)?; + let ReaderResponse::Delivery { + lease, + content_type, + payload, + } = response + else { + match response { + ReaderResponse::Empty => eprintln!("inbox empty"), + ReaderResponse::Refused { reason } => return Err(reason.into()), + _ => return Err("unexpected reader response".into()), + } + return Ok(()); + }; + // This example's application processing is writing and flushing stdout. + // A real application ACKs only after its own durable processing completes. + io::stdout().write_all(&payload)?; + io::stdout().flush()?; + eprintln!( + "received {} bytes as {} (message {:?}, delivery {:?})", + payload.len(), + content_type.name(), + lease.lease.message_id, + lease.lease.delivery_id + ); + if ack { + let mut stream = connect()?; + write_application_request( + &mut stream, + &ApplicationReaderRequest::Ack { + lease: lease.clone(), + }, + )?; + match read_response(&mut stream)? { + ReaderResponse::Acknowledged { ack: signed } if signed.ack.lease == *lease => { + eprintln!("locally acknowledged by MVM"); + } + ReaderResponse::Refused { reason } => return Err(reason.into()), + _ => return Err("ACK response did not identify this lease".into()), + } + } + Ok(()) +} diff --git a/crates/mvm-agentd/examples/mailbox_writer_client.rs b/crates/mvm-agentd/examples/mailbox_writer_client.rs new file mode 100644 index 0000000000..b5ba1b9c4b --- /dev/null +++ b/crates/mvm-agentd/examples/mailbox_writer_client.rs @@ -0,0 +1,22 @@ +//! Stage one guest-to-own-mailbox message. Local staging is not a signed +//! destination acceptance receipt; inspect the host journal for durability. + +use std::path::Path; + +use mvm_agentd::mailbox_outbox::{OUTBOX_PATH, write_outbox}; +use mvm_contract::protocol::mailbox::PrincipalId; +use mvm_core::mailbox::frame::{OWN_MAILBOX_RECIPIENT, OutboxMessage}; + +fn main() -> Result<(), Box> { + let body = std::env::args() + .nth(1) + .filter(|body| !body.is_empty()) + .ok_or("usage: mailbox_writer_client ")?; + let message = OutboxMessage::new( + PrincipalId::from_bytes(OWN_MAILBOX_RECIPIENT), + body.into_bytes(), + ); + write_outbox(Path::new(OUTBOX_PATH), &message)?; + eprintln!("staged locally: message {:?}", message.message_id); + Ok(()) +} diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent.rs index 1cce3dc768..4e2dc7bbdc 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent.rs @@ -34,6 +34,8 @@ mod init; mod interactive; #[path = "mvm-guest-agent/mailbox.rs"] mod mailbox; +#[path = "mvm-guest-agent/mailbox_reader.rs"] +mod mailbox_reader; #[path = "mvm-guest-agent/monitoring.rs"] mod monitoring; #[path = "mvm-guest-agent/port_forward.rs"] diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/init.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/init.rs index 37e79eb0fd..1e627894ba 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/init.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/init.rs @@ -186,6 +186,7 @@ pub(crate) fn apply_activation( crate::telemetry::capture_session_keys_before_privilege_drop(); crate::mailbox::capture_ack_key_before_privilege_drop(); guest_mount::drop_guest_agent_privilege(guest_mount::WORKLOAD_UID, guest_mount::WORKLOAD_GID)?; + crate::mailbox::protect_ack_key_after_privilege_drop()?; boot_state.set_activation(ActivationState::Activated); Ok(()) diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs index e7239e5cb8..5157fdf35a 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs @@ -14,7 +14,7 @@ use std::time::{Duration, Instant}; use ed25519_dalek::SigningKey; use mvm_agentd::flowmux_drive::{MAILBOX_ACK_SIGNING_KEY_FILE, RUN_MVM_DIR}; -use mvm_agentd::mailbox_outbox::{MAILBOX_CMDLINE_TOKEN, OUTBOX_PATH, wait_and_forward_outbox}; +use mvm_agentd::mailbox_outbox::{MAILBOX_CMDLINE_TOKEN, OUTBOX_PATH, forward_outbox}; use mvm_agentd::telemetry_capture::session::TimedStream; use mvm_contract::protocol::mailbox::MAILBOX_PORT; use mvm_core::mailbox::frame::Refusal; @@ -59,6 +59,23 @@ pub(crate) fn capture_ack_key_before_privilege_drop() { } } +/// The workload runs under the same UID as the agent. A captured ACK key must +/// not become readable through ptrace or `/proc//mem` after the UID drop. +pub(crate) fn protect_ack_key_after_privilege_drop() +-> Result<(), mvm_agentd::guest_mount::MountError> { + #[cfg(target_os = "linux")] + if ACK_SEED.get().is_some() { + // SAFETY: prctl receives only integer arguments and affects this process. + if unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) } != 0 { + return Err(mvm_agentd::guest_mount::MountError::InvalidConfig(format!( + "protect mailbox ACK key after privilege drop: {}", + io::Error::last_os_error() + ))); + } + } + Ok(()) +} + fn load_ack_seed(path: &Path, owner_uid: u32) -> io::Result> { use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _}; let mut file = std::fs::OpenOptions::new() @@ -184,18 +201,30 @@ pub(crate) fn init_mailbox() { if !asserted || unix_transport_selected() { return; } - let _ = mvm_agentd::detached::spawn_named("mailbox-listener", || { - match bind_vsock_listener(MAILBOX_PORT) { - Ok(fd) => accept_loop(fd), - Err(e) => eprintln!( - "mvm-guest-agent: mailbox listener bind failed (port {MAILBOX_PORT}): {e}" - ), - } - }); + let reader_boot = READER_BOOT.get().cloned(); + let ack_seed = ACK_SEED.get(); + let _ = + mvm_agentd::detached::spawn_named("mailbox-listener", move || { + match bind_vsock_listener(MAILBOX_PORT) { + Ok(fd) => { + let reader = match (reader_boot, ack_seed) { + (Some(binding), Some(seed)) => crate::mailbox_reader::start(binding, seed), + _ => None, + }; + accept_loop(fd, reader); + } + Err(e) => eprintln!( + "mvm-guest-agent: mailbox listener bind failed (port {MAILBOX_PORT}): {e}" + ), + } + }); } /// One host connection at a time, each answered from the outbox. -fn accept_loop(listener_fd: RawFd) { +fn accept_loop( + listener_fd: RawFd, + reader: Option>, +) { let mut refused = 0u32; while !SHUTDOWN_REQUESTED.load(Ordering::Acquire) { let Some(cfd) = accept_vsock(listener_fd, "mailbox") else { @@ -208,10 +237,9 @@ fn accept_loop(listener_fd: RawFd) { eprintln!("mvm-guest-agent: mailbox connection has no bounded I/O: {error}"); continue; } - match wait_and_forward_outbox( - &mut MailboxDeadline::new(&mut stream), - Path::new(OUTBOX_PATH), - ) { + let mut deadline = MailboxDeadline::new(&mut stream); + let result = wait_for_operation(&mut deadline, reader.as_ref(), Path::new(OUTBOX_PATH)); + match result { Ok(()) | Err(Refusal::Unavailable) => {} Err(refusal) => { if refused < 3 || refused.is_multiple_of(100) { @@ -223,6 +251,25 @@ fn accept_loop(listener_fd: RawFd) { } } +fn wait_for_operation( + stream: &mut (impl Read + Write), + reader: Option<&std::sync::mpsc::Receiver>, + outbox: &Path, +) -> Result<(), Refusal> { + for _ in 0..16 { + // A pending Send always gets the existing port first; reader traffic + // cannot starve a workload's outbox. + if outbox.exists() { + return forward_outbox(stream, outbox); + } + if let Some(Ok(pending)) = reader.map(std::sync::mpsc::Receiver::try_recv) { + return crate::mailbox_reader::exchange(stream, pending); + } + std::thread::sleep(Duration::from_millis(250)); + } + Err(Refusal::Unavailable) +} + #[cfg(test)] mod tests { use super::*; @@ -268,6 +315,34 @@ mod tests { ); } + #[test] + fn a_pending_send_still_uses_the_existing_port_when_receive_is_enabled() { + use mvm_contract::protocol::mailbox::PrincipalId; + use mvm_core::mailbox::frame::OutboxMessage; + let dir = tempfile::tempdir().unwrap(); + let outbox = dir.path().join("outbox"); + mvm_agentd::mailbox_outbox::write_outbox( + &outbox, + &OutboxMessage::new(PrincipalId::from_bytes([0; 32]), b"hello".to_vec()), + ) + .unwrap(); + let (mut guest, mut host) = std::os::unix::net::UnixStream::pair().unwrap(); + let (_sender, reader) = std::sync::mpsc::sync_channel(1); + let peer = std::thread::spawn(move || { + let mut len = [0; 4]; + host.read_exact(&mut len).unwrap(); + let mut body = vec![0; u32::from_be_bytes(len) as usize]; + host.read_exact(&mut body).unwrap(); + assert!(!mvm_core::mailbox::reader_wire::is_reader_frame(&body)); + assert_eq!(OutboxMessage::decode(&body).unwrap().body, b"hello"); + host.write_all(&serde_json::to_vec(&Ok::<(), Refusal>(())).unwrap()) + .unwrap(); + }); + wait_for_operation(&mut guest, Some(&reader), &outbox).unwrap(); + peer.join().unwrap(); + assert!(!outbox.exists(), "accepted Send is removed as before"); + } + #[test] fn ack_seed_loader_requires_exact_private_regular_file() { use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs new file mode 100644 index 0000000000..9d555e4806 --- /dev/null +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs @@ -0,0 +1,382 @@ +//! Explicit guest-local Receive and ACK-after-processing over the existing +//! mailbox vsock port. The workload can read the socket, not the ACK key. + +use std::io::{Read, Write}; +use std::os::unix::fs::{FileTypeExt as _, PermissionsExt as _}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::Path; +use std::sync::{Arc, Mutex, mpsc}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use ed25519_dalek::{SigningKey, VerifyingKey}; +use mvm_contract::protocol::mailbox::{ + DeliveryAckNonce, MailboxDeliveryAck, MailboxGrant, MailboxGuestReaderScope, ProtocolVersion, + SignatureAlgorithm, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, +}; +use mvm_core::mailbox::frame::Refusal; +use mvm_core::mailbox::reader::{ + MailboxReaderBootBinding, sign_delivery_ack, sign_reader_possession, + verify_delivery_lease_signature, verify_guest_reader_enrollment, +}; +use mvm_core::mailbox::reader_wire::{ + ApplicationReaderRequest, GUEST_READER_SOCKET_PATH, ReaderRequest, ReaderResponse, + read_application_request, read_response, write_request, write_response, +}; +use zeroize::Zeroizing; + +const LOCAL_TIMEOUT: Duration = Duration::from_secs(12); + +struct ActiveLease { + lease: SignedMailboxDeliveryLease, + ack: Option, +} + +struct GuestReader { + boot: MailboxReaderBootBinding, + ack_seed: &'static Zeroizing<[u8; 32]>, + active: Mutex>, +} + +pub(crate) struct PendingReader { + request: ReaderRequest, + reply: mpsc::SyncSender, + reader: Arc, +} + +fn now_unix_ms() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| Refusal::Unavailable)? + .as_millis() + .try_into() + .map_err(|_| Refusal::Unavailable) +} + +/// Only start the local endpoint when the independently provisioned, root-only +/// boot key was captured by the agent. Send works without a reader. +pub(crate) fn start( + boot: MailboxReaderBootBinding, + ack_seed: &'static Zeroizing<[u8; 32]>, +) -> Option> { + let path = Path::new(GUEST_READER_SOCKET_PATH); + let result = (|| { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.file_type().is_socket() => std::fs::remove_file(path)?, + Ok(_) => { + return Err(std::io::Error::other( + "refusing to replace a non-socket guest reader path", + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let listener = UnixListener::bind(path)?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; + Ok::<_, std::io::Error>(listener) + })(); + let listener = match result { + Ok(listener) => listener, + Err(error) => { + eprintln!("mvm-guest-agent: mailbox reader socket unavailable: {error}"); + return None; + } + }; + let (tx, rx) = mpsc::sync_channel(1); + let state = Arc::new(GuestReader { + boot, + ack_seed, + active: Mutex::new(None), + }); + std::thread::spawn(move || { + for client in listener.incoming() { + let Ok(mut client) = client else { break }; + let _ = client.set_read_timeout(Some(LOCAL_TIMEOUT)); + let _ = client.set_write_timeout(Some(LOCAL_TIMEOUT)); + let response = application_request(&mut client, &tx, &state) + .unwrap_or_else(|reason| ReaderResponse::Refused { reason }); + let _ = write_response(&mut client, &response); + } + }); + Some(rx) +} + +fn application_request( + stream: &mut UnixStream, + tx: &mpsc::SyncSender, + reader: &Arc, +) -> Result { + let request = match read_application_request(stream)? { + ApplicationReaderRequest::Receive => ReaderRequest::Receive, + ApplicationReaderRequest::Ack { lease } => { + let mut active = reader.active.lock().expect("mailbox reader lock poisoned"); + let active = active.as_mut().ok_or(Refusal::NotAuthorized)?; + if active.lease != *lease { + return Err(Refusal::NotAuthorized); + } + let ack = if let Some(ack) = &active.ack { + ack.clone() + } else { + verify_delivery_lease_signature( + &active.lease, + &VerifyingKey::from_bytes(&reader.boot.authority_key) + .map_err(|_| Refusal::NotAuthorized)?, + ) + .map_err(|_| Refusal::NotAuthorized)?; + let ack = sign_delivery_ack( + MailboxDeliveryAck { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + lease: active.lease.clone(), + ack_nonce: DeliveryAckNonce::from_bytes(rand::random()), + acknowledged_at_unix_ms: now_unix_ms()?, + }, + &SigningKey::from_bytes(reader.ack_seed), + ) + .map_err(|_| Refusal::NotAuthorized)?; + active.ack = Some(ack.clone()); + ack + }; + ReaderRequest::Ack { + signed: Box::new(ack), + } + } + }; + let (reply, received) = mpsc::sync_channel(1); + tx.try_send(PendingReader { + request, + reply, + reader: reader.clone(), + }) + .map_err(|_| Refusal::Unavailable)?; + received + .recv_timeout(LOCAL_TIMEOUT) + .map_err(|_| Refusal::Unavailable) +} + +/// The mailbox listener calls this only when an application operation is +/// pending; idle host dials continue to carry Send without an extra handshake. +pub(crate) fn exchange( + stream: &mut (impl Read + Write), + pending: PendingReader, +) -> Result<(), Refusal> { + let response = match perform_exchange(stream, &pending) { + Ok(response) => response, + Err(reason) => ReaderResponse::Refused { reason }, + }; + let _ = pending.reply.send(response); + Ok(()) +} + +fn perform_exchange( + stream: &mut (impl Read + Write), + pending: &PendingReader, +) -> Result { + write_request(stream, &pending.request)?; + match &pending.request { + ReaderRequest::Receive => { + let ReaderResponse::Challenge { + enrollment, + capability, + challenge, + } = read_response(stream)? + else { + return Err(Refusal::NotAuthorized); + }; + let boot = &pending.reader.boot; + let grant = MailboxGrant { + mailbox: boot.mailbox, + authority_key: boot.authority_key, + epoch: boot.execution_epoch, + }; + let authority = VerifyingKey::from_bytes(&boot.authority_key) + .map_err(|_| Refusal::NotAuthorized)?; + let scope = MailboxGuestReaderScope { + plan: &grant, + plan_id: &boot.plan_id, + plan_nonce: &boot.plan_nonce, + receive: &capability.grant, + current_boot: boot.boot, + current_session: enrollment.enrollment.consumer_session, + now_unix_ms: now_unix_ms()?, + }; + verify_guest_reader_enrollment(&enrollment, &capability, &scope, &authority, &[]) + .map_err(|_| Refusal::NotAuthorized)?; + if enrollment.enrollment.ack_public_key != boot.ack_public_key { + return Err(Refusal::NotAuthorized); + } + let possession = sign_reader_possession( + &enrollment, + &challenge, + &SigningKey::from_bytes(pending.reader.ack_seed), + ) + .map_err(|_| Refusal::NotAuthorized)?; + write_request( + stream, + &ReaderRequest::Possession { + signature: possession.to_vec(), + }, + )?; + let response = read_response(stream)?; + if let ReaderResponse::Delivery { + lease, + content_type, + payload, + } = &response + { + verify_delivery_lease_signature(lease, &authority) + .map_err(|_| Refusal::NotAuthorized)?; + lease + .check_scope( + &grant, + &capability.grant, + scope.current_session, + now_unix_ms()?, + ) + .map_err(|_| Refusal::NotAuthorized)?; + if payload.len() > capability.grant.max_payload_bytes as usize { + return Err(Refusal::NotAuthorized); + } + if !capability.grant.content_types.contains(content_type) { + return Err(Refusal::NotAuthorized); + } + *pending + .reader + .active + .lock() + .expect("mailbox reader lock poisoned") = Some(ActiveLease { + lease: (**lease).clone(), + ack: None, + }); + } + Ok(response) + } + ReaderRequest::Ack { signed } => correlate_ack(read_response(stream)?, signed), + ReaderRequest::Possession { .. } => Err(Refusal::NotAuthorized), + } +} + +fn correlate_ack( + response: ReaderResponse, + signed: &SignedMailboxDeliveryAck, +) -> Result { + match response { + ReaderResponse::Acknowledged { ack } if *ack == *signed => { + Ok(ReaderResponse::Acknowledged { ack }) + } + refused @ ReaderResponse::Refused { .. } => Ok(refused), + _ => Err(Refusal::NotAuthorized), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mvm_contract::plan::{Nonce, PlanId}; + use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, BootId, CapabilityId, ConsumerSessionId, DeliveryId, + DeliveryLeaseNonce, ExecutionEpoch, MailboxAddress, MailboxDeliveryLease, MailboxId, + MessageId, PrincipalId, TenantId, + }; + use mvm_core::mailbox::reader::sign_delivery_lease; + use mvm_core::mailbox::reader_wire::write_application_request; + use std::sync::OnceLock; + + static SEED: OnceLock> = OnceLock::new(); + + #[test] + fn local_receive_is_explicit_and_an_empty_reply_does_not_sign_an_ack() { + let ack_seed = SEED.get_or_init(|| Zeroizing::new([7; 32])); + let reader = Arc::new(GuestReader { + boot: MailboxReaderBootBinding { + plan_id: PlanId("current".into()), + plan_nonce: Nonce::from_bytes([1; 16]), + mailbox: MailboxId::from_bytes([2; 32]), + authority_key: [3; 32], + execution_epoch: ExecutionEpoch::new(1).unwrap(), + boot: BootId::from_bytes([4; 32]), + guest_identity_key: [5; 32], + ack_public_key: SigningKey::from_bytes(ack_seed).verifying_key().to_bytes(), + }, + ack_seed, + active: Mutex::new(None), + }); + let (mut client, mut server) = UnixStream::pair().unwrap(); + let (sender, pending) = mpsc::sync_channel(1); + let state = reader.clone(); + let worker = std::thread::spawn(move || application_request(&mut server, &sender, &state)); + write_application_request(&mut client, &ApplicationReaderRequest::Receive).unwrap(); + let next = pending.recv_timeout(LOCAL_TIMEOUT).unwrap(); + assert_eq!(next.request, ReaderRequest::Receive); + next.reply.send(ReaderResponse::Empty).unwrap(); + assert_eq!(worker.join().unwrap(), Ok(ReaderResponse::Empty)); + assert!(reader.active.lock().unwrap().is_none(), "no automatic ACK"); + } + + #[test] + fn ack_confirmation_must_echo_the_exact_signed_operation() { + let authority = SigningKey::from_bytes(&[9; 32]); + let guest = SigningKey::from_bytes(&[10; 32]); + let lease = sign_delivery_lease( + MailboxDeliveryLease { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + address: MailboxAddress { + tenant: TenantId::from_bytes([1; 32]), + authorization_domain: AuthorizationDomainId::from_bytes([2; 32]), + mailbox: MailboxId::from_bytes([3; 32]), + recipient: PrincipalId::from_bytes([4; 32]), + }, + sender: PrincipalId::from_bytes([5; 32]), + sender_boot: BootId::from_bytes([6; 32]), + message_id: MessageId::from_bytes([7; 32]), + delivery_id: DeliveryId::from_bytes([8; 32]), + execution_epoch: ExecutionEpoch::new(1).unwrap(), + consumer: PrincipalId::from_bytes([4; 32]), + consumer_session: ConsumerSessionId::from_bytes([11; 32]), + receive_capability_id: CapabilityId::from_bytes([12; 32]), + policy_epoch: 1, + ciphertext_commitment: [13; 32], + issued_at_unix_ms: 100, + expires_at_unix_ms: 200, + lease_nonce: DeliveryLeaseNonce::from_bytes([14; 16]), + }, + &authority, + ) + .unwrap(); + let ack = sign_delivery_ack( + MailboxDeliveryAck { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + lease, + ack_nonce: DeliveryAckNonce::from_bytes([15; 16]), + acknowledged_at_unix_ms: 150, + }, + &guest, + ) + .unwrap(); + let same = ReaderResponse::Acknowledged { + ack: Box::new(ack.clone()), + }; + let mut wire = Vec::new(); + write_response(&mut wire, &same).unwrap(); + assert_eq!(read_response(&mut wire.as_slice()), Ok(same.clone())); + assert_eq!(correlate_ack(same.clone(), &ack), Ok(same)); + let mut changed = ack.ack.clone(); + changed.ack_nonce = DeliveryAckNonce::from_bytes([16; 16]); + let changed = sign_delivery_ack(changed, &guest).unwrap(); + assert_eq!( + correlate_ack( + ReaderResponse::Acknowledged { + ack: Box::new(changed) + }, + &ack, + ), + Err(Refusal::NotAuthorized), + "a delayed success for a different signed ACK cannot confirm this one", + ); + assert_eq!( + correlate_ack(ReaderResponse::Empty, &ack), + Err(Refusal::NotAuthorized) + ); + } +} diff --git a/crates/mvm-agentd/src/bin/syscall-probe.rs b/crates/mvm-agentd/src/bin/syscall-probe.rs index 7e9acba754..1e37c05154 100644 --- a/crates/mvm-agentd/src/bin/syscall-probe.rs +++ b/crates/mvm-agentd/src/bin/syscall-probe.rs @@ -3,7 +3,7 @@ //! Test-only binary. Spawned by `tests/seccomp_apply.rs` under //! `mvm-seccomp-apply --` to verify that tier allowlists //! actually deny what they claim. The optional first argument names the -//! probe: `inet` (the default), `vsock`, or `groups`. +//! probe: `inet` (the default), `unix`, `vsock`, or `groups`. //! //! An optional second argument names a tier the probe installs on itself //! before probing (`syscall-probe groups essential`). Self-confinement is @@ -48,9 +48,12 @@ fn main() { } let family = match probe_name.as_str() { "inet" => libc::AF_INET, + "unix" => libc::AF_UNIX, "vsock" => libc::AF_VSOCK, other => { - eprintln!("syscall-probe: unknown probe {other:?} (expected inet, vsock or groups)"); + eprintln!( + "syscall-probe: unknown probe {other:?} (expected inet, unix, vsock or groups)" + ); std::process::exit(64); } }; @@ -72,6 +75,18 @@ fn main() { /// and skips the confirmation. #[cfg(target_os = "linux")] fn confine_to(tier_name: &str) { + if tier_name == "standard+mailbox" { + if let Err(error) = + mvm_agentd::workload_seccomp::admit_from_cmdline("mvm.seccomp=standard mvm.mailbox=1") + .and_then(|_| { + mvm_agentd::workload_seccomp::install_admitted().map_err(|e| e.to_string()) + }) + { + eprintln!("syscall-probe: install standard+mailbox: {error}"); + std::process::exit(2); + } + return; + } let tier: mvm_core::crypto::seccomp::SeccompTier = match tier_name.parse() { Ok(tier) => tier, Err(error) => { diff --git a/crates/mvm-agentd/src/workload_seccomp.rs b/crates/mvm-agentd/src/workload_seccomp.rs index 599e1ef80d..0acdecde09 100644 --- a/crates/mvm-agentd/src/workload_seccomp.rs +++ b/crates/mvm-agentd/src/workload_seccomp.rs @@ -25,6 +25,9 @@ //! ([`admit_restored`]), before the claim commits and so before any workload //! process starts. A cold boot from a host that predates the token is the one //! launch whose workloads still run unfiltered. +//! A cold plan's exact `mvm.mailbox=1` assertion adds only AF_UNIX socket +//! creation and stream I/O to the admitted tier, so its workload can use its +//! own guest-local reader socket without gaining AF_INET or AF_VSOCK. use mvm_core::crypto::seccomp::SeccompTier; use std::sync::OnceLock; @@ -62,6 +65,7 @@ pub fn tier_from_cmdline(cmdline: &str) -> Result, String> { /// `None` for `unrestricted`, which installs nothing. struct Admitted { tier: SeccompTier, + local_mailbox: bool, #[cfg(target_os = "linux")] program: Option, } @@ -80,28 +84,36 @@ static ADMITTED: OnceLock = OnceLock::new(); /// filter is fixed for the life of the guest; a second activation may repeat /// the tier but not change it. pub fn admit(tier: SeccompTier) -> Result<(), String> { + admit_with_local_mailbox(tier, false) +} + +fn admit_with_local_mailbox(tier: SeccompTier, local_mailbox: bool) -> Result<(), String> { if let Some(admitted) = ADMITTED.get() { - return if admitted.tier == tier { + return if admitted.tier == tier && admitted.local_mailbox == local_mailbox { Ok(()) } else { Err(format!( - "workload seccomp tier is already {}; refusing {tier}", - admitted.tier + "workload seccomp tier is already {} (mailbox={}); refusing {tier} (mailbox={local_mailbox})", + admitted.tier, admitted.local_mailbox )) }; } let admitted = Admitted { tier, + local_mailbox, #[cfg(target_os = "linux")] - program: compile(tier).map_err(|error| format!("compile seccomp tier {tier}: {error}"))?, + program: compile_with_local_mailbox(tier, local_mailbox) + .map_err(|error| format!("compile seccomp tier {tier}: {error}"))?, }; // Activation is single-threaded; losing a race here would mean two // activations at once, which the boot state already refuses. let _ = ADMITTED.set(admitted); match ADMITTED.get() { - Some(admitted) if admitted.tier == tier => Ok(()), + Some(admitted) if admitted.tier == tier && admitted.local_mailbox == local_mailbox => { + Ok(()) + } _ => Err(format!( - "workload seccomp tier changed while admitting {tier}" + "workload seccomp tier changed while admitting {tier} (mailbox={local_mailbox})" )), } } @@ -115,10 +127,21 @@ pub fn admit_from_cmdline(cmdline: &str) -> Result, String> let Some(tier) = tier_from_cmdline(cmdline)? else { return Ok(None); }; - admit(tier)?; + admit_with_local_mailbox(tier, local_mailbox_from_cmdline(cmdline)?)?; Ok(Some(tier)) } +fn local_mailbox_from_cmdline(cmdline: &str) -> Result { + let count = cmdline + .split_whitespace() + .filter(|token| *token == crate::mailbox_outbox::MAILBOX_CMDLINE_TOKEN) + .count(); + if count > 1 { + return Err("mvm.mailbox=1 is named more than once".into()); + } + Ok(count == 1) +} + /// Why a restored child cannot take `tier`, or `None` when it can. /// /// `admitted` is what the guest already holds; `workers_running` says whether @@ -225,7 +248,16 @@ pub fn confine(command: &mut std::process::Command) { /// The program fails to compile. #[cfg(target_os = "linux")] pub fn compile(tier: SeccompTier) -> Result, seccompiler::Error> { + compile_with_local_mailbox(tier, false) +} + +#[cfg(target_os = "linux")] +fn compile_with_local_mailbox( + tier: SeccompTier, + local_mailbox: bool, +) -> Result, seccompiler::Error> { use mvm_core::crypto::seccomp::syscall_table::syscall_number; + use mvm_core::crypto::seccomp::{AF_VSOCK, ArgumentScopedSyscall}; use seccompiler::{SeccompAction, SeccompFilter, SeccompRule}; if tier.is_unrestricted() { @@ -244,6 +276,34 @@ pub fn compile(tier: SeccompTier) -> Result, sec }; rules.insert(nr, argument_rules(scoped)?); } + if local_mailbox { + // The signed mailbox launch token grants this workload access to its + // own guest-local reader socket, not general networking. Standard + // still cannot create AF_INET or AF_VSOCK sockets; vsock retains only + // its explicitly admitted family alongside AF_UNIX. + if tier != SeccompTier::Network { + let allowed: &[u64] = if tier == SeccompTier::Vsock { + &[libc::AF_UNIX as u64, AF_VSOCK] + } else { + &[libc::AF_UNIX as u64] + }; + if let Some(nr) = syscall_number("socket") { + rules.insert( + nr, + argument_rules(&ArgumentScopedSyscall { + name: "socket", + arg_index: 0, + allowed, + })?, + ); + } + } + for syscall in ["connect", "sendto", "recvfrom"] { + if let Some(nr) = syscall_number(syscall) { + rules.entry(nr).or_default(); + } + } + } let filter = SeccompFilter::new( rules, SeccompAction::Errno(libc::EPERM as u32), @@ -336,6 +396,15 @@ mod tests { assert_eq!(tier_from_cmdline("mvm.seccomp_debug=1"), Ok(None)); } + #[test] + fn only_one_exact_signed_mailbox_token_admits_local_socket_access() { + assert_eq!(local_mailbox_from_cmdline("mvm.mailbox=1"), Ok(true)); + for cmdline in ["", "mvm.mailbox=0", "xmvm.mailbox=1"] { + assert_eq!(local_mailbox_from_cmdline(cmdline), Ok(false)); + } + assert!(local_mailbox_from_cmdline("mvm.mailbox=1 mvm.mailbox=1").is_err()); + } + /// The tier activation admits is the tier every later spawn reads, and a /// later activation cannot swap it. One test, because the admitted tier is /// process state. It admits `unrestricted`, which installs nothing, so a @@ -443,6 +512,18 @@ mod tests { assert!(program.len() > unscoped.len()); } + #[cfg(target_os = "linux")] + #[test] + fn mailbox_grant_adds_only_local_socket_access_to_standard() { + let standard = compile(SeccompTier::Standard).unwrap().unwrap(); + let mailbox = compile_with_local_mailbox(SeccompTier::Standard, true) + .unwrap() + .unwrap(); + assert!(mailbox.len() > standard.len()); + assert_eq!(libc::AF_UNIX, 1); + assert_ne!(libc::AF_VSOCK, libc::AF_UNIX); + } + #[cfg(target_os = "linux")] #[test] fn unrestricted_compiles_to_no_program() { diff --git a/crates/mvm-agentd/tests/seccomp_apply.rs b/crates/mvm-agentd/tests/seccomp_apply.rs index a0bd1af1b6..0a676ae5d3 100644 --- a/crates/mvm-agentd/tests/seccomp_apply.rs +++ b/crates/mvm-agentd/tests/seccomp_apply.rs @@ -63,6 +63,14 @@ fn run_groups_self_confined(tier: &str) -> Output { .expect("spawn syscall-probe") } +fn run_self_confined(tier: &str, family: &str) -> Output { + Command::new(probe()) + .arg(family) + .arg(tier) + .output() + .expect("spawn syscall-probe") +} + /// The filter let `socket` through. The probe exits 0 when the socket opened /// and with the errno otherwise; a kernel without the vsock transport loaded /// answers `AF_VSOCK` with `EAFNOSUPPORT`, which is the kernel speaking after @@ -158,6 +166,24 @@ fn standard_tier_denies_an_af_vsock_socket_with_eperm() { ); } +#[test] +fn own_mailbox_grant_adds_af_unix_not_networking_to_standard() { + assert_denied_by_the_filter( + &run_self_confined("standard", "unix"), + "socket(AF_UNIX) without a mailbox grant", + ); + assert_passed_the_filter( + &run_self_confined("standard+mailbox", "unix"), + "socket(AF_UNIX) with this workload's mailbox grant", + ); + for family in ["inet", "vsock"] { + assert_denied_by_the_filter( + &run_self_confined("standard+mailbox", family), + "mailbox grant must not enable external socket families", + ); + } +} + /// A GPU launch that also needs `network` keeps its vsock socket. #[test] fn network_tier_allows_an_af_vsock_socket() { diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 19c1d3dc31..9b75446a8a 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -16,9 +16,9 @@ use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; use ed25519_dalek::{SigningKey, VerifyingKey}; use hmac::{Hmac, Mac, digest::KeyInit}; use mvm_contract::protocol::mailbox::{ - DeliveryLeaseNonce, DeliveryOutcome, DestinationReceipt, MailboxAddress, MailboxDeliveryLease, - MailboxEnvelope, MailboxGrant, MailboxGuestReaderScope, MailboxOperation, ProtocolVersion, - SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, + ContentType, DeliveryLeaseNonce, DeliveryOutcome, DestinationReceipt, MailboxAddress, + MailboxDeliveryLease, MailboxEnvelope, MailboxGrant, MailboxGuestReaderScope, MailboxOperation, + ProtocolVersion, SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, }; use serde::{Deserialize, Serialize}; @@ -44,6 +44,7 @@ const MAX_ANCHOR_BYTES: u64 = 1024; const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; const MAX_RECORDS: usize = 100_000; const MAX_JOURNAL_BYTES: u64 = 1024 * 1024 * 1024; +const MAX_LEASE_RETRIES: u8 = 3; /// Limits are checked both before admission and while reconstructing state. #[derive(Debug, Clone, Copy)] @@ -128,6 +129,10 @@ enum JournalEvent { enrollment: Box, receive_capability: Box, }, + LeasedRetry { + identity: ReplayIdentity, + challenge: [u8; 32], + }, AcknowledgedSigned { identity: ReplayIdentity, ack: Box, @@ -135,7 +140,7 @@ enum JournalEvent { } struct LeaseState { - challenge: [u8; 32], + retry_count: u8, signed: SignedMailboxDeliveryLease, enrollment: SignedMailboxGuestReaderEnrollment, receive_capability: SignedMailboxCapability, @@ -210,6 +215,7 @@ pub struct JournalGuestLease<'a> { } /// One guest-authored ACK of the previously persisted active lease. +#[derive(Clone, Copy)] pub struct JournalGuestAck<'a> { pub receive: JournalReceive<'a>, pub plan: &'a MailboxGrant, @@ -234,6 +240,11 @@ impl LeasedMailboxMessage { pub fn lease(&self) -> &SignedMailboxDeliveryLease { &self.lease } + + #[must_use] + pub fn content_type(&self) -> &ContentType { + &self.message.opened.header().content_type + } } /// A pending, independently authorized message. Debug omits its plaintext. @@ -570,7 +581,7 @@ impl MailboxJournal { return Err(JournalError::Corrupt); } entry.lease = Some(LeaseState { - challenge, + retry_count: 0, signed: *lease, enrollment: *enrollment, receive_capability: *receive_capability, @@ -579,6 +590,25 @@ impl MailboxJournal { return Err(JournalError::Corrupt); } } + JournalEvent::LeasedRetry { + identity, + challenge, + } => { + let active = journal + .seen + .get_mut(&identity) + .filter(|entry| !entry.acknowledged) + .and_then(|entry| entry.lease.as_mut()) + .ok_or(JournalError::Corrupt)?; + active.retry_count = active + .retry_count + .checked_add(1) + .filter(|count| *count <= MAX_LEASE_RETRIES) + .ok_or(JournalError::Corrupt)?; + if challenge == [0; 32] || !journal.used_challenges.insert(challenge) { + return Err(JournalError::Corrupt); + } + } JournalEvent::AcknowledgedSigned { identity, ack } => { let entry = journal .seen @@ -814,15 +844,37 @@ impl MailboxJournal { if let Some(active) = &entry.lease && active.signed.lease.expires_at_unix_ms > now { - if active.challenge != *request.challenge - || active.enrollment != *request.enrollment + if active.enrollment != *request.enrollment || active.receive_capability != *request.receive.capability { return Err(JournalError::Busy); } + if self.used_challenges.contains(request.challenge) { + return Err(JournalError::ConflictingRetry); + } + if active.retry_count >= MAX_LEASE_RETRIES { + return Err(JournalError::Busy); + } + let signed = active.signed.clone(); + // A lost delivery response may be retried by this same enrolled + // guest, but every fresh possession proof gets its own anchored + // challenge event before plaintext is released again. + self.append( + JournalEvent::LeasedRetry { + identity, + challenge: *request.challenge, + }, + anchor, + )?; + self.used_challenges.insert(*request.challenge); + self.seen + .get_mut(&identity) + .and_then(|entry| entry.lease.as_mut()) + .ok_or(JournalError::Corrupt)? + .retry_count += 1; return Ok(Some(LeasedMailboxMessage { message, - lease: active.signed.clone(), + lease: signed, })); } if self.used_challenges.contains(request.challenge) { @@ -887,7 +939,7 @@ impl MailboxJournal { .get_mut(&identity) .ok_or(JournalError::Corrupt)? .lease = Some(LeaseState { - challenge: *request.challenge, + retry_count: 0, signed: signed.clone(), enrollment: request.enrollment.clone(), receive_capability: request.receive.capability.clone(), diff --git a/crates/mvm-core/src/mailbox/reader.rs b/crates/mvm-core/src/mailbox/reader.rs index 685d903d42..8d7dbb0cd2 100644 --- a/crates/mvm-core/src/mailbox/reader.rs +++ b/crates/mvm-core/src/mailbox/reader.rs @@ -116,7 +116,8 @@ impl MailboxReaderBootBinding { .mailbox .context("reader boot requires a mailbox grant")?; anyhow::ensure!( - self.plan_id == plan.plan_id + plan.plan_id == crate::plan::compute_plan_id(plan) + && self.plan_id == plan.plan_id && self.plan_nonce == plan.nonce && self.mailbox == grant.mailbox && self.authority_key == grant.authority_key @@ -323,6 +324,26 @@ pub fn sign_delivery_lease( Ok(signed) } +/// Verify the destination's signature before a guest accepts or acknowledges +/// a lease. Scope checks against the enrolled boot are still required. +pub fn verify_delivery_lease_signature( + signed: &SignedMailboxDeliveryLease, + authority: &VerifyingKey, +) -> Result<(), MailboxError> { + signed.encode()?; + let signature: [u8; 64] = signed + .signature + .as_slice() + .try_into() + .map_err(|_| MailboxError::DeliveryLeaseSignatureInvalid)?; + authority + .verify( + &canonical_bytes(DELIVERY_LEASE_SIGNATURE_DOMAIN, &signed.lease)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| MailboxError::DeliveryLeaseSignatureInvalid) +} + /// Already-authenticated destination facts for one leased ciphertext. /// The runtime must derive these from its verified plan, active reader /// session, current revocation state, and committed journal entry. @@ -382,17 +403,7 @@ pub fn verify_delivery_lease( { return Err(MailboxError::DeliveryLeaseMismatch); } - let signature: [u8; 64] = signed - .signature - .as_slice() - .try_into() - .map_err(|_| MailboxError::DeliveryLeaseSignatureInvalid)?; - authority - .verify( - &canonical_bytes(DELIVERY_LEASE_SIGNATURE_DOMAIN, lease)?, - &Signature::from_bytes(&signature), - ) - .map_err(|_| MailboxError::DeliveryLeaseSignatureInvalid) + verify_delivery_lease_signature(signed, authority) } /// Sign one acknowledgement with the guest's independently enrolled mailbox diff --git a/crates/mvm-core/src/mailbox/reader_wire.rs b/crates/mvm-core/src/mailbox/reader_wire.rs index a91114cc56..087d6da774 100644 --- a/crates/mvm-core/src/mailbox/reader_wire.rs +++ b/crates/mvm-core/src/mailbox/reader_wire.rs @@ -5,13 +5,13 @@ //! Decoding never supplies authority: the destination must verify the signed //! Receive capability, current boot, possession proof, lease and ACK against //! its journal before releasing plaintext or committing an acknowledgement. -//! The guest listener and destination service do not serve these frames yet. +//! The discriminator is inspected only after the outer length has been bounded. use std::io::{Read, Write}; use base64::Engine as _; use mvm_contract::protocol::mailbox::{ - SignedMailboxCapability, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, + ContentType, SignedMailboxCapability, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, }; use serde::{Deserialize, Serialize, de::DeserializeOwned}; @@ -22,6 +22,9 @@ const READER_MAGIC: &[u8; 5] = b"MVMR\x01"; const MAX_REQUEST_BYTES: usize = 16 * 1024; const MAX_RESPONSE_BYTES: usize = 64 * 1024; +/// Guest-local socket for explicit application Receive and ACK-after-processing. +pub const GUEST_READER_SOCKET_PATH: &str = "/run/mvm/mailbox/receive.sock"; + /// One on-demand guest request. `Possession` answers the immediately preceding /// destination challenge on the same connection, not an arbitrary old one. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -36,6 +39,17 @@ pub enum ReaderRequest { }, } +/// The guest application asks for a delivery or explicitly acknowledges one +/// after processing. It never receives the per-boot ACK signing key. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)] +pub enum ApplicationReaderRequest { + Receive, + Ack { + lease: Box, + }, +} + /// Destination response; a delivery is never sent before its lease is durable. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "outcome", rename_all = "snake_case", deny_unknown_fields)] @@ -47,11 +61,16 @@ pub enum ReaderResponse { }, Delivery { lease: Box, + content_type: ContentType, #[serde(with = "payload_base64")] payload: Vec, }, Empty, - Acknowledged, + Acknowledged { + /// The exact guest-signed operation committed and anchored locally. + /// This is correlation, not a destination-signed exportable receipt. + ack: Box, + }, Refused { reason: Refusal, }, @@ -74,6 +93,41 @@ pub fn read_request(stream: &mut impl Read) -> Result { read_frame(stream, MAX_REQUEST_BYTES) } +/// Distinguish the reader plane from an outbox body after bounding the outer frame. +/// The legacy outbox begins with an arbitrary recipient ID, so a four-byte +/// prefix alone would misclassify valid Send operations. Unknown versions with +/// the versioned JSON envelope still fail closed in `decode_request_body`. +#[must_use] +pub fn is_reader_frame(body: &[u8]) -> bool { + body.starts_with(b"MVMR") + && body + .get(READER_MAGIC.len()..) + .is_some_and(|json| json.starts_with(b"{\"version\":")) +} + +/// Decode a request when the shared mailbox listener has already read its length. +pub fn decode_request_body(body: &[u8]) -> Result { + if body.len() > MAX_REQUEST_BYTES { + return Err(Refusal::Oversized(MAX_REQUEST_BYTES)); + } + decode_body(body) +} + +/// Read one bounded, explicit application operation from the guest-local socket. +pub fn read_application_request( + stream: &mut impl Read, +) -> Result { + read_frame(stream, MAX_REQUEST_BYTES) +} + +/// Send an explicit application operation to the guest-local socket. +pub fn write_application_request( + stream: &mut impl Write, + request: &ApplicationReaderRequest, +) -> Result<(), Refusal> { + write_frame(stream, request, MAX_REQUEST_BYTES) +} + /// Write one destination response, including at most a bounded payload. pub fn write_response(stream: &mut impl Write, response: &ReaderResponse) -> Result<(), Refusal> { write_frame(stream, response, MAX_RESPONSE_BYTES) @@ -149,18 +203,16 @@ fn read_frame(stream: &mut impl Read, limit: usize) -> Resu if !(READER_MAGIC.len()..=limit).contains(&len) { return Err(Refusal::Oversized(limit)); } - let mut magic = [0u8; 5]; + let mut body = vec![0; len]; stream - .read_exact(&mut magic) + .read_exact(&mut body) .map_err(|_| Refusal::Malformed)?; - if &magic != READER_MAGIC { - return Err(Refusal::Malformed); - } - let mut json = vec![0; len - magic.len()]; - stream - .read_exact(&mut json) - .map_err(|_| Refusal::Malformed)?; - let frame: Versioned = serde_json::from_slice(&json).map_err(|_| Refusal::Malformed)?; + decode_body(&body) +} + +fn decode_body(body: &[u8]) -> Result { + let json = body.strip_prefix(READER_MAGIC).ok_or(Refusal::Malformed)?; + let frame: Versioned = serde_json::from_slice(json).map_err(|_| Refusal::Malformed)?; if frame.version != 1 { return Err(Refusal::Malformed); } diff --git a/crates/mvm-core/tests/mailbox_reader_wire.rs b/crates/mvm-core/tests/mailbox_reader_wire.rs index 2128e3898a..9209044010 100644 --- a/crates/mvm-core/tests/mailbox_reader_wire.rs +++ b/crates/mvm-core/tests/mailbox_reader_wire.rs @@ -1,6 +1,10 @@ +use mvm_contract::protocol::mailbox::PrincipalId; +use mvm_core::mailbox::frame::OutboxMessage; use mvm_core::mailbox::frame::Refusal; use mvm_core::mailbox::reader_wire::{ - ReaderRequest, ReaderResponse, read_request, read_response, write_request, write_response, + ApplicationReaderRequest, ReaderRequest, ReaderResponse, decode_request_body, is_reader_frame, + read_application_request, read_request, read_response, write_application_request, + write_request, write_response, }; const MAX_REQUEST_BYTES: usize = 16 * 1024; @@ -13,6 +17,8 @@ fn reader_request_is_versioned_and_bounded_before_parsing() { read_request(&mut frame.as_slice()), Ok(ReaderRequest::Receive) ); + assert!(is_reader_frame(&frame[4..])); + assert_eq!(decode_request_body(&frame[4..]), Ok(ReaderRequest::Receive)); let mut bad_version = frame.clone(); let one = bad_version @@ -24,6 +30,11 @@ fn reader_request_is_versioned_and_bounded_before_parsing() { read_request(&mut bad_version.as_slice()), Err(Refusal::Malformed) ); + assert!(is_reader_frame(&bad_version[4..])); + assert_eq!( + decode_request_body(&bad_version[4..]), + Err(Refusal::Malformed) + ); let mut wrong_plane = frame.clone(); wrong_plane[4] = b'X'; @@ -35,6 +46,18 @@ fn reader_request_is_versioned_and_bounded_before_parsing() { read_request(&mut ((MAX_REQUEST_BYTES + 1) as u32).to_be_bytes().as_slice()), Err(Refusal::Oversized(MAX_REQUEST_BYTES)) ); + assert_eq!( + decode_request_body(&vec![0; MAX_REQUEST_BYTES + 1]), + Err(Refusal::Oversized(MAX_REQUEST_BYTES)) + ); + assert!(!is_reader_frame(b"MVMO")); + let mut recipient = [7u8; 32]; + recipient[..5].copy_from_slice(b"MVMR\x01"); + let send = OutboxMessage::new(PrincipalId::from_bytes(recipient), b"send".to_vec()); + assert!( + !is_reader_frame(&send.encode()), + "a legacy Send recipient with the same magic prefix is still Send" + ); } #[test] @@ -90,3 +113,18 @@ fn a_refusal_is_not_an_ack_or_an_empty_mailbox() { Ok(ReaderResponse::Empty) ); } + +#[test] +fn application_receive_is_bounded_and_does_not_carry_an_ack_key() { + let mut frame = Vec::new(); + write_application_request(&mut frame, &ApplicationReaderRequest::Receive).unwrap(); + assert_eq!( + read_application_request(&mut frame.as_slice()), + Ok(ApplicationReaderRequest::Receive) + ); + frame[4] = b'X'; + assert_eq!( + read_application_request(&mut frame.as_slice()), + Err(Refusal::Malformed) + ); +} diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index db9a16d9d3..f730b0d100 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -38,6 +38,8 @@ use crate::mailbox_launch::{DerivedMailboxKeys, LAUNCH_LIMITS, derive_launch_key use crate::mailbox_service::durable::DurableMailboxService; use crate::vm_lifetime::{endpoint_needed, launcher_is_alive}; +mod reader; + /// The helper binary this module spawns. pub const MAILBOX_BIN: &str = "mvm-mailbox"; /// Path-override variable for the helper binary. @@ -118,6 +120,11 @@ fn serve_durable(config: MailboxProcessConfig) -> Result<()> { &anchor, ) .map_err(Error::other)?; + let enrolled = reader_boot + .map(|binding| { + reader::load_or_enroll(&vm_state_dir, &grant, binding, &sealer.capability, &signer) + }) + .transpose()?; let service = DurableMailboxService::bind( &socket, LAUNCH_LIMITS, @@ -126,6 +133,9 @@ fn serve_durable(config: MailboxProcessConfig) -> Result<()> { journal, anchor, )?; + if let Some(enrolled) = enrolled { + service.install_reader(enrolled)?; + } std::io::stdout().write_all(READY_MARKER)?; let mut sealer = sealer; while vm_state_dir.exists() @@ -499,6 +509,12 @@ mod tests { load_reader_boot_for_plan(state.path(), &plan).unwrap(), Some(binding.clone()) ); + let mut tampered = plan.clone(); + tampered.audit_labels.insert("changed".into(), "yes".into()); + assert!( + load_reader_boot_for_plan(state.path(), &tampered).is_err(), + "a changed plan body cannot reuse its old content address" + ); let mut stale = plan.clone(); stale.nonce = mvm_core::plan::Nonce::from_bytes([99; 16]); assert!(load_reader_boot_for_plan(state.path(), &stale).is_err()); diff --git a/crates/mvm-hostd/src/mailbox_process/reader.rs b/crates/mvm-hostd/src/mailbox_process/reader.rs new file mode 100644 index 0000000000..e8f30771b0 --- /dev/null +++ b/crates/mvm-hostd/src/mailbox_process/reader.rs @@ -0,0 +1,289 @@ +//! Persist the exact destination-signed reader authority across helper restarts. + +use std::io::{self, Read as _}; +use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use ed25519_dalek::SigningKey; +use mvm_contract::protocol::mailbox::{ + ConsumerSessionId, MailboxCapabilityGrant, MailboxGrant, MailboxGuestReaderEnrollment, + MailboxGuestReaderScope, ProtocolVersion, SignatureAlgorithm, SignedMailboxCapability, + SignedMailboxGuestReaderEnrollment, +}; +use mvm_core::mailbox::reader::{ + MailboxReaderBootBinding, mint_guest_receive_capability, sign_guest_reader_enrollment, + verify_guest_reader_enrollment, +}; +use mvm_core::mailbox::sign_capability; +use mvm_core::util::atomic_io::atomic_write_new; +use sha2::{Digest as _, Sha256}; + +use crate::mailbox_service::durable::GuestReaderAuthority; + +const READER_FILE: &str = "mailbox-reader-authority.json"; +const MAX_READER_FILE_BYTES: u64 = 64 * 1024; + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct ReaderRecord { + version: u8, + boot: MailboxReaderBootBinding, + send: SignedMailboxCapability, + receive: SignedMailboxCapability, + enrollment: SignedMailboxGuestReaderEnrollment, +} + +fn invalid(reason: impl std::fmt::Display) -> io::Error { + io::Error::new( + io::ErrorKind::PermissionDenied, + format!("mailbox reader authority refused: {reason}"), + ) +} + +fn session_for(binding: &MailboxReaderBootBinding) -> ConsumerSessionId { + let mut digest = Sha256::new(); + digest.update(b"mvm.mailbox.guest-session.v1\0"); + digest.update(binding.boot.as_bytes()); + digest.update(binding.plan_id.0.as_bytes()); + digest.update(binding.plan_nonce.as_hex().as_bytes()); + digest.update(binding.ack_public_key); + ConsumerSessionId::from_bytes(digest.finalize().into()) +} + +fn now_unix_ms() -> io::Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(io::Error::other)? + .as_millis() + .try_into() + .map_err(io::Error::other) +} + +fn read_record(path: &Path) -> io::Result> { + let file = match std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let meta = file.metadata()?; + if !meta.is_file() || meta.nlink() != 1 || meta.len() > MAX_READER_FILE_BYTES { + return Err(invalid("record is not a bounded single-link regular file")); + } + let mut bytes = Vec::new(); + file.take(MAX_READER_FILE_BYTES + 1) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_READER_FILE_BYTES { + return Err(invalid("record exceeds the encoded bound")); + } + serde_json::from_slice(&bytes).map(Some).map_err(invalid) +} + +pub(super) fn load_or_enroll( + state_dir: &Path, + grant: &MailboxGrant, + binding: MailboxReaderBootBinding, + send_grant: &MailboxCapabilityGrant, + signer: &SigningKey, +) -> io::Result { + // Re-read the host's persisted boot identity: stdin alone cannot change + // which cold guest this per-VM helper is authorized to serve. + if MailboxReaderBootBinding::load(state_dir).map_err(invalid)? != binding + || binding.mailbox != grant.mailbox + || binding.execution_epoch != grant.epoch + || binding.authority_key != grant.authority_key + || signer.verifying_key().to_bytes() != grant.authority_key + { + return Err(invalid("boot, signed grant or authority changed")); + } + // The reader wire caps the entire JSON/base64 delivery at 64 KiB. A + // larger granted message could otherwise acquire a durable lease that + // the guest can never receive or ACK. + if send_grant.max_payload_bytes > 32 * 1024 { + return Err(invalid("granted payload exceeds the guest reader frame")); + } + let now = now_unix_ms()?; + let path = state_dir.join(READER_FILE); + let record = match read_record(&path)? { + Some(record) => record, + None => { + let send = sign_capability(send_grant.clone(), signer).map_err(invalid)?; + let receive = + mint_guest_receive_capability(grant, &binding, &send, send_grant.address, signer) + .map_err(invalid)?; + let enrollment = sign_guest_reader_enrollment( + MailboxGuestReaderEnrollment { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + plan_id: binding.plan_id.clone(), + plan_nonce: binding.plan_nonce.clone(), + address: receive.grant.address, + consumer: receive.grant.subject, + consumer_boot: binding.boot, + consumer_session: session_for(&binding), + receive_capability_id: receive.grant.capability_id, + ack_public_key: binding.ack_public_key, + execution_epoch: grant.epoch, + policy_epoch: receive.grant.policy_epoch, + issued_at_unix_ms: now.max(receive.grant.not_before_unix_ms), + expires_at_unix_ms: receive.grant.expires_at_unix_ms, + }, + signer, + ) + .map_err(invalid)?; + let record = ReaderRecord { + version: 1, + boot: binding.clone(), + send, + receive, + enrollment, + }; + let bytes = serde_json::to_vec(&record).map_err(invalid)?; + if bytes.len() as u64 > MAX_READER_FILE_BYTES { + return Err(invalid("record exceeds the encoded bound")); + } + atomic_write_new(&path, &bytes).map_err(io::Error::other)?; + record + } + }; + if record.version != 1 + || record.boot != binding + || record.enrollment.enrollment.consumer_session != session_for(&binding) + { + return Err(invalid("reader belongs to a different boot or session")); + } + // The default sender template has a new time window when the helper is + // restarted. Do not silently mint a new Receive lease: retain the exact + // signed original, while requiring all non-time policy fields to agree. + let mut expected_send = send_grant.clone(); + expected_send.not_before_unix_ms = record.send.grant.not_before_unix_ms; + expected_send.expires_at_unix_ms = record.send.grant.expires_at_unix_ms; + if record.send.grant != expected_send { + return Err(invalid("sender policy changed across helper restart")); + } + let expected_receive = + mint_guest_receive_capability(grant, &binding, &record.send, send_grant.address, signer) + .map_err(invalid)?; + if record.receive != expected_receive + || record.enrollment.enrollment.ack_public_key != binding.ack_public_key + { + return Err(invalid("signed Receive grant or enrolled key changed")); + } + let scope = MailboxGuestReaderScope { + plan: grant, + plan_id: &binding.plan_id, + plan_nonce: &binding.plan_nonce, + receive: &record.receive.grant, + current_boot: binding.boot, + current_session: session_for(&binding), + now_unix_ms: now, + }; + verify_guest_reader_enrollment( + &record.enrollment, + &record.receive, + &scope, + &signer.verifying_key(), + &[], + ) + .map_err(invalid)?; + Ok(GuestReaderAuthority { + grant: *grant, + boot: binding, + capability: record.receive, + enrollment: record.enrollment, + revocations_path: state_dir.join("mailbox-reader-revocations.json"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use ed25519_dalek::SigningKey; + use mvm_contract::protocol::mailbox::{ExecutionEpoch, MailboxId}; + + #[test] + fn helper_restart_reuses_exact_signed_receive_and_enrollment() { + let keys = tempfile::tempdir().unwrap(); + let state = tempfile::tempdir().unwrap(); + crate::audit::host_keypair::load_or_init_at(keys.path()).unwrap(); + let mailbox = MailboxId::from_bytes([42; 32]); + let grant = MailboxGrant { + mailbox, + authority_key: crate::mailbox_launch::authority_key_for(keys.path(), &mailbox) + .unwrap() + .to_bytes(), + epoch: ExecutionEpoch::new(1).unwrap(), + }; + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(grant); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + let ack = SigningKey::from_bytes(&[44; 32]); + let binding = MailboxReaderBootBinding::mint( + &plan, + SigningKey::from_bytes(&[45; 32]).verifying_key(), + ack.verifying_key().to_bytes(), + ) + .unwrap(); + binding.persist(state.path()).unwrap(); + let derive = || crate::mailbox_launch::derive_launch_keys(keys.path(), &grant).unwrap(); + let first_keys = derive(); + let first_send = first_keys.trust().1.capability; + let first = load_or_enroll( + state.path(), + &grant, + binding.clone(), + &first_send, + &first_keys.durable_material().0, + ) + .unwrap(); + let second_keys = derive(); + let second_send = second_keys.trust().1.capability; + let second = load_or_enroll( + state.path(), + &grant, + binding.clone(), + &second_send, + &second_keys.durable_material().0, + ) + .unwrap(); + assert_eq!(first.capability, second.capability); + assert_eq!(first.enrollment, second.enrollment); + assert_ne!( + first.capability.grant.capability_id, + first_send.capability_id + ); + + let path = state.path().join(READER_FILE); + let mut forged: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + forged["receive"]["signature"] = serde_json::json!("AAAAAAAA"); + std::fs::write(&path, serde_json::to_vec(&forged).unwrap()).unwrap(); + assert!( + load_or_enroll( + state.path(), + &grant, + binding.clone(), + &second_send, + &second_keys.durable_material().0, + ) + .is_err(), + "a changed signed Receive grant must fail closed after restart" + ); + let mut other_boot = binding; + other_boot.boot = mvm_contract::protocol::mailbox::BootId::from_bytes([99; 32]); + assert!( + load_or_enroll( + state.path(), + &grant, + other_boot, + &second_send, + &second_keys.durable_material().0, + ) + .is_err() + ); + } +} diff --git a/crates/mvm-hostd/src/mailbox_service.rs b/crates/mvm-hostd/src/mailbox_service.rs index c625d3ed38..4ca60ddea3 100644 --- a/crates/mvm-hostd/src/mailbox_service.rs +++ b/crates/mvm-hostd/src/mailbox_service.rs @@ -648,7 +648,7 @@ fn reply_with(stream: &mut UnixStream, reply: &Result Result, Refusal> { +fn read_frame(max: usize, stream: &mut impl Read) -> Result, Refusal> { let mut len = [0u8; 4]; stream .read_exact(&mut len) diff --git a/crates/mvm-hostd/src/mailbox_service/durable.rs b/crates/mvm-hostd/src/mailbox_service/durable.rs index 59c707a34b..3cb5c61aea 100644 --- a/crates/mvm-hostd/src/mailbox_service/durable.rs +++ b/crates/mvm-hostd/src/mailbox_service/durable.rs @@ -5,22 +5,29 @@ //! authorization, journal fsync, and external anchor publication precede the //! destination signature being returned to the caller. -use std::io::Write; +use std::io::{Read, Write}; +use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; use std::os::unix::net::{UnixListener, UnixStream}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use std::thread::JoinHandle; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use ed25519_dalek::{SigningKey, VerifyingKey}; use mvm_contract::protocol::mailbox::{ - MailboxEnvelope, MailboxEnvelopeHeader, SignedDestinationReceipt, SignedMailboxCapability, + MailboxEnvelope, MailboxEnvelopeHeader, MailboxGrant, MailboxGuestReaderScope, RevocationId, + SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxGuestReaderEnrollment, }; use mvm_core::mailbox::frame::{OutboxMessage, Refusal, decode_frame, send_receipted_frame}; use mvm_core::mailbox::journal::{ - FileJournalAnchor, JournalAdmission, JournalError, JournalReceive, JournalTip, MailboxJournal, - ReceivedMailboxMessage, + FileJournalAnchor, JournalAdmission, JournalError, JournalGuestAck, JournalGuestLease, + JournalReceive, JournalTip, MailboxJournal, ReceivedMailboxMessage, +}; +use mvm_core::mailbox::reader::{MailboxReaderBootBinding, verify_guest_reader_enrollment}; +use mvm_core::mailbox::reader_wire::{ + ReaderRequest, ReaderResponse, decode_request_body, is_reader_frame, read_request, + write_response, }; use mvm_core::mailbox::{MailboxError, OpenContext, ReplayIdentity}; use mvm_core::vm_backend::HostDialRoute; @@ -28,6 +35,44 @@ use mvm_core::vm_backend::HostDialRoute; use super::{GuestSealer, IO_TIMEOUT, MailboxLimits, MailboxPolicy, read_frame}; const ACCEPT_POLL: std::time::Duration = std::time::Duration::from_millis(20); +const GUEST_EXCHANGE_TIMEOUT: Duration = Duration::from_secs(12); + +struct GuestDeadline<'a> { + stream: &'a mut UnixStream, + until: Instant, +} + +impl GuestDeadline<'_> { + fn remaining(&self) -> std::io::Result { + let remaining = self.until.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "mailbox guest exchange deadline", + )) + } else { + Ok(remaining) + } + } +} + +impl Read for GuestDeadline<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.stream.set_read_timeout(Some(self.remaining()?))?; + self.stream.read(buf) + } +} + +impl Write for GuestDeadline<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.stream.set_write_timeout(Some(self.remaining()?))?; + self.stream.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.stream.flush() + } +} /// A trusted anchor advances outside the journal's rollback domain. pub trait JournalAnchor: Send + Sync { @@ -59,6 +104,17 @@ struct Shared { state: Mutex, signer: SigningKey, anchor: Box, + reader: OnceLock, +} + +/// Enrollment pinned to the signed grant and current guest boot by the launcher. +pub(crate) struct GuestReaderAuthority { + pub(crate) grant: MailboxGrant, + pub(crate) boot: MailboxReaderBootBinding, + pub(crate) capability: SignedMailboxCapability, + pub(crate) enrollment: SignedMailboxGuestReaderEnrollment, + /// Re-read at each operation so revocation works without helper restart. + pub(crate) revocations_path: PathBuf, } /// One destination authority's bounded, journal-backed local admission socket. @@ -100,6 +156,7 @@ impl DurableMailboxService { state: Mutex::new(State { policy, journal }), signer, anchor: Box::new(anchor), + reader: OnceLock::new(), }); let stop = Arc::new(AtomicBool::new(false)); let worker = { @@ -133,6 +190,13 @@ impl DurableMailboxService { }) } + /// Install the separately verified reader identity before advertising readiness. + pub(crate) fn install_reader(&self, reader: GuestReaderAuthority) -> std::io::Result<()> { + self.shared.reader.set(reader).map_err(|_| { + std::io::Error::new(std::io::ErrorKind::AlreadyExists, "reader already enrolled") + }) + } + /// Read one guest outbox operation over its dedicated mailbox port. The /// guest sees success only after the canonical ciphertext, dedup identity, /// and anchor have committed; an idle connection returns `false`. @@ -145,10 +209,19 @@ impl DurableMailboxService { route, mvm_contract::protocol::mailbox::MAILBOX_PORT, )?; - stream.set_read_timeout(Some(std::time::Duration::from_secs(7)))?; - stream.set_write_timeout(Some(IO_TIMEOUT))?; + let mut stream = GuestDeadline { + stream: &mut stream, + until: Instant::now() + GUEST_EXCHANGE_TIMEOUT, + }; let body = read_frame(self.shared.limits.max_frame_bytes, &mut stream); let received = body.is_ok(); + let body = match body { + Ok(body) if is_reader_frame(&body) => { + serve_reader(&self.shared, &mut stream, &body); + return Ok(true); + } + other => other, + }; let verdict: Result<(), Refusal> = body .and_then(|body| OutboxMessage::decode(&body)) .and_then(|message| sealer.seal(message)) @@ -210,6 +283,183 @@ impl DurableMailboxService { } } +fn serve_reader(shared: &Shared, stream: &mut (impl Read + Write), body: &[u8]) { + let result = decode_request_body(body).and_then(|request| { + let reader = shared.reader.get().ok_or(Refusal::NotAuthorized)?; + match request { + ReaderRequest::Receive => { + { + let state = shared.state.lock().expect("mailbox journal lock poisoned"); + let context = guest_context(&state.policy, reader)?; + verify_guest_reader_enrollment( + &reader.enrollment, + &reader.capability, + &guest_scope(reader, &context), + &shared.signer.verifying_key(), + &context.revoked_capabilities, + ) + .map_err(|_| Refusal::NotAuthorized)?; + } + let challenge: [u8; 32] = rand::random(); + if challenge == [0; 32] { + return Err(Refusal::Unavailable); + } + write_response( + stream, + &ReaderResponse::Challenge { + enrollment: Box::new(reader.enrollment.clone()), + capability: Box::new(reader.capability.clone()), + challenge, + }, + )?; + let ReaderRequest::Possession { signature } = read_request(stream)? else { + return Err(Refusal::NotAuthorized); + }; + let mut state = shared.state.lock().expect("mailbox journal lock poisoned"); + let context = guest_context(&state.policy, reader)?; + let scope = guest_scope(reader, &context); + let State { policy, journal } = &mut *state; + let leased = journal + .lease_next( + JournalGuestLease { + receive: JournalReceive { + capability: &reader.capability, + context: &context, + encryption_key: &policy.encryption_key, + sender_key: &policy.sender_key, + }, + plan: &reader.grant, + scope, + enrollment: &reader.enrollment, + challenge: &challenge, + possession_signature: &signature, + signer: &shared.signer, + }, + |tip| shared.anchor.commit(tip), + ) + .map_err(reader_refusal)?; + Ok(match leased { + Some(message) => ReaderResponse::Delivery { + lease: Box::new(message.lease().clone()), + content_type: message.content_type().clone(), + payload: message.payload().to_vec(), + }, + None => ReaderResponse::Empty, + }) + } + ReaderRequest::Ack { signed } => { + let mut state = shared.state.lock().expect("mailbox journal lock poisoned"); + let context = guest_context(&state.policy, reader)?; + let scope = guest_scope(reader, &context); + let State { policy, journal } = &mut *state; + journal + .acknowledge_signed( + JournalGuestAck { + receive: JournalReceive { + capability: &reader.capability, + context: &context, + encryption_key: &policy.encryption_key, + sender_key: &policy.sender_key, + }, + plan: &reader.grant, + scope, + enrollment: &reader.enrollment, + signed: &signed, + }, + |tip| shared.anchor.commit(tip), + ) + .map_err(reader_refusal)?; + Ok(ReaderResponse::Acknowledged { ack: signed }) + } + ReaderRequest::Possession { .. } => Err(Refusal::NotAuthorized), + } + }); + let response = result.unwrap_or_else(|reason| ReaderResponse::Refused { reason }); + let _ = write_response(stream, &response); +} + +fn guest_context( + policy: &MailboxPolicy, + reader: &GuestReaderAuthority, +) -> Result { + if policy.context.expected_mailbox != reader.grant.mailbox + || policy.context.current_execution_epoch != reader.grant.epoch + || policy.context.expected_recipient != reader.capability.grant.address.recipient + || policy.authority_key.to_bytes() != reader.grant.authority_key + { + return Err(Refusal::NotAuthorized); + } + let mut context = policy.context.clone(); + context.acting_principal = reader.capability.grant.subject; + context.expected_capability_issuer = reader.capability.grant.issuer; + context.required_operation = mvm_contract::protocol::mailbox::MailboxOperation::Receive; + context.revoked_capabilities = reader_revocations(&reader.revocations_path)?; + context.now_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| Refusal::Unavailable)? + .as_millis() + .try_into() + .map_err(|_| Refusal::Unavailable)?; + Ok(context) +} + +fn reader_revocations(path: &Path) -> Result, Refusal> { + let file = match std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(_) => return Err(Refusal::Unavailable), + }; + let meta = file.metadata().map_err(|_| Refusal::Unavailable)?; + if !meta.is_file() || meta.nlink() != 1 || meta.len() > 4096 { + return Err(Refusal::NotAuthorized); + } + let mut body = Vec::new(); + file.take(4097) + .read_to_end(&mut body) + .map_err(|_| Refusal::Unavailable)?; + if body.len() > 4096 { + return Err(Refusal::NotAuthorized); + } + let revoked: Vec = + serde_json::from_slice(&body).map_err(|_| Refusal::NotAuthorized)?; + if revoked.len() > 32 { + return Err(Refusal::NotAuthorized); + } + Ok(revoked) +} + +fn guest_scope<'a>( + reader: &'a GuestReaderAuthority, + context: &OpenContext, +) -> MailboxGuestReaderScope<'a> { + MailboxGuestReaderScope { + plan: &reader.grant, + plan_id: &reader.boot.plan_id, + plan_nonce: &reader.boot.plan_nonce, + receive: &reader.capability.grant, + current_boot: reader.boot.boot, + current_session: reader.enrollment.enrollment.consumer_session, + now_unix_ms: context.now_unix_ms, + } +} + +fn reader_refusal(error: JournalError) -> Refusal { + match error { + JournalError::Full => Refusal::StorageFull, + JournalError::Mailbox(_) + | JournalError::WrongAuthority + | JournalError::ConflictingRetry + | JournalError::UnknownMessage + | JournalError::NoActiveLease + | JournalError::Busy => Refusal::NotAuthorized, + _ => Refusal::Unavailable, + } +} + fn reader_context( policy: &MailboxPolicy, reader: &OpenContext, @@ -318,3 +568,42 @@ pub fn send( destination_key, ) } + +#[cfg(test)] +mod reader_tests { + use super::*; + + #[test] + fn an_expired_guest_exchange_refuses_more_reads_and_writes() { + let (mut socket, _peer) = UnixStream::pair().unwrap(); + let mut expired = GuestDeadline { + stream: &mut socket, + until: Instant::now() - Duration::from_millis(1), + }; + assert_eq!( + expired.read(&mut [0; 1]).unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); + assert_eq!( + expired.write(&[0; 1]).unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); + } + + #[test] + fn revocations_are_reread_and_corrupt_or_oversized_state_fails_closed() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("revoked.json"); + let revoked = RevocationId::from_bytes([5; 32]); + assert!(reader_revocations(&file).unwrap().is_empty()); + std::fs::write(&file, serde_json::to_vec(&vec![revoked]).unwrap()).unwrap(); + assert_eq!(reader_revocations(&file).unwrap(), vec![revoked]); + std::fs::write(&file, b"not a revocation list").unwrap(); + assert_eq!(reader_revocations(&file), Err(Refusal::NotAuthorized)); + std::fs::write(&file, vec![b' '; 4097]).unwrap(); + assert_eq!(reader_revocations(&file), Err(Refusal::NotAuthorized)); + std::fs::remove_file(&file).unwrap(); + std::os::unix::fs::symlink(dir.path().join("missing"), &file).unwrap(); + assert!(reader_revocations(&file).is_err()); + } +} diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index 3cad888445..e5157d33ff 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1188,6 +1188,346 @@ fn durable_local_destination_signs_only_after_journal_and_anchor_commit() { assert_eq!(anchor.tip(), tip, "retry must not append a record"); } +#[test] +fn signed_guest_lease_and_exact_ack_survive_journal_restart() { + use mvm_contract::plan::{Nonce, PlanId}; + use mvm_contract::protocol::mailbox::{ + ConsumerSessionId, DeliveryAckNonce, MailboxDeliveryAck, MailboxGrant, + MailboxGuestReaderEnrollment, MailboxGuestReaderScope, + }; + use mvm_core::mailbox::journal::{ + FileJournalAnchor, JournalAdmission, JournalError, JournalGuestAck, JournalGuestLease, + JournalLimits, JournalReceive, MailboxJournal, + }; + use mvm_core::mailbox::reader::{ + sign_delivery_ack, sign_guest_reader_enrollment, sign_reader_possession, + }; + + let (node, channel) = channel_with_one_node(); + let root = node.home.path(); + let signer = &channel.authority_key; + let authority = signer.verifying_key(); + let guest = SigningKey::from_bytes(&[78; 32]); + let mut message = header(1, 3); + message.expires_at_unix_ms = 4_000_000_000_000; + let mut route = routing(); + route.expires_at_unix_ms = message.expires_at_unix_ms; + let mut send_grant = grant(&[MailboxOperation::Send]); + send_grant.expires_at_unix_ms = message.expires_at_unix_ms; + let send = sign_capability(send_grant, signer).unwrap(); + let envelope = seal_envelope( + SealRequest { + routing: route, + header: message, + plaintext: b"hello", + }, + &channel.encryption_key, + &channel.sender_key, + ) + .unwrap(); + let plan = MailboxGrant { + mailbox: address().mailbox, + authority_key: authority.to_bytes(), + epoch: MbEpoch::new(1).unwrap(), + }; + let mut reader_grant = grant(&[MailboxOperation::Receive]); + reader_grant.subject = aircraft_principal(); + reader_grant.capability_id = CapabilityId::from_bytes([88; 32]); + reader_grant.minimum_execution_epoch = plan.epoch; + reader_grant.maximum_execution_epoch = plan.epoch; + reader_grant.expires_at_unix_ms = 4_000_000_000_000; + let receive = sign_capability(reader_grant, signer).unwrap(); + let plan_id = PlanId("reader-plan".into()); + let plan_nonce = Nonce::from_bytes([79; 16]); + let boot = BootId::from_bytes([80; 32]); + let session = ConsumerSessionId::from_bytes([81; 32]); + let enrollment = sign_guest_reader_enrollment( + MailboxGuestReaderEnrollment { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + plan_id: plan_id.clone(), + plan_nonce: plan_nonce.clone(), + address: address(), + consumer: aircraft_principal(), + consumer_boot: boot, + consumer_session: session, + receive_capability_id: receive.grant.capability_id, + ack_public_key: guest.verifying_key().to_bytes(), + execution_epoch: plan.epoch, + policy_epoch: receive.grant.policy_epoch, + issued_at_unix_ms: 1_100, + expires_at_unix_ms: 4_000_000_000_000, + }, + signer, + ) + .unwrap(); + let mut reader = context(1); + reader.acting_principal = aircraft_principal(); + reader.required_operation = MailboxOperation::Receive; + let scope = MailboxGuestReaderScope { + plan: &plan, + plan_id: &plan_id, + plan_nonce: &plan_nonce, + receive: &receive.grant, + current_boot: boot, + current_session: session, + now_unix_ms: reader.now_unix_ms, + }; + let read = JournalReceive { + capability: &receive, + context: &reader, + encryption_key: &channel.encryption_key, + sender_key: &channel.sender_key.verifying_key(), + }; + let anchor_path = root.join("anchor"); + let journal_path = root.join("journal"); + let limits = JournalLimits { + max_records: 6, + max_bytes: 64 * 1024, + }; + let open = || { + let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]).unwrap(); + let journal = MailboxJournal::open_reconciling( + &journal_path, + address(), + authority, + [90; 32], + limits, + &anchor, + ) + .unwrap(); + (anchor, journal) + }; + let (anchor, mut journal) = open(); + journal + .admit( + JournalAdmission { + envelope: &envelope, + capability: &send, + context: &context(1), + encryption_key: &channel.encryption_key, + sender_key: &channel.sender_key.verifying_key(), + signer, + }, + |tip| anchor.advance(tip), + ) + .unwrap(); + let challenge = [82; 32]; + let proof = sign_reader_possession(&enrollment, &challenge, &guest).unwrap(); + let mut wrong_tenant = reader.clone(); + wrong_tenant.expected_tenant = TenantId::from_bytes([98; 32]); + assert!( + journal + .lease_next( + JournalGuestLease { + receive: JournalReceive { + context: &wrong_tenant, + ..read + }, + plan: &plan, + scope, + enrollment: &enrollment, + challenge: &challenge, + possession_signature: &proof, + signer, + }, + |tip| anchor.advance(tip), + ) + .is_err() + ); + let mut revoked = reader.clone(); + revoked + .revoked_capabilities + .push(receive.grant.revocation_id); + assert!( + journal + .lease_next( + JournalGuestLease { + receive: JournalReceive { + context: &revoked, + ..read + }, + plan: &plan, + scope, + enrollment: &enrollment, + challenge: &challenge, + possession_signature: &proof, + signer, + }, + |tip| anchor.advance(tip), + ) + .is_err() + ); + assert_eq!(anchor.tip().cursor, 1, "refusals did not lease plaintext"); + let leased = journal + .lease_next( + JournalGuestLease { + receive: read, + plan: &plan, + scope, + enrollment: &enrollment, + challenge: &challenge, + possession_signature: &proof, + signer, + }, + |tip| anchor.advance(tip), + ) + .unwrap() + .unwrap(); + assert_eq!(leased.payload(), b"hello"); + assert_eq!(leased.content_type(), &content_type()); + drop(journal); + drop(anchor); + let (anchor, mut journal) = open(); + assert_eq!( + anchor.tip().cursor, + 2, + "signed lease survives helper restart" + ); + let retry_challenge = [86; 32]; + let retry_proof = sign_reader_possession(&enrollment, &retry_challenge, &guest).unwrap(); + let retried = journal + .lease_next( + JournalGuestLease { + receive: read, + plan: &plan, + scope, + enrollment: &enrollment, + challenge: &retry_challenge, + possession_signature: &retry_proof, + signer, + }, + |tip| anchor.advance(tip), + ) + .unwrap() + .unwrap(); + assert_eq!( + retried.lease(), + leased.lease(), + "lost delivery keeps its lease" + ); + assert_eq!(anchor.tip().cursor, 3, "fresh challenge is durable first"); + assert!(matches!( + journal.lease_next( + JournalGuestLease { + receive: read, + plan: &plan, + scope, + enrollment: &enrollment, + challenge: &retry_challenge, + possession_signature: &retry_proof, + signer, + }, + |tip| anchor.advance(tip), + ), + Err(JournalError::ConflictingRetry) + )); + for n in [87, 88] { + let next = [n; 32]; + let proof = sign_reader_possession(&enrollment, &next, &guest).unwrap(); + let delivered = journal + .lease_next( + JournalGuestLease { + receive: read, + plan: &plan, + scope, + enrollment: &enrollment, + challenge: &next, + possession_signature: &proof, + signer, + }, + |tip| anchor.advance(tip), + ) + .unwrap() + .unwrap(); + assert_eq!(delivered.lease(), leased.lease()); + } + let exhausted = [89; 32]; + let proof = sign_reader_possession(&enrollment, &exhausted, &guest).unwrap(); + assert!(matches!( + journal.lease_next( + JournalGuestLease { + receive: read, + plan: &plan, + scope, + enrollment: &enrollment, + challenge: &exhausted, + possession_signature: &proof, + signer, + }, + |tip| anchor.advance(tip), + ), + Err(JournalError::Busy) + )); + assert_eq!(anchor.tip().cursor, 5, "retry exhaustion does not append"); + let ack = sign_delivery_ack( + MailboxDeliveryAck { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + lease: leased.lease().clone(), + ack_nonce: DeliveryAckNonce::from_bytes([83; 16]), + acknowledged_at_unix_ms: reader.now_unix_ms, + }, + &guest, + ) + .unwrap(); + let request = JournalGuestAck { + receive: read, + plan: &plan, + scope, + enrollment: &enrollment, + signed: &ack, + }; + let wrong_boot = JournalGuestAck { + scope: MailboxGuestReaderScope { + current_boot: BootId::from_bytes([84; 32]), + ..scope + }, + ..request + }; + assert!( + journal + .acknowledge_signed(wrong_boot, |tip| anchor.advance(tip)) + .is_err() + ); + let tip = journal + .acknowledge_signed(request, |tip| anchor.advance(tip)) + .unwrap(); + assert_eq!(tip.cursor, 6, "admission, leases and ACK committed"); + assert_eq!( + journal + .acknowledge_signed(request, |tip| anchor.advance(tip)) + .unwrap(), + tip, + "a lost ACK response retries the exact bytes without a second commit" + ); + let mut changed = ack.clone(); + changed.ack.ack_nonce = DeliveryAckNonce::from_bytes([85; 16]); + let changed = sign_delivery_ack(changed.ack, &guest).unwrap(); + assert!(matches!( + journal.acknowledge_signed( + JournalGuestAck { + signed: &changed, + ..request + }, + |tip| anchor.advance(tip), + ), + Err(JournalError::ConflictingRetry) + )); + drop(journal); + drop(anchor); + let (anchor, mut journal) = open(); + assert_eq!(anchor.tip(), tip); + assert!(journal.receive_next(read).unwrap().is_none()); + assert_eq!( + journal + .acknowledge_signed(request, |tip| anchor.advance(tip)) + .unwrap(), + tip, + "the exact ACK survives helper restart" + ); +} + #[test] fn durable_mailbox_helper_process() { let Ok(root) = std::env::var("MVM_MAILBOX_TEST_ROOT") else { From 3d3d8600ad3bacd173bc0d8051d16e1cee037cfb Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 22:52:57 -0800 Subject: [PATCH 09/51] fix(mailbox): gate durable verdict behind independent authority --- crates/mvm-agentd/README.md | 12 +- crates/mvm-cli/src/commands/vm/exec.rs | 1 + crates/mvm-cli/src/exec.rs | 7 + crates/mvm-cli/src/exec/transient.rs | 13 +- crates/mvm-client/src/admission/mod.rs | 1 + crates/mvm-client/src/entrypoint/admission.rs | 1 + crates/mvm-client/src/entrypoint/boot.rs | 3 + crates/mvm-client/src/launch/persistent.rs | 16 +- crates/mvm-hostd/src/mailbox_launch.rs | 1 + crates/mvm-hostd/src/mailbox_process.rs | 392 +++++++++++++++--- crates/mvm-hostd/src/mailbox_service.rs | 33 ++ .../mvm-hostd/src/mailbox_service/durable.rs | 7 - crates/mvm-hostd/src/plan_admission.rs | 124 +++++- 13 files changed, 533 insertions(+), 78 deletions(-) diff --git a/crates/mvm-agentd/README.md b/crates/mvm-agentd/README.md index d1cd4b1e13..ba44a8332c 100644 --- a/crates/mvm-agentd/README.md +++ b/crates/mvm-agentd/README.md @@ -40,11 +40,13 @@ ACK signing key inside the guest agent, not in the workload. ## Guest mailbox reader -Only a signed plan granting this workload's own mailbox enables Receive. The -agent exposes `/run/mvm/mailbox/receive.sock` to that workload when its separate -boot ACK key was provisioned. The local protocol is versioned, length-prefixed, -and bounded; use `ApplicationReaderRequest` and `ReaderResponse` from -`mvm_core::mailbox::reader_wire`. `Receive` returns a destination-signed lease, +Only a signed plan granting this workload's own mailbox can enable Receive. +The agent binds `/run/mvm/mailbox/receive.sock` when its separate boot ACK key +was provisioned, but the current Dev host helper refuses reader frames until +independent journal authority and key custody are deployed. The local protocol +is versioned, length-prefixed, and bounded; use `ApplicationReaderRequest` and +`ReaderResponse` from `mvm_core::mailbox::reader_wire`. Once backed by an +independent destination, `Receive` returns a signed lease, content type, and payload. Process the payload **before** sending `Ack { lease }` on a new connection. An empty inbox and a refusal are not successful delivery; reading never automatically ACKs. diff --git a/crates/mvm-cli/src/commands/vm/exec.rs b/crates/mvm-cli/src/commands/vm/exec.rs index 1c21d7126a..6667b37fa1 100644 --- a/crates/mvm-cli/src/commands/vm/exec.rs +++ b/crates/mvm-cli/src/commands/vm/exec.rs @@ -877,6 +877,7 @@ pub(in crate::commands) fn run_secure_started( let substrate = crate::exec::SessionAuditSubstrate { tenant_id: c.admitted.plan().tenant.0.clone(), plan_json, + mailbox_posture: c.admitted.mailbox_posture(), built_pack_check: c.admitted.built_pack_boot_check()?, caller_registration: None, bundle_json, diff --git a/crates/mvm-cli/src/exec.rs b/crates/mvm-cli/src/exec.rs index f340c2d31a..3bc535946c 100644 --- a/crates/mvm-cli/src/exec.rs +++ b/crates/mvm-cli/src/exec.rs @@ -623,6 +623,7 @@ fn run_inner( let ResolvedLaunch { backend, start_config, + mailbox_posture, use_snapshot, root_strategy, image: resolved, @@ -656,6 +657,7 @@ fn run_inner( let mut boot_attempt = BootAttempt { backend: &backend, start_config: &start_config, + mailbox_posture: mailbox_posture.as_ref(), resolved: &resolved, user_named: req.name.is_some(), writable_images: &mut disk_images, @@ -1203,6 +1205,8 @@ pub struct ResolvedLaunch { /// which backend's capabilities shaped the config. pub backend: AnyBackend, pub start_config: VmStartConfig, + /// Only admission may supply Dev posture for an opted-in mailbox. + pub mailbox_posture: Option, /// Whether snapshot restore survives admission (an admitted workload /// always cold-boots). pub use_snapshot: bool, @@ -1298,6 +1302,7 @@ pub fn resolve_launch( ); let mut use_snapshot = boot.use_snapshot; let mut built_pack_check = None; + let mut mailbox_posture = None; // Attach everything the boot waits on before admitting. Admission starts // the plan's validity window and the verb grant minted from it expires @@ -1362,6 +1367,7 @@ pub fn resolve_launch( { let mut sub = sub; built_pack_check = sub.built_pack_check.take(); + mailbox_posture = sub.mailbox_posture.take(); let read_only_materialized_images = sub.apply_to(&mut start_config); for volume in &mut start_config.volumes { if read_only_materialized_images.contains(&volume.host) @@ -1430,6 +1436,7 @@ pub fn resolve_launch( Ok(ResolvedLaunch { backend, start_config, + mailbox_posture, use_snapshot, root_strategy: boot.root_strategy, image, diff --git a/crates/mvm-cli/src/exec/transient.rs b/crates/mvm-cli/src/exec/transient.rs index 119101775c..76ae9cbc54 100644 --- a/crates/mvm-cli/src/exec/transient.rs +++ b/crates/mvm-cli/src/exec/transient.rs @@ -13,6 +13,7 @@ use super::*; pub(super) struct BootAttempt<'a> { pub(super) backend: &'a AnyBackend, pub(super) start_config: &'a VmStartConfig, + pub(super) mailbox_posture: Option<&'a mvm_hostd::plan_admission::MailboxLaunchPosture>, pub(super) resolved: &'a ResolvedImage, /// The caller named this VM. A named run has an identity a shared standby /// cannot take on, so it keeps its cold path instead of requiring a claim. @@ -164,7 +165,11 @@ pub(super) fn boot_transient_vm( }; if !booted { - mvm_hostd::mailbox_process::require_servable(attempt.start_config, attempt.backend)?; + mvm_hostd::mailbox_process::require_servable( + attempt.start_config, + attempt.backend, + attempt.mailbox_posture, + )?; ui::info(&format!("Booting transient VM '{vm_name}'...")); if mvm_runtime::catalog::descriptor(attempt.backend.kind()).is_workload && let Err(error) = @@ -194,6 +199,7 @@ pub(super) fn boot_transient_vm( attempt.backend, &VmId(vm_name.clone()), attempt.start_config, + attempt.mailbox_posture, ) { drop(phase); remove_transient_state_dir(&mvm_core::config::vm_state_dir(&vm_name).to_string_lossy()); @@ -702,6 +708,7 @@ mod tests { let mut attempt = BootAttempt { backend: &backend, start_config: &start_config, + mailbox_posture: None, resolved: &resolved, user_named: true, writable_images: &mut images, @@ -768,6 +775,7 @@ mod tests { let mut attempt = BootAttempt { backend: &backend, start_config: &start_config, + mailbox_posture: None, resolved: &resolved, user_named: true, writable_images: &mut images, @@ -806,6 +814,7 @@ mod tests { let mut attempt = BootAttempt { backend: &backend, start_config: &start_config, + mailbox_posture: None, resolved: &resolved, user_named: false, writable_images: &mut images, @@ -827,6 +836,7 @@ mod tests { let mut attempt = BootAttempt { backend: &backend, start_config: &start_config, + mailbox_posture: None, resolved: &resolved, user_named: false, writable_images: &mut images, @@ -926,6 +936,7 @@ mod tests { let mut attempt = BootAttempt { backend: &backend, start_config: &start_config, + mailbox_posture: None, resolved: &resolved, user_named: false, writable_images: &mut images, diff --git a/crates/mvm-client/src/admission/mod.rs b/crates/mvm-client/src/admission/mod.rs index 85fb518894..2307aeb1ca 100644 --- a/crates/mvm-client/src/admission/mod.rs +++ b/crates/mvm-client/src/admission/mod.rs @@ -1505,6 +1505,7 @@ pub(crate) mod admit_plan_tests { let mut substrate = crate::entrypoint::SessionAuditSubstrate { tenant_id: admitted.admitted.plan().tenant.0.clone(), plan_json: serde_json::to_string(admitted.admitted.signed()).unwrap(), + mailbox_posture: admitted.admitted.mailbox_posture(), caller_registration: None, bundle_json: None, config_files: Vec::new(), diff --git a/crates/mvm-client/src/entrypoint/admission.rs b/crates/mvm-client/src/entrypoint/admission.rs index 85d933bd5e..fc141a23c7 100644 --- a/crates/mvm-client/src/entrypoint/admission.rs +++ b/crates/mvm-client/src/entrypoint/admission.rs @@ -254,6 +254,7 @@ impl EntrypointAdmission { substrate: SessionAuditSubstrate { tenant_id: ctx.admitted.plan().tenant.0.clone(), plan_json, + mailbox_posture: ctx.admitted.mailbox_posture(), caller_registration, bundle_json, built_pack_check: ctx.admitted.built_pack_boot_check()?, diff --git a/crates/mvm-client/src/entrypoint/boot.rs b/crates/mvm-client/src/entrypoint/boot.rs index 97a54feb4f..927b943636 100644 --- a/crates/mvm-client/src/entrypoint/boot.rs +++ b/crates/mvm-client/src/entrypoint/boot.rs @@ -438,6 +438,8 @@ pub struct SessionAuditSubstrate { pub tenant_id: String, /// The signed plan, serialized. pub plan_json: String, + /// Verified launch posture retained outside caller-authored plan JSON. + pub mailbox_posture: Option, /// Trusted admission's optional caller registration, never a producer ACK. pub caller_registration: Option, /// The tenant policy bundle the plan resolved, serialized, when it has one. @@ -961,6 +963,7 @@ mod tests { let mut substrate = SessionAuditSubstrate { tenant_id: "tenant".into(), plan_json: "signed-plan".into(), + mailbox_posture: None, caller_registration: None, bundle_json: Some("policy-bundle".into()), config_files: Vec::new(), diff --git a/crates/mvm-client/src/launch/persistent.rs b/crates/mvm-client/src/launch/persistent.rs index 33ddb535bd..9ece9fdebb 100644 --- a/crates/mvm-client/src/launch/persistent.rs +++ b/crates/mvm-client/src/launch/persistent.rs @@ -488,13 +488,18 @@ fn finish_legacy_persistent( name: &str, backend_name: &str, ) -> Result { + let mailbox_posture = admission.admitted.mailbox_posture(); // VMM selection + workload-support check + start move behind the facade; the // admission gate (above) and the launched/failed emits stay here. #[cfg(unix)] let started = match (|| -> Result<_> { let mut images = prepared_volumes.take_writable_images(); complete_writable_images(&start_config, &mut images)?; - mvm_hostd::mailbox_process::require_servable(&start_config, &backend)?; + mvm_hostd::mailbox_process::require_servable( + &start_config, + &backend, + mailbox_posture.as_ref(), + )?; Ok(crate::boot::start_prepared_custodied( backend, &start_config, @@ -519,9 +524,12 @@ fn finish_legacy_persistent( }; // The guest asserts a mailbox exactly when the plan grants one, so the // host side starts here or the launch is rolled back. - if let Err(error) = - mvm_hostd::mailbox_process::serve_started(started.backend(), started.vm_id(), &start_config) - { + if let Err(error) = mvm_hostd::mailbox_process::serve_started( + started.backend(), + started.vm_id(), + &start_config, + mailbox_posture.as_ref(), + ) { emit_failed(&admission, "mailbox-process", &error); if let Err(cleanup_error) = crate::local::remove_stopped_runtime_state(name) { tracing::warn!(error = %cleanup_error, machine = name, "removing runtime state after a refused mailbox failed"); diff --git a/crates/mvm-hostd/src/mailbox_launch.rs b/crates/mvm-hostd/src/mailbox_launch.rs index f0dc1237aa..8a7d431a36 100644 --- a/crates/mvm-hostd/src/mailbox_launch.rs +++ b/crates/mvm-hostd/src/mailbox_launch.rs @@ -156,6 +156,7 @@ impl Drop for DerivedMailboxKeys { impl DerivedMailboxKeys { /// Mailbox-scoped destination signer and separate journal/anchor MAC keys. + #[cfg(test)] pub(crate) fn durable_material(&self) -> (SigningKey, [u8; 32], [u8; 32]) { ( SigningKey::from_bytes(&self.authority), diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index f730b0d100..baba30c38b 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -18,6 +18,11 @@ //! bridges it to a per-port socket, or reaches the guest through its vsock mux, //! declares the capability. A grant on one that does not refuses the launch //! rather than boot a guest that asserts a mailbox nobody serves. +//! +//! Until an independent checkpoint and persistent history-key provider exist, +//! only admitted Dev launches serve the ephemeral Send queue. Production +//! mailbox launches refuse before boot, and reader frames fail closed. +//! The local file anchor never authorizes a signed `AcceptedDurable` claim. use std::io::{Error, ErrorKind, Result, Write as _}; use std::os::unix::process::CommandExt as _; @@ -27,17 +32,18 @@ use std::sync::mpsc::sync_channel; use std::time::Duration; use mvm_contract::protocol::mailbox::{MAILBOX_PORT, MailboxGrant}; -use mvm_core::mailbox::journal::{FileJournalAnchor, JournalLimits, MailboxJournal}; -use mvm_core::plan::ExecutionPlan; +use mvm_core::plan::{ExecutionPlan, Variant}; use mvm_core::vm_backend::{HostDialRoute, VmId, VmStartConfig}; use mvm_runtime::backend::AnyBackend; use mvm_vmm::host::aux_bin::AuxBin; use mvm_vmm::host::flowmux_identity::{MAILBOX_READER_BOOT_FILE, MailboxReaderBootBinding}; use crate::mailbox_launch::{DerivedMailboxKeys, LAUNCH_LIMITS, derive_launch_keys}; -use crate::mailbox_service::durable::DurableMailboxService; +use crate::mailbox_service::MailboxService; +use crate::plan_admission::{MailboxLaunchPosture, MailboxPostureHandoff}; use crate::vm_lifetime::{endpoint_needed, launcher_is_alive}; +#[cfg(test)] mod reader; /// The helper binary this module spawns. @@ -49,6 +55,7 @@ pub use mvm_vmm::host::vm_helpers::MAILBOX_PID_FILE; const POLL_INTERVAL: Duration = Duration::from_millis(500); const READY_TIMEOUT: Duration = Duration::from_secs(10); const READY_MARKER: &[u8; 6] = b"READY\n"; +const MAX_HELPER_PLAN_BYTES: u64 = 1024 * 1024; /// What the process reads on stdin. `deny_unknown_fields`, so a field a newer /// launcher adds fails closed here. @@ -59,6 +66,8 @@ pub struct MailboxProcessConfig { pub keys: DerivedMailboxKeys, pub socket: PathBuf, pub guest_route: HostDialRoute, + /// Absent on an old or unauthenticated restart: fail closed, never default Dev. + pub posture: Option, /// Public ACK key and boot identity pinned to the admitted cold plan. /// Absent on a warm child until it rekeys and enrolls. pub reader_boot: Option, @@ -71,9 +80,39 @@ pub fn parse(raw: &[u8]) -> serde_json::Result { serde_json::from_slice(raw) } +fn load_admitted_plan(state_dir: &Path) -> Result { + use std::io::Read as _; + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(state_dir.join("plan.json"))?; + let meta = file.metadata()?; + if !meta.is_file() || meta.nlink() != 1 || meta.len() > MAX_HELPER_PLAN_BYTES { + return Err(Error::new( + ErrorKind::InvalidData, + "mailbox helper plan is not a bounded single-link regular file", + )); + } + let mut bytes = Vec::new(); + file.take(MAX_HELPER_PLAN_BYTES + 1) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_HELPER_PLAN_BYTES { + return Err(Error::new( + ErrorKind::InvalidData, + "mailbox helper plan is oversized", + )); + } + mvm_core::plan::plan_from_admitted_json( + std::str::from_utf8(&bytes).map_err(|error| Error::new(ErrorKind::InvalidData, error))?, + ) + .map_err(|error| Error::new(ErrorKind::InvalidData, error)) +} + /// Serve until neither the launcher nor the VM is left; the exit code. pub fn serve(config: MailboxProcessConfig) -> i32 { - match serve_durable(config) { + match serve_ephemeral(config) { Ok(()) => 0, Err(error) => { tracing::error!(%error, "mailbox service failed closed"); @@ -82,7 +121,7 @@ pub fn serve(config: MailboxProcessConfig) -> i32 { } } -fn serve_durable(config: MailboxProcessConfig) -> Result<()> { +fn serve_ephemeral(config: MailboxProcessConfig) -> Result<()> { // SAFETY: `getppid` is async-signal-safe and infallible. let launcher = unsafe { libc::getppid() }; let MailboxProcessConfig { @@ -90,58 +129,52 @@ fn serve_durable(config: MailboxProcessConfig) -> Result<()> { keys, socket, guest_route, + posture, reader_boot, vm_state_dir, } = config; - if let Some(binding) = &reader_boot - && (binding.mailbox != grant.mailbox || binding.execution_epoch != grant.epoch) - { + let posture = posture.ok_or_else(|| { + Error::new( + ErrorKind::PermissionDenied, + "mailbox helper has no admitted posture", + ) + })?; + let plan = load_admitted_plan(&vm_state_dir)?; + let vm_name = vm_state_dir + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| Error::new(ErrorKind::InvalidData, "mailbox VM identity is invalid"))?; + if reader_boot != load_reader_boot_for_plan(&vm_state_dir, &plan)? { return Err(Error::new( ErrorKind::PermissionDenied, - "mailbox reader boot does not match the helper's signed grant", + "mailbox reader boot changed since the admitted helper handoff", )); } - let (policy, mut sealer) = keys.trust(); - let (signer, journal_key, anchor_key) = keys.durable_material(); - sealer.intent_dir = Some(vm_state_dir.join("mailbox-intents")); - let sealer = sealer.with_grant(&grant)?; - let address = sealer.capability.address; - let anchor = FileJournalAnchor::open(&vm_state_dir.join("mailbox-anchor"), address, anchor_key) + posture + .verify( + &plan, + vm_name, + reader_boot.as_ref().map(|binding| binding.boot), + ) .map_err(Error::other)?; - let journal = MailboxJournal::open_reconciling( - &vm_state_dir.join("mailbox-journal"), - address, - signer.verifying_key(), - journal_key, - JournalLimits { - max_records: 64, - max_bytes: 4 * 1024 * 1024, - }, - &anchor, - ) - .map_err(Error::other)?; - let enrolled = reader_boot - .map(|binding| { - reader::load_or_enroll(&vm_state_dir, &grant, binding, &sealer.capability, &signer) - }) - .transpose()?; - let service = DurableMailboxService::bind( - &socket, - LAUNCH_LIMITS, - policy.with_grant(&grant)?, - signer, - journal, - anchor, - )?; - if let Some(enrolled) = enrolled { - service.install_reader(enrolled)?; + if posture.variant() != Variant::Dev || plan.mailbox != Some(grant) { + return Err(Error::new( + ErrorKind::PermissionDenied, + "production mailbox requires an independent checkpoint and key custody", + )); } + let (policy, sealer) = keys.trust(); + let sealer = sealer.with_grant(&grant)?; + let service = MailboxService::bind_live(&socket, LAUNCH_LIMITS, policy.with_grant(&grant)?)?; std::io::stdout().write_all(READY_MARKER)?; let mut sealer = sealer; while vm_state_dir.exists() && endpoint_needed(launcher_is_alive(launcher), vm_is_alive(&vm_state_dir)) { - if service.serve_guest(&guest_route, &mut sealer).is_err() { + if service + .serve_guest_route(&guest_route, &mut sealer) + .is_err() + { std::thread::sleep(POLL_INTERVAL); } } @@ -172,6 +205,7 @@ pub fn build_config( "backend cannot dial the guest mailbox port", ) })?, + posture: None, reader_boot: None, vm_state_dir: state_dir.to_path_buf(), }) @@ -274,9 +308,29 @@ fn granted_plan(config: &VmStartConfig) -> Option { /// Refuse, before the VM boots, a config whose guest would be told to serve a /// mailbox this backend cannot. -pub fn require_servable(config: &VmStartConfig, backend: &AnyBackend) -> anyhow::Result<()> { +pub fn require_servable( + config: &VmStartConfig, + backend: &AnyBackend, + posture: Option<&MailboxLaunchPosture>, +) -> anyhow::Result<()> { match granted_plan(config) { - Some(plan) => Ok(require_wired(&plan, backend)?), + Some(plan) => { + let posture = posture.ok_or_else(|| { + Error::new( + ErrorKind::PermissionDenied, + "mailbox launch has no admitted posture", + ) + })?; + posture.verify(&plan, &config.name)?; + if posture.variant() != Variant::Dev { + return Err(Error::new( + ErrorKind::PermissionDenied, + "production mailbox needs an independent checkpoint and persistent key custody", + ) + .into()); + } + Ok(require_wired(&plan, backend)?) + } None => Ok(()), } } @@ -289,26 +343,36 @@ pub fn serve_started( backend: &AnyBackend, id: &VmId, config: &VmStartConfig, + posture: Option<&MailboxLaunchPosture>, ) -> anyhow::Result<()> { - serve_started_with(backend, id, config, spawn_for_launch) + serve_started_with(backend, id, config, posture, spawn_for_launch) } fn serve_started_with( backend: &AnyBackend, id: &VmId, config: &VmStartConfig, - spawn: impl FnOnce(&ExecutionPlan, &AnyBackend, &str) -> Result>, + posture: Option<&MailboxLaunchPosture>, + spawn: impl FnOnce( + &ExecutionPlan, + &AnyBackend, + &str, + Option<&MailboxLaunchPosture>, + ) -> Result>, ) -> anyhow::Result<()> { let Some(plan) = granted_plan(config) else { return Ok(()); }; - if let Err(error) = spawn(&plan, backend, &id.0) { + let outcome = (|| -> anyhow::Result<()> { + require_servable(config, backend, posture)?; + spawn(&plan, backend, &id.0, posture)?; + Ok(()) + })(); + if let Err(error) = outcome { if let Err(stop) = backend.stop(id) { tracing::warn!(vm = %id.0, error = %stop, "stopping a VM whose mailbox could not be served failed"); } - return Err( - anyhow::Error::new(error).context("the granted guest mailbox could not be served") - ); + return Err(error.context("the granted guest mailbox could not be served")); } Ok(()) } @@ -319,10 +383,24 @@ pub fn spawn_for_launch( plan: &ExecutionPlan, backend: &AnyBackend, vm_name: &str, + posture: Option<&MailboxLaunchPosture>, ) -> Result> { let Some(grant) = &plan.mailbox else { return Ok(None); }; + let posture = posture.ok_or_else(|| { + Error::new( + ErrorKind::PermissionDenied, + "mailbox launch has no admitted posture", + ) + })?; + posture.verify(plan, vm_name).map_err(Error::other)?; + if posture.variant() != Variant::Dev { + return Err(Error::new( + ErrorKind::PermissionDenied, + "production mailbox needs independent durability", + )); + } require_wired(plan, backend)?; let state_dir = mvm_core::config::vm_state_dir(vm_name); let mut config = build_config( @@ -332,6 +410,7 @@ pub fn spawn_for_launch( backend, )?; config.reader_boot = load_reader_boot_for_plan(&state_dir, plan)?; + config.posture = Some(posture.handoff(config.reader_boot.as_ref().map(|binding| binding.boot))); let bin = mvm_vmm::host::broker_services_spawn::resolve_subprocess_bin_to_spawn(&AuxBin::new( MAILBOX_BIN, MAILBOX_BIN_ENV, @@ -419,7 +498,8 @@ mod tests { fn an_ungranted_plan_spawns_nothing() { let plan = mvm_core::plan::test_support::PlanFixture::new().build(); assert!(plan.mailbox.is_none()); - let started = spawn_for_launch(&plan, &backend_named("firecracker"), "no-such-vm").unwrap(); + let started = + spawn_for_launch(&plan, &backend_named("firecracker"), "no-such-vm", None).unwrap(); assert_eq!(started, None); } @@ -432,7 +512,16 @@ mod tests { let dir = keys_dir(); let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); plan.mailbox = Some(grant(dir.path())); - let error = spawn_for_launch(&plan, &backend_named("libkrun"), "no-such-vm").unwrap_err(); + plan.image.name = "no-such-vm".into(); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + let posture = MailboxLaunchPosture::for_test(&plan, Variant::Dev); + let error = spawn_for_launch( + &plan, + &backend_named("libkrun"), + "no-such-vm", + Some(&posture), + ) + .unwrap_err(); assert_eq!(error.kind(), ErrorKind::Unsupported); } @@ -579,6 +668,8 @@ mod tests { fn config_for(grant: Option) -> VmStartConfig { let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); plan.mailbox = grant; + plan.image.name = "vm-under-test".into(); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); VmStartConfig { name: "vm-under-test".into(), plan_json: Some(serde_json::to_string(&plan).unwrap()), @@ -588,17 +679,23 @@ mod tests { type Spawns = std::cell::RefCell>; + fn posture_for(config: &VmStartConfig, variant: Variant) -> Option { + granted_plan(config).map(|plan| MailboxLaunchPosture::for_test(&plan, variant)) + } + fn serve_with( config: &VmStartConfig, spawns: &Spawns, outcome: Result>, ) -> anyhow::Result<()> { let backend = AnyBackend::from_hypervisor("firecracker"); + let posture = posture_for(config, Variant::Dev); serve_started_with( &backend, &VmId("vm-under-test".into()), config, - |_, _, name| { + posture.as_ref(), + |_, _, name, _| { spawns.borrow_mut().push(name.to_string()); outcome }, @@ -636,11 +733,25 @@ mod tests { let grant = grant(dir.path()); let state = tempfile::tempdir().unwrap(); let state_path = state.path().to_path_buf(); + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(grant); + plan.image.name = state_path + .file_name() + .unwrap() + .to_string_lossy() + .into_owned(); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + std::fs::write( + state_path.join("plan.json"), + serde_json::to_vec(&plan).unwrap(), + ) + .unwrap(); let config = MailboxProcessConfig { keys: derive_launch_keys(dir.path(), &grant).unwrap(), grant, socket: dir.path().join("mailbox.sock"), guest_route: HostDialRoute::PortSocket(dir.path().join("absent-port")), + posture: Some(MailboxLaunchPosture::for_test(&plan, Variant::Dev).handoff(None)), reader_boot: None, vm_state_dir: state_path, }; @@ -660,6 +771,98 @@ mod tests { assert_eq!(serving.join().unwrap(), 0); } + #[test] + fn dev_send_stays_live_without_a_false_durable_receipt() { + use mvm_contract::protocol::mailbox::PrincipalId; + use mvm_core::mailbox::frame::{OWN_MAILBOX_RECIPIENT, OutboxMessage}; + let keys = keys_dir(); + let state = tempfile::tempdir().unwrap(); + let grant = grant(keys.path()); + let vm_name = state + .path() + .file_name() + .unwrap() + .to_string_lossy() + .into_owned(); + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(grant); + plan.image.name = vm_name; + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + std::fs::write( + state.path().join("plan.json"), + serde_json::to_vec(&plan).unwrap(), + ) + .unwrap(); + + let guest_port = keys.path().join("guest-port.sock"); + let listener = std::os::unix::net::UnixListener::bind(&guest_port).unwrap(); + let outbox = keys.path().join("outbox"); + mvm_agentd::mailbox_outbox::write_outbox( + &outbox, + &OutboxMessage::new( + PrincipalId::from_bytes(OWN_MAILBOX_RECIPIENT), + b"ephemeral dev send".to_vec(), + ), + ) + .unwrap(); + let socket = keys.path().join("mailbox.sock"); + let config = MailboxProcessConfig { + keys: derive_launch_keys(keys.path(), &grant).unwrap(), + grant, + socket: socket.clone(), + guest_route: HostDialRoute::PortSocket(guest_port), + posture: Some(MailboxLaunchPosture::for_test(&plan, Variant::Dev).handoff(None)), + reader_boot: None, + vm_state_dir: state.path().to_path_buf(), + }; + let helper = std::thread::spawn(move || serve(config)); + let ready = std::time::Instant::now() + Duration::from_secs(5); + while !socket.exists() { + assert!( + std::time::Instant::now() < ready, + "dev mailbox did not bind" + ); + std::thread::sleep(Duration::from_millis(10)); + } + let guest = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + mvm_agentd::mailbox_outbox::wait_and_forward_outbox(&mut stream, &outbox).unwrap(); + let (mut reader, _) = listener.accept().unwrap(); + reader + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + mvm_core::mailbox::reader_wire::write_request( + &mut reader, + &mvm_core::mailbox::reader_wire::ReaderRequest::Receive, + ) + .unwrap(); + assert_eq!( + mvm_core::mailbox::reader_wire::read_response(&mut reader), + Ok(mvm_core::mailbox::reader_wire::ReaderResponse::Refused { + reason: mvm_core::mailbox::frame::Refusal::Unavailable + }), + "ephemeral Send cannot grant a guest Receive lease" + ); + }); + guest.join().unwrap(); + assert_eq!( + crate::mailbox_service::recv(&socket).unwrap().message, + Some(hex::encode(b"ephemeral dev send")) + ); + assert!(!state.path().join("mailbox-journal").exists()); + assert!(!state.path().join("mailbox-anchor").exists()); + drop(state); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !helper.is_finished() { + assert!( + std::time::Instant::now() < deadline, + "ephemeral helper did not stop" + ); + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!(helper.join().unwrap(), 0); + } + #[test] fn a_grant_that_cannot_be_served_fails_the_launch() { let dir = keys_dir(); @@ -677,19 +880,94 @@ mod tests { fn a_grant_on_an_unwired_backend_is_refused_before_boot() { let dir = keys_dir(); let granted = config_for(Some(grant(dir.path()))); + let posture = posture_for(&granted, Variant::Dev); for name in ["firecracker", "hvf"] { let backend = backend_named(name); assert!(backend.capabilities().guest_mailbox, "{name}"); - assert!(require_servable(&granted, &backend).is_ok(), "{name}"); + assert!( + require_servable(&granted, &backend, posture.as_ref()).is_ok(), + "{name}" + ); } for name in ["libkrun", "qemu", "wasm"] { let backend = backend_named(name); assert!(!backend.capabilities().guest_mailbox, "{name}"); - assert!(require_servable(&granted, &backend).is_err(), "{name}"); assert!( - require_servable(&config_for(None), &backend).is_ok(), + require_servable(&granted, &backend, posture.as_ref()).is_err(), + "{name}" + ); + assert!( + require_servable(&config_for(None), &backend, None).is_ok(), "{name}" ); } } + + #[test] + fn mailbox_never_defaults_missing_or_production_posture_to_dev() { + let dir = keys_dir(); + let config = config_for(Some(grant(dir.path()))); + let backend = backend_named("firecracker"); + assert!(require_servable(&config, &backend, None).is_err()); + let prod = posture_for(&config, Variant::Prod); + assert!(require_servable(&config, &backend, prod.as_ref()).is_err()); + let dev = posture_for(&config, Variant::Dev); + assert!(require_servable(&config, &backend, dev.as_ref()).is_ok()); + + let mut forged = serde_json::to_value(dev.unwrap().handoff(None)).unwrap(); + forged["plan_id"] = serde_json::json!("sha256:wrong"); + let forged: MailboxPostureHandoff = serde_json::from_value(forged).unwrap(); + let plan = granted_plan(&config).unwrap(); + assert!(forged.verify(&plan, &config.name, None).is_err()); + let stale_boot = posture_for(&config, Variant::Dev).unwrap().handoff(Some( + mvm_contract::protocol::mailbox::BootId::from_bytes([8; 32]), + )); + assert!(stale_boot.verify(&plan, &config.name, None).is_err()); + } + + #[test] + fn helper_restart_without_admitted_posture_refuses_before_binding() { + let dir = keys_dir(); + let state = tempfile::tempdir().unwrap(); + let grant = grant(dir.path()); + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(grant); + plan.image.name = state + .path() + .file_name() + .unwrap() + .to_string_lossy() + .into_owned(); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + std::fs::write( + state.path().join("plan.json"), + serde_json::to_vec(&plan).unwrap(), + ) + .unwrap(); + let config = MailboxProcessConfig { + keys: derive_launch_keys(dir.path(), &grant).unwrap(), + grant, + socket: dir.path().join("mailbox.sock"), + guest_route: HostDialRoute::PortSocket(dir.path().join("absent-port")), + posture: None, + reader_boot: None, + vm_state_dir: state.path().to_path_buf(), + }; + assert_eq!(serve(config), 1); + assert!(!dir.path().join("mailbox.sock").exists()); + } + + #[test] + fn helper_refuses_oversized_or_symlinked_plan_state() { + let state = tempfile::tempdir().unwrap(); + let path = state.path().join("plan.json"); + std::fs::write(&path, vec![b' '; MAX_HELPER_PLAN_BYTES as usize + 1]).unwrap(); + assert_eq!( + load_admitted_plan(state.path()).err().unwrap().kind(), + ErrorKind::InvalidData + ); + std::fs::remove_file(&path).unwrap(); + std::os::unix::fs::symlink(state.path().join("missing"), &path).unwrap(); + assert!(load_admitted_plan(state.path()).is_err()); + } } diff --git a/crates/mvm-hostd/src/mailbox_service.rs b/crates/mvm-hostd/src/mailbox_service.rs index 4ca60ddea3..a7555afc93 100644 --- a/crates/mvm-hostd/src/mailbox_service.rs +++ b/crates/mvm-hostd/src/mailbox_service.rs @@ -542,6 +542,39 @@ impl MailboxService { })) } + /// Dev-only Send on the existing mailbox port. A reader frame is refused + /// explicitly until independent durable authority can back guest leases. + pub(crate) fn serve_guest_route( + &self, + route: &mvm_core::vm_backend::HostDialRoute, + sealer: &mut GuestSealer, + ) -> std::io::Result { + let mut stream = crate::telemetry_collector::dial_guest_route( + route, + mvm_contract::protocol::mailbox::MAILBOX_PORT, + )?; + stream.set_read_timeout(Some(Duration::from_secs(7)))?; + stream.set_write_timeout(Some(IO_TIMEOUT))?; + let frame = read_frame(self.shared.limits.max_frame_bytes, &mut stream); + if let Ok(body) = &frame + && mvm_core::mailbox::reader_wire::is_reader_frame(body) + { + mvm_core::mailbox::reader_wire::write_response( + &mut stream, + &mvm_core::mailbox::reader_wire::ReaderResponse::Refused { + reason: Refusal::Unavailable, + }, + ) + .map_err(std::io::Error::other)?; + return Ok(true); + } + let received = frame.is_ok(); + answer(&self.shared, &mut stream, frame, |body| { + sealer.seal(OutboxMessage::decode(body)?) + }); + Ok(received) + } + /// Take the oldest accepted plaintext, if any. pub fn receive(&self) -> Option> { self.shared.take().0 diff --git a/crates/mvm-hostd/src/mailbox_service/durable.rs b/crates/mvm-hostd/src/mailbox_service/durable.rs index 3cb5c61aea..baa6d078e5 100644 --- a/crates/mvm-hostd/src/mailbox_service/durable.rs +++ b/crates/mvm-hostd/src/mailbox_service/durable.rs @@ -190,13 +190,6 @@ impl DurableMailboxService { }) } - /// Install the separately verified reader identity before advertising readiness. - pub(crate) fn install_reader(&self, reader: GuestReaderAuthority) -> std::io::Result<()> { - self.shared.reader.set(reader).map_err(|_| { - std::io::Error::new(std::io::ErrorKind::AlreadyExists, "reader already enrolled") - }) - } - /// Read one guest outbox operation over its dedicated mailbox port. The /// guest sees success only after the canonical ciphertext, dedup identity, /// and anchor have committed; an idle connection returns `false`. diff --git a/crates/mvm-hostd/src/plan_admission.rs b/crates/mvm-hostd/src/plan_admission.rs index 4998565b67..e5723e3624 100644 --- a/crates/mvm-hostd/src/plan_admission.rs +++ b/crates/mvm-hostd/src/plan_admission.rs @@ -154,6 +154,7 @@ pub use registry_pack::{BuiltPackBootCheck, PreparedBuiltPackRuntime, prepare_bu pub struct AdmittedPlan { plan: ExecutionPlan, plan_id: PlanId, + variant: Variant, signer_id: String, signed: SignedExecutionPlan, extensions: Vec, @@ -166,6 +167,103 @@ pub struct AdmittedPlan { dispatched: bool, } +/// Launch-only posture retained from host admission, never inferred from plan labels. +#[derive(Debug, Clone)] +pub struct MailboxLaunchPosture { + variant: Variant, + plan_id: PlanId, + plan_nonce: mvm_core::plan::Nonce, + vm_name: String, + grant: mvm_contract::protocol::mailbox::MailboxGrant, +} + +/// The private-pipe projection; unlike the launch token, parsing this is not +/// admission authority. Only the launcher holding a verified token sends it. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MailboxPostureHandoff { + variant: Variant, + plan_id: PlanId, + plan_nonce: mvm_core::plan::Nonce, + vm_name: String, + grant: mvm_contract::protocol::mailbox::MailboxGrant, + reader_boot: Option, +} + +impl MailboxPostureHandoff { + pub(crate) fn verify( + &self, + plan: &ExecutionPlan, + vm_name: &str, + reader_boot: Option, + ) -> Result<()> { + MailboxLaunchPosture { + variant: self.variant, + plan_id: self.plan_id.clone(), + plan_nonce: self.plan_nonce.clone(), + vm_name: self.vm_name.clone(), + grant: self.grant, + } + .verify(plan, vm_name)?; + anyhow::ensure!( + self.reader_boot == reader_boot, + "mailbox helper posture belongs to another guest boot" + ); + Ok(()) + } + + pub(crate) const fn variant(&self) -> Variant { + self.variant + } +} + +impl MailboxLaunchPosture { + #[cfg(test)] + pub(crate) fn for_test(plan: &ExecutionPlan, variant: Variant) -> Self { + Self { + variant, + plan_id: plan.plan_id.clone(), + plan_nonce: plan.nonce.clone(), + vm_name: plan.image.name.clone(), + grant: plan.mailbox.expect("mailbox posture fixture has a grant"), + } + } + + /// Check the private helper handoff against this VM's admitted plan. + pub fn verify(&self, plan: &ExecutionPlan, vm_name: &str) -> Result<()> { + anyhow::ensure!( + plan.plan_id == mvm_core::plan::compute_plan_id(plan) + && self.plan_id == plan.plan_id + && self.plan_nonce == plan.nonce + && self.vm_name == vm_name + && plan.image.name == vm_name + && plan.mailbox == Some(self.grant), + "mailbox helper posture does not match the admitted VM and plan" + ); + Ok(()) + } + + /// The posture admission selected; not a hint from the image or guest. + #[must_use] + pub const fn variant(&self) -> Variant { + self.variant + } + + pub(crate) fn handoff( + &self, + reader_boot: Option, + ) -> MailboxPostureHandoff { + MailboxPostureHandoff { + variant: self.variant, + plan_id: self.plan_id.clone(), + plan_nonce: self.plan_nonce.clone(), + vm_name: self.vm_name.clone(), + grant: self.grant, + reader_boot, + } + } +} + /// Re-verified host artifact behind one extension binding in the signed plan. #[derive(Debug, Clone)] pub struct AdmittedExtension { @@ -187,6 +285,18 @@ impl AdmittedPlan { &self.plan_id } + /// Retain this host admission's mailbox posture for the exact VM it admitted. + #[must_use] + pub fn mailbox_posture(&self) -> Option { + self.plan.mailbox.map(|grant| MailboxLaunchPosture { + variant: self.variant, + plan_id: self.plan_id.clone(), + plan_nonce: self.plan.nonce.clone(), + vm_name: self.plan.image.name.clone(), + grant, + }) + } + /// The signer whose key the plan verified under — this host's own signer /// for a synthesized run, or the pinned external signer for a /// fleet-issued one. @@ -224,6 +334,7 @@ impl AdmittedPlan { ) -> Self { Self { plan_id: plan.plan_id.clone(), + variant: Variant::Dev, signer_id, plan, signed, @@ -615,6 +726,7 @@ fn finish_admission(parts: VerifiedAdmission<'_>) -> Result { Ok(AdmittedPlan { plan_id: verified.plan_id.clone(), + variant: posture.variant(), signer_id, plan: verified, signed, @@ -2102,6 +2214,7 @@ pub fn start_admitted(params: StartAdmittedParams<'_>) -> Result let backend_name = backend.name().to_string(); let vm_name = config.name.clone(); + let mailbox_posture = admitted.mailbox_posture(); registry_pack::verify_runtime_at_dispatch( _built_runtime.as_ref(), &config, @@ -2110,7 +2223,7 @@ pub fn start_admitted(params: StartAdmittedParams<'_>) -> Result )?; // Before the boot, so a grant this backend cannot serve never reaches a // guest that would assert it. - crate::mailbox_process::require_servable(&config, backend)?; + crate::mailbox_process::require_servable(&config, backend, mailbox_posture.as_ref())?; // Spent before the backend starts, so an admission that reached dispatch // can never authorize another launch, whatever the start's outcome. admitted.dispatched = true; @@ -2170,9 +2283,12 @@ pub fn start_admitted(params: StartAdmittedParams<'_>) -> Result // Last, so no earlier rollback leaves a mailbox process behind. // A grant that cannot be served is refused rather than booting a // guest whose mailbox port nobody answers. - if let Err(err) = - crate::mailbox_process::spawn_for_launch(admitted.plan(), backend, &vm_name) - { + if let Err(err) = crate::mailbox_process::spawn_for_launch( + admitted.plan(), + backend, + &vm_name, + mailbox_posture.as_ref(), + ) { return Err(undo_launch(UndoLaunch { backend, vm_id: &vm_id, From 2dbda40ba2e2b82f60380abcb237c496b0fc531f Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 22:53:38 -0800 Subject: [PATCH 10/51] feat(mailbox): gate journal acceptance on independent anchor Persist unsigned receipt state and a destination-signed admission intent in versioned encrypted records. Sign durable receipts only after journal and anchor commitment; refuse the legacy callback path. Add recovery and failure-path witnesses without claiming a deployed checkpoint or history-key provider. --- crates/mvm-core/README.md | 36 ++ crates/mvm-core/src/mailbox/journal.rs | 778 ++++++++++++++++++++++++- 2 files changed, 790 insertions(+), 24 deletions(-) diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index 82380fd035..b56279a99c 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -56,6 +56,42 @@ ownership decision. | Runtime abstraction | `build_env`, `client`, `util` test support | | Observability data | `observability`, `span_timing`, `usage_capture` | +## Mailbox journal boundary + +`mailbox::journal` stores bounded, authenticated records containing the +canonical encrypted mailbox frame and its deduplication identity. Its +`JournalAnchor` interface requires an independently trusted authority to read +the current `(cursor, head)` and atomically compare-and-advance it after a +journal write. The journal persists unsigned receipt state in record version 2 +with a destination-signed, domain-separated admission intent binding the +verified identity, canonical frame, and receipt state. The destination signs +the actual acceptance receipt only after both commits. Earlier private-handoff +version-1 records are refused, not silently reinterpreted. Recovery verifies +the intent, refuses a missing or divergent anchored prefix, and may reconcile +only an intact contiguous suffix written before an interrupted anchor commit. + +`FileJournalAnchor` is a local development aid, **not** that independent +authority: restoring both the journal and its host-local anchor can hide +rollback. The journal's current raw key input also does not establish +persistent, versioned history-key custody or recover the separately encrypted +payload after the per-launch mailbox secret is erased. Until those providers +and the existing service's admission handoff are integrated and tested, local +journal writes must not be presented as `accepted_durable`. The trusted-host +profile can access plaintext; host-blind confidentiality requires a different +attested key-release and protected-memory architecture. + +The intended trusted-host custody boundary is a separately retained, versioned +mailbox-history root under host key custody. Each record must bind its key ID +and tenant/domain/mailbox scope; reopening existing history must **load the +existing version**, never mint a replacement or fall back to the signing seed +or erased per-launch secret. Durable payloads also need an explicitly +recoverable recipient encryption key: persisting the current per-launch key +would silently negate its erasure guarantee. Rotation retains old versions +only until their anchored history is retired. An independently operated +checkpoint must atomically compare the anchored `(mailbox, cursor, head)` with +the expected prior value; an unavailable or divergent checkpoint refuses +durable acceptance. These custody and checkpoint providers are not wired yet. + ## Features The default feature set is deliberately small. Notable opt-ins include diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 9b75446a8a..0d90848b8e 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -1,10 +1,10 @@ //! Destination-owned, bounded ciphertext admission journal. //! //! Each immutable entry is published with a no-clobber rename and a directory -//! sync. The signed acceptance receipt and dedup identity live in the same -//! authenticated record as the canonical envelope. A caller must retain the -//! returned tip in an independent trusted anchor to detect whole-journal -//! rollback; a hash chain alone cannot detect deletion of its suffix. +//! sync. The unsigned receipt state and dedup identity live in the same +//! authenticated record as the canonical envelope. The destination signs +//! acceptance only after the journal and independent anchor commit. A hash +//! chain alone cannot detect deletion of its suffix. use std::collections::{BTreeMap, BTreeSet}; use std::fs; @@ -13,7 +13,7 @@ use std::path::{Path, PathBuf}; use std::sync::Mutex; use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; -use ed25519_dalek::{SigningKey, VerifyingKey}; +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; use hmac::{Hmac, Mac, digest::KeyInit}; use mvm_contract::protocol::mailbox::{ ContentType, DeliveryLeaseNonce, DeliveryOutcome, DestinationReceipt, MailboxAddress, @@ -31,7 +31,7 @@ use super::reader::{ }; use super::{ MailboxError, OpenContext, ReplayIdentity, ReplayProtector, authorize_receive, frame, - open_authenticated_envelope, sign_receipt, verify_receipt, + open_authenticated_envelope, sign_receipt, }; use crate::crypto::aead::{self, Key}; use crate::private_fs; @@ -40,6 +40,7 @@ use crate::util::atomic_io::{FileLock, atomic_write_durable, atomic_write_new, s const DOMAIN: &[u8] = b"mvm.mailbox.journal.v1\0"; const ENCRYPTION_DOMAIN: &[u8] = b"mvm.mailbox.journal.encryption.v1\0"; const ANCHOR_DOMAIN: &[u8] = b"mvm.mailbox.anchor.v1\0"; +const ACCEPTANCE_INTENT_DOMAIN: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v2\0"; const MAX_ANCHOR_BYTES: u64 = 1024; const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; const MAX_RECORDS: usize = 100_000; @@ -71,6 +72,25 @@ impl JournalTip { } } +/// Independent, authenticated high-water authority for one mailbox. +/// +/// Implementations must persist outside the journal host's rollback domain. +/// A local file, an in-memory value, or a best-effort transport acknowledgement +/// cannot authorize durable acceptance. Compare-and-advance must be atomic, +/// refuse divergent tips, and return only after the new tip is durable. +pub trait JournalAnchor { + /// Read the authoritative tip, including after a host restart. + fn load(&self, address: MailboxAddress) -> Result; + + /// Commit exactly one successor of `expected`, or refuse a stale writer. + fn compare_and_advance( + &self, + address: MailboxAddress, + expected: JournalTip, + next: JournalTip, + ) -> Result<(), JournalError>; +} + /// Fail-closed admission and recovery failures; no plaintext is included. #[derive(Debug, thiserror::Error)] pub enum JournalError { @@ -94,6 +114,8 @@ pub enum JournalError { NoActiveLease, #[error("journal cannot be used after an uncertain write")] Uncertain, + #[error("independent rollback anchor and recoverable key custody are required")] + DurableAuthorityUnavailable, #[error("mailbox authorization refused: {0}")] Mailbox(#[from] MailboxError), #[error("journal storage is unavailable: {0}")] @@ -111,12 +133,13 @@ struct Record { } #[derive(Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] enum JournalEvent { Accepted { identity: ReplayIdentity, frame: String, - receipt: Box, + receipt: Box, + authorization_signature: Vec, }, Acknowledged { identity: ReplayIdentity, @@ -381,6 +404,30 @@ fn anchor_mac(key: &[u8], body: &[u8]) -> [u8; 32] { mac.finalize().into_bytes().into() } +#[derive(Serialize)] +struct AcceptanceIntent<'a> { + identity: ReplayIdentity, + frame: &'a str, + receipt: &'a DestinationReceipt, +} + +fn acceptance_intent_bytes( + identity: ReplayIdentity, + frame: &str, + receipt: &DestinationReceipt, +) -> Result, JournalError> { + let mut bytes = ACCEPTANCE_INTENT_DOMAIN.to_vec(); + bytes.extend( + serde_jcs::to_vec(&AcceptanceIntent { + identity, + frame, + receipt, + }) + .map_err(|_| JournalError::Corrupt)?, + ); + Ok(bytes) +} + impl MailboxJournal { /// Reconstruct under an exclusive lock and require the separately trusted /// high-water mark. Genesis is valid only for an empty journal. @@ -416,6 +463,29 @@ impl MailboxJournal { }) } + /// Recover against an independently trusted high-water mark. + /// + /// An intact, authenticated journal suffix may be advanced after a crash. + /// A missing, divergent, or rolled-back anchored prefix is never repaired. + /// The caller must obtain `key` from persistent custody, not the erased + /// per-launch mailbox secret. + pub fn open_with_anchor( + root: &Path, + address: MailboxAddress, + authority: VerifyingKey, + key: [u8; 32], + limits: JournalLimits, + anchor: &impl JournalAnchor, + ) -> Result { + let expected = anchor.load(address)?; + let mut previous = expected; + Self::open_with_recovery(root, address, authority, key, limits, expected, |next| { + anchor.compare_and_advance(address, previous, next)?; + previous = next; + Ok(()) + }) + } + fn open_with_recovery( root: &Path, address: MailboxAddress, @@ -466,6 +536,7 @@ impl MailboxJournal { } } paths.sort(); + let mut suffix_synced = false; for path in paths { let cursor = journal .tip @@ -496,7 +567,7 @@ impl MailboxJournal { ); let record: Record = serde_json::from_slice(&plaintext).map_err(|_| JournalError::Corrupt)?; - if record.version != 1 + if record.version != 2 || record.cursor != cursor || record.previous != journal.tip.head || record.address != address @@ -508,22 +579,32 @@ impl MailboxJournal { identity, frame: encoded_frame, receipt, + authorization_signature, } => { - if receipt.receipt.address != address - || receipt.receipt.outcome != DeliveryOutcome::AcceptedDurable - || receipt.receipt.message_id != identity.message_id - || receipt.receipt.execution_epoch != identity.execution_epoch + if receipt.address != address + || receipt.outcome != DeliveryOutcome::AcceptedDurable + || receipt.message_id != identity.message_id + || receipt.execution_epoch != identity.execution_epoch { return Err(JournalError::Corrupt); } - verify_receipt(&receipt, &authority).map_err(|_| JournalError::Corrupt)?; + let signature: [u8; 64] = authorization_signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + authority + .verify( + &acceptance_intent_bytes(identity, &encoded_frame, &receipt)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; let raw = STANDARD_NO_PAD .decode(&encoded_frame) .map_err(|_| JournalError::Corrupt)?; let (capability, envelope) = frame::decode_frame(&raw).map_err(|_| JournalError::Corrupt)?; - if receipt.receipt.delivery_id != envelope.routing.delivery_id - || receipt.receipt.ciphertext_commitment != envelope.ciphertext_commitment + if receipt.delivery_id != envelope.routing.delivery_id + || receipt.ciphertext_commitment != envelope.ciphertext_commitment || journal .seen .insert( @@ -631,6 +712,14 @@ impl MailboxJournal { return Err(JournalError::Rollback); } if cursor > expected.cursor { + if !suffix_synced { + // A predecessor could crash after publishing a record but + // before syncing its directory entry. Recovery must not + // advance an independent anchor ahead of that entry. + sync_dir(root) + .map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; + suffix_synced = true; + } reconcile(next)?; } journal.tip = next; @@ -641,10 +730,20 @@ impl MailboxJournal { Ok(journal) } - /// Verify the canonical envelope and capability, commit ciphertext and - /// dedup identity, then sign a destination verdict. `anchor` must commit the - /// new tip in a separate rollback domain before this returns success. + /// The legacy local callback cannot prove an independent anchor or + /// recoverable key custody, so it cannot issue durable acceptance. + /// + /// Use [`Self::admit_with_anchor`] only after provisioning a trusted + /// external checkpoint and durable mailbox-history keys. pub fn admit( + &mut self, + _request: JournalAdmission<'_>, + _anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, + ) -> Result { + Err(JournalError::DurableAuthorityUnavailable) + } + + fn admit_verified( &mut self, request: JournalAdmission<'_>, anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, @@ -699,14 +798,24 @@ impl MailboxJournal { if self.tip.cursor >= self.limits.max_records as u64 { return Err(JournalError::Full); } - let signed = sign_receipt(receipt(DeliveryOutcome::AcceptedDurable), signer)?; + let accepted = receipt(DeliveryOutcome::AcceptedDurable); let encoded_frame = frame::encode_frame(capability, envelope).map_err(|_| JournalError::Corrupt)?; + let encoded_frame = STANDARD_NO_PAD.encode(encoded_frame); + let authorization_signature = signer + .sign(&acceptance_intent_bytes( + identity, + &encoded_frame, + &accepted, + )?) + .to_bytes() + .to_vec(); self.append( JournalEvent::Accepted { identity, - frame: STANDARD_NO_PAD.encode(encoded_frame), - receipt: Box::new(signed.clone()), + frame: encoded_frame, + receipt: Box::new(accepted.clone()), + authorization_signature, }, anchor, )?; @@ -721,7 +830,26 @@ impl MailboxJournal { }, ); self.order.push(identity); - Ok(signed) + sign_receipt(accepted, signer).map_err(JournalError::from) + } + + /// Admit only against the current independently anchored tip. + /// + /// The legacy local-file callback API refuses admission. A service may + /// use this path only with independent checkpoint and history-key custody. + pub fn admit_with_anchor( + &mut self, + request: JournalAdmission<'_>, + anchor: &impl JournalAnchor, + ) -> Result { + let expected = self.tip; + if anchor.load(self.address)? != expected { + return Err(JournalError::Rollback); + } + let address = self.address; + self.admit_verified(request, |next| { + anchor.compare_and_advance(address, expected, next) + }) } fn append( @@ -737,7 +865,7 @@ impl MailboxJournal { return Err(JournalError::Full); } let record = Record { - version: 1, + version: 2, cursor, previous: self.tip.head, address: self.address, @@ -1145,3 +1273,605 @@ fn read_record(path: &Path) -> Result, JournalError> { } Ok(bytes) } + +#[cfg(test)] +mod anchor_tests { + use super::*; + use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, AuthorizationDomainRouteLabel, BootId, CapabilityId, + CapabilityOperations, CausationId, CiphertextPadding, ContentType, CorrelationId, + DeliveryId, DestinationRouteToken, EncryptionAlgorithm, EncryptionContext, ExecutionEpoch, + MailboxCapabilityGrant, MailboxEnvelopeHeader, MailboxId, MailboxMessageKind, + MailboxRoutingFrame, MessageId, OrderingScope, PrincipalId, ReplayNonce, RevocationId, + SequenceNumber, TenantId, TransportBindingId, + }; + use std::sync::atomic::{AtomicBool, Ordering}; + + struct IndependentAnchor { + address: MailboxAddress, + tip: Mutex, + } + + impl IndependentAnchor { + fn new(address: MailboxAddress) -> Self { + Self { + address, + tip: Mutex::new(JournalTip::genesis()), + } + } + } + + impl JournalAnchor for IndependentAnchor { + fn load(&self, address: MailboxAddress) -> Result { + if address != self.address { + return Err(JournalError::WrongAuthority); + } + Ok(*self.tip.lock().expect("test anchor lock")) + } + + fn compare_and_advance( + &self, + address: MailboxAddress, + expected: JournalTip, + next: JournalTip, + ) -> Result<(), JournalError> { + if address != self.address { + return Err(JournalError::WrongAuthority); + } + let mut tip = self.tip.lock().expect("test anchor lock"); + if *tip != expected || expected.cursor.checked_add(1) != Some(next.cursor) { + return Err(JournalError::Rollback); + } + *tip = next; + Ok(()) + } + } + + fn address() -> MailboxAddress { + MailboxAddress { + tenant: TenantId::from_bytes([1; 32]), + authorization_domain: AuthorizationDomainId::from_bytes([2; 32]), + mailbox: MailboxId::from_bytes([3; 32]), + recipient: PrincipalId::from_bytes([4; 32]), + } + } + + fn limits() -> JournalLimits { + JournalLimits { + max_records: 4, + max_bytes: 64 * 1024, + } + } + + #[test] + fn admission_intent_signature_is_bound_to_its_own_domain() { + let (envelope, capability, context, _, _, signer) = message(); + let identity = ReplayIdentity { + sender: context.expected_sender, + sender_boot: BootId::from_bytes([8; 32]), + execution_epoch: context.current_execution_epoch, + message_id: MessageId::from_bytes([9; 32]), + }; + let frame = STANDARD_NO_PAD + .encode(frame::encode_frame(&capability, &envelope).expect("canonical frame")); + let receipt = DestinationReceipt { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: envelope.routing.delivery_id, + message_id: MessageId::from_bytes([9; 32]), + address: address(), + execution_epoch: context.current_execution_epoch, + outcome: DeliveryOutcome::AcceptedDurable, + observed_at_unix_ms: context.now_unix_ms, + ciphertext_commitment: envelope.ciphertext_commitment, + }; + let intent = acceptance_intent_bytes(identity, &frame, &receipt).expect("intent"); + let signature = signer.sign(&intent); + signer + .verifying_key() + .verify(&intent, &signature) + .expect("intended domain"); + let bare = serde_jcs::to_vec(&AcceptanceIntent { + identity, + frame: &frame, + receipt: &receipt, + }) + .expect("canonical intent"); + assert!(signer.verifying_key().verify(&bare, &signature).is_err()); + let wrong_domain = [b"mvm.mailbox.receipt.v1\0".as_slice(), bare.as_slice()].concat(); + assert!( + signer + .verifying_key() + .verify(&wrong_domain, &signature) + .is_err() + ); + } + + fn message() -> ( + MailboxEnvelope, + SignedMailboxCapability, + OpenContext, + Key, + VerifyingKey, + SigningKey, + ) { + let destination = address(); + let signer = SigningKey::from_bytes(&[6; 32]); + let sender = SigningKey::from_bytes(&[7; 32]); + let sender_id = PrincipalId::from_bytes(sender.verifying_key().to_bytes()); + let content_type = ContentType::parse("application/vnd.mvm.test", 1).expect("content type"); + let epoch = ExecutionEpoch::new(1).expect("epoch"); + let now = 1_800_000_000_000; + let header = MailboxEnvelopeHeader { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + tenant: destination.tenant, + authorization_domain: destination.authorization_domain, + sender: sender_id, + recipient: destination.recipient, + mailbox: destination.mailbox, + sender_boot: BootId::from_bytes([8; 32]), + execution_epoch: epoch, + message_id: MessageId::from_bytes([9; 32]), + correlation_id: CorrelationId::from_bytes([10; 32]), + causation_id: CausationId::from_bytes([11; 32]), + kind: MailboxMessageKind::Cast, + reply_to: None, + ordering_scope: OrderingScope::parse("test").expect("scope"), + sequence: SequenceNumber::new(1).expect("sequence"), + issued_at_unix_ms: now - 1, + expires_at_unix_ms: now + 10_000, + nonce: ReplayNonce::from_bytes([12; 16]), + content_type: content_type.clone(), + capability_id: CapabilityId::from_bytes([13; 32]), + policy_epoch: 1, + encryption: EncryptionContext { + algorithm: EncryptionAlgorithm::AES_256_GCM, + key_id: [14; 32], + }, + plaintext_len: 5, + }; + let routing = MailboxRoutingFrame { + protocol_version: ProtocolVersion::V1, + authorization_domain_route_label: AuthorizationDomainRouteLabel::from_bytes([15; 32]), + destination_route_token: DestinationRouteToken::from_bytes([16; 32]), + delivery_id: DeliveryId::from_bytes([17; 32]), + ciphertext_len: 0, + padding: CiphertextPadding::Exact, + expires_at_unix_ms: header.expires_at_unix_ms, + hop_limit: 4, + transport_binding_id: TransportBindingId::from_bytes([18; 32]), + }; + let payload_key = Key::from_bytes([19; 32]); + let envelope = super::super::seal_envelope( + super::super::SealRequest { + routing, + header: header.clone(), + plaintext: b"hello", + }, + &payload_key, + &sender, + ) + .expect("seal canonical frame"); + let grant = MailboxCapabilityGrant { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + capability_id: header.capability_id, + issuer: PrincipalId::from_bytes(signer.verifying_key().to_bytes()), + subject: sender_id, + address: destination, + operations: CapabilityOperations::from_operations(&[MailboxOperation::Send]), + message_kinds: vec![MailboxMessageKind::Cast], + content_types: vec![content_type.clone()], + max_payload_bytes: 64, + max_messages_per_minute: 60, + max_outstanding_calls: 1, + max_queue_contribution_bytes: 1024, + minimum_execution_epoch: epoch, + maximum_execution_epoch: epoch, + policy_epoch: 1, + not_before_unix_ms: now - 10, + expires_at_unix_ms: now + 10_000, + revocation_id: RevocationId::from_bytes([20; 32]), + delegation_depth: 0, + }; + let capability = super::super::sign_capability(grant, &signer).expect("grant"); + let context = OpenContext { + expected_tenant: destination.tenant, + expected_authorization_domain: destination.authorization_domain, + expected_mailbox: destination.mailbox, + expected_sender: sender_id, + expected_recipient: destination.recipient, + acting_principal: sender_id, + expected_capability_issuer: PrincipalId::from_bytes(signer.verifying_key().to_bytes()), + current_execution_epoch: epoch, + current_policy_epoch: 1, + required_operation: MailboxOperation::Send, + now_unix_ms: now, + accepted_content_types: vec![content_type], + revoked_capabilities: Vec::new(), + }; + ( + envelope, + capability, + context, + payload_key, + sender.verifying_key(), + signer, + ) + } + + fn admission<'a>( + envelope: &'a MailboxEnvelope, + capability: &'a SignedMailboxCapability, + context: &'a OpenContext, + payload_key: &'a Key, + sender_key: &'a VerifyingKey, + signer: &'a SigningKey, + ) -> JournalAdmission<'a> { + JournalAdmission { + envelope, + capability, + context, + encryption_key: payload_key, + sender_key, + signer, + } + } + + #[test] + fn anchored_acceptance_recovers_ciphertext_and_dedup_without_plaintext_on_disk() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let anchor = IndependentAnchor::new(address()); + let mut journal = MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ) + .expect("genesis"); + let accepted = journal + .admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("anchor committed before receipt"); + assert_eq!(accepted.receipt.outcome, DeliveryOutcome::AcceptedDurable); + assert_eq!(journal.tip(), anchor.load(address()).expect("remote tip")); + let record = fs::read(record_path(&root, 1)).expect("encrypted record"); + assert!( + !record + .windows(b"hello".len()) + .any(|bytes| bytes == b"hello") + ); + let decrypted = aead::open( + &journal.record_key(), + &record[32..], + &[DOMAIN, &JournalTip::genesis().head].concat(), + ) + .expect("authenticated journal record"); + let parsed: Record = serde_json::from_slice(&decrypted).expect("record body"); + let JournalEvent::Accepted { + receipt, + authorization_signature, + .. + } = parsed.event + else { + panic!("the first record must be an admission"); + }; + assert_eq!(*receipt, accepted.receipt); + assert_eq!(authorization_signature.len(), 64); + assert!( + !decrypted + .windows(accepted.signature.len()) + .any(|bytes| bytes == accepted.signature.as_slice()), + "an actual acceptance signature must not be persisted before the anchor commit" + ); + drop(journal); + + let mut recovered = MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ) + .expect("restart recovers exact anchored history"); + let duplicate = recovered + .admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("retry is deduplicated"); + assert_eq!(duplicate.receipt.outcome, DeliveryOutcome::Duplicate); + assert_eq!(anchor.load(address()).expect("unchanged tip").cursor, 1); + } + + #[test] + fn interrupted_anchor_commit_reconciles_only_intact_suffix() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let anchor = IndependentAnchor::new(address()); + let mut journal = MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ) + .expect("genesis"); + let interrupted = AtomicBool::new(true); + assert!(matches!( + journal.admit_verified( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + |tip| { + if interrupted.swap(false, Ordering::SeqCst) { + return Err(JournalError::Uncertain); + } + anchor.compare_and_advance(address(), JournalTip::genesis(), tip) + } + ), + Err(JournalError::Uncertain) + )); + assert_eq!( + anchor.load(address()).expect("no receipt"), + JournalTip::genesis() + ); + assert!(matches!( + journal.admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor + ), + Err(JournalError::Rollback) + )); + drop(journal); + let recovered = MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ) + .expect("verified unanchored suffix is reconciled"); + assert_eq!( + recovered.tip(), + anchor.load(address()).expect("advanced tip") + ); + } + + #[test] + fn legacy_callback_cannot_issue_durable_acceptance() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let anchor = IndependentAnchor::new(address()); + let mut journal = MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ) + .expect("genesis"); + assert!(matches!( + journal.admit( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + |_| Ok(()) + ), + Err(JournalError::DurableAuthorityUnavailable) + )); + assert_eq!(journal.tip(), JournalTip::genesis()); + assert_eq!( + anchor.load(address()).expect("no commit"), + JournalTip::genesis() + ); + } + + #[test] + fn wrong_key_tamper_and_truncation_fail_closed_without_changing_anchor() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let anchor = IndependentAnchor::new(address()); + let mut journal = MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ) + .expect("genesis"); + journal + .admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("admission"); + let tip = anchor.load(address()).expect("committed tip"); + drop(journal); + + assert!(matches!( + MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [22; 32], + limits(), + &anchor, + ), + Err(JournalError::Corrupt) + )); + assert!(matches!( + MailboxJournal::open_with_anchor( + &root, + address(), + SigningKey::from_bytes(&[23; 32]).verifying_key(), + [21; 32], + limits(), + &anchor, + ), + Err(JournalError::Corrupt) + )); + let path = record_path(&root, 1); + let original = fs::read(&path).expect("stored ciphertext"); + let mut tampered = original.clone(); + tampered[40] ^= 1; + fs::write(&path, &tampered).expect("tamper record"); + assert!(matches!( + MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ), + Err(JournalError::Corrupt) + )); + fs::write(&path, &original[..40]).expect("interrupted write"); + assert!(matches!( + MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ), + Err(JournalError::Corrupt) + )); + assert_eq!(anchor.load(address()).expect("unchanged anchor"), tip); + fs::remove_file(&path).expect("simulate journal rollback"); + assert!(matches!( + MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ), + Err(JournalError::Rollback) + )); + } + + #[test] + fn independent_anchor_refuses_wrong_domain_and_stale_competing_writers() { + let anchor = IndependentAnchor::new(address()); + let old = JournalTip::genesis(); + let next = JournalTip { + cursor: 1, + head: [9; 32], + }; + let mut foreign = address(); + foreign.authorization_domain = AuthorizationDomainId::from_bytes([5; 32]); + assert!(matches!( + anchor.load(foreign), + Err(JournalError::WrongAuthority) + )); + anchor + .compare_and_advance(address(), old, next) + .expect("first writer advances"); + assert!(matches!( + anchor.compare_and_advance(address(), old, next), + Err(JournalError::Rollback) + )); + assert_eq!(anchor.load(address()).expect("authoritative tip"), next); + } + + #[test] + fn recovery_refuses_missing_anchored_history_and_wrong_domain() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let authority = SigningKey::from_bytes(&[6; 32]).verifying_key(); + let anchor = IndependentAnchor::new(address()); + let journal = MailboxJournal::open_with_anchor( + &root, + address(), + authority, + [7; 32], + limits(), + &anchor, + ) + .expect("empty history agrees with genesis"); + assert_eq!(journal.tip(), JournalTip::genesis()); + drop(journal); + + let anchored = JournalTip { + cursor: 1, + head: [8; 32], + }; + anchor + .compare_and_advance(address(), JournalTip::genesis(), anchored) + .expect("external anchor survives local loss"); + assert!(matches!( + MailboxJournal::open_with_anchor( + &root, + address(), + authority, + [7; 32], + limits(), + &anchor, + ), + Err(JournalError::Rollback) + )); + let mut foreign = address(); + foreign.tenant = TenantId::from_bytes([9; 32]); + assert!(matches!( + MailboxJournal::open_with_anchor(&root, foreign, authority, [7; 32], limits(), &anchor), + Err(JournalError::WrongAuthority) + )); + } +} From 4b60a0bd0a8ddc20582f6b099f0daa8737fdc0e0 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 23:06:56 -0800 Subject: [PATCH 11/51] test(mailbox): refuse unanchored durable verdicts Replace local FileJournalAnchor acceptance expectations with explicit refusal across service and process restart. Exercise signed lease and ACK recovery only behind a test-scoped compare-and-advance anchor; the real service remains dev-ephemeral and production-refusing. --- crates/mvm-hostd/tests/mission_demo.rs | 162 +++++++++++++------------ 1 file changed, 84 insertions(+), 78 deletions(-) diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index e5157d33ff..493fff96c8 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1064,8 +1064,7 @@ fn a_running_guest_submits_again_after_its_first_message_was_consumed() { } #[test] -fn durable_local_destination_signs_only_after_journal_and_anchor_commit() { - use mvm_contract::protocol::mailbox::DeliveryOutcome; +fn local_file_anchor_cannot_issue_durable_acceptance() { use mvm_core::mailbox::journal::{FileJournalAnchor, JournalLimits, MailboxJournal}; use mvm_hostd::mailbox_service::durable::{DurableMailboxService, send as send_durable}; @@ -1085,9 +1084,6 @@ fn durable_local_destination_signs_only_after_journal_and_anchor_commit() { routing.expires_at_unix_ms = header.expires_at_unix_ms; let mut capability_grant = grant(&[MailboxOperation::Send]); capability_grant.expires_at_unix_ms = header.expires_at_unix_ms; - let mut undersized_grant = capability_grant.clone(); - undersized_grant.max_payload_bytes = 4; - let undersized = sign_capability(undersized_grant, &signer).unwrap(); let capability = sign_capability(capability_grant, &signer).unwrap(); let envelope = seal_envelope( SealRequest { @@ -1130,78 +1126,87 @@ fn durable_local_destination_signs_only_after_journal_and_anchor_commit() { let service = start(); assert_eq!( - send_durable(&socket, &undersized, &envelope, &header, &authority), - Err(Refusal::Oversized(4)), - "the signed payload ceiling is a terminal refusal, not a durable receipt" + send_durable(&socket, &capability, &envelope, &header, &authority), + Err(Refusal::Unavailable), + "a local anchor cannot authorize a signed durable verdict" ); - let accepted = send_durable(&socket, &capability, &envelope, &header, &authority) - .expect("destination signed durable verdict"); - assert_eq!(accepted.receipt.outcome, DeliveryOutcome::AcceptedDurable); - let mut receive_grant = grant(&[MailboxOperation::Receive]); - receive_grant.subject = aircraft_principal(); - receive_grant.capability_id = CapabilityId::from_bytes([88; 32]); - receive_grant.expires_at_unix_ms = header.expires_at_unix_ms; - let receive_capability = sign_capability(receive_grant, &signer).unwrap(); - let mut reader = context(1); - reader.acting_principal = aircraft_principal(); - reader.required_operation = MailboxOperation::Receive; - assert!(matches!( - service.receive_next(&capability, &reader), - Err(mvm_core::mailbox::journal::JournalError::Mailbox( - MailboxError::WrongCapabilitySubject - )) - )); - let pending = service - .receive_next(&receive_capability, &reader) - .expect("independent Receive authority") - .expect("pending message"); - assert_eq!(pending.payload(), b"hello"); - let identity = pending.identity(); - service - .acknowledge(identity, &receive_capability, &reader) - .expect("durable ACK"); - assert!( - service - .receive_next(&receive_capability, &reader) - .expect("after ACK") - .is_none() + assert_eq!( + std::fs::read_dir(&journal_dir).unwrap().count(), + 1, + "lock only" ); drop(service); let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]) .expect("anchor survived destination restart"); - let tip = anchor.tip(); - assert_eq!(tip.cursor, 2, "acceptance and ACK each advanced the anchor"); + assert_eq!( + anchor.tip().cursor, + 0, + "refused admission cannot advance a local anchor" + ); drop(anchor); let service = start(); - assert!( - service - .receive_next(&receive_capability, &reader) - .expect("ACK survived restart") - .is_none() + assert_eq!( + send_durable(&socket, &capability, &envelope, &header, &authority), + Err(Refusal::Unavailable), + "restart does not upgrade an untrusted local anchor" ); - let duplicate = send_durable(&socket, &capability, &envelope, &header, &authority) - .expect("retry after service restart"); - assert_eq!(duplicate.receipt.outcome, DeliveryOutcome::Duplicate); drop(service); - let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]) - .expect("retry did not corrupt the anchor"); - assert_eq!(anchor.tip(), tip, "retry must not append a record"); } #[test] -fn signed_guest_lease_and_exact_ack_survive_journal_restart() { +fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { use mvm_contract::plan::{Nonce, PlanId}; use mvm_contract::protocol::mailbox::{ ConsumerSessionId, DeliveryAckNonce, MailboxDeliveryAck, MailboxGrant, MailboxGuestReaderEnrollment, MailboxGuestReaderScope, }; use mvm_core::mailbox::journal::{ - FileJournalAnchor, JournalAdmission, JournalError, JournalGuestAck, JournalGuestLease, - JournalLimits, JournalReceive, MailboxJournal, + JournalAdmission, JournalAnchor, JournalError, JournalGuestAck, JournalGuestLease, + JournalLimits, JournalReceive, JournalTip, MailboxJournal, }; use mvm_core::mailbox::reader::{ sign_delivery_ack, sign_guest_reader_enrollment, sign_reader_possession, }; + use std::sync::{Arc, Mutex}; + + #[derive(Clone)] + struct TestAnchor(Arc>); + + impl JournalAnchor for TestAnchor { + fn load(&self, target: MailboxAddress) -> Result { + if target != address() { + return Err(JournalError::WrongAuthority); + } + Ok(*self.0.lock().expect("mock anchor lock")) + } + + fn compare_and_advance( + &self, + target: MailboxAddress, + expected: JournalTip, + next: JournalTip, + ) -> Result<(), JournalError> { + if target != address() { + return Err(JournalError::WrongAuthority); + } + let mut current = self.0.lock().expect("mock anchor lock"); + if *current != expected || expected.cursor.checked_add(1) != Some(next.cursor) { + return Err(JournalError::Rollback); + } + *current = next; + Ok(()) + } + } + + impl TestAnchor { + fn tip(&self) -> JournalTip { + self.load(address()).expect("mock anchor tip") + } + + fn advance(&self, next: JournalTip) -> Result<(), JournalError> { + self.compare_and_advance(address(), self.tip(), next) + } + } let (node, channel) = channel_with_one_node(); let root = node.home.path(); @@ -1279,15 +1284,15 @@ fn signed_guest_lease_and_exact_ack_survive_journal_restart() { encryption_key: &channel.encryption_key, sender_key: &channel.sender_key.verifying_key(), }; - let anchor_path = root.join("anchor"); let journal_path = root.join("journal"); let limits = JournalLimits { max_records: 6, max_bytes: 64 * 1024, }; + let anchor_state = Arc::new(Mutex::new(JournalTip::genesis())); let open = || { - let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]).unwrap(); - let journal = MailboxJournal::open_reconciling( + let anchor = TestAnchor(Arc::clone(&anchor_state)); + let journal = MailboxJournal::open_with_anchor( &journal_path, address(), authority, @@ -1300,7 +1305,7 @@ fn signed_guest_lease_and_exact_ack_survive_journal_restart() { }; let (anchor, mut journal) = open(); journal - .admit( + .admit_with_anchor( JournalAdmission { envelope: &envelope, capability: &send, @@ -1309,7 +1314,7 @@ fn signed_guest_lease_and_exact_ack_survive_journal_restart() { sender_key: &channel.sender_key.verifying_key(), signer, }, - |tip| anchor.advance(tip), + &anchor, ) .unwrap(); let challenge = [82; 32]; @@ -1573,10 +1578,9 @@ fn durable_mailbox_helper_process() { } #[test] -fn a_killed_destination_reopens_its_anchor_and_deduplicates_the_retry() { - use mvm_contract::protocol::mailbox::DeliveryOutcome; +fn a_killed_destination_still_refuses_unanchored_durable_acceptance() { + use mvm_core::mailbox::frame::send_receipted_frame; use mvm_core::mailbox::journal::FileJournalAnchor; - use mvm_hostd::mailbox_service::durable::send as send_durable; let dir = tempfile::tempdir_in("/tmp").expect("short Unix socket path"); let socket = dir.path().join("durable.sock"); @@ -1612,22 +1616,24 @@ fn a_killed_destination_reopens_its_anchor_and_deduplicates_the_retry() { ) }; let submit = || { - (0..100) - .find_map(|_| { - match send_durable(&socket, &capability, &envelope, &header, &authority) { - Ok(receipt) => Some(receipt), - Err(Refusal::Unavailable) => { - std::thread::sleep(std::time::Duration::from_millis(20)); - None - } - Err(other) => panic!("unexpected destination refusal: {other}"), - } + let mut stream = [0, 100, 250, 500, 1_000] + .into_iter() + .find_map(|delay_ms| { + std::thread::sleep(std::time::Duration::from_millis(delay_ms)); + std::os::unix::net::UnixStream::connect(&socket).ok() }) - .expect("destination was ready within two seconds") + .expect("destination was ready within two seconds"); + stream + .set_read_timeout(Some(std::time::Duration::from_secs(5))) + .unwrap(); + stream + .set_write_timeout(Some(std::time::Duration::from_secs(5))) + .unwrap(); + send_receipted_frame(&mut stream, &capability, &envelope, &header, &authority) }; let mut process = launch(); - assert_eq!(submit().receipt.outcome, DeliveryOutcome::AcceptedDurable); + assert_eq!(submit(), Err(Refusal::Unavailable)); process.0.kill().expect("crash destination"); process.0.wait().expect("reap crashed process"); drop(process); @@ -1635,12 +1641,12 @@ fn a_killed_destination_reopens_its_anchor_and_deduplicates_the_retry() { .expect("anchor survives process crash"); assert_eq!( anchor.tip().cursor, - 1, - "the signed verdict followed durable commit" + 0, + "refusal cannot publish a local anchor even before a process crash" ); drop(anchor); let _restarted = launch(); - assert_eq!(submit().receipt.outcome, DeliveryOutcome::Duplicate); + assert_eq!(submit(), Err(Refusal::Unavailable)); } /// One framed request to the mailbox socket as a host peer; the raw reply. From 784e7adb558b224a999477a03eb3401740e42829 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 23:09:30 -0800 Subject: [PATCH 12/51] test(mailbox): name missing posture refusal accurately The old corrupt-anchor fixture could only fail because its helper lacked admitted posture. Assert that actual fail-closed boundary instead of claiming local-anchor verification that never ran. --- crates/mvm-hostd/tests/mission_demo.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index 493fff96c8..c11767ec9c 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1915,7 +1915,7 @@ fn served_mailbox_process(dir: &std::path::Path) -> (KillOnDrop, std::path::Path } #[test] -fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { +fn missing_verified_posture_prevents_the_helper_from_reporting_ready() { use mvm_hostd::mailbox_process::build_config; let dir = tempfile::tempdir_in("/tmp").unwrap(); @@ -1930,7 +1930,10 @@ fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { let grant = granted_plan(authority).mailbox.unwrap(); let backend = mvm_runtime::backend::AnyBackend::from_hypervisor("hvf"); let config = build_config(&keys, &grant, &state_dir, &backend).unwrap(); - std::fs::write(state_dir.join("mailbox-anchor"), b"{").unwrap(); + assert!( + config.posture.is_none(), + "unadmitted helper has no Dev token" + ); let mut process = spawn_mailbox_process(&serde_json::to_vec(&config).unwrap()); let status = (0..100).find_map(|_| { let exited = process.0.try_wait().unwrap(); @@ -1942,12 +1945,12 @@ fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { assert_eq!( status.and_then(|status| status.code()), Some(1), - "a corrupt anchor must fail closed; stderr:\n{}", + "missing verified posture must fail closed; stderr:\n{}", process.stderr() ); assert!( !config.socket.exists(), - "no mailbox endpoint may be published from a corrupt anchor" + "no mailbox endpoint may be published without verified admission" ); } From 6e75656a2113a4e09c8e03644bbdb37921615cb1 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 23:10:44 -0800 Subject: [PATCH 13/51] test(mailbox): retire obsolete durable helper fixture --- crates/mvm-hostd/tests/mission_demo.rs | 151 ++----------------------- 1 file changed, 7 insertions(+), 144 deletions(-) diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index e5157d33ff..45671e2e90 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1769,22 +1769,6 @@ impl KillOnDrop { fn stderr(&self) -> String { std::fs::read_to_string(&self.1).unwrap_or_default() } - - /// Accept one connection within 20s, or fail with the child's stderr. - fn accept( - &mut self, - listener: &std::os::unix::net::UnixListener, - ) -> std::os::unix::net::UnixStream { - listener.set_nonblocking(true).unwrap(); - for _ in 0..400 { - if let Ok((stream, _)) = listener.accept() { - stream.set_nonblocking(false).unwrap(); - return stream; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - panic!("the process never dialed; stderr:\n{}", self.stderr()); - } } impl Drop for KillOnDrop { fn drop(&mut self) { @@ -1806,110 +1790,8 @@ fn spawn_mailbox_process(config: &[u8]) -> KillOnDrop { KillOnDrop(child, log) } -/// A mailbox process under `dir`, started for a granted mailbox, whose guest -/// has delivered its one message. Returns the process and the VM state dir. -fn served_mailbox_process(dir: &std::path::Path) -> (KillOnDrop, std::path::PathBuf) { - use mvm_core::vm_backend::HostDialRoute; - use mvm_hostd::mailbox_process::build_config; - use std::os::unix::net::UnixListener; - - let keys = dir.join("keys"); - std::fs::create_dir_all(&keys).unwrap(); - host_keys(&keys, 5); - let state_dir = dir.join("vm"); - std::fs::create_dir_all(&state_dir).unwrap(); - let authority = mvm_hostd::mailbox_launch::authority_key_for(&keys, &address().mailbox) - .unwrap() - .to_bytes(); - let grant = granted_plan(authority).mailbox.unwrap(); - let backend = mvm_runtime::backend::AnyBackend::from_hypervisor("hvf"); - let config = build_config(&keys, &grant, &state_dir, &backend).unwrap(); - let HostDialRoute::PortSocket(ref guest_port) = config.guest_route else { - panic!("HVF must bridge the host-dialed mailbox port"); - }; - std::fs::create_dir_all(guest_port.parent().unwrap()).unwrap(); - let listener = UnixListener::bind(guest_port).unwrap(); - let wire = serde_json::to_vec(&config).unwrap(); - let seed = std::fs::read(keys.join("host-signer.ed25519")).unwrap(); - assert!( - !String::from_utf8_lossy(&wire).contains(&hex::encode(&seed)), - "the seed is not in the process config" - ); - - let outbox = dir.join("outbox"); - std::fs::write(&outbox, outbox_message().encode()).unwrap(); - let mut process = spawn_mailbox_process(&wire); - let mut stream = process.accept(&listener); - assert_eq!( - mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &outbox), - Ok(()) - ); - assert!( - (0..100).any(|_| { - std::thread::sleep(std::time::Duration::from_millis(50)); - mvm_core::config::vm_mailbox_socket_at(&state_dir).exists() - }), - "the host service bound its socket; stderr:\n{}", - process.stderr() - ); - assert!( - !outbox.exists(), - "guest received a durable admission verdict" - ); - assert!(state_dir.join("mailbox-anchor").exists()); - let record = std::fs::read( - state_dir - .join("mailbox-journal") - .join("entry-00000000000000000001.bin"), - ) - .expect("accepted canonical ciphertext is durable"); - assert!( - !record.windows(5).any(|w| w == b"hello"), - "the durable record never stores guest plaintext" - ); - let later = OutboxMessage::new(aircraft_principal(), b"after-start".to_vec()); - mvm_agentd::mailbox_outbox::write_outbox(&outbox, &later).unwrap(); - let mut stream = process.accept(&listener); - assert_eq!( - mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &outbox), - Ok(()), - "the running process keeps serving the live guest" - ); - assert!( - state_dir - .join("mailbox-journal") - .join("entry-00000000000000000002.bin") - .exists(), - "a post-start operation committed durably" - ); - process.0.kill().expect("crash the mailbox helper"); - process.0.wait().expect("reap the crashed helper"); - drop(process); - mvm_agentd::mailbox_outbox::write_outbox(&outbox, &outbox_message()).unwrap(); - let mut restarted = spawn_mailbox_process(&wire); - let forwarded = (0..8).any(|_| { - let mut stream = restarted.accept(&listener); - matches!( - mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &outbox), - Ok(()) - ) - }); - assert!( - forwarded, - "the same immutable guest operation must be reconciled after restart" - ); - assert!( - !state_dir - .join("mailbox-journal") - .join("entry-00000000000000000003.bin") - .exists(), - "a recovered duplicate does not create a second durable acceptance" - ); - (restarted, state_dir) -} - #[test] -fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { +fn missing_verified_posture_prevents_the_helper_from_reporting_ready() { use mvm_hostd::mailbox_process::build_config; let dir = tempfile::tempdir_in("/tmp").unwrap(); @@ -1924,7 +1806,10 @@ fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { let grant = granted_plan(authority).mailbox.unwrap(); let backend = mvm_runtime::backend::AnyBackend::from_hypervisor("hvf"); let config = build_config(&keys, &grant, &state_dir, &backend).unwrap(); - std::fs::write(state_dir.join("mailbox-anchor"), b"{").unwrap(); + assert!( + config.posture.is_none(), + "build_config cannot mint admission" + ); let mut process = spawn_mailbox_process(&serde_json::to_vec(&config).unwrap()); let status = (0..100).find_map(|_| { let exited = process.0.try_wait().unwrap(); @@ -1936,37 +1821,15 @@ fn a_corrupt_mailbox_anchor_prevents_the_helper_from_reporting_ready() { assert_eq!( status.and_then(|status| status.code()), Some(1), - "a corrupt anchor must fail closed; stderr:\n{}", + "missing admitted posture must fail closed; stderr:\n{}", process.stderr() ); assert!( !config.socket.exists(), - "no mailbox endpoint may be published from a corrupt anchor" + "no mailbox endpoint may be published without admitted posture" ); } -/// Removing the VM's state directory, and with it the mailbox socket path, -/// ends a serving process: stopping its service must not need that path. -#[test] -fn the_mailbox_process_exits_when_its_vm_state_dir_is_removed() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - let (mut process, state_dir) = served_mailbox_process(dir.path()); - std::fs::remove_dir_all(&state_dir).unwrap(); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); - let status = loop { - if let Some(status) = process.0.try_wait().unwrap() { - break status; - } - assert!( - std::time::Instant::now() < deadline, - "the process outlived its VM; stderr:\n{}", - process.stderr() - ); - std::thread::sleep(std::time::Duration::from_millis(50)); - }; - assert!(status.success(), "{status}"); -} - // ---- two nodes over a loopback TCP PeerTransport ---------------------------- fn tcp_limits() -> TcpListenerLimits { From 5d50caa29b81bfb158719eabf53cdf382da620e2 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 23:12:14 -0800 Subject: [PATCH 14/51] test(mailbox): retire obsolete durable helper fixture The old integration fixture required unverified helper posture and local-file accepted_durable. Preserve its missing-posture refusal; the verified-token mailbox-process unit test covers lifecycle exit. Based on guest owner test handoff 6e75656a21. --- crates/mvm-hostd/tests/mission_demo.rs | 140 ------------------------- 1 file changed, 140 deletions(-) diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index c11767ec9c..e750d5199f 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1775,22 +1775,6 @@ impl KillOnDrop { fn stderr(&self) -> String { std::fs::read_to_string(&self.1).unwrap_or_default() } - - /// Accept one connection within 20s, or fail with the child's stderr. - fn accept( - &mut self, - listener: &std::os::unix::net::UnixListener, - ) -> std::os::unix::net::UnixStream { - listener.set_nonblocking(true).unwrap(); - for _ in 0..400 { - if let Ok((stream, _)) = listener.accept() { - stream.set_nonblocking(false).unwrap(); - return stream; - } - std::thread::sleep(std::time::Duration::from_millis(50)); - } - panic!("the process never dialed; stderr:\n{}", self.stderr()); - } } impl Drop for KillOnDrop { fn drop(&mut self) { @@ -1812,108 +1796,6 @@ fn spawn_mailbox_process(config: &[u8]) -> KillOnDrop { KillOnDrop(child, log) } -/// A mailbox process under `dir`, started for a granted mailbox, whose guest -/// has delivered its one message. Returns the process and the VM state dir. -fn served_mailbox_process(dir: &std::path::Path) -> (KillOnDrop, std::path::PathBuf) { - use mvm_core::vm_backend::HostDialRoute; - use mvm_hostd::mailbox_process::build_config; - use std::os::unix::net::UnixListener; - - let keys = dir.join("keys"); - std::fs::create_dir_all(&keys).unwrap(); - host_keys(&keys, 5); - let state_dir = dir.join("vm"); - std::fs::create_dir_all(&state_dir).unwrap(); - let authority = mvm_hostd::mailbox_launch::authority_key_for(&keys, &address().mailbox) - .unwrap() - .to_bytes(); - let grant = granted_plan(authority).mailbox.unwrap(); - let backend = mvm_runtime::backend::AnyBackend::from_hypervisor("hvf"); - let config = build_config(&keys, &grant, &state_dir, &backend).unwrap(); - let HostDialRoute::PortSocket(ref guest_port) = config.guest_route else { - panic!("HVF must bridge the host-dialed mailbox port"); - }; - std::fs::create_dir_all(guest_port.parent().unwrap()).unwrap(); - let listener = UnixListener::bind(guest_port).unwrap(); - let wire = serde_json::to_vec(&config).unwrap(); - let seed = std::fs::read(keys.join("host-signer.ed25519")).unwrap(); - assert!( - !String::from_utf8_lossy(&wire).contains(&hex::encode(&seed)), - "the seed is not in the process config" - ); - - let outbox = dir.join("outbox"); - std::fs::write(&outbox, outbox_message().encode()).unwrap(); - let mut process = spawn_mailbox_process(&wire); - let mut stream = process.accept(&listener); - assert_eq!( - mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &outbox), - Ok(()) - ); - assert!( - (0..100).any(|_| { - std::thread::sleep(std::time::Duration::from_millis(50)); - mvm_core::config::vm_mailbox_socket_at(&state_dir).exists() - }), - "the host service bound its socket; stderr:\n{}", - process.stderr() - ); - assert!( - !outbox.exists(), - "guest received a durable admission verdict" - ); - assert!(state_dir.join("mailbox-anchor").exists()); - let record = std::fs::read( - state_dir - .join("mailbox-journal") - .join("entry-00000000000000000001.bin"), - ) - .expect("accepted canonical ciphertext is durable"); - assert!( - !record.windows(5).any(|w| w == b"hello"), - "the durable record never stores guest plaintext" - ); - let later = OutboxMessage::new(aircraft_principal(), b"after-start".to_vec()); - mvm_agentd::mailbox_outbox::write_outbox(&outbox, &later).unwrap(); - let mut stream = process.accept(&listener); - assert_eq!( - mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &outbox), - Ok(()), - "the running process keeps serving the live guest" - ); - assert!( - state_dir - .join("mailbox-journal") - .join("entry-00000000000000000002.bin") - .exists(), - "a post-start operation committed durably" - ); - process.0.kill().expect("crash the mailbox helper"); - process.0.wait().expect("reap the crashed helper"); - drop(process); - mvm_agentd::mailbox_outbox::write_outbox(&outbox, &outbox_message()).unwrap(); - let mut restarted = spawn_mailbox_process(&wire); - let forwarded = (0..8).any(|_| { - let mut stream = restarted.accept(&listener); - matches!( - mvm_agentd::mailbox_outbox::forward_outbox(&mut stream, &outbox), - Ok(()) - ) - }); - assert!( - forwarded, - "the same immutable guest operation must be reconciled after restart" - ); - assert!( - !state_dir - .join("mailbox-journal") - .join("entry-00000000000000000003.bin") - .exists(), - "a recovered duplicate does not create a second durable acceptance" - ); - (restarted, state_dir) -} - #[test] fn missing_verified_posture_prevents_the_helper_from_reporting_ready() { use mvm_hostd::mailbox_process::build_config; @@ -1954,28 +1836,6 @@ fn missing_verified_posture_prevents_the_helper_from_reporting_ready() { ); } -/// Removing the VM's state directory, and with it the mailbox socket path, -/// ends a serving process: stopping its service must not need that path. -#[test] -fn the_mailbox_process_exits_when_its_vm_state_dir_is_removed() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - let (mut process, state_dir) = served_mailbox_process(dir.path()); - std::fs::remove_dir_all(&state_dir).unwrap(); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); - let status = loop { - if let Some(status) = process.0.try_wait().unwrap() { - break status; - } - assert!( - std::time::Instant::now() < deadline, - "the process outlived its VM; stderr:\n{}", - process.stderr() - ); - std::thread::sleep(std::time::Duration::from_millis(50)); - }; - assert!(status.success(), "{status}"); -} - // ---- two nodes over a loopback TCP PeerTransport ---------------------------- fn tcp_limits() -> TcpListenerLimits { From 771a822f73d675116977b71102e6c1474cc0e208 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 23:30:05 -0800 Subject: [PATCH 15/51] fix(mailbox): verify recovered reader controls and ACK retries Recheck retained authority-signed enrollment, Receive capability and lease, plus the guest ACK signature, before recovery advances the anchor. Require current boot, session and expiry authorization on an exact committed ACK retry. Exercise malicious history-key record rewrites and stale duplicate attempts under a test-only CAS anchor. --- crates/mvm-core/README.md | 4 + crates/mvm-core/src/mailbox/journal.rs | 103 +++++++++++++++--- crates/mvm-hostd/tests/mission_demo.rs | 140 +++++++++++++++++++++++++ 3 files changed, 233 insertions(+), 14 deletions(-) diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index b56279a99c..fccb2bdf7b 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -69,6 +69,10 @@ the actual acceptance receipt only after both commits. Earlier private-handoff version-1 records are refused, not silently reinterpreted. Recovery verifies the intent, refuses a missing or divergent anchored prefix, and may reconcile only an intact contiguous suffix written before an interrupted anchor commit. +Retained guest-reader controls are checked against the historical signed +Receive capability, enrollment, lease and guest ACK before recovery accepts +their delivery state. An exact ACK retry still needs current boot, session, +policy and expiry authorization; an old committed ACK is not a new grant. `FileJournalAnchor` is a local development aid, **not** that independent authority: restoring both the journal and its host-local anchor can hide diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 0d90848b8e..944b605796 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -16,10 +16,11 @@ use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; use hmac::{Hmac, Mac, digest::KeyInit}; use mvm_contract::protocol::mailbox::{ - ContentType, DeliveryLeaseNonce, DeliveryOutcome, DestinationReceipt, MailboxAddress, - MailboxDeliveryLease, MailboxEnvelope, MailboxGrant, MailboxGuestReaderScope, MailboxOperation, - ProtocolVersion, SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, - SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, + ContentType, DELIVERY_ACK_SIGNATURE_DOMAIN, DeliveryLeaseNonce, DeliveryOutcome, + DestinationReceipt, MailboxAddress, MailboxDeliveryLease, MailboxEnvelope, MailboxGrant, + MailboxGuestReaderScope, MailboxOperation, ProtocolVersion, SignatureAlgorithm, + SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxDeliveryAck, + SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, }; use serde::{Deserialize, Serialize}; use sha2::Sha256; @@ -27,11 +28,11 @@ use zeroize::Zeroizing; use super::reader::{ ReaderDeliveryContext, sign_delivery_lease, verify_delivery_ack, verify_delivery_lease, - verify_guest_reader_enrollment, verify_reader_possession, + verify_delivery_lease_signature, verify_guest_reader_enrollment, verify_reader_possession, }; use super::{ - MailboxError, OpenContext, ReplayIdentity, ReplayProtector, authorize_receive, frame, - open_authenticated_envelope, sign_receipt, + MailboxError, OpenContext, ReplayIdentity, ReplayProtector, authorize_receive, canonical_bytes, + frame, open_authenticated_envelope, sign_receipt, }; use crate::crypto::aead::{self, Key}; use crate::private_fs; @@ -428,6 +429,50 @@ fn acceptance_intent_bytes( Ok(bytes) } +/// Authenticate retained reader controls at their original issuance time. +/// Current boot, policy and revocation are checked separately before use. +fn verify_recovered_lease( + entry: &Seen, + identity: ReplayIdentity, + lease: &SignedMailboxDeliveryLease, + enrollment: &SignedMailboxGuestReaderEnrollment, + capability: &SignedMailboxCapability, + authority: &VerifyingKey, + now_unix_ms: i64, +) -> Result { + let plan = MailboxGrant { + mailbox: lease.lease.address.mailbox, + authority_key: authority.to_bytes(), + epoch: lease.lease.execution_epoch, + }; + let scope = MailboxGuestReaderScope { + plan: &plan, + plan_id: &enrollment.enrollment.plan_id, + plan_nonce: &enrollment.enrollment.plan_nonce, + receive: &capability.grant, + current_boot: enrollment.enrollment.consumer_boot, + current_session: enrollment.enrollment.consumer_session, + now_unix_ms, + }; + let guest_key = verify_guest_reader_enrollment(enrollment, capability, &scope, authority, &[]) + .map_err(|_| JournalError::Corrupt)?; + lease + .check_scope(&plan, &capability.grant, scope.current_session, now_unix_ms) + .map_err(|_| JournalError::Corrupt)?; + verify_delivery_lease_signature(lease, authority).map_err(|_| JournalError::Corrupt)?; + if lease.lease.sender != identity.sender + || lease.lease.sender_boot != identity.sender_boot + || lease.lease.message_id != identity.message_id + || lease.lease.execution_epoch != identity.execution_epoch + || lease.lease.delivery_id != entry.envelope.routing.delivery_id + || lease.lease.ciphertext_commitment != entry.envelope.ciphertext_commitment + || lease.lease.expires_at_unix_ms > entry.envelope.routing.expires_at_unix_ms + { + return Err(JournalError::Corrupt); + } + Ok(guest_key) +} + impl MailboxJournal { /// Reconstruct under an exclusive lock and require the separately trusted /// high-water mark. Genesis is valid only for an empty journal. @@ -661,6 +706,15 @@ impl MailboxJournal { { return Err(JournalError::Corrupt); } + verify_recovered_lease( + entry, + identity, + &lease, + &enrollment, + &receive_capability, + &authority, + lease.lease.issued_at_unix_ms, + )?; entry.lease = Some(LeaseState { retry_count: 0, signed: *lease, @@ -700,6 +754,27 @@ impl MailboxJournal { { return Err(JournalError::Corrupt); } + let guest_key = verify_recovered_lease( + entry, + identity, + &active.signed, + &active.enrollment, + &active.receive_capability, + &authority, + ack.ack.acknowledged_at_unix_ms, + )?; + let signature: [u8; 64] = ack + .signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + guest_key + .verify( + &canonical_bytes(DELIVERY_ACK_SIGNATURE_DOMAIN, &ack.ack) + .map_err(|_| JournalError::Corrupt)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; entry.acknowledged = true; entry.signed_ack = Some(*ack); } @@ -1153,13 +1228,6 @@ impl MailboxJournal { { return Err(JournalError::Mailbox(MailboxError::CapabilityRevoked)); } - if entry.acknowledged { - return if entry.signed_ack.as_ref() == Some(request.signed) { - Ok(self.tip) - } else { - Err(JournalError::ConflictingRetry) - }; - } let opened = self.open_for_reader(identity, entry, &request.receive)?; let guest_key = VerifyingKey::from_bytes(&request.enrollment.enrollment.ack_public_key) .map_err(|_| JournalError::Mailbox(MailboxError::ReaderKeyInvalid))?; @@ -1180,6 +1248,13 @@ impl MailboxJournal { request.enrollment, &guest_key, )?; + if entry.acknowledged { + return if entry.signed_ack.as_ref() == Some(request.signed) { + Ok(self.tip) + } else { + Err(JournalError::ConflictingRetry) + }; + } let tip = self.append( JournalEvent::AcknowledgedSigned { identity, diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index e750d5199f..80970d7357 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1208,6 +1208,45 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { } } + fn forge_record_signature( + root: &std::path::Path, + cursor: u64, + previous: [u8; 32], + signature_path: &[&str], + ) -> (Vec, JournalTip) { + use hmac::{Hmac, Mac, digest::KeyInit}; + use sha2::Sha256; + + let path = root.join(format!("entry-{cursor:020}.bin")); + let original = std::fs::read(&path).expect("record"); + let journal_key = [90; 32]; + let mut derivation = + as KeyInit>::new_from_slice(&journal_key).expect("journal key"); + derivation.update(b"mvm.mailbox.journal.encryption.v1\0"); + let record_key = Key::from_bytes(derivation.finalize().into_bytes().into()); + let aad = [b"mvm.mailbox.journal.v1\0".as_slice(), &previous].concat(); + let plaintext = + mvm_core::crypto::aead::open(&record_key, &original[32..], &aad).expect("open record"); + let mut record: serde_json::Value = + serde_json::from_slice(&plaintext).expect("record JSON"); + let mut field = &mut record["event"]; + for component in signature_path { + field = &mut field[*component]; + } + let signature = field.as_str().expect("base64 signature"); + let replacement = if signature.starts_with('A') { 'B' } else { 'A' }; + *field = serde_json::Value::String(format!("{replacement}{}", &signature[1..])); + let body = serde_json::to_vec(&record).expect("modified record"); + let ciphertext = mvm_core::crypto::aead::seal(&record_key, &body, &aad); + let mut mac = as KeyInit>::new_from_slice(&journal_key).expect("journal key"); + mac.update(b"mvm.mailbox.journal.v1\0"); + mac.update(&previous); + mac.update(&ciphertext); + let head: [u8; 32] = mac.finalize().into_bytes().into(); + std::fs::write(&path, [&head[..], &ciphertext].concat()).expect("rewrite sealed record"); + (original, JournalTip { cursor, head }) + } + let (node, channel) = channel_with_one_node(); let root = node.home.path(); let signer = &channel.authority_key; @@ -1317,6 +1356,7 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { &anchor, ) .unwrap(); + let accepted_tip = anchor.tip(); let challenge = [82; 32]; let proof = sign_reader_possession(&enrollment, &challenge, &guest).unwrap(); let mut wrong_tenant = reader.clone(); @@ -1381,8 +1421,38 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { .unwrap(); assert_eq!(leased.payload(), b"hello"); assert_eq!(leased.content_type(), &content_type()); + let lease_tip = anchor.tip(); drop(journal); drop(anchor); + for signature_path in [ + &["lease", "signature"][..], + &["enrollment", "signature"], + &["receive_capability", "signature"], + ] { + let (original, forged) = + forge_record_signature(&journal_path, 2, accepted_tip.head, signature_path); + *anchor_state.lock().expect("mock anchor lock") = forged; + assert!( + matches!( + MailboxJournal::open_with_anchor( + &journal_path, + address(), + authority, + [90; 32], + limits, + &TestAnchor(Arc::clone(&anchor_state)), + ), + Err(JournalError::Corrupt) + ), + "a history-key holder cannot forge a signed lease, enrollment or Receive grant" + ); + std::fs::write( + journal_path.join("entry-00000000000000000002.bin"), + original, + ) + .expect("restore valid lease"); + *anchor_state.lock().expect("mock anchor lock") = lease_tip; + } let (anchor, mut journal) = open(); assert_eq!( anchor.tip().cursor, @@ -1495,6 +1565,7 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { .acknowledge_signed(wrong_boot, |tip| anchor.advance(tip)) .is_err() ); + let before_ack = anchor.tip(); let tip = journal .acknowledge_signed(request, |tip| anchor.advance(tip)) .unwrap(); @@ -1506,6 +1577,52 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { tip, "a lost ACK response retries the exact bytes without a second commit" ); + let stale_boot = JournalGuestAck { + scope: MailboxGuestReaderScope { + current_boot: BootId::from_bytes([84; 32]), + ..scope + }, + ..request + }; + assert!( + journal + .acknowledge_signed(stale_boot, |tip| anchor.advance(tip)) + .is_err(), + "a committed ACK cannot bypass current boot authority on retry" + ); + let stale_session = JournalGuestAck { + scope: MailboxGuestReaderScope { + current_session: ConsumerSessionId::from_bytes([85; 32]), + ..scope + }, + ..request + }; + assert!( + journal + .acknowledge_signed(stale_session, |tip| anchor.advance(tip)) + .is_err(), + "a committed ACK cannot bypass current session authority on retry" + ); + let mut expired_reader = reader.clone(); + expired_reader.now_unix_ms = 4_000_000_000_000; + let expired = JournalGuestAck { + receive: JournalReceive { + context: &expired_reader, + ..read + }, + scope: MailboxGuestReaderScope { + now_unix_ms: expired_reader.now_unix_ms, + ..scope + }, + ..request + }; + assert!( + journal + .acknowledge_signed(expired, |tip| anchor.advance(tip)) + .is_err(), + "an expired message cannot borrow a prior ACK to appear authorized" + ); + assert_eq!(anchor.tip(), tip, "refused retries do not append"); let mut changed = ack.clone(); changed.ack.ack_nonce = DeliveryAckNonce::from_bytes([85; 16]); let changed = sign_delivery_ack(changed.ack, &guest).unwrap(); @@ -1521,6 +1638,29 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { )); drop(journal); drop(anchor); + let (original, forged) = + forge_record_signature(&journal_path, 6, before_ack.head, &["ack", "signature"]); + *anchor_state.lock().expect("mock anchor lock") = forged; + assert!( + matches!( + MailboxJournal::open_with_anchor( + &journal_path, + address(), + authority, + [90; 32], + limits, + &TestAnchor(Arc::clone(&anchor_state)), + ), + Err(JournalError::Corrupt) + ), + "a history-key holder cannot fabricate a guest-signed ACK" + ); + std::fs::write( + journal_path.join("entry-00000000000000000006.bin"), + original, + ) + .expect("restore valid ACK"); + *anchor_state.lock().expect("mock anchor lock") = tip; let (anchor, mut journal) = open(); assert_eq!(anchor.tip(), tip); assert!(journal.receive_next(read).unwrap().is_none()); From cac0122af2f8141d01396ed123b2bcd8d1f988e5 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 23:46:47 -0800 Subject: [PATCH 16/51] fix(mailbox): require signed ACK after guest lease Refuse the legacy unsigned host ACK for any guest-leased message at admission and replay, while retaining it for never-leased host messages. Exercise the live refusal and a forged encrypted recovery record with matching MAC and mock anchor. --- crates/mvm-core/README.md | 4 +- crates/mvm-core/src/mailbox/journal.rs | 81 +++++++++++++++++++- crates/mvm-hostd/tests/mission_demo.rs | 100 ++++++++++++++++++++----- 3 files changed, 164 insertions(+), 21 deletions(-) diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index fccb2bdf7b..71cba037c4 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -72,7 +72,9 @@ only an intact contiguous suffix written before an interrupted anchor commit. Retained guest-reader controls are checked against the historical signed Receive capability, enrollment, lease and guest ACK before recovery accepts their delivery state. An exact ACK retry still needs current boot, session, -policy and expiry authorization; an old committed ACK is not a new grant. +policy and expiry authorization; an old committed ACK is not a new grant. A +message ever leased to a guest cannot be closed by the legacy unsigned host +ACK, including during recovery; it requires that guest's signed ACK. `FileJournalAnchor` is a local development aid, **not** that independent authority: restoring both the journal and its host-local anchor can hide diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 944b605796..50d3c184e9 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -113,6 +113,8 @@ pub enum JournalError { UnknownMessage, #[error("mailbox message has no active delivery lease")] NoActiveLease, + #[error("guest-leased message requires a signed guest acknowledgement")] + GuestAckRequired, #[error("journal cannot be used after an uncertain write")] Uncertain, #[error("independent rollback anchor and recoverable key custody are required")] @@ -675,7 +677,7 @@ impl MailboxJournal { let Some(entry) = journal.seen.get_mut(&identity) else { return Err(JournalError::Corrupt); }; - if entry.acknowledged || acknowledged_at_unix_ms < 0 { + if entry.acknowledged || entry.lease.is_some() || acknowledged_at_unix_ms < 0 { return Err(JournalError::Corrupt); } entry.acknowledged = true; @@ -1169,6 +1171,9 @@ impl MailboxJournal { .get(&identity) .ok_or(JournalError::UnknownMessage)?; let _opened = self.open_for_reader(identity, entry, &request)?; + if entry.lease.is_some() { + return Err(JournalError::GuestAckRequired); + } if entry.acknowledged { return Ok(self.tip); } @@ -1681,6 +1686,80 @@ mod anchor_tests { assert_eq!(anchor.load(address()).expect("unchanged tip").cursor, 1); } + #[test] + fn unsigned_host_ack_remains_available_only_before_a_guest_lease() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let anchor = IndependentAnchor::new(address()); + let mut journal = MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ) + .expect("genesis"); + journal + .admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("accepted under test anchor"); + let mut receive_grant = capability.grant.clone(); + receive_grant.subject = address().recipient; + receive_grant.capability_id = CapabilityId::from_bytes([31; 32]); + receive_grant.operations = + CapabilityOperations::from_operations(&[MailboxOperation::Receive]); + let receive = super::super::sign_capability(receive_grant, &signer).expect("Receive grant"); + let mut reader_context = context; + reader_context.acting_principal = address().recipient; + reader_context.required_operation = MailboxOperation::Receive; + let reader = JournalReceive { + capability: &receive, + context: &reader_context, + encryption_key: &payload_key, + sender_key: &sender_key, + }; + let identity = ReplayIdentity { + sender: reader_context.expected_sender, + sender_boot: BootId::from_bytes([8; 32]), + execution_epoch: reader_context.current_execution_epoch, + message_id: MessageId::from_bytes([9; 32]), + }; + let previous = journal.tip(); + let tip = journal + .acknowledge(identity, reader, |next| { + anchor.compare_and_advance(address(), previous, next) + }) + .expect("never-leased host message can be acknowledged"); + assert_eq!(tip.cursor, 2); + drop(journal); + let recovered = MailboxJournal::open_with_anchor( + &root, + address(), + signer.verifying_key(), + [21; 32], + limits(), + &anchor, + ) + .expect("host ACK survives restart"); + assert!( + recovered + .receive_next(reader) + .expect("authorized reader") + .is_none() + ); + } + #[test] fn interrupted_anchor_commit_reconciles_only_intact_suffix() { let dir = tempfile::tempdir().expect("journal root"); diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index 80970d7357..1a03b080a7 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1208,25 +1208,52 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { } } - fn forge_record_signature( + fn test_record_key() -> Key { + use hmac::{Hmac, Mac, digest::KeyInit}; + use sha2::Sha256; + + let mut derivation = + as KeyInit>::new_from_slice(&[90; 32]).expect("journal key"); + derivation.update(b"mvm.mailbox.journal.encryption.v1\0"); + Key::from_bytes(derivation.finalize().into_bytes().into()) + } + + fn write_resealed_record( root: &std::path::Path, cursor: u64, previous: [u8; 32], - signature_path: &[&str], - ) -> (Vec, JournalTip) { + record: &serde_json::Value, + ) -> JournalTip { use hmac::{Hmac, Mac, digest::KeyInit}; use sha2::Sha256; + let aad = [b"mvm.mailbox.journal.v1\0".as_slice(), &previous].concat(); + let body = serde_json::to_vec(record).expect("modified record"); + let ciphertext = mvm_core::crypto::aead::seal(&test_record_key(), &body, &aad); + let mut mac = as KeyInit>::new_from_slice(&[90; 32]).expect("journal key"); + mac.update(b"mvm.mailbox.journal.v1\0"); + mac.update(&previous); + mac.update(&ciphertext); + let head: [u8; 32] = mac.finalize().into_bytes().into(); + std::fs::write( + root.join(format!("entry-{cursor:020}.bin")), + [&head[..], &ciphertext].concat(), + ) + .expect("rewrite sealed record"); + JournalTip { cursor, head } + } + + fn forge_record_signature( + root: &std::path::Path, + cursor: u64, + previous: [u8; 32], + signature_path: &[&str], + ) -> (Vec, JournalTip) { let path = root.join(format!("entry-{cursor:020}.bin")); let original = std::fs::read(&path).expect("record"); - let journal_key = [90; 32]; - let mut derivation = - as KeyInit>::new_from_slice(&journal_key).expect("journal key"); - derivation.update(b"mvm.mailbox.journal.encryption.v1\0"); - let record_key = Key::from_bytes(derivation.finalize().into_bytes().into()); let aad = [b"mvm.mailbox.journal.v1\0".as_slice(), &previous].concat(); - let plaintext = - mvm_core::crypto::aead::open(&record_key, &original[32..], &aad).expect("open record"); + let plaintext = mvm_core::crypto::aead::open(&test_record_key(), &original[32..], &aad) + .expect("open record"); let mut record: serde_json::Value = serde_json::from_slice(&plaintext).expect("record JSON"); let mut field = &mut record["event"]; @@ -1236,15 +1263,10 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { let signature = field.as_str().expect("base64 signature"); let replacement = if signature.starts_with('A') { 'B' } else { 'A' }; *field = serde_json::Value::String(format!("{replacement}{}", &signature[1..])); - let body = serde_json::to_vec(&record).expect("modified record"); - let ciphertext = mvm_core::crypto::aead::seal(&record_key, &body, &aad); - let mut mac = as KeyInit>::new_from_slice(&journal_key).expect("journal key"); - mac.update(b"mvm.mailbox.journal.v1\0"); - mac.update(&previous); - mac.update(&ciphertext); - let head: [u8; 32] = mac.finalize().into_bytes().into(); - std::fs::write(&path, [&head[..], &ciphertext].concat()).expect("rewrite sealed record"); - (original, JournalTip { cursor, head }) + ( + original, + write_resealed_record(root, cursor, previous, &record), + ) } let (node, channel) = channel_with_one_node(); @@ -1422,6 +1444,16 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { assert_eq!(leased.payload(), b"hello"); assert_eq!(leased.content_type(), &content_type()); let lease_tip = anchor.tip(); + let identity = journal.receive_next(read).unwrap().unwrap().identity(); + assert!(matches!( + journal.acknowledge(identity, read, |tip| anchor.advance(tip)), + Err(JournalError::GuestAckRequired) + )); + assert_eq!( + anchor.tip(), + lease_tip, + "host ACK cannot close a guest lease" + ); drop(journal); drop(anchor); for signature_path in [ @@ -1453,6 +1485,36 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { .expect("restore valid lease"); *anchor_state.lock().expect("mock anchor lock") = lease_tip; } + let forged_host_ack = serde_json::json!({ + "version": 2, + "cursor": 3, + "previous": lease_tip.head, + "address": address(), + "event": { + "kind": "acknowledged", + "identity": identity, + "acknowledged_at_unix_ms": reader.now_unix_ms, + } + }); + let forged = write_resealed_record(&journal_path, 3, lease_tip.head, &forged_host_ack); + *anchor_state.lock().expect("mock anchor lock") = forged; + assert!( + matches!( + MailboxJournal::open_with_anchor( + &journal_path, + address(), + authority, + [90; 32], + limits, + &TestAnchor(Arc::clone(&anchor_state)), + ), + Err(JournalError::Corrupt) + ), + "an encrypted unsigned host ACK cannot close a guest-leased delivery" + ); + std::fs::remove_file(journal_path.join("entry-00000000000000000003.bin")) + .expect("remove forged host ACK"); + *anchor_state.lock().expect("mock anchor lock") = lease_tip; let (anchor, mut journal) = open(); assert_eq!( anchor.tip().cursor, From 2f01b6298ad65a1e3de5d3e1f7d2a5eb4e4b0222 Mon Sep 17 00:00:00 2001 From: Ari Date: Tue, 10 Nov 2026 18:04:56 -0800 Subject: [PATCH 17/51] feat(mailbox): bind report Cast policy to signed plan --- crates/mvm-cli/src/exec/transient.rs | 1 + .../mvm-client/src/admission/mailbox_grant.rs | 1 + crates/mvm-contract/src/protocol/mailbox.rs | 104 ++++++++++++++++++ crates/mvm-contract/tests/mailbox_delivery.rs | 81 +++++++++++++- crates/mvm-core/src/plan/signing.rs | 44 ++++++++ crates/mvm-core/src/plan/synthesis.rs | 14 ++- .../src/workload_runner/runner/sockets.rs | 1 + crates/mvm-vmm/src/host/cmdline.rs | 1 + 8 files changed, 243 insertions(+), 4 deletions(-) diff --git a/crates/mvm-cli/src/exec/transient.rs b/crates/mvm-cli/src/exec/transient.rs index 76ae9cbc54..ecb0daf716 100644 --- a/crates/mvm-cli/src/exec/transient.rs +++ b/crates/mvm-cli/src/exec/transient.rs @@ -720,6 +720,7 @@ mod tests { mailbox: MailboxId::from_bytes([1; 32]), authority_key: [2; 32], epoch: ExecutionEpoch::new(1).unwrap(), + report: None, }; let error = boot(Some(granted)).err().expect("the grant is refused"); assert!(format!("{error:#}").contains("mailbox"), "{error:#}"); diff --git a/crates/mvm-client/src/admission/mailbox_grant.rs b/crates/mvm-client/src/admission/mailbox_grant.rs index 6b7f9558b9..2d076516b7 100644 --- a/crates/mvm-client/src/admission/mailbox_grant.rs +++ b/crates/mvm-client/src/admission/mailbox_grant.rs @@ -26,6 +26,7 @@ pub fn grant_for(mailbox: MailboxId, authority: &VerifyingKey) -> MailboxGrant { mailbox, authority_key: authority.to_bytes(), epoch: ExecutionEpoch::new(1).expect("epoch 1 is nonzero"), + report: None, } } diff --git a/crates/mvm-contract/src/protocol/mailbox.rs b/crates/mvm-contract/src/protocol/mailbox.rs index d6c8e3fdcf..29e62d97b2 100644 --- a/crates/mvm-contract/src/protocol/mailbox.rs +++ b/crates/mvm-contract/src/protocol/mailbox.rs @@ -182,6 +182,59 @@ impl From for u64 { } } +/// Versioned ceiling on application reports, separate from the signed Send capability. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + try_from = "MailboxReportSendPolicyWire", + into = "MailboxReportSendPolicyWire" +)] +pub struct MailboxReportSendPolicy { + max_payload_bytes: u32, +} + +impl MailboxReportSendPolicy { + /// Refuses zero or values beyond the canonical plaintext ceiling. + pub fn new(max_payload_bytes: u32) -> Result { + if max_payload_bytes == 0 || u64::from(max_payload_bytes) > MAX_MAILBOX_PAYLOAD_BYTES as u64 + { + return Err(MailboxContractError::InvalidReportPolicyBounds); + } + Ok(Self { max_payload_bytes }) + } + + #[must_use] + pub const fn max_payload_bytes(self) -> u32 { + self.max_payload_bytes + } +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct MailboxReportSendPolicyWire { + version: u16, + max_payload_bytes: u32, +} + +impl TryFrom for MailboxReportSendPolicy { + type Error = MailboxContractError; + + fn try_from(wire: MailboxReportSendPolicyWire) -> Result { + if wire.version != 1 { + return Err(MailboxContractError::UnsupportedReportPolicyVersion); + } + Self::new(wire.max_payload_bytes) + } +} + +impl From for MailboxReportSendPolicyWire { + fn from(policy: MailboxReportSendPolicy) -> Self { + Self { + version: 1, + max_payload_bytes: policy.max_payload_bytes, + } + } +} + /// What a signed `ExecutionPlan` admits for one workload's mailbox: which /// mailbox, the key whose capabilities are honored, and the execution epoch. /// Presence grants that workload its own mailbox, including the ability for @@ -197,6 +250,39 @@ pub struct MailboxGrant { /// Ed25519 verifying key that must have signed any accepted capability. pub authority_key: [u8; 32], pub epoch: ExecutionEpoch, + /// An absent policy does not authorize a typed application report. Omitted + /// on older signed plans so their canonical bytes remain unchanged. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub report: Option, +} + +impl MailboxGrant { + /// Check an application report against the signed plan's exact Cast shape. + /// + /// This does not verify the plan signature or the narrower signed Send + /// capability, destination mailbox, current epoch, frame cap, or journal + /// quota. Runtime must intersect all of those before admission. + pub fn check_report_send( + &self, + kind: MailboxMessageKind, + content_type: &ContentType, + reply_to: Option<&ReplyRoute>, + encoded_payload_bytes: usize, + ) -> Result<(), MailboxContractError> { + let policy = self.report.ok_or(MailboxContractError::ReportNotGranted)?; + if kind != MailboxMessageKind::Cast || reply_to.is_some() { + return Err(MailboxContractError::InvalidReportMessageKind); + } + if content_type.name() != crate::protocol::workload_report::WORKLOAD_REPORT_CONTENT_TYPE + || content_type.version() != crate::protocol::workload_report::WORKLOAD_REPORT_VERSION + { + return Err(MailboxContractError::InvalidReportContentType); + } + if encoded_payload_bytes > policy.max_payload_bytes as usize { + return Err(MailboxContractError::ReportPayloadTooLarge); + } + Ok(()) + } } /// Monotonic sequence within one declared ordering scope. @@ -1151,6 +1237,18 @@ pub enum MailboxContractError { InvalidCapabilityBounds, #[error("capability execution epoch range is invalid")] InvalidEpochRange, + #[error("workload report policy version is unsupported")] + UnsupportedReportPolicyVersion, + #[error("workload report policy bounds are invalid")] + InvalidReportPolicyBounds, + #[error("workload report is not granted by the signed plan")] + ReportNotGranted, + #[error("workload report must be a cast without a reply route")] + InvalidReportMessageKind, + #[error("workload report content type is not granted")] + InvalidReportContentType, + #[error("workload report exceeds the signed plan ceiling")] + ReportPayloadTooLarge, } impl MailboxContractError { @@ -1173,6 +1271,12 @@ impl MailboxContractError { Self::CiphertextLengthMismatch => "ciphertext_length_mismatch", Self::InvalidCapabilityBounds => "invalid_capability_bounds", Self::InvalidEpochRange => "invalid_epoch_range", + Self::UnsupportedReportPolicyVersion => "unsupported_report_policy_version", + Self::InvalidReportPolicyBounds => "invalid_report_policy_bounds", + Self::ReportNotGranted => "report_not_granted", + Self::InvalidReportMessageKind => "invalid_report_message_kind", + Self::InvalidReportContentType => "invalid_report_content_type", + Self::ReportPayloadTooLarge => "report_payload_too_large", } } } diff --git a/crates/mvm-contract/tests/mailbox_delivery.rs b/crates/mvm-contract/tests/mailbox_delivery.rs index e1769a7ec2..00e88a4c31 100644 --- a/crates/mvm-contract/tests/mailbox_delivery.rs +++ b/crates/mvm-contract/tests/mailbox_delivery.rs @@ -4,9 +4,9 @@ use mvm_contract::protocol::mailbox::{ ContentType, DeliveryAckNonce, DeliveryId, DeliveryLeaseNonce, ExecutionEpoch, MailboxAddress, MailboxCapabilityGrant, MailboxDeliveryAck, MailboxDeliveryError, MailboxDeliveryLease, MailboxGrant, MailboxGuestReaderEnrollment, MailboxGuestReaderScope, MailboxId, - MailboxMessageKind, MailboxOperation, MessageId, PrincipalId, ProtocolVersion, RevocationId, - SignatureAlgorithm, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, - SignedMailboxGuestReaderEnrollment, TenantId, + MailboxMessageKind, MailboxOperation, MailboxReportSendPolicy, MessageId, PrincipalId, + ProtocolVersion, ReplyRoute, RevocationId, SignatureAlgorithm, SignedMailboxDeliveryAck, + SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, TenantId, }; fn address() -> MailboxAddress { @@ -23,9 +23,84 @@ fn plan_grant() -> MailboxGrant { mailbox: address().mailbox, authority_key: [5; 32], epoch: ExecutionEpoch::new(7).expect("epoch"), + report: None, } } +#[test] +fn report_policy_is_signed_plan_only_and_legacy_grants_stay_byte_identical() { + let old = plan_grant(); + let encoded = serde_json::to_vec(&old).expect("serialize legacy grant"); + assert!(!String::from_utf8_lossy(&encoded).contains("report")); + assert_eq!( + serde_json::from_slice::(&encoded).expect("old plan grant"), + old + ); + let mut enabled = old; + enabled.report = Some(MailboxReportSendPolicy::new(16 * 1024).expect("bound")); + let encoded = serde_json::to_vec(&enabled).expect("serialize plan grant"); + assert_eq!( + serde_json::from_slice::(&encoded).expect("report plan grant"), + enabled + ); + assert!(String::from_utf8_lossy(&encoded).contains("\"report\"")); + assert_ne!(old, enabled, "missing policy cannot authorize a report"); + for (version, max_payload_bytes) in [(2, 100), (1, 0), (1, 1024 * 1024 + 1)] { + let mut changed: serde_json::Value = serde_json::from_slice(&encoded).expect("JSON"); + changed["report"]["version"] = version.into(); + changed["report"]["max_payload_bytes"] = max_payload_bytes.into(); + assert!( + serde_json::from_value::(changed).is_err(), + "unsupported version or payload bound must refuse" + ); + } +} + +#[test] +fn report_policy_allows_only_its_signed_cast_type_and_bounded_body() { + use mvm_contract::protocol::mailbox::MailboxContractError; + use mvm_contract::protocol::workload_report::{ + WORKLOAD_REPORT_CONTENT_TYPE, WORKLOAD_REPORT_VERSION, + }; + + let report_type = ContentType::parse(WORKLOAD_REPORT_CONTENT_TYPE, WORKLOAD_REPORT_VERSION) + .expect("static content type"); + let ordinary_type = + ContentType::parse("application/vnd.mvm.guest-message", 1).expect("static content type"); + let old = plan_grant(); + assert_eq!( + old.check_report_send(MailboxMessageKind::Cast, &report_type, None, 1), + Err(MailboxContractError::ReportNotGranted) + ); + + let mut enabled = old; + enabled.report = Some(MailboxReportSendPolicy::new(16 * 1024).expect("bound")); + assert_eq!( + enabled.check_report_send(MailboxMessageKind::Cast, &report_type, None, 16 * 1024), + Ok(()) + ); + assert_eq!( + enabled.check_report_send(MailboxMessageKind::Cast, &report_type, None, 16 * 1024 + 1), + Err(MailboxContractError::ReportPayloadTooLarge) + ); + assert_eq!( + enabled.check_report_send(MailboxMessageKind::Call, &report_type, None, 1), + Err(MailboxContractError::InvalidReportMessageKind) + ); + let reply = ReplyRoute { + address: address(), + capability_id: CapabilityId::from_bytes([19; 32]), + }; + assert_eq!( + enabled.check_report_send(MailboxMessageKind::Cast, &report_type, Some(&reply), 1), + Err(MailboxContractError::InvalidReportMessageKind) + ); + assert_eq!( + enabled.check_report_send(MailboxMessageKind::Cast, &ordinary_type, None, 1), + Err(MailboxContractError::InvalidReportContentType) + ); +} + fn receive_capability() -> MailboxCapabilityGrant { MailboxCapabilityGrant { protocol_version: ProtocolVersion::V1, diff --git a/crates/mvm-core/src/plan/signing.rs b/crates/mvm-core/src/plan/signing.rs index d472a2a2b6..19ca9071b3 100644 --- a/crates/mvm-core/src/plan/signing.rs +++ b/crates/mvm-core/src/plan/signing.rs @@ -503,6 +503,7 @@ mod tests { mailbox: MailboxId::from_bytes([7; 32]), authority_key: [9; 32], epoch: ExecutionEpoch::new(3).unwrap(), + report: None, } } @@ -547,6 +548,49 @@ mod tests { )); } + #[test] + fn a_report_ceiling_cannot_be_added_or_widened_after_plan_signing() { + use mvm_contract::protocol::mailbox::MailboxReportSendPolicy; + + let (sk, vk) = fresh_key(); + let mut plan = sample_plan(); + plan.mailbox = Some(mailbox_grant()); + let signed_legacy = sign_plan(&plan, &sk, "test-signer"); + let old: serde_json::Value = serde_json::from_slice(&signed_legacy.0.payload).unwrap(); + assert!(old["mailbox"].get("report").is_none()); + + let mut forged = signed_legacy.clone(); + let mut payload = old; + payload["mailbox"]["report"] = serde_json::json!({ + "version": 1, + "max_payload_bytes": 16384 + }); + forged.0.payload = serde_json::to_vec(&payload).unwrap(); + assert!(matches!( + verify_plan(&forged, &[("test-signer", &vk)]), + Err(PlanVerifyError::SignatureInvalid(_)) + )); + + let mut report_grant = mailbox_grant(); + report_grant.report = Some(MailboxReportSendPolicy::new(16 * 1024).unwrap()); + plan.mailbox = Some(report_grant); + let signed = sign_plan(&plan, &sk, "test-signer"); + assert_eq!( + verify_plan(&signed, &[("test-signer", &vk)]) + .unwrap() + .mailbox, + Some(report_grant) + ); + let mut forged = signed; + let mut payload: serde_json::Value = serde_json::from_slice(&forged.0.payload).unwrap(); + payload["mailbox"]["report"]["max_payload_bytes"] = serde_json::json!(32768); + forged.0.payload = serde_json::to_vec(&payload).unwrap(); + assert!(matches!( + verify_plan(&forged, &[("test-signer", &vk)]), + Err(PlanVerifyError::SignatureInvalid(_)) + )); + } + #[test] fn secret_approval_requirement_is_signature_bound() { let mut plan = sample_plan(); diff --git a/crates/mvm-core/src/plan/synthesis.rs b/crates/mvm-core/src/plan/synthesis.rs index a638958ca6..b394974431 100644 --- a/crates/mvm-core/src/plan/synthesis.rs +++ b/crates/mvm-core/src/plan/synthesis.rs @@ -1617,13 +1617,16 @@ mod tests { #[test] fn a_mailbox_grant_is_minted_into_the_signed_plan_and_absence_stays_none() { - use mvm_contract::protocol::mailbox::{ExecutionEpoch, MailboxGrant, MailboxId}; + use mvm_contract::protocol::mailbox::{ + ExecutionEpoch, MailboxGrant, MailboxId, MailboxReportSendPolicy, + }; assert!(synthesize_plan(&input("vm")).unwrap().mailbox.is_none()); let grant = MailboxGrant { mailbox: MailboxId::from_bytes([7; 32]), authority_key: [9; 32], epoch: ExecutionEpoch::new(1).unwrap(), + report: None, }; let mut inp = input("vm"); inp.mailbox = Some(grant); @@ -1634,6 +1637,15 @@ mod tests { synthesize_plan(&input("vm")).unwrap().plan_id, "the grant is part of the content address" ); + let mut report = grant; + report.report = Some(MailboxReportSendPolicy::new(16 * 1024).unwrap()); + inp.mailbox = Some(report); + let with_report = synthesize_plan(&inp).unwrap(); + assert_eq!(with_report.mailbox, Some(report)); + assert_ne!( + with_report.plan_id, plan.plan_id, + "a changed report authority must move the signed plan identity" + ); } #[test] diff --git a/crates/mvm-runtime/src/workload_runner/runner/sockets.rs b/crates/mvm-runtime/src/workload_runner/runner/sockets.rs index 8387ee5e7b..f528927cf1 100644 --- a/crates/mvm-runtime/src/workload_runner/runner/sockets.rs +++ b/crates/mvm-runtime/src/workload_runner/runner/sockets.rs @@ -136,6 +136,7 @@ mod tests { mailbox: mvm_contract::protocol::mailbox::MailboxId::from_bytes([4; 32]), authority_key: [0; 32], epoch: mvm_contract::protocol::mailbox::ExecutionEpoch::new(1).unwrap(), + report: None, }); let config = VmStartConfig { plan_json: Some(serde_json::to_string(&plan).unwrap()), diff --git a/crates/mvm-vmm/src/host/cmdline.rs b/crates/mvm-vmm/src/host/cmdline.rs index 38c207b6b2..bf44c9dcbc 100644 --- a/crates/mvm-vmm/src/host/cmdline.rs +++ b/crates/mvm-vmm/src/host/cmdline.rs @@ -551,6 +551,7 @@ mod tests { mailbox: MailboxId::from_bytes([1; 32]), authority_key: [2; 32], epoch: ExecutionEpoch::new(1).unwrap(), + report: None, })); let absent = build(None); let tokens = |c: &str| c.split_whitespace().map(str::to_owned).collect::>(); From 9112d8abd5da3b573d645b9ff471f120a5ac7e42 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 08:47:57 -0800 Subject: [PATCH 18/51] test(mailbox): retain legacy grant fixtures without report policy --- crates/mvm-core/src/mailbox/transport/tests.rs | 1 + crates/mvm-core/tests/mailbox_reader.rs | 1 + crates/mvm-hostd/src/mailbox_launch.rs | 1 + crates/mvm-hostd/src/mailbox_process.rs | 1 + crates/mvm-hostd/src/mailbox_process/reader.rs | 1 + crates/mvm-hostd/tests/mission_demo.rs | 3 +++ crates/mvm-runtime/src/workload_runner/runner/spawner.rs | 1 + crates/mvm-vmm/src/host/flowmux_identity.rs | 2 ++ 8 files changed, 11 insertions(+) diff --git a/crates/mvm-core/src/mailbox/transport/tests.rs b/crates/mvm-core/src/mailbox/transport/tests.rs index a2e4974aa2..cbbad85b0d 100644 --- a/crates/mvm-core/src/mailbox/transport/tests.rs +++ b/crates/mvm-core/src/mailbox/transport/tests.rs @@ -507,6 +507,7 @@ fn guest_lease_and_signed_ack_are_durable_replay_fenced_and_boot_bound() { mailbox: address().mailbox, authority_key: signer.verifying_key().to_bytes(), epoch: epoch(), + report: None, }; let plan_id = PlanId(format!("sha256:{}", "a".repeat(64))); let plan_nonce = Nonce::from_bytes(id(52)); diff --git a/crates/mvm-core/tests/mailbox_reader.rs b/crates/mvm-core/tests/mailbox_reader.rs index fca1890af4..9a77d3a823 100644 --- a/crates/mvm-core/tests/mailbox_reader.rs +++ b/crates/mvm-core/tests/mailbox_reader.rs @@ -42,6 +42,7 @@ fn fixture() -> ( mailbox: address.mailbox, authority_key: authority.verifying_key().to_bytes(), epoch, + report: None, }; let plan_id = PlanId(format!("sha256:{}", "a".repeat(64))); let plan_nonce = Nonce::from_bytes([7; 16]); diff --git a/crates/mvm-hostd/src/mailbox_launch.rs b/crates/mvm-hostd/src/mailbox_launch.rs index 8a7d431a36..ddb5e24bb3 100644 --- a/crates/mvm-hostd/src/mailbox_launch.rs +++ b/crates/mvm-hostd/src/mailbox_launch.rs @@ -454,6 +454,7 @@ mod tests { mailbox, authority_key: authority_key_for(dir, &mailbox).unwrap().to_bytes(), epoch: ExecutionEpoch::new(1).unwrap(), + report: None, } } diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index baba30c38b..fd713bd193 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -491,6 +491,7 @@ mod tests { .unwrap() .to_bytes(), epoch: ExecutionEpoch::new(1).unwrap(), + report: None, } } diff --git a/crates/mvm-hostd/src/mailbox_process/reader.rs b/crates/mvm-hostd/src/mailbox_process/reader.rs index e8f30771b0..3e9901a848 100644 --- a/crates/mvm-hostd/src/mailbox_process/reader.rs +++ b/crates/mvm-hostd/src/mailbox_process/reader.rs @@ -217,6 +217,7 @@ mod tests { .unwrap() .to_bytes(), epoch: ExecutionEpoch::new(1).unwrap(), + report: None, }; let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); plan.mailbox = Some(grant); diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index 45671e2e90..3bf458aa7a 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -773,6 +773,7 @@ fn plan_and_trust(channel: &Channel, granted: bool) -> (ExecutionPlan, MailboxPo mailbox: address().mailbox, authority_key: channel.authority_key.verifying_key().to_bytes(), epoch: MbEpoch::new(1).unwrap(), + report: None, }); } let trust = MailboxPolicy { @@ -912,6 +913,7 @@ fn granted_plan(authority_key: [u8; 32]) -> ExecutionPlan { mailbox: address().mailbox, authority_key, epoch: MbEpoch::new(1).unwrap(), + report: None, }); plan } @@ -1229,6 +1231,7 @@ fn signed_guest_lease_and_exact_ack_survive_journal_restart() { mailbox: address().mailbox, authority_key: authority.to_bytes(), epoch: MbEpoch::new(1).unwrap(), + report: None, }; let mut reader_grant = grant(&[MailboxOperation::Receive]); reader_grant.subject = aircraft_principal(); diff --git a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs index adab036ff2..e7a691b721 100644 --- a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs +++ b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs @@ -769,6 +769,7 @@ mod tests { mailbox: MailboxId::from_bytes([9; 32]), authority_key: host.verifying_key().to_bytes(), epoch: ExecutionEpoch::new(3).unwrap(), + report: None, }); plan.plan_id = mvm_core::plan::compute_plan_id(&plan); let redaction = RedactionPolicy::default(); diff --git a/crates/mvm-vmm/src/host/flowmux_identity.rs b/crates/mvm-vmm/src/host/flowmux_identity.rs index 5a8d590c0d..3101660fa3 100644 --- a/crates/mvm-vmm/src/host/flowmux_identity.rs +++ b/crates/mvm-vmm/src/host/flowmux_identity.rs @@ -544,6 +544,7 @@ mod tests { mailbox: MailboxId::from_bytes([8; 32]), authority_key: anchor.to_bytes(), epoch: ExecutionEpoch::new(3).unwrap(), + report: None, }); plan.plan_id = mvm_core::plan::compute_plan_id(&plan); let binding = MailboxReaderBootBinding::mint( @@ -595,6 +596,7 @@ mod tests { mailbox: MailboxId::from_bytes([8; 32]), authority_key: host_key().verifying_key().to_bytes(), epoch: ExecutionEpoch::new(3).unwrap(), + report: None, }); plan.plan_id = mvm_core::plan::compute_plan_id(&plan); let dir = tempfile::tempdir().unwrap(); From a7731950b6f92ef21328efaee223b34fb2b430e2 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 08:41:39 -0800 Subject: [PATCH 19/51] feat(mailbox): distinguish local durable receipt from anchored acceptance --- crates/mvm-contract/src/protocol/mailbox.rs | 84 +++++++++++++++++- crates/mvm-contract/tests/mailbox_receipt.rs | 87 +++++++++++++++++++ crates/mvm-core/tests/mailbox_crypto.rs | 8 ++ .../057-mailbox-semantics-and-threat-model.md | 32 +++++-- 4 files changed, 201 insertions(+), 10 deletions(-) create mode 100644 crates/mvm-contract/tests/mailbox_receipt.rs diff --git a/crates/mvm-contract/src/protocol/mailbox.rs b/crates/mvm-contract/src/protocol/mailbox.rs index 29e62d97b2..60fa3403e0 100644 --- a/crates/mvm-contract/src/protocol/mailbox.rs +++ b/crates/mvm-contract/src/protocol/mailbox.rs @@ -7,7 +7,8 @@ use alloc::string::{String, ToString}; use alloc::vec::Vec; -use serde::{Deserialize, Serialize}; +use serde::ser::SerializeStruct; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::plan::{Nonce, PlanId}; @@ -17,6 +18,10 @@ use crate::plan::{Nonce, PlanId}; pub const MAILBOX_PORT: u32 = 5257; /// Current mailbox protocol version. pub const MAILBOX_PROTOCOL_VERSION: u16 = 1; +/// Receipt-only version for a locally fsynced, trusted-host acceptance. +pub const LOCAL_DURABLE_RECEIPT_VERSION: u16 = 2; +/// Local receipt signatures must not reuse the V1 destination-receipt domain. +pub const LOCAL_DURABLE_RECEIPT_SIGNATURE_DOMAIN: &[u8] = b"mvm.mailbox.receipt.v2\0"; /// Current signature algorithm identifier (Ed25519). pub const ED25519_ALGORITHM: u16 = 1; /// Current payload-encryption algorithm identifier (AES-256-GCM). @@ -129,6 +134,8 @@ pub struct ProtocolVersion(pub u16); impl ProtocolVersion { pub const V1: Self = Self(MAILBOX_PROTOCOL_VERSION); + /// Only local-durable destination receipts use version two. + pub const V2: Self = Self(LOCAL_DURABLE_RECEIPT_VERSION); } /// Signature algorithm carried on every signed object. @@ -712,6 +719,8 @@ pub struct SignedMailboxCapability { pub enum DeliveryOutcome { AcceptedEphemeral, AcceptedDurable, + /// Local crash durability only; no independent whole-host rollback witness. + AcceptedLocalDurable, Duplicate, Expired, Refused, @@ -722,8 +731,7 @@ pub enum DeliveryOutcome { } /// Receipt body signed only by the destination mailbox authority. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct DestinationReceipt { pub protocol_version: ProtocolVersion, pub signature_algorithm: SignatureAlgorithm, @@ -736,6 +744,73 @@ pub struct DestinationReceipt { pub ciphertext_commitment: [u8; 32], } +impl DestinationReceipt { + /// Keeps V1 receipt claims distinct from the V2 local-only durability claim. + pub fn validate_versioned_outcome(&self) -> Result<(), MailboxContractError> { + let expected = match self.outcome { + DeliveryOutcome::AcceptedLocalDurable => ProtocolVersion::V2, + _ => ProtocolVersion::V1, + }; + if self.protocol_version == expected { + Ok(()) + } else { + Err(MailboxContractError::InvalidReceiptVersion) + } + } +} + +impl Serialize for DestinationReceipt { + fn serialize(&self, serializer: S) -> Result { + self.validate_versioned_outcome() + .map_err(serde::ser::Error::custom)?; + let mut receipt = serializer.serialize_struct("DestinationReceipt", 9)?; + receipt.serialize_field("protocol_version", &self.protocol_version)?; + receipt.serialize_field("signature_algorithm", &self.signature_algorithm)?; + receipt.serialize_field("delivery_id", &self.delivery_id)?; + receipt.serialize_field("message_id", &self.message_id)?; + receipt.serialize_field("address", &self.address)?; + receipt.serialize_field("execution_epoch", &self.execution_epoch)?; + receipt.serialize_field("outcome", &self.outcome)?; + receipt.serialize_field("observed_at_unix_ms", &self.observed_at_unix_ms)?; + receipt.serialize_field("ciphertext_commitment", &self.ciphertext_commitment)?; + receipt.end() + } +} + +impl<'de> Deserialize<'de> for DestinationReceipt { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Wire { + protocol_version: ProtocolVersion, + signature_algorithm: SignatureAlgorithm, + delivery_id: DeliveryId, + message_id: MessageId, + address: MailboxAddress, + execution_epoch: ExecutionEpoch, + outcome: DeliveryOutcome, + observed_at_unix_ms: i64, + ciphertext_commitment: [u8; 32], + } + let wire = Wire::deserialize(deserializer)?; + let receipt = Self { + protocol_version: wire.protocol_version, + signature_algorithm: wire.signature_algorithm, + delivery_id: wire.delivery_id, + message_id: wire.message_id, + address: wire.address, + execution_epoch: wire.execution_epoch, + outcome: wire.outcome, + observed_at_unix_ms: wire.observed_at_unix_ms, + ciphertext_commitment: wire.ciphertext_commitment, + }; + receipt + .validate_versioned_outcome() + .map_err(serde::de::Error::custom)?; + Ok(receipt) + } +} + /// Signed destination receipt. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -1249,6 +1324,8 @@ pub enum MailboxContractError { InvalidReportContentType, #[error("workload report exceeds the signed plan ceiling")] ReportPayloadTooLarge, + #[error("receipt outcome is unsupported for its protocol version")] + InvalidReceiptVersion, } impl MailboxContractError { @@ -1277,6 +1354,7 @@ impl MailboxContractError { Self::InvalidReportMessageKind => "invalid_report_message_kind", Self::InvalidReportContentType => "invalid_report_content_type", Self::ReportPayloadTooLarge => "report_payload_too_large", + Self::InvalidReceiptVersion => "invalid_receipt_version", } } } diff --git a/crates/mvm-contract/tests/mailbox_receipt.rs b/crates/mvm-contract/tests/mailbox_receipt.rs new file mode 100644 index 0000000000..2c5c6a01ee --- /dev/null +++ b/crates/mvm-contract/tests/mailbox_receipt.rs @@ -0,0 +1,87 @@ +use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, DeliveryId, DeliveryOutcome, DestinationReceipt, ExecutionEpoch, + MailboxAddress, MailboxContractError, MailboxId, MessageId, PrincipalId, ProtocolVersion, + SignatureAlgorithm, TenantId, +}; + +fn receipt(outcome: DeliveryOutcome, protocol_version: ProtocolVersion) -> DestinationReceipt { + DestinationReceipt { + protocol_version, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: DeliveryId::from_bytes([1; 32]), + message_id: MessageId::from_bytes([2; 32]), + address: MailboxAddress { + tenant: TenantId::from_bytes([3; 32]), + authorization_domain: AuthorizationDomainId::from_bytes([4; 32]), + mailbox: MailboxId::from_bytes([5; 32]), + recipient: PrincipalId::from_bytes([6; 32]), + }, + execution_epoch: ExecutionEpoch::new(1).expect("nonzero epoch"), + outcome, + observed_at_unix_ms: 100, + ciphertext_commitment: [7; 32], + } +} + +#[test] +fn local_durable_is_a_distinct_v2_receipt_not_a_v1_durable_alias() { + let local = receipt(DeliveryOutcome::AcceptedLocalDurable, ProtocolVersion::V2); + let wire = serde_json::to_vec(&local).expect("encode local receipt"); + assert_eq!( + serde_json::from_slice::(&wire).expect("decode local receipt"), + local + ); + assert_eq!( + serde_json::from_slice::(&wire).expect("inspect local receipt")["outcome"], + "accepted_local_durable" + ); + assert_ne!( + local.outcome, + DeliveryOutcome::AcceptedDurable, + "a local crash witness is not independent rollback evidence" + ); +} + +#[test] +fn old_v1_receipts_keep_their_bytes_and_reject_a_local_claim() { + let old = receipt(DeliveryOutcome::AcceptedDurable, ProtocolVersion::V1); + let wire = serde_json::to_string(&old).expect("encode old receipt"); + assert!(wire.starts_with("{\"protocol_version\":1,\"signature_algorithm\":1,\"delivery_id\":")); + assert!(wire.contains("\"outcome\":\"accepted_durable\"")); + assert_eq!( + serde_json::from_str::(&wire).expect("decode old receipt"), + old + ); + + let mut forged = old.clone(); + forged.outcome = DeliveryOutcome::AcceptedLocalDurable; + assert_eq!( + forged.validate_versioned_outcome(), + Err(MailboxContractError::InvalidReceiptVersion) + ); + assert!( + serde_json::to_vec(&forged).is_err(), + "do not sign a V1 local claim" + ); + let mut value: serde_json::Value = serde_json::from_str(&wire).expect("old JSON"); + value["outcome"] = "accepted_local_durable".into(); + assert!(serde_json::from_value::(value).is_err()); +} + +#[test] +fn v2_cannot_relabel_an_old_outcome_or_admit_unknown_versions() { + let mut local = receipt(DeliveryOutcome::AcceptedLocalDurable, ProtocolVersion::V2); + local.outcome = DeliveryOutcome::AcceptedDurable; + assert!(serde_json::to_vec(&local).is_err()); + + let mut value = serde_json::to_value(receipt( + DeliveryOutcome::AcceptedLocalDurable, + ProtocolVersion::V2, + )) + .expect("valid V2"); + value["outcome"] = "accepted_durable".into(); + assert!(serde_json::from_value::(value.clone()).is_err()); + value["outcome"] = "accepted_local_durable".into(); + value["protocol_version"] = 3.into(); + assert!(serde_json::from_value::(value).is_err()); +} diff --git a/crates/mvm-core/tests/mailbox_crypto.rs b/crates/mvm-core/tests/mailbox_crypto.rs index 448442b5b4..a5e3d9a6f3 100644 --- a/crates/mvm-core/tests/mailbox_crypto.rs +++ b/crates/mvm-core/tests/mailbox_crypto.rs @@ -408,6 +408,12 @@ fn destination_receipt_wire_roundtrip_and_signature_cover_outcome_and_identity() ciphertext_commitment: [33; 32], }; let mut signed = sign_receipt(receipt, &destination_key).expect("sign receipt"); + let mut invalid = signed.receipt.clone(); + invalid.outcome = DeliveryOutcome::AcceptedLocalDurable; + assert!( + sign_receipt(invalid, &destination_key).is_err(), + "a V1 signer must not label local acceptance as durable" + ); let wire = serde_jcs::to_vec(&signed).expect("encode receipt"); let decoded: mvm_contract::protocol::mailbox::SignedDestinationReceipt = serde_json::from_slice(&wire).expect("decode receipt"); @@ -1064,6 +1070,7 @@ fn the_delivery_and_message_vocabularies_are_closed() { match outcome { DeliveryOutcome::AcceptedEphemeral => "accepted_ephemeral", DeliveryOutcome::AcceptedDurable => "accepted_durable", + DeliveryOutcome::AcceptedLocalDurable => "accepted_local_durable", DeliveryOutcome::Duplicate => "duplicate", DeliveryOutcome::Expired => "expired", DeliveryOutcome::Refused => "refused", @@ -1083,6 +1090,7 @@ fn the_delivery_and_message_vocabularies_are_closed() { for outcome in [ DeliveryOutcome::AcceptedEphemeral, DeliveryOutcome::AcceptedDurable, + DeliveryOutcome::AcceptedLocalDurable, DeliveryOutcome::Duplicate, DeliveryOutcome::Expired, DeliveryOutcome::Refused, diff --git a/specs/adrs/057-mailbox-semantics-and-threat-model.md b/specs/adrs/057-mailbox-semantics-and-threat-model.md index afcabb5440..993b6f2e05 100644 --- a/specs/adrs/057-mailbox-semantics-and-threat-model.md +++ b/specs/adrs/057-mailbox-semantics-and-threat-model.md @@ -244,6 +244,7 @@ Receipts distinguish: ```text accepted_ephemeral +accepted_local_durable accepted_durable duplicate expired @@ -262,13 +263,22 @@ epoch, and ordering scope. Gaps block only that scope until the missing sequence arrives, expires, or receives a terminal disposition. There is no order across senders, mailboxes, or epochs and no global total order. -`accepted_durable` means the encrypted envelope and deduplication identity are -durably committed before the receipt is signed. `accepted_ephemeral` means the -message is owned by the named live mailbox epoch. If that epoch terminates, -the signed epoch-terminal observation closes the volatile queue; senders that -need recovery use stable IDs and retry into a newly authorized epoch. A -partitioned observer may remain uncertain until reconciliation; the protocol -does not invent a terminal fact it cannot observe. +`accepted_local_durable` is a distinct version-two destination receipt: the +encrypted envelope, deduplication identity, and required delivery state are +fsynced outside the source VM's lifetime before signing. Its locally fsynced +chain head permits ordinary process-crash recovery but does not detect a whole +host rollback or survive destruction of all host storage. It is available to +standalone MVM without a fleet service or hardware dependency. The existing +version-one `accepted_durable` remains reserved for independently witnessed +rollback protection; an old receipt must never be relabeled as local durability, +and version-one readers reject the new outcome. Neither receipt implies that an +application processed or ACKed the message. + +`accepted_ephemeral` means the message is owned by the named live mailbox +epoch. If that epoch terminates, the signed epoch-terminal observation closes +the volatile queue; senders that need recovery use stable IDs and retry into a +newly authorized epoch. A partitioned observer may remain uncertain until +reconciliation; the protocol does not invent a terminal fact it cannot observe. ### 7. Durability, retry, poison messages, and dead letters @@ -357,6 +367,14 @@ that migration is implemented. | Isolated production on ordinary virtualization | Sender and recipient guests; transport and stores hold ciphertext, but the hypervisor owner can inspect guest memory. | Relay- and service-process-blind payloads, not malicious-host confidentiality. | | Host-blind sealed execution | Only measured sender/recipient confidential guests after attestation-bound key release; ordinary host and `mvmd` processes never receive decryption material. | Host-blind payload confidentiality after hardware, attestation, rollback, and key-release witnesses pass. | +The trusted-host local-durable receipt is an acceptance class, not an assurance +tier upgrade: the same host remains trusted for key custody and rollback. A +missing or rotated history/payload key must refuse recovery rather than mint a +replacement. Standalone MVM may keep versioned, purpose-separated keys in its +existing protected host key directory; an independently operated checkpoint +and optional confidential hardware remain upgrades, never implicit dependencies +of a local mailbox launch. + Encryption at rest, TLS, a microVM boundary, Keychain, TPM storage, or an honest host does not by itself establish host-blind execution. A platform without confidential-computing memory protection must refuse a requested From 62451aeb8db53ee040d5fdd9a6327ffacf89cbb2 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 08:44:09 -0800 Subject: [PATCH 20/51] docs(mailbox): preserve local acceptance proof on exact retry --- crates/mvm-contract/tests/mailbox_receipt.rs | 9 +++++++-- specs/adrs/057-mailbox-semantics-and-threat-model.md | 7 +++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/mvm-contract/tests/mailbox_receipt.rs b/crates/mvm-contract/tests/mailbox_receipt.rs index 2c5c6a01ee..9fae95c0a9 100644 --- a/crates/mvm-contract/tests/mailbox_receipt.rs +++ b/crates/mvm-contract/tests/mailbox_receipt.rs @@ -79,8 +79,13 @@ fn v2_cannot_relabel_an_old_outcome_or_admit_unknown_versions() { ProtocolVersion::V2, )) .expect("valid V2"); - value["outcome"] = "accepted_durable".into(); - assert!(serde_json::from_value::(value.clone()).is_err()); + for old_outcome in ["accepted_durable", "duplicate", "accepted_ephemeral"] { + value["outcome"] = old_outcome.into(); + assert!( + serde_json::from_value::(value.clone()).is_err(), + "V2 cannot relabel a local acceptance as {old_outcome}" + ); + } value["outcome"] = "accepted_local_durable".into(); value["protocol_version"] = 3.into(); assert!(serde_json::from_value::(value).is_err()); diff --git a/specs/adrs/057-mailbox-semantics-and-threat-model.md b/specs/adrs/057-mailbox-semantics-and-threat-model.md index 993b6f2e05..22508312e9 100644 --- a/specs/adrs/057-mailbox-semantics-and-threat-model.md +++ b/specs/adrs/057-mailbox-semantics-and-threat-model.md @@ -274,6 +274,13 @@ rollback protection; an old receipt must never be relabeled as local durability, and version-one readers reject the new outcome. Neither receipt implies that an application processed or ACKed the message. +An exact same-ID, same-ciphertext retry of locally accepted history returns the +original signed version-two local receipt, including its original observation +time, after current authority and committed-state checks. Re-signing that body +must reproduce the original bytes; an unprovable acceptance is unknown, not a +new version-one `duplicate` that loses the acceptance tier. Different bytes +under the same ID conflict. Version-one `duplicate` behavior is unchanged. + `accepted_ephemeral` means the message is owned by the named live mailbox epoch. If that epoch terminates, the signed epoch-terminal observation closes the volatile queue; senders that need recovery use stable IDs and retry into a From d7ec5c8c6cb43e3d68f2c79b93fe7ded813bb450 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 08:58:31 -0800 Subject: [PATCH 21/51] feat(mailbox): bind journal history to versioned custody Authenticate mailbox, epoch and history-key version in v3 records and admission intents. Keep constructors test-scoped until verified restart authority exists; refuse raw-key durable admission. Preserve original accepted receipt state for future local lost-verdict reconciliation and reject silent v2-to-v3 reinterpretation. --- crates/mvm-core/README.md | 55 +- crates/mvm-core/src/mailbox/journal.rs | 695 +++++++++++++++++++++---- crates/mvm-hostd/tests/mission_demo.rs | 46 +- 3 files changed, 671 insertions(+), 125 deletions(-) diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index 71cba037c4..be45bcef2c 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -62,13 +62,22 @@ ownership decision. canonical encrypted mailbox frame and its deduplication identity. Its `JournalAnchor` interface requires an independently trusted authority to read the current `(cursor, head)` and atomically compare-and-advance it after a -journal write. The journal persists unsigned receipt state in record version 2 -with a destination-signed, domain-separated admission intent binding the -verified identity, canonical frame, and receipt state. The destination signs -the actual acceptance receipt only after both commits. Earlier private-handoff -version-1 records are refused, not silently reinterpreted. Recovery verifies -the intent, refuses a missing or divergent anchored prefix, and may reconcile -only an intact contiguous suffix written before an interrupted anchor commit. +journal write. The journal persists unsigned receipt state with a +destination-signed, domain-separated admission intent binding the verified +identity, canonical frame, and receipt state. The test-scoped version 3 also +binds a `HistoryKeyRef` (mailbox address, epoch, AES-256-GCM key ID and nonzero +version) to the encrypted record and signed intent. Its recovery obtains the +exact key through an existing-only `HistoryKeyCustody` lookup and refuses +changed key identity, wrong key, or missing custody; it does not create a +replacement key. The raw-key opener cannot issue an `accepted_durable` receipt. +The destination signs an actual acceptance receipt only after both commits. +Earlier private-handoff version-1 records are refused, not silently +reinterpreted. Version-2 test/local records have no custody reference and are +not silently promoted to version 3. Recovery verifies the intent, refuses a +missing or divergent anchored prefix, and may reconcile only an intact +contiguous suffix written before an interrupted anchor commit. +The original accepted receipt body and observation time are retained with +deduplication state; V1 retries still use their existing `duplicate` outcome. Retained guest-reader controls are checked against the historical signed Receive capability, enrollment, lease and guest ACK before recovery accepts their delivery state. An exact ACK retry still needs current boot, session, @@ -76,27 +85,27 @@ policy and expiry authorization; an old committed ACK is not a new grant. A message ever leased to a guest cannot be closed by the legacy unsigned host ACK, including during recovery; it requires that guest's signed ACK. -`FileJournalAnchor` is a local development aid, **not** that independent +`FileJournalAnchor` is a local development aid, **not** an independent authority: restoring both the journal and its host-local anchor can hide -rollback. The journal's current raw key input also does not establish -persistent, versioned history-key custody or recover the separately encrypted -payload after the per-launch mailbox secret is erased. Until those providers -and the existing service's admission handoff are integrated and tested, local +rollback. The custody interface has only a test provider: it does not supply +a retained host key, rotation, or the separately encrypted payload's opening +key after the per-launch mailbox secret is erased. Until real providers and +the existing service's admission handoff are integrated and tested, local journal writes must not be presented as `accepted_durable`. The trusted-host profile can access plaintext; host-blind confidentiality requires a different attested key-release and protected-memory architecture. -The intended trusted-host custody boundary is a separately retained, versioned -mailbox-history root under host key custody. Each record must bind its key ID -and tenant/domain/mailbox scope; reopening existing history must **load the -existing version**, never mint a replacement or fall back to the signing seed -or erased per-launch secret. Durable payloads also need an explicitly -recoverable recipient encryption key: persisting the current per-launch key -would silently negate its erasure guarantee. Rotation retains old versions -only until their anchored history is retired. An independently operated -checkpoint must atomically compare the anchored `(mailbox, cursor, head)` with -the expected prior value; an unavailable or divergent checkpoint refuses -durable acceptance. These custody and checkpoint providers are not wired yet. +The planned standalone `accepted_local_durable` tier has a distinct versioned +receipt contract. Its history and payload-purpose keys belong in the existing +protected host keys directory, scoped to tenant/domain/mailbox/epoch and key +version, with an authority signer pinned for retained history. Recovery must +load the existing key version; it must not mint a new signer or use the erased +per-launch secret as its only opening key. Rotation and deletion need explicit +retention and refusal rules. A host-local chain can witness ordinary crash and +process restart, **not** rollback of the entire host. Neither this local tier +nor its key provider is enabled by the journal primitive. The stronger +`accepted_durable` outcome continues to require a separately trusted +compare-and-advance checkpoint; no such provider is wired. ## Features diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 50d3c184e9..a9a449bde6 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -17,10 +17,10 @@ use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; use hmac::{Hmac, Mac, digest::KeyInit}; use mvm_contract::protocol::mailbox::{ ContentType, DELIVERY_ACK_SIGNATURE_DOMAIN, DeliveryLeaseNonce, DeliveryOutcome, - DestinationReceipt, MailboxAddress, MailboxDeliveryLease, MailboxEnvelope, MailboxGrant, - MailboxGuestReaderScope, MailboxOperation, ProtocolVersion, SignatureAlgorithm, - SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxDeliveryAck, - SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, + DestinationReceipt, EncryptionAlgorithm, ExecutionEpoch, MailboxAddress, MailboxDeliveryLease, + MailboxEnvelope, MailboxGrant, MailboxGuestReaderScope, MailboxOperation, ProtocolVersion, + SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, + SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, }; use serde::{Deserialize, Serialize}; use sha2::Sha256; @@ -41,7 +41,8 @@ use crate::util::atomic_io::{FileLock, atomic_write_durable, atomic_write_new, s const DOMAIN: &[u8] = b"mvm.mailbox.journal.v1\0"; const ENCRYPTION_DOMAIN: &[u8] = b"mvm.mailbox.journal.encryption.v1\0"; const ANCHOR_DOMAIN: &[u8] = b"mvm.mailbox.anchor.v1\0"; -const ACCEPTANCE_INTENT_DOMAIN: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v2\0"; +const ACCEPTANCE_INTENT_DOMAIN_V2: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v2\0"; +const ACCEPTANCE_INTENT_DOMAIN_V3: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v3\0"; const MAX_ANCHOR_BYTES: u64 = 1024; const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; const MAX_RECORDS: usize = 100_000; @@ -92,6 +93,95 @@ pub trait JournalAnchor { ) -> Result<(), JournalError>; } +/// Recoverable storage-key identity for one mailbox and execution epoch. +/// +/// This identifies the journal key, not the separately encrypted payload's +/// opening key. A later epoch or rotation needs an explicit anchored handoff. +#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HistoryKeyRef { + address: MailboxAddress, + execution_epoch: ExecutionEpoch, + algorithm: EncryptionAlgorithm, + key_id: [u8; 32], + version: u32, +} + +impl std::fmt::Debug for HistoryKeyRef { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("HistoryKeyRef") + .field("version", &self.version) + .finish_non_exhaustive() + } +} + +impl HistoryKeyRef { + /// Bind one synthetic key version in tests; production needs verified scope. + #[cfg(any(test, feature = "test-support"))] + pub fn new( + address: MailboxAddress, + execution_epoch: ExecutionEpoch, + key_id: [u8; 32], + version: u32, + ) -> Result { + if key_id == [0; 32] || version == 0 { + return Err(JournalError::HistoryKeyUnavailable); + } + Ok(Self { + address, + execution_epoch, + algorithm: EncryptionAlgorithm::AES_256_GCM, + key_id, + version, + }) + } + + #[must_use] + pub const fn address(&self) -> MailboxAddress { + self.address + } + + #[must_use] + pub const fn execution_epoch(&self) -> ExecutionEpoch { + self.execution_epoch + } + + #[must_use] + pub const fn algorithm(&self) -> EncryptionAlgorithm { + self.algorithm + } + + #[must_use] + pub const fn key_id(&self) -> &[u8; 32] { + &self.key_id + } + + #[must_use] + pub const fn version(&self) -> u32 { + self.version + } + + fn validate(&self) -> Result<(), JournalError> { + if self.key_id == [0; 32] + || self.version == 0 + || self.algorithm != EncryptionAlgorithm::AES_256_GCM + { + return Err(JournalError::HistoryKeyUnavailable); + } + Ok(()) + } +} + +/// Existing-only key lookup; recovery must never silently mint a new root. +/// +/// An implementation must persist the referenced history key independently +/// of the erased per-launch secret and deny missing, retired or revoked keys. +pub trait HistoryKeyCustody { + /// Load the exact retained version, or refuse without creating a key. + fn load_existing(&self, reference: &HistoryKeyRef) -> Result; +} + /// Fail-closed admission and recovery failures; no plaintext is included. #[derive(Debug, thiserror::Error)] pub enum JournalError { @@ -119,6 +209,10 @@ pub enum JournalError { Uncertain, #[error("independent rollback anchor and recoverable key custody are required")] DurableAuthorityUnavailable, + #[error("mailbox history key is unavailable")] + HistoryKeyUnavailable, + #[error("mailbox history key identity does not match retained records")] + HistoryKeyMismatch, #[error("mailbox authorization refused: {0}")] Mailbox(#[from] MailboxError), #[error("journal storage is unavailable: {0}")] @@ -132,9 +226,25 @@ struct Record { cursor: u64, previous: [u8; 32], address: MailboxAddress, + #[serde(default, skip_serializing_if = "Option::is_none")] + history_key: Option, event: JournalEvent, } +struct JournalKey { + bytes: Zeroizing<[u8; 32]>, + reference: Option, +} + +impl JournalKey { + fn unversioned(bytes: [u8; 32]) -> Self { + Self { + bytes: Zeroizing::new(bytes), + reference: None, + } + } +} + #[derive(Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] enum JournalEvent { @@ -175,6 +285,7 @@ struct LeaseState { struct Seen { envelope: MailboxEnvelope, capability: SignedMailboxCapability, + accepted_receipt: DestinationReceipt, acknowledged: bool, lease: Option, signed_ack: Option, @@ -197,6 +308,7 @@ pub struct MailboxJournal { address: MailboxAddress, authority: VerifyingKey, key: Zeroizing<[u8; 32]>, + history_key: Option, limits: JournalLimits, tip: JournalTip, bytes: u64, @@ -412,19 +524,28 @@ struct AcceptanceIntent<'a> { identity: ReplayIdentity, frame: &'a str, receipt: &'a DestinationReceipt, + #[serde(skip_serializing_if = "Option::is_none")] + history_key: Option<&'a HistoryKeyRef>, } fn acceptance_intent_bytes( identity: ReplayIdentity, frame: &str, receipt: &DestinationReceipt, + history_key: Option<&HistoryKeyRef>, ) -> Result, JournalError> { - let mut bytes = ACCEPTANCE_INTENT_DOMAIN.to_vec(); + let domain = if history_key.is_some() { + ACCEPTANCE_INTENT_DOMAIN_V3 + } else { + ACCEPTANCE_INTENT_DOMAIN_V2 + }; + let mut bytes = domain.to_vec(); bytes.extend( serde_jcs::to_vec(&AcceptanceIntent { identity, frame, receipt, + history_key, }) .map_err(|_| JournalError::Corrupt)?, ); @@ -486,9 +607,15 @@ impl MailboxJournal { limits: JournalLimits, expected: JournalTip, ) -> Result { - Self::open_with_recovery(root, address, authority, key, limits, expected, |_| { - Err(JournalError::Rollback) - }) + Self::open_with_recovery( + root, + address, + authority, + JournalKey::unversioned(key), + limits, + expected, + |_| Err(JournalError::Rollback), + ) } /// Recover a journal fsynced before its anchor during an interrupted @@ -505,17 +632,23 @@ impl MailboxJournal { if anchor.address != address { return Err(JournalError::WrongAuthority); } - Self::open_with_recovery(root, address, authority, key, limits, anchor.tip(), |tip| { - anchor.advance(tip) - }) + Self::open_with_recovery( + root, + address, + authority, + JournalKey::unversioned(key), + limits, + anchor.tip(), + |tip| anchor.advance(tip), + ) } - /// Recover against an independently trusted high-water mark. + /// Recover a raw-key preproduction journal against an anchor. /// /// An intact, authenticated journal suffix may be advanced after a crash. /// A missing, divergent, or rolled-back anchored prefix is never repaired. - /// The caller must obtain `key` from persistent custody, not the erased - /// per-launch mailbox secret. + /// This opener does not establish retained key custody and cannot issue + /// a durable receipt; use [`Self::open_with_custody`] for versioned history. pub fn open_with_anchor( root: &Path, address: MailboxAddress, @@ -523,6 +656,53 @@ impl MailboxJournal { key: [u8; 32], limits: JournalLimits, anchor: &impl JournalAnchor, + ) -> Result { + Self::open_against_anchor( + root, + address, + authority, + JournalKey::unversioned(key), + limits, + anchor, + ) + } + + /// Reopen a test-scoped versioned history with an existing key. + /// + /// The production seam remains unavailable until a verified admission + /// scope and authenticated restart manifest can supply this reference. + /// This test path must not derive its key from the erased launch secret. + #[cfg(any(test, feature = "test-support"))] + pub fn open_with_custody( + root: &Path, + reference: HistoryKeyRef, + authority: VerifyingKey, + limits: JournalLimits, + custody: &impl HistoryKeyCustody, + anchor: &impl JournalAnchor, + ) -> Result { + reference.validate()?; + let key = custody.load_existing(&reference)?; + Self::open_against_anchor( + root, + reference.address, + authority, + JournalKey { + bytes: Zeroizing::new(*key.expose_bytes()), + reference: Some(reference), + }, + limits, + anchor, + ) + } + + fn open_against_anchor( + root: &Path, + address: MailboxAddress, + authority: VerifyingKey, + key: JournalKey, + limits: JournalLimits, + anchor: &impl JournalAnchor, ) -> Result { let expected = anchor.load(address)?; let mut previous = expected; @@ -537,7 +717,7 @@ impl MailboxJournal { root: &Path, address: MailboxAddress, authority: VerifyingKey, - key: [u8; 32], + key: JournalKey, limits: JournalLimits, expected: JournalTip, mut reconcile: impl FnMut(JournalTip) -> Result<(), JournalError>, @@ -562,7 +742,8 @@ impl MailboxJournal { root: root.to_owned(), address, authority, - key: Zeroizing::new(key), + key: key.bytes, + history_key: key.reference, limits, tip: JournalTip::genesis(), bytes: 0, @@ -614,13 +795,20 @@ impl MailboxJournal { ); let record: Record = serde_json::from_slice(&plaintext).map_err(|_| JournalError::Corrupt)?; - if record.version != 2 + let expected_version = if journal.history_key.is_some() { 3 } else { 2 }; + if record.version != expected_version || record.cursor != cursor || record.previous != journal.tip.head || record.address != address { return Err(JournalError::Corrupt); } + if let Some(reference) = &record.history_key { + reference.validate().map_err(|_| JournalError::Corrupt)?; + } + if record.history_key != journal.history_key { + return Err(JournalError::HistoryKeyMismatch); + } match record.event { JournalEvent::Accepted { identity, @@ -641,7 +829,12 @@ impl MailboxJournal { .map_err(|_| JournalError::Corrupt)?; authority .verify( - &acceptance_intent_bytes(identity, &encoded_frame, &receipt)?, + &acceptance_intent_bytes( + identity, + &encoded_frame, + &receipt, + journal.history_key.as_ref(), + )?, &Signature::from_bytes(&signature), ) .map_err(|_| JournalError::Corrupt)?; @@ -659,6 +852,7 @@ impl MailboxJournal { Seen { envelope, capability, + accepted_receipt: *receipt, acknowledged: false, lease: None, signed_ack: None, @@ -844,6 +1038,12 @@ impl MailboxJournal { { return Err(JournalError::WrongAuthority); } + if self.history_key.is_some_and(|reference| { + reference.address != self.address + || reference.execution_epoch != context.current_execution_epoch + }) { + return Err(JournalError::HistoryKeyMismatch); + } let mut probe = Probe::default(); let opened = open_authenticated_envelope( envelope, @@ -870,6 +1070,13 @@ impl MailboxJournal { if previous.envelope != *envelope || previous.capability != *capability { return Err(JournalError::ConflictingRetry); } + if previous.accepted_receipt.outcome != DeliveryOutcome::AcceptedDurable + || previous.accepted_receipt.address != self.address + || previous.accepted_receipt.message_id != opened.header().message_id + || previous.accepted_receipt.ciphertext_commitment != envelope.ciphertext_commitment + { + return Err(JournalError::Corrupt); + } return Ok(sign_receipt(receipt(DeliveryOutcome::Duplicate), signer)?); } if self.tip.cursor >= self.limits.max_records as u64 { @@ -884,6 +1091,7 @@ impl MailboxJournal { identity, &encoded_frame, &accepted, + self.history_key.as_ref(), )?) .to_bytes() .to_vec(); @@ -901,6 +1109,7 @@ impl MailboxJournal { Seen { envelope: envelope.clone(), capability: capability.clone(), + accepted_receipt: accepted.clone(), acknowledged: false, lease: None, signed_ack: None, @@ -919,6 +1128,9 @@ impl MailboxJournal { request: JournalAdmission<'_>, anchor: &impl JournalAnchor, ) -> Result { + if self.history_key.is_none() { + return Err(JournalError::DurableAuthorityUnavailable); + } let expected = self.tip; if anchor.load(self.address)? != expected { return Err(JournalError::Rollback); @@ -942,10 +1154,11 @@ impl MailboxJournal { return Err(JournalError::Full); } let record = Record { - version: 2, + version: if self.history_key.is_some() { 3 } else { 2 }, cursor, previous: self.tip.head, address: self.address, + history_key: self.history_key, event, }; let body = Zeroizing::new(serde_json::to_vec(&record).map_err(|_| JournalError::Corrupt)?); @@ -1407,6 +1620,22 @@ mod anchor_tests { } } + struct TestCustody { + reference: HistoryKeyRef, + bytes: Option<[u8; 32]>, + } + + impl HistoryKeyCustody for TestCustody { + fn load_existing(&self, reference: &HistoryKeyRef) -> Result { + if reference != &self.reference { + return Err(JournalError::HistoryKeyUnavailable); + } + self.bytes + .map(Key::from_bytes) + .ok_or(JournalError::HistoryKeyUnavailable) + } + } + fn address() -> MailboxAddress { MailboxAddress { tenant: TenantId::from_bytes([1; 32]), @@ -1423,6 +1652,23 @@ mod anchor_tests { } } + fn versioned_custody() -> (HistoryKeyRef, TestCustody) { + let reference = HistoryKeyRef::new( + address(), + ExecutionEpoch::new(1).expect("test epoch"), + [30; 32], + 1, + ) + .expect("versioned key"); + ( + reference, + TestCustody { + reference, + bytes: Some([21; 32]), + }, + ) + } + #[test] fn admission_intent_signature_is_bound_to_its_own_domain() { let (envelope, capability, context, _, _, signer) = message(); @@ -1445,7 +1691,7 @@ mod anchor_tests { observed_at_unix_ms: context.now_unix_ms, ciphertext_commitment: envelope.ciphertext_commitment, }; - let intent = acceptance_intent_bytes(identity, &frame, &receipt).expect("intent"); + let intent = acceptance_intent_bytes(identity, &frame, &receipt, None).expect("intent"); let signature = signer.sign(&intent); signer .verifying_key() @@ -1455,6 +1701,7 @@ mod anchor_tests { identity, frame: &frame, receipt: &receipt, + history_key: None, }) .expect("canonical intent"); assert!(signer.verifying_key().verify(&bare, &signature).is_err()); @@ -1605,12 +1852,13 @@ mod anchor_tests { let root = dir.path().join("journal"); let (envelope, capability, context, payload_key, sender_key, signer) = message(); let anchor = IndependentAnchor::new(address()); - let mut journal = MailboxJournal::open_with_anchor( + let (reference, custody) = versioned_custody(); + let mut journal = MailboxJournal::open_with_custody( &root, - address(), + reference, signer.verifying_key(), - [21; 32], limits(), + &custody, &anchor, ) .expect("genesis"); @@ -1660,12 +1908,12 @@ mod anchor_tests { ); drop(journal); - let mut recovered = MailboxJournal::open_with_anchor( + let mut recovered = MailboxJournal::open_with_custody( &root, - address(), + reference, signer.verifying_key(), - [21; 32], limits(), + &custody, &anchor, ) .expect("restart recovers exact anchored history"); @@ -1687,12 +1935,246 @@ mod anchor_tests { } #[test] - fn unsigned_host_ack_remains_available_only_before_a_guest_lease() { + fn scoped_history_key_survives_restart_and_refuses_missing_or_wrong_versions() { let dir = tempfile::tempdir().expect("journal root"); let root = dir.path().join("journal"); let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let reference = HistoryKeyRef::new(address(), context.current_execution_epoch, [30; 32], 1) + .expect("versioned reference"); + let roundtrip: HistoryKeyRef = + serde_json::from_slice(&serde_json::to_vec(&reference).expect("encode reference")) + .expect("decode reference"); + assert_eq!(roundtrip, reference); + assert!(matches!( + HistoryKeyRef::new(address(), context.current_execution_epoch, [0; 32], 1), + Err(JournalError::HistoryKeyUnavailable) + )); + assert!(matches!( + HistoryKeyRef::new(address(), context.current_execution_epoch, [30; 32], 0), + Err(JournalError::HistoryKeyUnavailable) + )); + let custody = TestCustody { + reference, + bytes: Some([21; 32]), + }; let anchor = IndependentAnchor::new(address()); - let mut journal = MailboxJournal::open_with_anchor( + let missing = TestCustody { + reference, + bytes: None, + }; + assert!(matches!( + MailboxJournal::open_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &missing, + &anchor, + ), + Err(JournalError::HistoryKeyUnavailable) + )); + assert!( + !root.exists(), + "a recovery read must not create missing keys or history" + ); + + let mut journal = MailboxJournal::open_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ) + .expect("new history uses retained test key"); + let accepted = journal + .admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("record and dedup anchored"); + let record = fs::read(record_path(&root, 1)).expect("history record"); + let body = aead::open( + &journal.record_key(), + &record[32..], + &[DOMAIN, &JournalTip::genesis().head].concat(), + ) + .expect("open authenticated record"); + let parsed: Record = serde_json::from_slice(&body).expect("versioned record"); + assert_eq!(parsed.version, 3); + assert_eq!(parsed.history_key, Some(reference)); + let JournalEvent::Accepted { + identity, + frame, + receipt, + authorization_signature, + } = parsed.event + else { + panic!("versioned record must be an admission"); + }; + let signature: [u8; 64] = authorization_signature + .try_into() + .expect("destination intent signature"); + let signature = Signature::from_bytes(&signature); + signer + .verifying_key() + .verify( + &acceptance_intent_bytes(identity, &frame, &receipt, Some(&reference)) + .expect("v3 intent"), + &signature, + ) + .expect("key reference is signed under the v3 domain"); + assert!( + signer + .verifying_key() + .verify( + &acceptance_intent_bytes(identity, &frame, &receipt, None).expect("v2 intent"), + &signature, + ) + .is_err(), + "a v3 key reference cannot borrow the old intent domain" + ); + drop(journal); + + let mut recovered = MailboxJournal::open_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ) + .expect("exact historical key and ref recover"); + let retained = recovered + .seen + .get(&identity) + .expect("recovered dedup identity") + .accepted_receipt + .clone(); + assert_eq!( + retained, accepted.receipt, + "original acceptance time is retained" + ); + assert_eq!( + sign_receipt(retained, &signer) + .expect("deterministic original signature") + .signature, + accepted.signature, + "an authorized signer can reproduce the original proof after a lost response" + ); + assert_eq!( + recovered + .admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("retry deduplicates") + .receipt + .outcome, + DeliveryOutcome::Duplicate + ); + drop(recovered); + + for changed in [ + HistoryKeyRef::new(address(), context.current_execution_epoch, [31; 32], 1) + .expect("other key id"), + HistoryKeyRef::new(address(), context.current_execution_epoch, [30; 32], 2) + .expect("other version"), + HistoryKeyRef::new( + address(), + ExecutionEpoch::new(2).expect("other epoch"), + [30; 32], + 1, + ) + .expect("other epoch reference"), + ] { + let same_bytes = TestCustody { + reference: changed, + bytes: Some([21; 32]), + }; + assert!(matches!( + MailboxJournal::open_with_custody( + &root, + changed, + signer.verifying_key(), + limits(), + &same_bytes, + &anchor, + ), + Err(JournalError::HistoryKeyMismatch) + )); + } + let wrong_bytes = TestCustody { + reference, + bytes: Some([22; 32]), + }; + assert!(matches!( + MailboxJournal::open_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &wrong_bytes, + &anchor, + ), + Err(JournalError::Corrupt) + )); + let mut foreign = address(); + foreign.authorization_domain = AuthorizationDomainId::from_bytes([55; 32]); + let foreign_ref = HistoryKeyRef::new(foreign, context.current_execution_epoch, [30; 32], 1) + .expect("other domain"); + assert!(matches!( + MailboxJournal::open_with_custody( + &root, + foreign_ref, + signer.verifying_key(), + limits(), + &TestCustody { + reference: foreign_ref, + bytes: Some([21; 32]), + }, + &anchor, + ), + Err(JournalError::WrongAuthority) + )); + let mut unsupported = serde_json::to_value(reference).expect("reference value"); + unsupported["algorithm"] = serde_json::json!(42); + let unsupported: HistoryKeyRef = serde_json::from_value(unsupported).expect("raw version"); + assert!(matches!( + MailboxJournal::open_with_custody( + &root, + unsupported, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ), + Err(JournalError::HistoryKeyUnavailable) + )); + assert_eq!(anchor.load(address()).expect("anchor unchanged").cursor, 1); + } + + #[test] + fn preproduction_unversioned_history_is_not_reinterpreted_under_v3_custody() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let anchor = IndependentAnchor::new(address()); + let mut legacy = MailboxJournal::open_with_anchor( &root, address(), signer.verifying_key(), @@ -1700,6 +2182,59 @@ mod anchor_tests { limits(), &anchor, ) + .expect("v2 test history"); + let prior = legacy.tip(); + legacy + .admit_verified( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + |next| anchor.compare_and_advance(address(), prior, next), + ) + .expect("model a previously written v2 record"); + let record = fs::read(record_path(&root, 1)).expect("v2 bytes"); + let tip = anchor.load(address()).expect("v2 anchored tip"); + drop(legacy); + + let (reference, custody) = versioned_custody(); + assert!(matches!( + MailboxJournal::open_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ), + Err(JournalError::Corrupt) + )); + assert_eq!( + fs::read(record_path(&root, 1)).expect("retained bytes"), + record + ); + assert_eq!(anchor.load(address()).expect("unchanged anchor"), tip); + } + + #[test] + fn unsigned_host_ack_remains_available_only_before_a_guest_lease() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let anchor = IndependentAnchor::new(address()); + let (reference, custody) = versioned_custody(); + let mut journal = MailboxJournal::open_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ) .expect("genesis"); journal .admit_with_anchor( @@ -1743,12 +2278,12 @@ mod anchor_tests { .expect("never-leased host message can be acknowledged"); assert_eq!(tip.cursor, 2); drop(journal); - let recovered = MailboxJournal::open_with_anchor( + let recovered = MailboxJournal::open_with_custody( &root, - address(), + reference, signer.verifying_key(), - [21; 32], limits(), + &custody, &anchor, ) .expect("host ACK survives restart"); @@ -1766,12 +2301,13 @@ mod anchor_tests { let root = dir.path().join("journal"); let (envelope, capability, context, payload_key, sender_key, signer) = message(); let anchor = IndependentAnchor::new(address()); - let mut journal = MailboxJournal::open_with_anchor( + let (reference, custody) = versioned_custody(); + let mut journal = MailboxJournal::open_with_custody( &root, - address(), + reference, signer.verifying_key(), - [21; 32], limits(), + &custody, &anchor, ) .expect("genesis"); @@ -1814,12 +2350,12 @@ mod anchor_tests { Err(JournalError::Rollback) )); drop(journal); - let recovered = MailboxJournal::open_with_anchor( + let recovered = MailboxJournal::open_with_custody( &root, - address(), + reference, signer.verifying_key(), - [21; 32], limits(), + &custody, &anchor, ) .expect("verified unanchored suffix is reconciled"); @@ -1858,6 +2394,20 @@ mod anchor_tests { ), Err(JournalError::DurableAuthorityUnavailable) )); + assert!(matches!( + journal.admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ), + Err(JournalError::DurableAuthorityUnavailable) + )); assert_eq!(journal.tip(), JournalTip::genesis()); assert_eq!( anchor.load(address()).expect("no commit"), @@ -1871,15 +2421,21 @@ mod anchor_tests { let root = dir.path().join("journal"); let (envelope, capability, context, payload_key, sender_key, signer) = message(); let anchor = IndependentAnchor::new(address()); - let mut journal = MailboxJournal::open_with_anchor( - &root, - address(), - signer.verifying_key(), - [21; 32], - limits(), - &anchor, - ) - .expect("genesis"); + let (reference, _) = versioned_custody(); + let open = |key_bytes, authority| { + MailboxJournal::open_with_custody( + &root, + reference, + authority, + limits(), + &TestCustody { + reference, + bytes: Some(key_bytes), + }, + &anchor, + ) + }; + let mut journal = open([21; 32], signer.verifying_key()).expect("genesis"); journal .admit_with_anchor( admission( @@ -1897,25 +2453,11 @@ mod anchor_tests { drop(journal); assert!(matches!( - MailboxJournal::open_with_anchor( - &root, - address(), - signer.verifying_key(), - [22; 32], - limits(), - &anchor, - ), + open([22; 32], signer.verifying_key()), Err(JournalError::Corrupt) )); assert!(matches!( - MailboxJournal::open_with_anchor( - &root, - address(), - SigningKey::from_bytes(&[23; 32]).verifying_key(), - [21; 32], - limits(), - &anchor, - ), + open([21; 32], SigningKey::from_bytes(&[23; 32]).verifying_key()), Err(JournalError::Corrupt) )); let path = record_path(&root, 1); @@ -1924,39 +2466,18 @@ mod anchor_tests { tampered[40] ^= 1; fs::write(&path, &tampered).expect("tamper record"); assert!(matches!( - MailboxJournal::open_with_anchor( - &root, - address(), - signer.verifying_key(), - [21; 32], - limits(), - &anchor, - ), + open([21; 32], signer.verifying_key()), Err(JournalError::Corrupt) )); fs::write(&path, &original[..40]).expect("interrupted write"); assert!(matches!( - MailboxJournal::open_with_anchor( - &root, - address(), - signer.verifying_key(), - [21; 32], - limits(), - &anchor, - ), + open([21; 32], signer.verifying_key()), Err(JournalError::Corrupt) )); assert_eq!(anchor.load(address()).expect("unchanged anchor"), tip); fs::remove_file(&path).expect("simulate journal rollback"); assert!(matches!( - MailboxJournal::open_with_anchor( - &root, - address(), - signer.verifying_key(), - [21; 32], - limits(), - &anchor, - ), + open([21; 32], signer.verifying_key()), Err(JournalError::Rollback) )); } diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index 1a03b080a7..58122ee98f 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1161,8 +1161,9 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { MailboxGuestReaderEnrollment, MailboxGuestReaderScope, }; use mvm_core::mailbox::journal::{ - JournalAdmission, JournalAnchor, JournalError, JournalGuestAck, JournalGuestLease, - JournalLimits, JournalReceive, JournalTip, MailboxJournal, + HistoryKeyCustody, HistoryKeyRef, JournalAdmission, JournalAnchor, JournalError, + JournalGuestAck, JournalGuestLease, JournalLimits, JournalReceive, JournalTip, + MailboxJournal, }; use mvm_core::mailbox::reader::{ sign_delivery_ack, sign_guest_reader_enrollment, sign_reader_possession, @@ -1208,6 +1209,17 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { } } + struct TestCustody(HistoryKeyRef); + + impl HistoryKeyCustody for TestCustody { + fn load_existing(&self, reference: &HistoryKeyRef) -> Result { + if reference != &self.0 { + return Err(JournalError::HistoryKeyUnavailable); + } + Ok(Key::from_bytes([90; 32])) + } + } + fn test_record_key() -> Key { use hmac::{Hmac, Mac, digest::KeyInit}; use sha2::Sha256; @@ -1350,15 +1362,18 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { max_records: 6, max_bytes: 64 * 1024, }; + let reference = + HistoryKeyRef::new(address(), plan.epoch, [91; 32], 1).expect("test history key version"); + let custody = TestCustody(reference); let anchor_state = Arc::new(Mutex::new(JournalTip::genesis())); let open = || { let anchor = TestAnchor(Arc::clone(&anchor_state)); - let journal = MailboxJournal::open_with_anchor( + let journal = MailboxJournal::open_with_custody( &journal_path, - address(), + reference, authority, - [90; 32], limits, + &custody, &anchor, ) .unwrap(); @@ -1466,12 +1481,12 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { *anchor_state.lock().expect("mock anchor lock") = forged; assert!( matches!( - MailboxJournal::open_with_anchor( + MailboxJournal::open_with_custody( &journal_path, - address(), + reference, authority, - [90; 32], limits, + &custody, &TestAnchor(Arc::clone(&anchor_state)), ), Err(JournalError::Corrupt) @@ -1486,10 +1501,11 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { *anchor_state.lock().expect("mock anchor lock") = lease_tip; } let forged_host_ack = serde_json::json!({ - "version": 2, + "version": 3, "cursor": 3, "previous": lease_tip.head, "address": address(), + "history_key": reference, "event": { "kind": "acknowledged", "identity": identity, @@ -1500,12 +1516,12 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { *anchor_state.lock().expect("mock anchor lock") = forged; assert!( matches!( - MailboxJournal::open_with_anchor( + MailboxJournal::open_with_custody( &journal_path, - address(), + reference, authority, - [90; 32], limits, + &custody, &TestAnchor(Arc::clone(&anchor_state)), ), Err(JournalError::Corrupt) @@ -1705,12 +1721,12 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { *anchor_state.lock().expect("mock anchor lock") = forged; assert!( matches!( - MailboxJournal::open_with_anchor( + MailboxJournal::open_with_custody( &journal_path, - address(), + reference, authority, - [90; 32], limits, + &custody, &TestAnchor(Arc::clone(&anchor_state)), ), Err(JournalError::Corrupt) From c56bf7b654d44898e7d6a2449debc5e05d36266a Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 09:04:15 -0800 Subject: [PATCH 22/51] test(mailbox): refuse forged serialized history references Demonstrate that decoded key IDs, versions, epochs and authorization domains are not authority even when a test custody source returns the same key bytes. Document the signed plan and restart-manifest precondition for any production opener. --- crates/mvm-core/README.md | 3 +++ crates/mvm-core/src/mailbox/journal.rs | 35 ++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index be45bcef2c..1bdf19e9bb 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -70,6 +70,9 @@ version) to the encrypted record and signed intent. Its recovery obtains the exact key through an existing-only `HistoryKeyCustody` lookup and refuses changed key identity, wrong key, or missing custody; it does not create a replacement key. The raw-key opener cannot issue an `accepted_durable` receipt. +Deserializing a `HistoryKeyRef` does not grant authority: production reopening +must verify its scope against the admitted plan and an authenticated retained +manifest before loading a key. The destination signs an actual acceptance receipt only after both commits. Earlier private-handoff version-1 records are refused, not silently reinterpreted. Version-2 test/local records have no custody reference and are diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index a9a449bde6..9f892c9947 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -96,7 +96,9 @@ pub trait JournalAnchor { /// Recoverable storage-key identity for one mailbox and execution epoch. /// /// This identifies the journal key, not the separately encrypted payload's -/// opening key. A later epoch or rotation needs an explicit anchored handoff. +/// opening key. Deserialization is not admission authority; production must +/// bind it to a verified plan and authenticated restart manifest. A later +/// epoch or rotation needs an explicit anchored handoff. #[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct HistoryKeyRef { @@ -2133,10 +2135,33 @@ mod anchor_tests { ), Err(JournalError::Corrupt) )); - let mut foreign = address(); - foreign.authorization_domain = AuthorizationDomainId::from_bytes([55; 32]); - let foreign_ref = HistoryKeyRef::new(foreign, context.current_execution_epoch, [30; 32], 1) - .expect("other domain"); + for (field, value) in [ + ("/version", serde_json::json!(2)), + ("/key_id", serde_json::json!(vec![31u8; 32])), + ("/execution_epoch", serde_json::json!(2)), + ] { + let mut forged = serde_json::to_value(reference).expect("reference JSON"); + *forged.pointer_mut(field).expect("reference field") = value; + let forged: HistoryKeyRef = serde_json::from_value(forged).expect("forged reference"); + assert!(matches!( + MailboxJournal::open_with_custody( + &root, + forged, + signer.verifying_key(), + limits(), + &TestCustody { + reference: forged, + bytes: Some([21; 32]), + }, + &anchor, + ), + Err(JournalError::HistoryKeyMismatch) + )); + } + let mut foreign = serde_json::to_value(reference).expect("reference JSON"); + foreign["address"]["authorization_domain"] = serde_json::json!(vec![55u8; 32]); + let foreign_ref: HistoryKeyRef = + serde_json::from_value(foreign).expect("forged foreign-domain reference"); assert!(matches!( MailboxJournal::open_with_custody( &root, From 037154bde50b94721b73109e4a1e2c6db1bc5f35 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 09:57:39 -0800 Subject: [PATCH 23/51] fix(mailbox): reconcile report grant in recovered lease --- crates/mvm-core/src/mailbox/journal.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 9f892c9947..d1d4d6e681 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -569,6 +569,7 @@ fn verify_recovered_lease( mailbox: lease.lease.address.mailbox, authority_key: authority.to_bytes(), epoch: lease.lease.execution_epoch, + report: None, }; let scope = MailboxGuestReaderScope { plan: &plan, From fc607ba9677475413b039a6a0334aa638295b96f Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 10:17:36 -0800 Subject: [PATCH 24/51] feat(mailbox): integrate verified envelope and Reply authority --- crates/mvm-core/src/mailbox.rs | 72 +++++++--- crates/mvm-core/src/mailbox/transport.rs | 22 ++- .../mvm-core/src/mailbox/transport/tests.rs | 125 ++++++++++++++++++ crates/mvm-core/tests/mailbox_crypto.rs | 91 +++++++++++++ 4 files changed, 285 insertions(+), 25 deletions(-) diff --git a/crates/mvm-core/src/mailbox.rs b/crates/mvm-core/src/mailbox.rs index f691339632..6129c0e965 100644 --- a/crates/mvm-core/src/mailbox.rs +++ b/crates/mvm-core/src/mailbox.rs @@ -488,45 +488,55 @@ fn verify_capability_scope( Ok(()) } -/// A message whose signed inner header, capability and replay identity passed -/// the destination's authority checks. The header can be journaled with the -/// unchanged ciphertext without parsing untrusted routing hints as identity. +/// Authenticated header and plaintext from one successful canonical open. +/// This is verification under the supplied context, not a delivery receipt or +/// permission to skip a later admission check. +/// +/// ```compile_fail +/// use mvm_contract::protocol::mailbox::MailboxEnvelopeHeader; +/// use mvm_core::mailbox::OpenedMailboxEnvelope; +/// fn forge(header: MailboxEnvelopeHeader, payload: Vec) -> OpenedMailboxEnvelope { +/// OpenedMailboxEnvelope { header, payload } +/// } +/// ``` #[derive(PartialEq, Eq)] -pub struct OpenedMailboxMessage { +pub struct OpenedMailboxEnvelope { header: MailboxEnvelopeHeader, payload: Vec, } -impl std::fmt::Debug for OpenedMailboxMessage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("OpenedMailboxMessage") - .field("payload_len", &self.payload.len()) +impl std::fmt::Debug for OpenedMailboxEnvelope { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OpenedMailboxEnvelope") .finish_non_exhaustive() } } -impl OpenedMailboxMessage { - /// The authenticated inner header used to bind journal and dedup records. +/// Compatibility name for a verified, opened mailbox message. +pub type OpenedMailboxMessage = OpenedMailboxEnvelope; + +impl OpenedMailboxEnvelope { + /// The exact signed header authenticated with these bytes. #[must_use] pub fn header(&self) -> &MailboxEnvelopeHeader { &self.header } - /// Plaintext for the authorized recipient, not for a router or store. + /// The verified plaintext without copying it. #[must_use] pub fn payload(&self) -> &[u8] { &self.payload } - /// Consume the opened message to hand plaintext to its authorized reader. + /// Consume the verified result, retaining the original plaintext allocation. #[must_use] pub fn into_payload(self) -> Vec { self.payload } } -/// Verify identities, signature, capability, current state, ciphertext, and -/// replay identity before returning plaintext. +/// Verify an envelope and return its plaintext through the canonical open path. pub fn open_envelope( envelope: &MailboxEnvelope, capability: &SignedMailboxCapability, @@ -536,7 +546,7 @@ pub fn open_envelope( authority_key: &VerifyingKey, replay: &mut impl ReplayProtector, ) -> Result, MailboxError> { - open_authenticated_envelope( + open_verified_envelope( envelope, capability, context, @@ -545,12 +555,13 @@ pub fn open_envelope( authority_key, replay, ) - .map(OpenedMailboxMessage::into_payload) + .map(OpenedMailboxEnvelope::into_payload) } -/// Open the envelope and retain the verified inner header for destination -/// admission. `replay` must stage its identity until any durable commit is -/// complete; an in-memory protector is appropriate only for ephemeral delivery. +/// Open the canonical envelope and retain its authenticated inner header. +/// +/// `replay` must stage identity until journal commit; in-memory replay is +/// appropriate only for ephemeral delivery. pub fn open_authenticated_envelope( envelope: &MailboxEnvelope, capability: &SignedMailboxCapability, @@ -560,6 +571,27 @@ pub fn open_authenticated_envelope( authority_key: &VerifyingKey, replay: &mut impl ReplayProtector, ) -> Result { + open_verified_envelope( + envelope, + capability, + context, + encryption_key, + sender_key, + authority_key, + replay, + ) +} + +/// Verify identities, signature, capability, current state, ciphertext, and replay. +pub fn open_verified_envelope( + envelope: &MailboxEnvelope, + capability: &SignedMailboxCapability, + context: &OpenContext, + encryption_key: &Key, + sender_key: &VerifyingKey, + authority_key: &VerifyingKey, + replay: &mut impl ReplayProtector, +) -> Result { envelope.validate_bounds()?; validate_outer_versions(envelope)?; let actual_commitment: [u8; 32] = Sha256::digest(&envelope.ciphertext).into(); @@ -601,7 +633,7 @@ pub fn open_authenticated_envelope( if !replay.check_and_record(ReplayIdentity::from_header(&message.header)) { return Err(MailboxError::Replay); } - Ok(OpenedMailboxMessage { + Ok(OpenedMailboxEnvelope { header: message.header, payload: message.payload, }) diff --git a/crates/mvm-core/src/mailbox/transport.rs b/crates/mvm-core/src/mailbox/transport.rs index 482bb5b3f2..649944c444 100644 --- a/crates/mvm-core/src/mailbox/transport.rs +++ b/crates/mvm-core/src/mailbox/transport.rs @@ -14,8 +14,8 @@ use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; use mvm_contract::protocol::mailbox::{ AuthorizationDomainRouteLabel, DestinationRouteToken, ED25519_SIGNATURE_BYTES, ExecutionEpoch, MAILBOX_PROTOCOL_VERSION, MAX_ENCODED_ENVELOPE_BYTES, MailboxAddress, MailboxEnvelope, - MailboxEnvelopeHeader, MailboxOperation, PrincipalId, ProtocolVersion, SignatureAlgorithm, - SignedMailboxCapability, TransportBindingId, + MailboxEnvelopeHeader, MailboxMessageKind, MailboxOperation, PrincipalId, ProtocolVersion, + SignatureAlgorithm, SignedMailboxCapability, TransportBindingId, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -580,11 +580,21 @@ impl AuthorizedMailboxFrame { pub fn authorize_outbound( request: OutboundAuthorization<'_>, ) -> Result { - if request.context.required_operation != MailboxOperation::Send { - return Err(OutboundAuthorizationError::SendOperationRequired); - } let envelope = &request.prepared.envelope; let header = &request.prepared.header; + match header.kind { + MailboxMessageKind::Call | MailboxMessageKind::Cast + if request.context.required_operation != MailboxOperation::Send => + { + return Err(OutboundAuthorizationError::SendOperationRequired); + } + MailboxMessageKind::Reply + if request.context.required_operation != MailboxOperation::Reply => + { + return Err(OutboundAuthorizationError::ReplyOperationRequired); + } + MailboxMessageKind::Call | MailboxMessageKind::Cast | MailboxMessageKind::Reply => {} + } envelope.validate_bounds().map_err(MailboxError::from)?; validate_outer_versions(envelope)?; validate_inner_versions(header)?; @@ -625,6 +635,8 @@ pub enum OutboundAuthorizationError { Mailbox(#[from] MailboxError), #[error("outbound transport authorization requires the send operation")] SendOperationRequired, + #[error("outbound reply authorization requires the reply operation")] + ReplyOperationRequired, #[error("mailbox envelope and supplied signed header do not match")] EnvelopeHeaderMismatch, #[error("mailbox envelope does not match the authorized transport route")] diff --git a/crates/mvm-core/src/mailbox/transport/tests.rs b/crates/mvm-core/src/mailbox/transport/tests.rs index cbbad85b0d..8f5b957d6e 100644 --- a/crates/mvm-core/src/mailbox/transport/tests.rs +++ b/crates/mvm-core/src/mailbox/transport/tests.rs @@ -1531,6 +1531,131 @@ fn authenticated_connection_is_not_mailbox_authority() { assert!(transport.drain(route.route_id()).expect("drain").is_empty()); } +fn fixture_for_operation(kind: MailboxMessageKind, operation: MailboxOperation) -> MailboxFixture { + let mut fixture = mailbox_fixture(); + let mut header = fixture.prepared.header.clone(); + header.kind = kind; + header.reply_to = match kind { + MailboxMessageKind::Call => Some(mvm_contract::protocol::mailbox::ReplyRoute { + address: address(), + capability_id: header.capability_id, + }), + MailboxMessageKind::Cast | MailboxMessageKind::Reply => None, + }; + fixture.prepared = prepare_mailbox_envelope( + SealRequest { + routing: fixture.prepared.envelope().routing.clone(), + header, + plaintext: b"hello", + }, + &Key::from_bytes(id(21)), + &SigningKey::from_bytes(&id(22)), + ) + .expect("message seals"); + let mut grant = fixture.capability.grant.clone(); + grant.operations = CapabilityOperations::from_operations(&[operation]); + grant.message_kinds = vec![kind]; + fixture.capability = sign_capability(grant, &fixture.capability_authority).expect("grant"); + fixture.context.required_operation = operation; + fixture +} + +#[test] +fn reply_only_grant_authorizes_the_exact_prepared_reply() { + let route_authority = SigningKey::from_bytes(&id(61)); + let (_, route) = signed_route(TransportPath::Direct, 62, &route_authority); + let fixture = fixture_for_operation(MailboxMessageKind::Reply, MailboxOperation::Reply); + let authorized = authorize_outbound(OutboundAuthorization { + prepared: &fixture.prepared, + capability: &fixture.capability, + context: &fixture.context, + capability_authority_key: &fixture.capability_authority.verifying_key(), + route: &route, + }) + .expect("reply-only authority"); + assert_eq!( + authorized.as_bytes(), + encode_envelope(fixture.prepared.envelope()).expect("canonical bytes") + ); +} + +#[test] +fn outbound_message_kind_cannot_borrow_another_operation() { + let authority = SigningKey::from_bytes(&id(63)); + let (_, route) = signed_route(TransportPath::Direct, 64, &authority); + for kind in [ + MailboxMessageKind::Call, + MailboxMessageKind::Cast, + MailboxMessageKind::Reply, + ] { + for operation in [ + MailboxOperation::Send, + MailboxOperation::Reply, + MailboxOperation::Receive, + ] { + let fixture = fixture_for_operation(kind, operation); + let result = authorize_outbound(OutboundAuthorization { + prepared: &fixture.prepared, + capability: &fixture.capability, + context: &fixture.context, + capability_authority_key: &fixture.capability_authority.verifying_key(), + route: &route, + }); + match kind { + MailboxMessageKind::Call | MailboxMessageKind::Cast + if operation == MailboxOperation::Send => + { + assert!(result.is_ok()); + } + MailboxMessageKind::Reply if operation == MailboxOperation::Reply => { + assert!(result.is_ok()); + } + MailboxMessageKind::Reply => assert_eq!( + result, + Err(OutboundAuthorizationError::ReplyOperationRequired) + ), + MailboxMessageKind::Call | MailboxMessageKind::Cast => assert_eq!( + result, + Err(OutboundAuthorizationError::SendOperationRequired) + ), + } + } + } +} + +#[test] +fn reply_context_still_requires_a_reply_capability_and_matching_route() { + let authority = SigningKey::from_bytes(&id(65)); + let (_, route) = signed_route(TransportPath::Direct, 66, &authority); + let mut fixture = fixture_for_operation(MailboxMessageKind::Reply, MailboxOperation::Send); + fixture.context.required_operation = MailboxOperation::Reply; + let request = |current: &MailboxFixture, selected: &AuthorizedRoute| { + authorize_outbound(OutboundAuthorization { + prepared: ¤t.prepared, + capability: ¤t.capability, + context: ¤t.context, + capability_authority_key: ¤t.capability_authority.verifying_key(), + route: selected, + }) + }; + assert_eq!( + request(&fixture, &route), + Err(MailboxError::OperationNotAuthorized.into()) + ); + let valid = fixture_for_operation(MailboxMessageKind::Reply, MailboxOperation::Reply); + let mut mismatched = claims(TransportPath::Direct, 67); + mismatched.address.recipient = PrincipalId::from_bytes(id(68)); + let signed = sign_transport_binding(mismatched, &authority).expect("different route"); + let mut context = binding_context(); + context.expected_address = signed.claims.address; + let foreign_route = authorize_route(&signed, &context, &authority.verifying_key()) + .expect("valid foreign route"); + assert_eq!( + request(&valid, &foreign_route), + Err(OutboundAuthorizationError::RouteMismatch) + ); +} + #[test] fn direct_delivery_is_pass_by_value_bounded_and_cancelable() { let authority = SigningKey::from_bytes(&id(46)); diff --git a/crates/mvm-core/tests/mailbox_crypto.rs b/crates/mvm-core/tests/mailbox_crypto.rs index a5e3d9a6f3..c36128258f 100644 --- a/crates/mvm-core/tests/mailbox_crypto.rs +++ b/crates/mvm-core/tests/mailbox_crypto.rs @@ -819,6 +819,97 @@ fn open_under( ) } +#[test] +fn verified_open_retains_authenticated_header_and_records_replay_once() { + #[derive(Default)] + struct CountingReplay(usize); + + impl mvm_core::mailbox::ReplayProtector for CountingReplay { + fn check_and_record(&mut self, _: mvm_core::mailbox::ReplayIdentity) -> bool { + self.0 += 1; + self.0 == 1 + } + } + + let (sender_key, authority_key, encryption_key) = contract_keys(); + let capability = sign_capability(grant(), &authority_key).expect("capability"); + let expected = header(); + let envelope = seal_with(expected.clone(), b"hello"); + let mut replay = CountingReplay::default(); + let opened = mvm_core::mailbox::open_verified_envelope( + &envelope, + &capability, + &context(), + &encryption_key, + &sender_key.verifying_key(), + &authority_key.verifying_key(), + &mut replay, + ) + .expect("verified header and bytes"); + assert_eq!(opened.header(), &expected); + assert_eq!(opened.payload(), b"hello"); + assert_eq!(replay.0, 1); + assert_eq!(opened.into_payload(), b"hello"); + assert_eq!( + open_envelope( + &envelope, + &capability, + &context(), + &encryption_key, + &sender_key.verifying_key(), + &authority_key.verifying_key(), + &mut replay, + ) + .expect_err("legacy open shares replay authority"), + MailboxError::Replay + ); + assert_eq!(replay.0, 2); +} + +#[test] +fn refused_verified_open_exposes_no_proof_and_preserves_replay_identity() { + let (sender_key, authority_key, encryption_key) = contract_keys(); + let capability = sign_capability(grant(), &authority_key).expect("capability"); + let envelope = seal_with(header(), b"hello"); + let open = |current: &OpenContext, replay: &mut InMemoryReplayProtector| { + mvm_core::mailbox::open_verified_envelope( + &envelope, + &capability, + current, + &encryption_key, + &sender_key.verifying_key(), + &authority_key.verifying_key(), + replay, + ) + }; + let mut recipient = context(); + recipient.expected_recipient = PrincipalId::from_bytes([41; 32]); + let mut epoch = context(); + epoch.current_execution_epoch = ExecutionEpoch::new(8).expect("epoch"); + let mut policy = context(); + policy.current_policy_epoch += 1; + let mut revoked = context(); + revoked.revoked_capabilities.push(grant().revocation_id); + let mut expired = context(); + expired.now_unix_ms = 2_000; + for (refused, expected) in [ + (recipient, MailboxError::WrongRecipient), + (epoch, MailboxError::StaleExecutionEpoch), + (policy, MailboxError::StalePolicyEpoch), + (revoked, MailboxError::CapabilityRevoked), + (expired, MailboxError::Expired), + ] { + let mut replay = InMemoryReplayProtector::default(); + assert_eq!(open(&refused, &mut replay).expect_err("no proof"), expected); + let accepted = open(&context(), &mut replay).expect("refusal did not consume identity"); + assert_eq!(accepted.header(), &header()); + assert_eq!(accepted.payload(), b"hello"); + let debug = format!("{accepted:?}"); + assert!(!debug.contains("hello")); + assert!(!debug.contains("capability_id")); + } +} + #[test] fn a_refused_message_does_not_consume_its_dedup_identity() { let envelope = seal_with(header(), b"hello"); From e436a6798b1c84ed8eea7ec4f51d58e18e8f8ece Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 10:17:45 -0800 Subject: [PATCH 25/51] feat(mailbox): integrate borrowed verified admission capsule --- crates/mvm-core/src/mailbox.rs | 291 ++++++++++++++++++++++++ crates/mvm-core/tests/mailbox_crypto.rs | 233 +++++++++++++++++++ 2 files changed, 524 insertions(+) diff --git a/crates/mvm-core/src/mailbox.rs b/crates/mvm-core/src/mailbox.rs index 6129c0e965..c7be7dff0c 100644 --- a/crates/mvm-core/src/mailbox.rs +++ b/crates/mvm-core/src/mailbox.rs @@ -536,6 +536,297 @@ impl OpenedMailboxEnvelope { } } +/// A fixed-size snapshot of the successful verifier's authenticated header and +/// supplied local context. This is historical verification data, not current +/// authorization, a journal commit, or permission to sign a durable receipt. +/// Accepted-content and revocation vectors are deliberately not retained. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct VerifiedMailboxScope { + tenant: TenantId, + authorization_domain: AuthorizationDomainId, + mailbox: MailboxId, + sender: PrincipalId, + recipient: PrincipalId, + sender_boot: BootId, + message_id: MessageId, + capability_id: mvm_contract::protocol::mailbox::CapabilityId, + encryption_key_id: [u8; 32], + execution_epoch: ExecutionEpoch, + policy_epoch: u64, + expires_at_unix_ms: i64, + acting_principal: PrincipalId, + required_operation: MailboxOperation, + capability_issuer: PrincipalId, + current_execution_epoch: ExecutionEpoch, + current_policy_epoch: u64, + verified_at_unix_ms: i64, +} + +impl std::fmt::Debug for VerifiedMailboxScope { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("VerifiedMailboxScope") + .finish_non_exhaustive() + } +} + +impl VerifiedMailboxScope { + #[must_use] + pub fn tenant(&self) -> TenantId { + self.tenant + } + #[must_use] + pub fn authorization_domain(&self) -> AuthorizationDomainId { + self.authorization_domain + } + #[must_use] + pub fn mailbox(&self) -> MailboxId { + self.mailbox + } + #[must_use] + pub fn sender(&self) -> PrincipalId { + self.sender + } + #[must_use] + pub fn recipient(&self) -> PrincipalId { + self.recipient + } + #[must_use] + pub fn sender_boot(&self) -> BootId { + self.sender_boot + } + #[must_use] + pub fn message_id(&self) -> MessageId { + self.message_id + } + #[must_use] + pub fn capability_id(&self) -> mvm_contract::protocol::mailbox::CapabilityId { + self.capability_id + } + #[must_use] + pub fn encryption_key_id(&self) -> [u8; 32] { + self.encryption_key_id + } + #[must_use] + pub fn execution_epoch(&self) -> ExecutionEpoch { + self.execution_epoch + } + #[must_use] + pub fn policy_epoch(&self) -> u64 { + self.policy_epoch + } + #[must_use] + pub fn expires_at_unix_ms(&self) -> i64 { + self.expires_at_unix_ms + } + #[must_use] + pub fn acting_principal(&self) -> PrincipalId { + self.acting_principal + } + #[must_use] + pub fn required_operation(&self) -> MailboxOperation { + self.required_operation + } + #[must_use] + pub fn capability_issuer(&self) -> PrincipalId { + self.capability_issuer + } + #[must_use] + pub fn current_execution_epoch(&self) -> ExecutionEpoch { + self.current_execution_epoch + } + #[must_use] + pub fn current_policy_epoch(&self) -> u64 { + self.current_policy_epoch + } + #[must_use] + pub fn verified_at_unix_ms(&self) -> i64 { + self.verified_at_unix_ms + } +} + +/// Original ciphertext ownership plus exactly one canonical verified-open result. +/// The service may inspect plaintext; the journal receives only a narrow view. +/// No constructor accepts caller assertions of verification. This capsule is +/// neither Clone nor serializable/deserializable. +/// +/// ```compile_fail +/// use mvm_core::mailbox::{OpenedMailboxEnvelope, VerifiedMailboxAdmission, VerifiedMailboxScope}; +/// use mvm_contract::protocol::mailbox::MailboxEnvelope; +/// fn forge(envelope: MailboxEnvelope, opened: OpenedMailboxEnvelope, scope: VerifiedMailboxScope) -> VerifiedMailboxAdmission { +/// VerifiedMailboxAdmission { envelope, opened, scope } +/// } +/// ``` +/// +/// ```compile_fail +/// use mvm_core::mailbox::VerifiedMailboxAdmission; +/// fn requires_clone() {} +/// requires_clone::(); +/// ``` +/// +/// ```compile_fail +/// use mvm_core::mailbox::VerifiedMailboxAdmission; +/// fn requires_serialize() {} +/// requires_serialize::(); +/// ``` +/// +/// ```compile_fail +/// use mvm_core::mailbox::VerifiedMailboxAdmission; +/// fn requires_deserialize() {} +/// requires_deserialize::(); +/// ``` +pub struct VerifiedMailboxAdmission { + envelope: MailboxEnvelope, + opened: OpenedMailboxEnvelope, + scope: VerifiedMailboxScope, +} + +impl std::fmt::Debug for VerifiedMailboxAdmission { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("VerifiedMailboxAdmission") + .finish_non_exhaustive() + } +} + +impl VerifiedMailboxAdmission { + /// Borrow only verified metadata and original ciphertext, never plaintext. + #[must_use] + pub fn journal_view(&self) -> JournalAdmissionView<'_> { + JournalAdmissionView { + envelope: &self.envelope, + header: self.opened.header(), + scope: &self.scope, + replay_identity: ReplayIdentity::from_header(self.opened.header()), + } + } + + /// Service-facing plaintext access. A journal view intentionally has none. + #[must_use] + pub fn payload(&self) -> &[u8] { + self.opened.payload() + } + + /// Consume the capsule, preserving both original ciphertext and verified + /// plaintext allocations. This does not create a journal or delivery result. + #[must_use] + pub fn into_parts(self) -> (MailboxEnvelope, OpenedMailboxEnvelope) { + (self.envelope, self.opened) + } +} + +/// Ciphertext-only journal handoff borrowed from a canonical verified capsule. +/// Header and commitment are authenticated by canonical verification; the outer +/// envelope remains the exact admitted representation, including routing fields. +/// This view does not turn transport-mutable routing data into route authority. +/// A journal must still require actual destination/Receive scope, current +/// authorization, resource admission, durable commit, anchor and key custody. +/// In particular, a view verified for Send is not Receive admission authority. +/// +/// ```compile_fail +/// use mvm_core::mailbox::{JournalAdmissionView, ReplayIdentity, VerifiedMailboxScope}; +/// use mvm_contract::protocol::mailbox::{MailboxEnvelope, MailboxEnvelopeHeader}; +/// fn forge<'a>(envelope: &'a MailboxEnvelope, header: &'a MailboxEnvelopeHeader, scope: &'a VerifiedMailboxScope) -> JournalAdmissionView<'a> { +/// JournalAdmissionView { envelope, header, scope, replay_identity: ReplayIdentity::from_header(header) } +/// } +/// ``` +/// +/// ```compile_fail +/// use mvm_core::mailbox::JournalAdmissionView; +/// fn plaintext<'a>(view: &'a JournalAdmissionView<'a>) -> &'a [u8] { +/// view.payload() +/// } +/// ``` +pub struct JournalAdmissionView<'a> { + envelope: &'a MailboxEnvelope, + header: &'a MailboxEnvelopeHeader, + scope: &'a VerifiedMailboxScope, + replay_identity: ReplayIdentity, +} + +impl std::fmt::Debug for JournalAdmissionView<'_> { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("JournalAdmissionView") + .finish_non_exhaustive() + } +} + +impl JournalAdmissionView<'_> { + #[must_use] + pub fn envelope(&self) -> &MailboxEnvelope { + self.envelope + } + #[must_use] + pub fn header(&self) -> &MailboxEnvelopeHeader { + self.header + } + /// Data identity, not a replay reservation or durable admission token. + #[must_use] + pub fn replay_identity(&self) -> ReplayIdentity { + self.replay_identity + } + #[must_use] + pub fn ciphertext_commitment(&self) -> &[u8; 32] { + &self.envelope.ciphertext_commitment + } + #[must_use] + pub fn verified_scope(&self) -> &VerifiedMailboxScope { + self.scope + } +} + +/// Take ownership of an envelope and run the existing canonical verifier once, +/// including its original replay decision. No ciphertext clone, second replay +/// check, alternative crypto path or context-vector clone is performed. +/// Verification under the supplied context is historical, not current permission. +/// No journal I/O, commit evidence or receipt authority is created here. +pub fn open_verified_admission( + envelope: MailboxEnvelope, + capability: &SignedMailboxCapability, + context: &OpenContext, + encryption_key: &Key, + sender_key: &VerifyingKey, + authority_key: &VerifyingKey, + replay: &mut impl ReplayProtector, +) -> Result { + let opened = open_verified_envelope( + &envelope, + capability, + context, + encryption_key, + sender_key, + authority_key, + replay, + )?; + let header = opened.header(); + let scope = VerifiedMailboxScope { + tenant: header.tenant, + authorization_domain: header.authorization_domain, + mailbox: header.mailbox, + sender: header.sender, + recipient: header.recipient, + sender_boot: header.sender_boot, + message_id: header.message_id, + capability_id: header.capability_id, + encryption_key_id: header.encryption.key_id, + execution_epoch: header.execution_epoch, + policy_epoch: header.policy_epoch, + expires_at_unix_ms: header.expires_at_unix_ms, + acting_principal: context.acting_principal, + required_operation: context.required_operation, + capability_issuer: context.expected_capability_issuer, + current_execution_epoch: context.current_execution_epoch, + current_policy_epoch: context.current_policy_epoch, + verified_at_unix_ms: context.now_unix_ms, + }; + Ok(VerifiedMailboxAdmission { + envelope, + opened, + scope, + }) +} + /// Verify an envelope and return its plaintext through the canonical open path. pub fn open_envelope( envelope: &MailboxEnvelope, diff --git a/crates/mvm-core/tests/mailbox_crypto.rs b/crates/mvm-core/tests/mailbox_crypto.rs index c36128258f..fd3b954108 100644 --- a/crates/mvm-core/tests/mailbox_crypto.rs +++ b/crates/mvm-core/tests/mailbox_crypto.rs @@ -1283,3 +1283,236 @@ fn only_the_destination_authority_can_sign_acceptance() { Err(MailboxError::ReceiptSignatureInvalid) )); } + +#[test] +fn verified_admission_retains_original_ciphertext_and_service_plaintext_allocations() { + use mvm_core::mailbox::{ReplayIdentity, ReplayProtector, open_verified_admission}; + #[derive(Default)] + struct CountingReplay(usize); + impl ReplayProtector for CountingReplay { + fn check_and_record(&mut self, _: ReplayIdentity) -> bool { + self.0 += 1; + self.0 == 1 + } + } + let (sender_key, authority_key, encryption_key) = contract_keys(); + let capability = sign_capability(grant(), &authority_key).unwrap(); + let expected = header(); + let envelope = seal_with(expected.clone(), b"hello"); + let ciphertext_ptr = envelope.ciphertext.as_ptr(); + let ciphertext_capacity = envelope.ciphertext.capacity(); + let mut replay = CountingReplay::default(); + let admission = open_verified_admission( + envelope, + &capability, + &context(), + &encryption_key, + &sender_key.verifying_key(), + &authority_key.verifying_key(), + &mut replay, + ) + .unwrap(); + assert_eq!(replay.0, 1); + { + let view = admission.journal_view(); + assert_eq!(view.envelope().ciphertext.as_ptr(), ciphertext_ptr); + assert_eq!(view.envelope().ciphertext.capacity(), ciphertext_capacity); + assert_eq!(view.header(), &expected); + assert_eq!( + view.replay_identity(), + ReplayIdentity::from_header(&expected) + ); + assert_eq!( + view.ciphertext_commitment(), + &view.envelope().ciphertext_commitment + ); + assert_eq!( + view.verified_scope().acting_principal(), + context().acting_principal + ); + assert_eq!( + view.verified_scope().required_operation(), + context().required_operation + ); + assert_eq!( + view.verified_scope().capability_issuer(), + context().expected_capability_issuer + ); + assert_eq!( + view.verified_scope().current_execution_epoch(), + context().current_execution_epoch + ); + assert_eq!( + view.verified_scope().current_policy_epoch(), + context().current_policy_epoch + ); + assert_eq!( + view.verified_scope().verified_at_unix_ms(), + context().now_unix_ms + ); + } + assert_eq!(admission.payload(), b"hello"); + let plaintext_ptr = admission.payload().as_ptr(); + let view = admission.journal_view(); + assert_eq!(view.header(), &expected); + assert_eq!(replay.0, 1, "views must not verify or consume replay again"); + let (envelope, opened) = admission.into_parts(); + assert_eq!(envelope.ciphertext.as_ptr(), ciphertext_ptr); + assert_eq!(opened.payload().as_ptr(), plaintext_ptr); + assert_eq!(opened.header(), &expected); + assert_eq!(opened.into_payload(), b"hello"); + assert_eq!( + open_envelope( + &envelope, + &capability, + &context(), + &encryption_key, + &sender_key.verifying_key(), + &authority_key.verifying_key(), + &mut replay + ) + .unwrap_err(), + MailboxError::Replay + ); + assert_eq!(replay.0, 2); +} + +#[test] +fn verified_admission_refusals_do_not_mint_capsule_or_record_replay() { + use mvm_core::mailbox::{ReplayIdentity, ReplayProtector, open_verified_admission}; + #[derive(Default)] + struct CountingReplay(usize); + impl ReplayProtector for CountingReplay { + fn check_and_record(&mut self, _: ReplayIdentity) -> bool { + self.0 += 1; + true + } + } + let (sender_key, authority_key, encryption_key) = contract_keys(); + let capability = sign_capability(grant(), &authority_key).unwrap(); + for failure in [ + "ciphertext", + "sender_key", + "authority_key", + "tenant", + "domain", + "recipient", + "acting_principal", + "operation", + "issuer", + "epoch", + "policy", + "expired", + "revoked", + ] { + let mut envelope = seal_with(header(), b"hello"); + let mut current = context(); + let mut sender_public = sender_key.verifying_key(); + let mut authority_public = authority_key.verifying_key(); + match failure { + "ciphertext" => envelope.ciphertext[0] ^= 1, + "sender_key" => sender_public = SigningKey::from_bytes(&[99; 32]).verifying_key(), + "authority_key" => authority_public = SigningKey::from_bytes(&[99; 32]).verifying_key(), + "tenant" => current.expected_tenant = TenantId::from_bytes([99; 32]), + "domain" => { + current.expected_authorization_domain = AuthorizationDomainId::from_bytes([99; 32]) + } + "recipient" => current.expected_recipient = PrincipalId::from_bytes([99; 32]), + "acting_principal" => current.acting_principal = PrincipalId::from_bytes([99; 32]), + "operation" => current.required_operation = MailboxOperation::Monitor, + "issuer" => current.expected_capability_issuer = PrincipalId::from_bytes([99; 32]), + "epoch" => current.current_execution_epoch = ExecutionEpoch::new(8).unwrap(), + "policy" => current.current_policy_epoch += 1, + "expired" => current.now_unix_ms = 2_000, + _ => current.revoked_capabilities.push(grant().revocation_id), + } + let mut replay = CountingReplay::default(); + assert!( + open_verified_admission( + envelope, + &capability, + ¤t, + &encryption_key, + &sender_public, + &authority_public, + &mut replay + ) + .is_err(), + "{failure}" + ); + assert_eq!(replay.0, 0, "{failure}"); + } +} + +#[test] +fn verified_admission_debug_never_contains_payload_or_ciphertext() { + use mvm_core::mailbox::open_verified_admission; + let (sender_key, authority_key, encryption_key) = contract_keys(); + let capability = sign_capability(grant(), &authority_key).unwrap(); + let admission = open_verified_admission( + seal_with(header(), b"hello"), + &capability, + &context(), + &encryption_key, + &sender_key.verifying_key(), + &authority_key.verifying_key(), + &mut InMemoryReplayProtector::default(), + ) + .unwrap(); + assert_eq!(format!("{admission:?}"), "VerifiedMailboxAdmission { .. }"); + assert_eq!( + format!("{:?}", admission.journal_view()), + "JournalAdmissionView { .. }" + ); +} + +#[test] +fn verified_admission_scope_preserves_supplied_operation_and_historical_context() { + use mvm_core::mailbox::{VerifiedMailboxScope, open_verified_admission}; + let (sender_key, authority_key, encryption_key) = contract_keys(); + let capability = sign_capability(grant(), &authority_key).unwrap(); + let mut current = context(); + let admission = open_verified_admission( + seal_with(header(), b"hello"), + &capability, + ¤t, + &encryption_key, + &sender_key.verifying_key(), + &authority_key.verifying_key(), + &mut InMemoryReplayProtector::default(), + ) + .unwrap(); + current.now_unix_ms += 100; + current.current_policy_epoch += 1; + current.required_operation = MailboxOperation::Receive; + current.revoked_capabilities.push(grant().revocation_id); + let view = admission.journal_view(); + let scope = view.verified_scope(); + assert_eq!( + scope.required_operation(), + MailboxOperation::Send, + "verification cannot become receive authority" + ); + assert_eq!(scope.verified_at_unix_ms(), 1_500); + assert_eq!(scope.current_policy_epoch(), 4); + assert_ne!(scope.current_policy_epoch(), current.current_policy_epoch); + assert_eq!(scope.tenant(), view.header().tenant); + assert_eq!( + scope.authorization_domain(), + view.header().authorization_domain + ); + assert_eq!(scope.mailbox(), view.header().mailbox); + assert_eq!(scope.sender(), view.header().sender); + assert_eq!(scope.recipient(), view.header().recipient); + assert_eq!(scope.sender_boot(), view.header().sender_boot); + assert_eq!(scope.message_id(), view.header().message_id); + assert_eq!(scope.capability_id(), view.header().capability_id); + assert_eq!(scope.encryption_key_id(), view.header().encryption.key_id); + assert_eq!(scope.execution_epoch(), view.header().execution_epoch); + assert_eq!(scope.policy_epoch(), view.header().policy_epoch); + assert_eq!(scope.expires_at_unix_ms(), view.header().expires_at_unix_ms); + assert!( + !std::mem::needs_drop::(), + "fixed scope must not retain owned context vectors" + ); +} From 629f98b4e3580819446e40d60d86b4b50fa85fe8 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 10:33:32 -0800 Subject: [PATCH 26/51] feat(mailbox): sign local receipts in V2 domain --- crates/mvm-core/src/mailbox.rs | 39 +++++++++++++-------- crates/mvm-core/tests/mailbox_crypto.rs | 45 ++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/crates/mvm-core/src/mailbox.rs b/crates/mvm-core/src/mailbox.rs index c7be7dff0c..89992c7e76 100644 --- a/crates/mvm-core/src/mailbox.rs +++ b/crates/mvm-core/src/mailbox.rs @@ -9,12 +9,12 @@ use std::collections::BTreeSet; use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; use mvm_contract::protocol::mailbox::{ AES_256_GCM_ALGORITHM, AuthorizationDomainId, BootId, ContentType, DestinationReceipt, - ED25519_ALGORITHM, ED25519_SIGNATURE_BYTES, ExecutionEpoch, MAILBOX_PROTOCOL_VERSION, - MAX_ENCODED_ENVELOPE_BYTES, MAX_MAILBOX_PAYLOAD_BYTES, MailboxCapabilityGrant, - MailboxContractError, MailboxDeliveryError, MailboxEnvelope, MailboxEnvelopeHeader, MailboxId, - MailboxMessage, MailboxOperation, MailboxRoutingFrame, MessageId, PrincipalId, ProtocolVersion, - RevocationId, SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxMessage, - TenantId, + ED25519_ALGORITHM, ED25519_SIGNATURE_BYTES, ExecutionEpoch, + LOCAL_DURABLE_RECEIPT_SIGNATURE_DOMAIN, MAILBOX_PROTOCOL_VERSION, MAX_ENCODED_ENVELOPE_BYTES, + MAX_MAILBOX_PAYLOAD_BYTES, MailboxCapabilityGrant, MailboxContractError, MailboxDeliveryError, + MailboxEnvelope, MailboxEnvelopeHeader, MailboxId, MailboxMessage, MailboxOperation, + MailboxRoutingFrame, MessageId, PrincipalId, ProtocolVersion, RevocationId, + SignedDestinationReceipt, SignedMailboxCapability, SignedMailboxMessage, TenantId, }; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -947,13 +947,27 @@ pub fn authorize_receive( verify_capability_scope(header, capability, context, false) } -/// Sign a destination receipt in a domain distinct from envelopes and grants. +fn receipt_signature_domain(receipt: &DestinationReceipt) -> Result<&'static [u8], MailboxError> { + receipt + .validate_versioned_outcome() + .map_err(MailboxError::Contract)?; + if receipt.signature_algorithm.0 != ED25519_ALGORITHM { + return Err(MailboxError::UnsupportedSignatureAlgorithm); + } + match receipt.protocol_version { + ProtocolVersion::V1 => Ok(RECEIPT_SIGNATURE_DOMAIN), + ProtocolVersion::V2 => Ok(LOCAL_DURABLE_RECEIPT_SIGNATURE_DOMAIN), + _ => Err(MailboxError::UnsupportedProtocolVersion), + } +} + +/// Sign a destination receipt in its versioned outcome domain. pub fn sign_receipt( receipt: DestinationReceipt, destination_key: &SigningKey, ) -> Result { - validate_signed_object_versions(receipt.protocol_version, receipt.signature_algorithm)?; - let signature = destination_key.sign(&canonical_bytes(RECEIPT_SIGNATURE_DOMAIN, &receipt)?); + let domain = receipt_signature_domain(&receipt)?; + let signature = destination_key.sign(&canonical_bytes(domain, &receipt)?); Ok(SignedDestinationReceipt { receipt, signature: signature.to_bytes().to_vec(), @@ -965,10 +979,7 @@ pub fn verify_receipt( signed: &SignedDestinationReceipt, destination_key: &VerifyingKey, ) -> Result<(), MailboxError> { - validate_signed_object_versions( - signed.receipt.protocol_version, - signed.receipt.signature_algorithm, - )?; + let domain = receipt_signature_domain(&signed.receipt)?; let signature: [u8; ED25519_SIGNATURE_BYTES] = signed .signature .as_slice() @@ -976,7 +987,7 @@ pub fn verify_receipt( .map_err(|_| MailboxError::ReceiptSignatureInvalid)?; destination_key .verify( - &canonical_bytes(RECEIPT_SIGNATURE_DOMAIN, &signed.receipt)?, + &canonical_bytes(domain, &signed.receipt)?, &Signature::from_bytes(&signature), ) .map_err(|_| MailboxError::ReceiptSignatureInvalid) diff --git a/crates/mvm-core/tests/mailbox_crypto.rs b/crates/mvm-core/tests/mailbox_crypto.rs index fd3b954108..e5f81b67c2 100644 --- a/crates/mvm-core/tests/mailbox_crypto.rs +++ b/crates/mvm-core/tests/mailbox_crypto.rs @@ -1,4 +1,4 @@ -use ed25519_dalek::SigningKey; +use ed25519_dalek::{Signer, SigningKey}; use mvm_contract::plan::{Nonce, PlanId}; use mvm_contract::protocol::mailbox::{ AuthorizationDomainId, AuthorizationDomainRouteLabel, BootId, CapabilityId, @@ -426,6 +426,49 @@ fn destination_receipt_wire_roundtrip_and_signature_cover_outcome_and_identity() )); } +#[test] +fn local_receipt_uses_only_its_v2_signature_domain() { + let destination_key = SigningKey::from_bytes(&[31; 32]); + let receipt = mvm_contract::protocol::mailbox::DestinationReceipt { + protocol_version: ProtocolVersion::V2, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: DeliveryId::from_bytes([32; 32]), + message_id: MessageId::from_bytes([6; 32]), + address: address(), + execution_epoch: ExecutionEpoch::new(7).expect("epoch"), + outcome: DeliveryOutcome::AcceptedLocalDurable, + observed_at_unix_ms: 1_600, + ciphertext_commitment: [33; 32], + }; + let signed = sign_receipt(receipt.clone(), &destination_key).expect("sign V2 local receipt"); + verify_receipt(&signed, &destination_key.verifying_key()).expect("verify V2 local receipt"); + assert_eq!( + sign_receipt(receipt, &destination_key) + .expect("deterministic local signature") + .signature, + signed.signature + ); + + let mut wrong_domain = signed.clone(); + let mut v1_bytes = b"mvm.mailbox.receipt.v1\0".to_vec(); + v1_bytes.extend(serde_jcs::to_vec(&signed.receipt).expect("canonical receipt")); + wrong_domain.signature = destination_key.sign(&v1_bytes).to_bytes().to_vec(); + assert_eq!( + verify_receipt(&wrong_domain, &destination_key.verifying_key()), + Err(MailboxError::ReceiptSignatureInvalid) + ); + assert_eq!( + verify_receipt(&signed, &SigningKey::from_bytes(&[34; 32]).verifying_key()), + Err(MailboxError::ReceiptSignatureInvalid) + ); + let mut old_outcome = signed.receipt.clone(); + old_outcome.outcome = DeliveryOutcome::Duplicate; + assert!(sign_receipt(old_outcome, &destination_key).is_err()); + let mut old_version = signed.receipt; + old_version.protocol_version = ProtocolVersion::V1; + assert!(sign_receipt(old_version, &destination_key).is_err()); +} + #[test] fn protocol_algorithm_signature_and_nonce_tampering_are_typed_failures() { let sender_key = SigningKey::from_bytes(&[21; 32]); From b1d351c01ca9159442ac411e0736cf6d3abc5419 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 10:33:32 -0800 Subject: [PATCH 27/51] feat(mailbox): stage local journal acceptance and exact retries --- crates/mvm-core/README.md | 13 +- crates/mvm-core/src/mailbox/journal.rs | 209 ++++++++++++++++++++++++- 2 files changed, 214 insertions(+), 8 deletions(-) diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index 1bdf19e9bb..73f7c82cc4 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -81,6 +81,12 @@ missing or divergent anchored prefix, and may reconcile only an intact contiguous suffix written before an interrupted anchor commit. The original accepted receipt body and observation time are retained with deduplication state; V1 retries still use their existing `duplicate` outcome. +The test-scoped local opener and `admit_local` sign a distinct V2 +`accepted_local_durable` receipt only after the encrypted record and local +anchor have been fsynced. An exact local retry re-verifies current admission +and the committed local tip before re-signing the original body/time, yielding +the same signed bytes after restart. Recovery refuses a local V2 receipt in an +independently anchored journal or a V1 strong receipt in a local journal. Retained guest-reader controls are checked against the historical signed Receive capability, enrollment, lease and guest ACK before recovery accepts their delivery state. An exact ACK retry still needs current boot, session, @@ -99,14 +105,15 @@ profile can access plaintext; host-blind confidentiality requires a different attested key-release and protected-memory architecture. The planned standalone `accepted_local_durable` tier has a distinct versioned -receipt contract. Its history and payload-purpose keys belong in the existing +receipt contract and signer. Its history and payload-purpose keys belong in the existing protected host keys directory, scoped to tenant/domain/mailbox/epoch and key version, with an authority signer pinned for retained history. Recovery must load the existing key version; it must not mint a new signer or use the erased per-launch secret as its only opening key. Rotation and deletion need explicit retention and refusal rules. A host-local chain can witness ordinary crash and -process restart, **not** rollback of the entire host. Neither this local tier -nor its key provider is enabled by the journal primitive. The stronger +process restart, **not** rollback of the entire host. The local journal test +path is not production activation: its custody is synthetic, with no retained +key provider, authenticated restart manifest, or service adapter. The stronger `accepted_durable` outcome continues to require a separately trusted compare-and-advance checkpoint; no such provider is wired. diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index d1d4d6e681..f036f36b40 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -236,6 +236,17 @@ struct Record { struct JournalKey { bytes: Zeroizing<[u8; 32]>, reference: Option, + tier: AcceptanceTier, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum AcceptanceTier { + Independent, + #[cfg_attr( + not(any(test, feature = "test-support")), + expect(dead_code, reason = "local custody opener is not production-wired yet") + )] + Local, } impl JournalKey { @@ -243,6 +254,7 @@ impl JournalKey { Self { bytes: Zeroizing::new(bytes), reference: None, + tier: AcceptanceTier::Independent, } } } @@ -311,6 +323,7 @@ pub struct MailboxJournal { authority: VerifyingKey, key: Zeroizing<[u8; 32]>, history_key: Option, + tier: AcceptanceTier, limits: JournalLimits, tip: JournalTip, bytes: u64, @@ -693,12 +706,46 @@ impl MailboxJournal { JournalKey { bytes: Zeroizing::new(*key.expose_bytes()), reference: Some(reference), + tier: AcceptanceTier::Independent, }, limits, anchor, ) } + /// Test the software-only local tier against a fsynced same-host anchor. + /// + /// Production must obtain the reference from an authenticated restart + /// manifest and verified mailbox scope, not a deserialized route hint. + #[cfg(any(test, feature = "test-support"))] + pub fn open_local_with_custody( + root: &Path, + reference: HistoryKeyRef, + authority: VerifyingKey, + limits: JournalLimits, + custody: &impl HistoryKeyCustody, + anchor: &FileJournalAnchor, + ) -> Result { + reference.validate()?; + if reference.address != anchor.address { + return Err(JournalError::WrongAuthority); + } + let key = custody.load_existing(&reference)?; + Self::open_with_recovery( + root, + reference.address, + authority, + JournalKey { + bytes: Zeroizing::new(*key.expose_bytes()), + reference: Some(reference), + tier: AcceptanceTier::Local, + }, + limits, + anchor.tip(), + |next| anchor.advance(next), + ) + } + fn open_against_anchor( root: &Path, address: MailboxAddress, @@ -747,6 +794,7 @@ impl MailboxJournal { authority, key: key.bytes, history_key: key.reference, + tier: key.tier, limits, tip: JournalTip::genesis(), bytes: 0, @@ -819,10 +867,15 @@ impl MailboxJournal { receipt, authorization_signature, } => { + let expected_outcome = match journal.tier { + AcceptanceTier::Independent => DeliveryOutcome::AcceptedDurable, + AcceptanceTier::Local => DeliveryOutcome::AcceptedLocalDurable, + }; if receipt.address != address - || receipt.outcome != DeliveryOutcome::AcceptedDurable + || receipt.outcome != expected_outcome || receipt.message_id != identity.message_id || receipt.execution_epoch != identity.execution_epoch + || receipt.validate_versioned_outcome().is_err() { return Err(JournalError::Corrupt); } @@ -1021,6 +1074,15 @@ impl MailboxJournal { &mut self, request: JournalAdmission<'_>, anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, + ) -> Result { + self.admit_at_tier(request, AcceptanceTier::Independent, anchor) + } + + fn admit_at_tier( + &mut self, + request: JournalAdmission<'_>, + tier: AcceptanceTier, + anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, ) -> Result { let JournalAdmission { envelope, @@ -1033,6 +1095,9 @@ impl MailboxJournal { if self.poisoned { return Err(JournalError::Uncertain); } + if self.tier != tier { + return Err(JournalError::DurableAuthorityUnavailable); + } if self.address.tenant != context.expected_tenant || self.address.authorization_domain != context.expected_authorization_domain || self.address.mailbox != context.expected_mailbox @@ -1059,7 +1124,10 @@ impl MailboxJournal { )?; let identity = probe.0.ok_or(JournalError::Corrupt)?; let receipt = |outcome| DestinationReceipt { - protocol_version: ProtocolVersion::V1, + protocol_version: match tier { + AcceptanceTier::Independent => ProtocolVersion::V1, + AcceptanceTier::Local => ProtocolVersion::V2, + }, signature_algorithm: SignatureAlgorithm::ED25519, delivery_id: envelope.routing.delivery_id, message_id: opened.header().message_id, @@ -1073,19 +1141,32 @@ impl MailboxJournal { if previous.envelope != *envelope || previous.capability != *capability { return Err(JournalError::ConflictingRetry); } - if previous.accepted_receipt.outcome != DeliveryOutcome::AcceptedDurable + let expected_outcome = match tier { + AcceptanceTier::Independent => DeliveryOutcome::AcceptedDurable, + AcceptanceTier::Local => DeliveryOutcome::AcceptedLocalDurable, + }; + if previous.accepted_receipt.outcome != expected_outcome || previous.accepted_receipt.address != self.address || previous.accepted_receipt.message_id != opened.header().message_id || previous.accepted_receipt.ciphertext_commitment != envelope.ciphertext_commitment { return Err(JournalError::Corrupt); } - return Ok(sign_receipt(receipt(DeliveryOutcome::Duplicate), signer)?); + return Ok(sign_receipt( + match tier { + AcceptanceTier::Independent => receipt(DeliveryOutcome::Duplicate), + AcceptanceTier::Local => previous.accepted_receipt.clone(), + }, + signer, + )?); } if self.tip.cursor >= self.limits.max_records as u64 { return Err(JournalError::Full); } - let accepted = receipt(DeliveryOutcome::AcceptedDurable); + let accepted = receipt(match tier { + AcceptanceTier::Independent => DeliveryOutcome::AcceptedDurable, + AcceptanceTier::Local => DeliveryOutcome::AcceptedLocalDurable, + }); let encoded_frame = frame::encode_frame(capability, envelope).map_err(|_| JournalError::Corrupt)?; let encoded_frame = STANDARD_NO_PAD.encode(encoded_frame); @@ -1144,6 +1225,23 @@ impl MailboxJournal { }) } + /// Commit local acceptance only after the ciphertext/dedup record and + /// same-host anchor are fsynced. This cannot detect whole-host rollback. + #[cfg(any(test, feature = "test-support"))] + pub fn admit_local( + &mut self, + request: JournalAdmission<'_>, + anchor: &FileJournalAnchor, + ) -> Result { + if self.tier != AcceptanceTier::Local || self.history_key.is_none() { + return Err(JournalError::DurableAuthorityUnavailable); + } + if anchor.address != self.address || anchor.tip() != self.tip { + return Err(JournalError::Rollback); + } + self.admit_at_tier(request, AcceptanceTier::Local, |next| anchor.advance(next)) + } + fn append( &mut self, event: JournalEvent, @@ -1849,6 +1947,107 @@ mod anchor_tests { } } + #[test] + fn local_acceptance_restarts_with_exact_receipt_and_refuses_rollback() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let anchor_path = dir.path().join("anchor").join("tip.json"); + let (envelope, capability, mut context, payload_key, sender_key, signer) = message(); + let (reference, custody) = versioned_custody(); + let anchor = FileJournalAnchor::open(&anchor_path, address(), [22; 32]) + .expect("local anchor genesis"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ) + .expect("empty local journal"); + let accepted = journal + .admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("local record and anchor fsynced before signature"); + assert_eq!(accepted.receipt.protocol_version, ProtocolVersion::V2); + assert_eq!( + accepted.receipt.outcome, + DeliveryOutcome::AcceptedLocalDurable + ); + super::super::verify_receipt(&accepted, &signer.verifying_key()) + .expect("local receipt signature"); + assert_eq!(journal.tip(), anchor.tip()); + drop(journal); + drop(anchor); + + let anchor = FileJournalAnchor::open(&anchor_path, address(), [22; 32]) + .expect("recover local anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ) + .expect("recover retained ciphertext and dedup"); + context.now_unix_ms += 1; + let duplicate = journal + .admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("recover original local receipt after lost reply"); + assert_eq!( + serde_jcs::to_vec(&duplicate).expect("signed retry bytes"), + serde_jcs::to_vec(&accepted).expect("signed original bytes") + ); + assert_eq!(journal.tip().cursor, 1, "retry must not append"); + assert!(matches!( + journal.admit_with_anchor( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &IndependentAnchor::new(address()), + ), + Err(JournalError::Rollback | JournalError::DurableAuthorityUnavailable) + )); + drop(journal); + fs::remove_file(record_path(&root, 1)).expect("simulate journal-only rollback"); + assert!(matches!( + MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor + ), + Err(JournalError::Rollback) + )); + } + #[test] fn anchored_acceptance_recovers_ciphertext_and_dedup_without_plaintext_on_disk() { let dir = tempfile::tempdir().expect("journal root"); From 434cca038bfc2325bf13b51e4080af9646dfc2b3 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 10:38:18 -0800 Subject: [PATCH 28/51] fix(mailbox): retain optional report field in guest reader grant --- crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs index 9d555e4806..e5909f88aa 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs @@ -187,6 +187,7 @@ fn perform_exchange( mailbox: boot.mailbox, authority_key: boot.authority_key, epoch: boot.execution_epoch, + report: None, }; let authority = VerifyingKey::from_bytes(&boot.authority_key) .map_err(|_| Refusal::NotAuthorized)?; From 54a47bbc9ceebee6432119d95fed240a84c12943 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 12:54:04 -0800 Subject: [PATCH 29/51] feat(mailbox): stage signed restart custody with separate purpose keys --- crates/mvm-core/README.md | 14 +- crates/mvm-core/src/mailbox/journal.rs | 6 +- crates/mvm-hostd/README.md | 10 + crates/mvm-hostd/src/lib.rs | 2 + crates/mvm-hostd/src/mailbox_custody.rs | 542 ++++++++++++++++++++++++ 5 files changed, 565 insertions(+), 9 deletions(-) create mode 100644 crates/mvm-hostd/src/mailbox_custody.rs diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index 73f7c82cc4..34313d738a 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -64,13 +64,13 @@ canonical encrypted mailbox frame and its deduplication identity. Its the current `(cursor, head)` and atomically compare-and-advance it after a journal write. The journal persists unsigned receipt state with a destination-signed, domain-separated admission intent binding the verified -identity, canonical frame, and receipt state. The test-scoped version 3 also +identity, canonical frame, and receipt state. The version-3 test path also binds a `HistoryKeyRef` (mailbox address, epoch, AES-256-GCM key ID and nonzero version) to the encrypted record and signed intent. Its recovery obtains the exact key through an existing-only `HistoryKeyCustody` lookup and refuses changed key identity, wrong key, or missing custody; it does not create a replacement key. The raw-key opener cannot issue an `accepted_durable` receipt. -Deserializing a `HistoryKeyRef` does not grant authority: production reopening +Constructing or deserializing a `HistoryKeyRef` does not grant authority: production reopening must verify its scope against the admitted plan and an authenticated retained manifest before loading a key. The destination signs an actual acceptance receipt only after both commits. @@ -94,12 +94,12 @@ policy and expiry authorization; an old committed ACK is not a new grant. A message ever leased to a guest cannot be closed by the legacy unsigned host ACK, including during recovery; it requires that guest's signed ACK. -`FileJournalAnchor` is a local development aid, **not** an independent +`FileJournalAnchor` is a host-local anchor, **not** an independent authority: restoring both the journal and its host-local anchor can hide -rollback. The custody interface has only a test provider: it does not supply -a retained host key, rotation, or the separately encrypted payload's opening -key after the per-launch mailbox secret is erased. Until real providers and -the existing service's admission handoff are integrated and tested, local +rollback. The hostd test-scoped custody prototype stores distinct retained +purpose keys under a signed restart binding, but it is not connected to the +production service. Until verified custody and the existing service's +admission handoff are integrated and tested, local journal writes must not be presented as `accepted_durable`. The trusted-host profile can access plaintext; host-blind confidentiality requires a different attested key-release and protected-memory architecture. diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index f036f36b40..6960e7afdb 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -119,8 +119,10 @@ impl std::fmt::Debug for HistoryKeyRef { } impl HistoryKeyRef { - /// Bind one synthetic key version in tests; production needs verified scope. - #[cfg(any(test, feature = "test-support"))] + /// Identify a custody key version; construction alone is not authority. + /// + /// The host must bind these fields to a verified plan and authenticated + /// restart manifest before using the reference to reopen retained history. pub fn new( address: MailboxAddress, execution_epoch: ExecutionEpoch, diff --git a/crates/mvm-hostd/README.md b/crates/mvm-hostd/README.md index 114978da57..c0e4ae34cb 100644 --- a/crates/mvm-hostd/README.md +++ b/crates/mvm-hostd/README.md @@ -41,6 +41,16 @@ signing, and audit emission each pass through their dedicated policy gate. Key material remains inside the smallest role that needs it; callers receive only the result. +The test-scoped `mailbox_custody` prototype retains three distinct random +purpose keys (journal history, recipient payload opening, local anchor) in a +private bundle under the existing host keys directory. A host-signed manifest +binds their IDs and version to the admitted plan, VM, address, epoch and +authority; recovery loads the existing signer and exact bundle, never mints a +replacement. An interrupted first provision leaves an orphan and refuses. +This is trusted-host local crash recovery only: it is not wired to the +production mailbox process, cannot detect whole-host rollback, and does not +provide host-blind confidentiality. + ## Main areas | Area | Representative modules | diff --git a/crates/mvm-hostd/src/lib.rs b/crates/mvm-hostd/src/lib.rs index a1dd12d882..e775dd1ca2 100644 --- a/crates/mvm-hostd/src/lib.rs +++ b/crates/mvm-hostd/src/lib.rs @@ -64,6 +64,8 @@ pub mod jailer; /// [`keyholder::SecretResolver`] trait + the single-host /// [`keyholder::LocalResolver`]. pub mod keyholder; +#[cfg(test)] +mod mailbox_custody; pub mod mailbox_launch; pub mod mailbox_process; pub mod mailbox_service; diff --git a/crates/mvm-hostd/src/mailbox_custody.rs b/crates/mvm-hostd/src/mailbox_custody.rs new file mode 100644 index 0000000000..0cd6acf830 --- /dev/null +++ b/crates/mvm-hostd/src/mailbox_custody.rs @@ -0,0 +1,542 @@ +//! Retained, host-local mailbox keys and their signed restart binding. +//! +//! This is a trusted-host crash-recovery primitive, not an independent +//! rollback witness or an attested key-release boundary. + +use std::fs::OpenOptions; +use std::io::{Error, ErrorKind, Read}; +use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context as _, Result, ensure}; +use ed25519_dalek::{Signature, Signer as _, SigningKey, Verifier as _}; +use mvm_contract::plan::{Nonce, PlanId}; +use mvm_contract::protocol::mailbox::{ + BootId, EncryptionAlgorithm, ExecutionEpoch, MailboxAddress, +}; +use mvm_core::crypto::aead::Key; +use mvm_core::mailbox::journal::{HistoryKeyCustody, HistoryKeyRef, JournalError}; +use mvm_core::plan::ExecutionPlan; +use mvm_core::util::atomic_io::{atomic_write_new, sync_dir}; +use rand::Rng as _; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use zeroize::Zeroizing; + +use crate::audit::host_keypair::{PUBLIC_FILENAME, SECRET_FILENAME}; +use crate::mailbox_launch::derive_launch_keys; +use crate::plan_admission::{MailboxLaunchPosture, MailboxPostureHandoff}; + +const DOMAIN: &[u8] = b"mvm.mailbox.local-custody.v1\0"; +const KEY_VERSION: u32 = 1; +const KEY_BYTES: usize = 96; +const MAX_MANIFEST_BYTES: u64 = 4096; + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Binding { + plan_id: PlanId, + plan_nonce: Nonce, + vm_name: String, + address: MailboxAddress, + epoch: ExecutionEpoch, + authority_key: [u8; 32], + host_signer: [u8; 32], + key_version: u32, + history: HistoryKeyRef, + payload_key_id: [u8; 32], + anchor_key_id: [u8; 32], +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SignedBinding { + binding: Binding, + signature: Vec, +} + +#[derive(Serialize)] +struct Identity<'a> { + plan_id: &'a PlanId, + plan_nonce: &'a Nonce, + vm_name: &'a str, + address: MailboxAddress, + epoch: ExecutionEpoch, +} + +/// Three independent purpose keys in one no-clobber, fsynced private bundle. +/// +/// Only the manifest is signed; the key IDs bind every bundle component to it. +/// Neither the bundle nor this type contains a per-launch mailbox secret. +pub(crate) struct RetainedMailboxKeys { + binding: Binding, + material: Zeroizing<[u8; KEY_BYTES]>, +} + +fn paths(keys_dir: &Path, identity: &Identity<'_>) -> Result<(PathBuf, PathBuf)> { + let digest = Sha256::digest(serde_jcs::to_vec(identity)?); + let name = hex::encode(digest); + let directory = keys_dir.join("mailbox-local-v1"); + Ok(( + directory.join(format!("{name}.keys")), + directory.join(format!("{name}.manifest")), + )) +} + +fn signed_bytes(binding: &Binding) -> Result> { + let mut bytes = DOMAIN.to_vec(); + bytes.extend(serde_jcs::to_vec(binding)?); + Ok(bytes) +} + +fn existing_host_signer(keys_dir: &Path) -> Result { + let (signer, _) = mvm_core::crypto::ed25519_keypair::load_existing( + &keys_dir.join(SECRET_FILENAME), + &keys_dir.join(PUBLIC_FILENAME), + ) + .context("retained mailbox custody needs the existing host signer")?; + Ok(signer) +} + +fn read_private(path: &Path, maximum: u64) -> Result>> { + let mut file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path)?; + let meta = file.metadata()?; + ensure!( + meta.is_file() + && meta.nlink() == 1 + && meta.len() <= maximum + && meta.permissions().mode() & 0o077 == 0, + "retained mailbox custody member is not a bounded private single-link file" + ); + let mut bytes = Zeroizing::new(Vec::new()); + file.by_ref().take(maximum + 1).read_to_end(&mut bytes)?; + ensure!( + bytes.len() as u64 <= maximum, + "mailbox custody member grew while reading" + ); + Ok(bytes) +} + +fn identity<'a>( + keys_dir: &Path, + plan: &'a ExecutionPlan, + vm_name: &'a str, + address: MailboxAddress, +) -> Result> { + let grant = plan + .mailbox + .context("admitted plan does not grant a mailbox")?; + ensure!( + address.mailbox == grant.mailbox && plan.image.name == vm_name, + "mailbox custody identity does not match the admitted VM" + ); + let (policy, _) = derive_launch_keys(keys_dir, &grant)?.trust(); + ensure!( + address + == MailboxAddress { + tenant: policy.context.expected_tenant, + authorization_domain: policy.context.expected_authorization_domain, + mailbox: policy.context.expected_mailbox, + recipient: policy.context.expected_recipient, + }, + "mailbox custody address does not match host-trusted template" + ); + Ok(Identity { + plan_id: &plan.plan_id, + plan_nonce: &plan.nonce, + vm_name, + address, + epoch: grant.epoch, + }) +} + +impl RetainedMailboxKeys { + /// Provision once under a verified launch posture; an interrupted first + /// provision leaves an orphan bundle and refuses instead of replacing it. + pub(crate) fn provision( + keys_dir: &Path, + plan: &ExecutionPlan, + vm_name: &str, + posture: &MailboxLaunchPosture, + address: MailboxAddress, + ) -> Result { + posture.verify(plan, vm_name)?; + let identity = identity(keys_dir, plan, vm_name, address)?; + let signer = existing_host_signer(keys_dir)?; + let (bundle_path, manifest_path) = paths(keys_dir, &identity)?; + mvm_core::private_fs::ensure_private_dir( + bundle_path.parent().context("mailbox custody directory")?, + )?; + sync_dir(keys_dir)?; + ensure!( + !bundle_path.exists() && !manifest_path.exists(), + "mailbox custody already exists; use existing-only recovery" + ); + let mut material = Zeroizing::new([0u8; KEY_BYTES]); + rand::rng().fill_bytes(&mut *material); + let history = HistoryKeyRef::new( + address, + identity.epoch, + Sha256::digest(&material[..32]).into(), + KEY_VERSION, + )?; + let binding = Binding { + plan_id: plan.plan_id.clone(), + plan_nonce: plan.nonce.clone(), + vm_name: vm_name.to_owned(), + address, + epoch: identity.epoch, + authority_key: plan.mailbox.context("mailbox grant")?.authority_key, + host_signer: signer.verifying_key().to_bytes(), + key_version: KEY_VERSION, + history, + payload_key_id: Sha256::digest(&material[32..64]).into(), + anchor_key_id: Sha256::digest(&material[64..]).into(), + }; + atomic_write_new(&bundle_path, &material[..])?; + sync_dir(bundle_path.parent().context("mailbox custody directory")?)?; + let signed = SignedBinding { + signature: signer.sign(&signed_bytes(&binding)?).to_bytes().to_vec(), + binding: binding.clone(), + }; + atomic_write_new(&manifest_path, &serde_json::to_vec(&signed)?)?; + sync_dir( + manifest_path + .parent() + .context("mailbox custody directory")?, + )?; + Ok(Self { binding, material }) + } + + /// Recover only an existing manifest and exact key bundle under the current + /// host signer. A missing/rotated signer, manifest or purpose key refuses. + pub(crate) fn open_existing( + keys_dir: &Path, + plan: &ExecutionPlan, + vm_name: &str, + posture: &MailboxPostureHandoff, + current_boot: Option, + address: MailboxAddress, + ) -> Result { + posture.verify(plan, vm_name, current_boot)?; + let identity = identity(keys_dir, plan, vm_name, address)?; + let signer = existing_host_signer(keys_dir)?; + let (bundle_path, manifest_path) = paths(keys_dir, &identity)?; + let directory = bundle_path.parent().context("mailbox custody directory")?; + let directory_meta = std::fs::symlink_metadata(directory)?; + ensure!( + directory_meta.file_type().is_dir() && directory_meta.permissions().mode() & 0o077 == 0, + "retained mailbox custody directory is not private" + ); + let manifest: SignedBinding = + serde_json::from_slice(&read_private(&manifest_path, MAX_MANIFEST_BYTES)?)?; + let signature: [u8; 64] = manifest.signature.as_slice().try_into().map_err(|_| { + Error::new(ErrorKind::InvalidData, "invalid mailbox manifest signature") + })?; + signer.verifying_key().verify( + &signed_bytes(&manifest.binding)?, + &Signature::from_bytes(&signature), + )?; + let binding = manifest.binding; + ensure!( + binding.plan_id == *identity.plan_id + && binding.plan_nonce == *identity.plan_nonce + && binding.vm_name == identity.vm_name + && binding.address == address + && binding.epoch == identity.epoch + && binding.authority_key == plan.mailbox.context("mailbox grant")?.authority_key + && binding.host_signer == signer.verifying_key().to_bytes() + && binding.key_version == KEY_VERSION + && binding.history.address() == address + && binding.history.execution_epoch() == identity.epoch + && binding.history.algorithm() == EncryptionAlgorithm::AES_256_GCM + && binding.history.version() == KEY_VERSION, + "retained mailbox custody scope or authority changed" + ); + let bytes = read_private(&bundle_path, KEY_BYTES as u64)?; + ensure!( + bytes.len() == KEY_BYTES, + "mailbox custody key bundle length changed" + ); + let mut material = Zeroizing::new([0u8; KEY_BYTES]); + material.copy_from_slice(&bytes); + let history_id: [u8; 32] = Sha256::digest(&material[..32]).into(); + let payload_id: [u8; 32] = Sha256::digest(&material[32..64]).into(); + let anchor_id: [u8; 32] = Sha256::digest(&material[64..]).into(); + ensure!( + binding.history.key_id() == &history_id + && binding.payload_key_id == payload_id + && binding.anchor_key_id == anchor_id, + "retained mailbox purpose key changed" + ); + Ok(Self { binding, material }) + } + + pub(crate) fn history_reference(&self) -> HistoryKeyRef { + self.binding.history + } + + pub(crate) fn payload_key(&self) -> Key { + Key::from_bytes( + self.material[32..64] + .try_into() + .expect("fixed payload key length"), + ) + } + + pub(crate) fn payload_key_id(&self) -> [u8; 32] { + self.binding.payload_key_id + } + + pub(crate) fn anchor_key(&self) -> [u8; 32] { + self.material[64..] + .try_into() + .expect("fixed anchor key length") + } +} + +impl HistoryKeyCustody for RetainedMailboxKeys { + fn load_existing(&self, reference: &HistoryKeyRef) -> Result { + if reference != &self.binding.history { + return Err(JournalError::HistoryKeyMismatch); + } + Ok(Key::from_bytes( + self.material[..32] + .try_into() + .expect("fixed history key length"), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mvm_contract::protocol::mailbox::{AuthorizationDomainId, MailboxGrant, MailboxId}; + use mvm_core::plan::{Variant, compute_plan_id, test_support::PlanFixture}; + use std::fs; + + fn setup() -> (tempfile::TempDir, ExecutionPlan, MailboxAddress) { + let dir = tempfile::tempdir_in("/tmp").expect("private test home"); + crate::audit::host_keypair::load_or_init_at(dir.path()).expect("host signer"); + let mailbox = MailboxId::from_bytes([4; 32]); + let mut plan = PlanFixture::new().build(); + plan.mailbox = Some(MailboxGrant { + mailbox, + authority_key: crate::mailbox_launch::authority_key_for(dir.path(), &mailbox) + .expect("mailbox authority") + .to_bytes(), + epoch: ExecutionEpoch::new(1).expect("epoch"), + report: None, + }); + plan.plan_id = compute_plan_id(&plan); + let (policy, _) = derive_launch_keys(dir.path(), &plan.mailbox.expect("grant")) + .expect("host mailbox keys") + .trust(); + let address = MailboxAddress { + tenant: policy.context.expected_tenant, + authorization_domain: policy.context.expected_authorization_domain, + mailbox: policy.context.expected_mailbox, + recipient: policy.context.expected_recipient, + }; + (dir, plan, address) + } + + #[test] + fn purpose_keys_and_signed_manifest_survive_restart_without_launch_secret() { + let (dir, plan, address) = setup(); + let vm = plan.image.name.as_str(); + let posture = MailboxLaunchPosture::for_test(&plan, Variant::Dev); + let source_state = dir.path().join("source-vm"); + fs::create_dir(&source_state).expect("source VM state"); + let before = RetainedMailboxKeys::provision(dir.path(), &plan, vm, &posture, address) + .expect("first provision"); + assert_ne!( + before.payload_key().expose_bytes(), + before + .load_existing(&before.history_reference()) + .unwrap() + .expose_bytes() + ); + assert_ne!( + before.anchor_key(), + *before.payload_key().expose_bytes(), + "anchor key must be a separate purpose" + ); + let payload_id: [u8; 32] = Sha256::digest(before.payload_key().expose_bytes()).into(); + assert_eq!(before.payload_key_id(), payload_id); + assert!( + RetainedMailboxKeys::provision(dir.path(), &plan, vm, &posture, address).is_err(), + "existing custody must never mint a replacement" + ); + let reference = before.history_reference(); + let history = *before + .load_existing(&reference) + .expect("history key") + .expose_bytes(); + let payload = *before.payload_key().expose_bytes(); + let anchor = before.anchor_key(); + drop(before); + fs::remove_dir(&source_state).expect("source teardown"); + + let recovered = RetainedMailboxKeys::open_existing( + dir.path(), + &plan, + vm, + &posture.handoff(None), + None, + address, + ) + .expect("restart loads only retained custody"); + assert_eq!(recovered.history_reference(), reference); + assert_eq!( + *recovered + .load_existing(&reference) + .expect("key") + .expose_bytes(), + history + ); + assert_eq!(*recovered.payload_key().expose_bytes(), payload); + assert_eq!(recovered.anchor_key(), anchor); + let wrong = HistoryKeyRef::new(address, plan.mailbox.expect("grant").epoch, [8; 32], 2) + .expect("foreign version"); + assert!(matches!( + recovered.load_existing(&wrong), + Err(JournalError::HistoryKeyMismatch) + )); + let mut next_epoch = plan.clone(); + let mut next_grant = next_epoch.mailbox.expect("grant"); + next_grant.epoch = ExecutionEpoch::new(2).expect("next epoch"); + next_epoch.mailbox = Some(next_grant); + next_epoch.plan_id = compute_plan_id(&next_epoch); + let next_posture = MailboxLaunchPosture::for_test(&next_epoch, Variant::Dev); + assert!( + RetainedMailboxKeys::open_existing( + dir.path(), + &next_epoch, + vm, + &next_posture.handoff(None), + None, + address, + ) + .is_err(), + "another execution epoch cannot inherit this key version" + ); + } + + #[test] + fn recovery_refuses_missing_or_changed_keys_and_signed_scope() { + let (dir, plan, address) = setup(); + let vm = plan.image.name.as_str(); + let posture = MailboxLaunchPosture::for_test(&plan, Variant::Dev); + let handoff = posture.handoff(None); + assert!( + RetainedMailboxKeys::open_existing(dir.path(), &plan, vm, &handoff, None, address) + .is_err(), + "recovery never creates missing key material" + ); + RetainedMailboxKeys::provision(dir.path(), &plan, vm, &posture, address) + .expect("first provision"); + let identity = identity(dir.path(), &plan, vm, address).expect("trusted identity"); + let (bundle, manifest) = paths(dir.path(), &identity).expect("scope path"); + let mut foreign = address; + foreign.authorization_domain = AuthorizationDomainId::from_bytes([9; 32]); + assert!( + RetainedMailboxKeys::open_existing(dir.path(), &plan, vm, &handoff, None, foreign) + .is_err(), + "a different domain cannot borrow this history" + ); + let mut changed = fs::read(&bundle).expect("bundle"); + changed[0] ^= 1; + fs::write(&bundle, changed).expect("tamper bundle"); + assert!( + RetainedMailboxKeys::open_existing(dir.path(), &plan, vm, &handoff, None, address) + .is_err(), + "recomputed key ID must reject changed material" + ); + fs::write(&manifest, b"{}").expect("tamper manifest"); + assert!( + RetainedMailboxKeys::open_existing(dir.path(), &plan, vm, &handoff, None, address) + .is_err(), + "unsigned metadata cannot authorize reopening" + ); + } + + #[test] + fn interrupted_provision_does_not_replace_an_orphan_key_bundle() { + let (dir, plan, address) = setup(); + let vm = plan.image.name.as_str(); + let posture = MailboxLaunchPosture::for_test(&plan, Variant::Dev); + let trusted = identity(dir.path(), &plan, vm, address).expect("trusted scope"); + let (bundle, manifest) = paths(dir.path(), &trusted).expect("scope path"); + mvm_core::private_fs::ensure_private_dir(bundle.parent().expect("bundle directory")) + .expect("private key directory"); + atomic_write_new(&bundle, &[7; KEY_BYTES]).expect("crash after key publication"); + sync_dir(bundle.parent().expect("bundle directory")).expect("durable orphan"); + + assert!( + RetainedMailboxKeys::provision(dir.path(), &plan, vm, &posture, address).is_err(), + "an orphan must not be overwritten with another key" + ); + assert!( + !manifest.exists(), + "failed provision must not claim custody" + ); + assert!( + RetainedMailboxKeys::open_existing( + dir.path(), + &plan, + vm, + &posture.handoff(None), + None, + address, + ) + .is_err(), + "recovery cannot adopt an unsigned orphan" + ); + assert_eq!(fs::read(bundle).expect("orphan retained"), [7; KEY_BYTES]); + } + + #[test] + fn recovery_refuses_missing_signer_and_symlinked_material() { + let (dir, plan, address) = setup(); + let vm = plan.image.name.as_str(); + let posture = MailboxLaunchPosture::for_test(&plan, Variant::Dev); + RetainedMailboxKeys::provision(dir.path(), &plan, vm, &posture, address) + .expect("first provision"); + let handoff = posture.handoff(None); + let identity = identity(dir.path(), &plan, vm, address).expect("trusted identity"); + let (bundle, _) = paths(dir.path(), &identity).expect("scope path"); + let copy = bundle.with_extension("copy"); + fs::rename(&bundle, ©).expect("relocate bundle"); + std::os::unix::fs::symlink(©, &bundle).expect("symlink bundle"); + assert!( + RetainedMailboxKeys::open_existing(dir.path(), &plan, vm, &handoff, None, address) + .is_err(), + "private material cannot be a symlink" + ); + fs::remove_file(&bundle).expect("remove test symlink"); + fs::rename(©, &bundle).expect("restore bundle"); + let rotated = SigningKey::from_bytes(&[66; 32]); + fs::write(dir.path().join(SECRET_FILENAME), rotated.to_bytes()) + .expect("rotate test signer secret"); + fs::write( + dir.path().join(PUBLIC_FILENAME), + rotated.verifying_key().to_bytes(), + ) + .expect("rotate test signer public key"); + assert!( + RetainedMailboxKeys::open_existing(dir.path(), &plan, vm, &handoff, None, address) + .is_err(), + "signer rotation cannot silently adopt retained history" + ); + fs::remove_file(dir.path().join(SECRET_FILENAME)).expect("remove test signer"); + assert!( + RetainedMailboxKeys::open_existing(dir.path(), &plan, vm, &handoff, None, address) + .is_err(), + "recovery cannot silently mint a replacement signer" + ); + } +} From baa6dd15bbd3a8ba9e662c644866a78fe652862b Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 13:30:49 -0800 Subject: [PATCH 30/51] feat(mailbox): commit signed terminal expiry before disclosure --- crates/mvm-core/README.md | 6 + crates/mvm-core/src/mailbox/journal.rs | 373 +++++++++++++++++++++++- crates/mvm-core/tests/mailbox_crypto.rs | 101 +++++++ 3 files changed, 474 insertions(+), 6 deletions(-) diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index 34313d738a..bf567f828d 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -87,6 +87,12 @@ anchor have been fsynced. An exact local retry re-verifies current admission and the committed local tip before re-signing the original body/time, yielding the same signed bytes after restart. Recovery refuses a local V2 receipt in an independently anchored journal or a V1 strong receipt in a local journal. +`expire_next` records an authority-signed terminal expiry only after the +message's absolute deadline; a lost response can be reconstructed after +recovery. Expired messages are not disclosed, and an uncertain expiry write +cannot mint a terminal receipt. This does not yet implement an anchored +retention/compaction boundary: exhausted record or byte quotas still refuse +new work instead of deleting dedup history. Retained guest-reader controls are checked against the historical signed Receive capability, enrollment, lease and guest ACK before recovery accepts their delivery state. An exact ACK retry still needs current boot, session, diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 6960e7afdb..29d8f498c0 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -43,6 +43,7 @@ const ENCRYPTION_DOMAIN: &[u8] = b"mvm.mailbox.journal.encryption.v1\0"; const ANCHOR_DOMAIN: &[u8] = b"mvm.mailbox.anchor.v1\0"; const ACCEPTANCE_INTENT_DOMAIN_V2: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v2\0"; const ACCEPTANCE_INTENT_DOMAIN_V3: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v3\0"; +const EXPIRY_INTENT_DOMAIN: &[u8] = b"mvm.mailbox.journal-expiry.v1\0"; const MAX_ANCHOR_BYTES: u64 = 1024; const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; const MAX_RECORDS: usize = 100_000; @@ -289,6 +290,11 @@ enum JournalEvent { identity: ReplayIdentity, ack: Box, }, + Expired { + identity: ReplayIdentity, + expired_at_unix_ms: i64, + authorization_signature: Vec, + }, } struct LeaseState { @@ -303,6 +309,7 @@ struct Seen { capability: SignedMailboxCapability, accepted_receipt: DestinationReceipt, acknowledged: bool, + expired_at_unix_ms: Option, lease: Option, signed_ack: Option, } @@ -569,6 +576,33 @@ fn acceptance_intent_bytes( Ok(bytes) } +#[derive(Serialize)] +struct ExpiryIntent { + address: MailboxAddress, + identity: ReplayIdentity, + ciphertext_commitment: [u8; 32], + expired_at_unix_ms: i64, +} + +fn expiry_intent_bytes( + address: MailboxAddress, + identity: ReplayIdentity, + entry: &Seen, + expired_at_unix_ms: i64, +) -> Result, JournalError> { + let mut bytes = EXPIRY_INTENT_DOMAIN.to_vec(); + bytes.extend( + serde_jcs::to_vec(&ExpiryIntent { + address, + identity, + ciphertext_commitment: entry.envelope.ciphertext_commitment, + expired_at_unix_ms, + }) + .map_err(|_| JournalError::Corrupt)?, + ); + Ok(bytes) +} + /// Authenticate retained reader controls at their original issuance time. /// Current boot, policy and revocation are checked separately before use. fn verify_recovered_lease( @@ -912,6 +946,7 @@ impl MailboxJournal { capability, accepted_receipt: *receipt, acknowledged: false, + expired_at_unix_ms: None, lease: None, signed_ack: None, }, @@ -929,7 +964,11 @@ impl MailboxJournal { let Some(entry) = journal.seen.get_mut(&identity) else { return Err(JournalError::Corrupt); }; - if entry.acknowledged || entry.lease.is_some() || acknowledged_at_unix_ms < 0 { + if entry.acknowledged + || entry.expired_at_unix_ms.is_some() + || entry.lease.is_some() + || acknowledged_at_unix_ms < 0 + { return Err(JournalError::Corrupt); } entry.acknowledged = true; @@ -946,6 +985,7 @@ impl MailboxJournal { .get_mut(&identity) .ok_or(JournalError::Corrupt)?; if entry.acknowledged + || entry.expired_at_unix_ms.is_some() || lease.encode().is_err() || enrollment.encode().is_err() || lease.lease.address != address @@ -986,7 +1026,7 @@ impl MailboxJournal { let active = journal .seen .get_mut(&identity) - .filter(|entry| !entry.acknowledged) + .filter(|entry| !entry.acknowledged && entry.expired_at_unix_ms.is_none()) .and_then(|entry| entry.lease.as_mut()) .ok_or(JournalError::Corrupt)?; active.retry_count = active @@ -1004,7 +1044,10 @@ impl MailboxJournal { .get_mut(&identity) .ok_or(JournalError::Corrupt)?; let active = entry.lease.as_ref().ok_or(JournalError::Corrupt)?; - if entry.acknowledged || ack.encode().is_err() || ack.ack.lease != active.signed + if entry.acknowledged + || entry.expired_at_unix_ms.is_some() + || ack.encode().is_err() + || ack.ack.lease != active.signed { return Err(JournalError::Corrupt); } @@ -1032,6 +1075,33 @@ impl MailboxJournal { entry.acknowledged = true; entry.signed_ack = Some(*ack); } + JournalEvent::Expired { + identity, + expired_at_unix_ms, + authorization_signature, + } => { + let entry = journal + .seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)?; + if entry.acknowledged + || entry.expired_at_unix_ms.is_some() + || expired_at_unix_ms < entry.envelope.routing.expires_at_unix_ms + { + return Err(JournalError::Corrupt); + } + let signature: [u8; 64] = authorization_signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + authority + .verify( + &expiry_intent_bytes(address, identity, entry, expired_at_unix_ms)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; + entry.expired_at_unix_ms = Some(expired_at_unix_ms); + } } let next = JournalTip { cursor, @@ -1143,6 +1213,9 @@ impl MailboxJournal { if previous.envelope != *envelope || previous.capability != *capability { return Err(JournalError::ConflictingRetry); } + if previous.expired_at_unix_ms.is_some() { + return Err(JournalError::Mailbox(MailboxError::Expired)); + } let expected_outcome = match tier { AcceptanceTier::Independent => DeliveryOutcome::AcceptedDurable, AcceptanceTier::Local => DeliveryOutcome::AcceptedLocalDurable, @@ -1197,6 +1270,7 @@ impl MailboxJournal { capability: capability.clone(), accepted_receipt: accepted.clone(), acknowledged: false, + expired_at_unix_ms: None, lease: None, signed_ack: None, }, @@ -1300,8 +1374,93 @@ impl MailboxJournal { Ok(tip) } - /// Inspect the oldest unacknowledged ciphertext under an independently - /// signed Receive grant. No journal state changes until `acknowledge`. + /// Commit one expired message's terminal state before returning its signed + /// outcome. A full journal refuses rather than dropping retained dedup data. + pub fn expire_next( + &mut self, + now_unix_ms: i64, + signer: &SigningKey, + anchor: impl FnOnce(JournalTip) -> Result<(), JournalError>, + ) -> Result, JournalError> { + if self.poisoned { + return Err(JournalError::Uncertain); + } + if now_unix_ms < 0 || signer.verifying_key() != self.authority { + return Err(JournalError::WrongAuthority); + } + let Some(identity) = self.order.iter().copied().find(|identity| { + self.seen.get(identity).is_some_and(|entry| { + !entry.acknowledged + && entry.expired_at_unix_ms.is_none() + && now_unix_ms >= entry.envelope.routing.expires_at_unix_ms + }) + }) else { + return Ok(None); + }; + let entry = self.seen.get(&identity).ok_or(JournalError::Corrupt)?; + let authorization_signature = signer + .sign(&expiry_intent_bytes( + self.address, + identity, + entry, + now_unix_ms, + )?) + .to_bytes() + .to_vec(); + self.append( + JournalEvent::Expired { + identity, + expired_at_unix_ms: now_unix_ms, + authorization_signature, + }, + anchor, + )?; + self.seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)? + .expired_at_unix_ms = Some(now_unix_ms); + Ok(Some(self.expired_receipt(identity, signer)?)) + } + + /// Reconstruct the original signed terminal expiry after a lost response, + /// but only from the verified and committed journal state. + pub fn expired_receipt( + &self, + identity: ReplayIdentity, + signer: &SigningKey, + ) -> Result { + if self.poisoned { + return Err(JournalError::Uncertain); + } + if signer.verifying_key() != self.authority { + return Err(JournalError::WrongAuthority); + } + let entry = self + .seen + .get(&identity) + .ok_or(JournalError::UnknownMessage)?; + let observed_at_unix_ms = entry + .expired_at_unix_ms + .ok_or(JournalError::UnknownMessage)?; + Ok(sign_receipt( + DestinationReceipt { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: entry.envelope.routing.delivery_id, + message_id: identity.message_id, + address: self.address, + execution_epoch: identity.execution_epoch, + outcome: DeliveryOutcome::Expired, + observed_at_unix_ms, + ciphertext_commitment: entry.envelope.ciphertext_commitment, + }, + signer, + )?) + } + + /// Inspect the oldest unacknowledged, unexpired ciphertext under a signed + /// Receive grant. Call `expire_next` first to durably settle expired entries; + /// an empty read alone is not an expiry receipt. pub fn receive_next( &self, request: JournalReceive<'_>, @@ -1311,7 +1470,10 @@ impl MailboxJournal { } for identity in &self.order { let entry = self.seen.get(identity).ok_or(JournalError::Corrupt)?; - if entry.acknowledged { + if entry.acknowledged + || entry.expired_at_unix_ms.is_some() + || request.context.now_unix_ms >= entry.envelope.routing.expires_at_unix_ms + { continue; } let opened = self.open_for_reader(*identity, entry, &request)?; @@ -1486,6 +1648,9 @@ impl MailboxJournal { .seen .get(&identity) .ok_or(JournalError::UnknownMessage)?; + if entry.expired_at_unix_ms.is_some() { + return Err(JournalError::Mailbox(MailboxError::Expired)); + } let _opened = self.open_for_reader(identity, entry, &request)?; if entry.lease.is_some() { return Err(JournalError::GuestAckRequired); @@ -1534,6 +1699,9 @@ impl MailboxJournal { .seen .get(&identity) .ok_or(JournalError::UnknownMessage)?; + if entry.expired_at_unix_ms.is_some() { + return Err(JournalError::Mailbox(MailboxError::Expired)); + } let active = entry.lease.as_ref().ok_or(JournalError::NoActiveLease)?; if active.signed != request.signed.ack.lease || active.enrollment != *request.enrollment @@ -2050,6 +2218,199 @@ mod anchor_tests { )); } + #[test] + fn interrupted_terminal_expiry_reconciles_without_an_early_receipt() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let anchor_path = dir.path().join("anchor").join("tip.json"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let (reference, custody) = versioned_custody(); + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("local anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ) + .expect("empty journal"); + journal + .admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("local acceptance"); + let identity = journal.order[0]; + let deadline = envelope.routing.expires_at_unix_ms; + assert!( + journal + .expire_next(deadline - 1, &signer, |_| Ok(())) + .expect("not expired yet") + .is_none() + ); + assert!(matches!( + journal.expire_next(deadline, &SigningKey::from_bytes(&[42; 32]), |_| Ok(())), + Err(JournalError::WrongAuthority) + )); + assert!(matches!( + journal.expire_next(deadline, &signer, |_| Err(JournalError::Rollback)), + Err(JournalError::Rollback) + )); + assert_eq!(anchor.tip().cursor, 1, "no unanchored terminal receipt"); + assert!(matches!( + journal.expired_receipt(identity, &signer), + Err(JournalError::Uncertain) + )); + drop(journal); + drop(anchor); + + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("retained anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ) + .expect("verify and reconcile terminal suffix"); + assert_eq!(anchor.tip().cursor, 2); + let expired = journal + .expired_receipt(identity, &signer) + .expect("terminal receipt after reconciliation"); + assert_eq!(expired.receipt.outcome, DeliveryOutcome::Expired); + assert_eq!(expired.receipt.observed_at_unix_ms, deadline); + super::super::verify_receipt(&expired, &signer.verifying_key()) + .expect("signed terminal outcome"); + assert!(matches!( + journal.admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ), + Err(JournalError::Mailbox(MailboxError::Expired)) + )); + assert!( + journal + .expire_next(deadline + 1, &signer, |_| Ok(())) + .expect("idempotent expiry") + .is_none() + ); + assert!(matches!( + journal.acknowledge( + identity, + JournalReceive { + capability: &capability, + context: &context, + encryption_key: &payload_key, + sender_key: &sender_key, + }, + |_| Ok(()) + ), + Err(JournalError::Mailbox(MailboxError::Expired)) + )); + assert_eq!(journal.tip(), anchor.tip(), "no duplicate terminal event"); + } + + #[test] + fn recovered_expiry_rejects_a_forged_authority_signature_even_with_a_valid_history_mac() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let anchor_path = dir.path().join("anchor").join("tip.json"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let (reference, custody) = versioned_custody(); + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("local anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ) + .expect("empty journal"); + journal + .admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("accepted"); + let accepted_tip = journal.tip(); + journal + .expire_next(envelope.routing.expires_at_unix_ms, &signer, |tip| { + anchor.advance(tip) + }) + .expect("expiry commit") + .expect("terminal receipt"); + let path = record_path(&root, 2); + let ciphertext = fs::read(&path).expect("expiry record"); + let plaintext = aead::open( + &journal.record_key(), + &ciphertext[32..], + &[DOMAIN, &accepted_tip.head].concat(), + ) + .expect("decrypt expiry"); + let mut record: Record = serde_json::from_slice(&plaintext).expect("parse expiry"); + let JournalEvent::Expired { + authorization_signature, + .. + } = &mut record.event + else { + panic!("the second record must be the terminal expiry"); + }; + authorization_signature[0] ^= 1; + let forged = aead::seal( + &journal.record_key(), + &serde_json::to_vec(&record).expect("encode forged record"), + &[DOMAIN, &accepted_tip.head].concat(), + ); + journal.tip = accepted_tip; + let mut bytes = journal.mac(&forged).to_vec(); + bytes.extend(forged); + fs::write(path, bytes).expect("install valid-MAC forged expiry"); + drop(journal); + drop(anchor); + + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("retained anchor"); + assert!(matches!( + MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + ), + Err(JournalError::Corrupt) + )); + assert_eq!(anchor.tip().cursor, 2, "recovery must not amend the anchor"); + } + #[test] fn anchored_acceptance_recovers_ciphertext_and_dedup_without_plaintext_on_disk() { let dir = tempfile::tempdir().expect("journal root"); diff --git a/crates/mvm-core/tests/mailbox_crypto.rs b/crates/mvm-core/tests/mailbox_crypto.rs index e5f81b67c2..0912e0d222 100644 --- a/crates/mvm-core/tests/mailbox_crypto.rs +++ b/crates/mvm-core/tests/mailbox_crypto.rs @@ -393,6 +393,107 @@ fn identity_scope_epoch_expiry_and_content_type_are_checked() { } } +#[cfg(feature = "test-support")] +#[test] +fn local_journal_expiry_recovers_after_interrupted_anchor_commit() { + use mvm_core::mailbox::journal::{ + FileJournalAnchor, HistoryKeyCustody, HistoryKeyRef, JournalAdmission, JournalError, + JournalLimits, MailboxJournal, + }; + + struct ExistingKey(HistoryKeyRef); + impl HistoryKeyCustody for ExistingKey { + fn load_existing(&self, reference: &HistoryKeyRef) -> Result { + if reference != &self.0 { + return Err(JournalError::HistoryKeyMismatch); + } + Ok(Key::from_bytes([41; 32])) + } + } + + let dir = tempfile::tempdir().expect("private journal fixture"); + let root = dir.path().join("journal"); + let anchor_path = dir.path().join("anchor").join("tip"); + let signer = SigningKey::from_bytes(&[22; 32]); + let sender_key = SigningKey::from_bytes(&[21; 32]); + let payload_key = Key::from_bytes([23; 32]); + let capability = sign_capability(grant(), &signer).expect("send grant"); + let envelope = seal_envelope( + SealRequest { + routing: routing(), + header: header(), + plaintext: b"hello", + }, + &payload_key, + &sender_key, + ) + .expect("canonical encrypted envelope"); + let context = context(); + let reference = HistoryKeyRef::new(address(), header().execution_epoch, [42; 32], 1) + .expect("history identity"); + let custody = ExistingKey(reference); + let limits = JournalLimits { + max_records: 4, + max_bytes: 64 * 1024, + }; + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [43; 32]).expect("local anchor genesis"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits, + &custody, + &anchor, + ) + .expect("empty local journal"); + let accepted = journal + .admit_local( + JournalAdmission { + envelope: &envelope, + capability: &capability, + context: &context, + encryption_key: &payload_key, + sender_key: &sender_key.verifying_key(), + signer: &signer, + }, + &anchor, + ) + .expect("fsynced local acceptance"); + assert_eq!( + accepted.receipt.outcome, + DeliveryOutcome::AcceptedLocalDurable + ); + let identity = mvm_core::mailbox::ReplayIdentity::from_header(&header()); + assert!(matches!( + journal.expire_next(routing().expires_at_unix_ms, &signer, |_| { + Err(JournalError::Rollback) + }), + Err(JournalError::Rollback) + )); + assert_eq!(anchor.tip().cursor, 1, "no terminal receipt before anchor"); + drop(journal); + drop(anchor); + + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [43; 32]).expect("recover local anchor"); + let journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits, + &custody, + &anchor, + ) + .expect("reconcile verified terminal state"); + let expired = journal + .expired_receipt(identity, &signer) + .expect("signed terminal outcome after restart"); + assert_eq!(expired.receipt.outcome, DeliveryOutcome::Expired); + verify_receipt(&expired, &signer.verifying_key()).expect("signed expiry"); + assert_eq!(anchor.tip().cursor, 2); +} + #[test] fn destination_receipt_wire_roundtrip_and_signature_cover_outcome_and_identity() { let destination_key = SigningKey::from_bytes(&[31; 32]); From 23e92584e9ea0b6e17be46483835778ed3567ac2 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 15:45:41 -0800 Subject: [PATCH 31/51] feat(mailbox): bind local receipts to existing journal service --- crates/mvm-core/README.md | 5 +- crates/mvm-core/src/mailbox/frame.rs | 42 ++++- crates/mvm-core/src/mailbox/journal.rs | 44 +++-- crates/mvm-hostd/README.md | 7 + .../mvm-hostd/src/mailbox_service/durable.rs | 138 +++++++++++---- crates/mvm-hostd/tests/mission_demo.rs | 161 ++++++++++++++++++ 6 files changed, 347 insertions(+), 50 deletions(-) diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index bf567f828d..423e66519d 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -81,12 +81,15 @@ missing or divergent anchored prefix, and may reconcile only an intact contiguous suffix written before an interrupted anchor commit. The original accepted receipt body and observation time are retained with deduplication state; V1 retries still use their existing `duplicate` outcome. -The test-scoped local opener and `admit_local` sign a distinct V2 +The local opener and `admit_local` sign a distinct V2 `accepted_local_durable` receipt only after the encrypted record and local anchor have been fsynced. An exact local retry re-verifies current admission and the committed local tip before re-signing the original body/time, yielding the same signed bytes after restart. Recovery refuses a local V2 receipt in an independently anchored journal or a V1 strong receipt in a local journal. +The hostd service can bind that local journal to its existing endpoint, but +the production launch path still refuses until verified custody, retention, +and guest receipt handoff are integrated end to end. `expire_next` records an authority-signed terminal expiry only after the message's absolute deadline; a lost response can be reconstructed after recovery. Expired messages are not disclosed, and an uncertain expiry write diff --git a/crates/mvm-core/src/mailbox/frame.rs b/crates/mvm-core/src/mailbox/frame.rs index b8ebc5d220..ce15587f1f 100644 --- a/crates/mvm-core/src/mailbox/frame.rs +++ b/crates/mvm-core/src/mailbox/frame.rs @@ -154,6 +154,43 @@ pub fn send_receipted_frame( envelope: &MailboxEnvelope, expected_header: &MailboxEnvelopeHeader, destination_key: &VerifyingKey, +) -> Result { + send_expected_receipt( + stream, + capability, + envelope, + expected_header, + destination_key, + &[DeliveryOutcome::AcceptedDurable, DeliveryOutcome::Duplicate], + ) +} + +/// Verify the separate trusted-host local receipt without upgrading it to the +/// stronger independently anchored acceptance tier. +pub fn send_local_receipted_frame( + stream: &mut S, + capability: &SignedMailboxCapability, + envelope: &MailboxEnvelope, + expected_header: &MailboxEnvelopeHeader, + destination_key: &VerifyingKey, +) -> Result { + send_expected_receipt( + stream, + capability, + envelope, + expected_header, + destination_key, + &[DeliveryOutcome::AcceptedLocalDurable], + ) +} + +fn send_expected_receipt( + stream: &mut S, + capability: &SignedMailboxCapability, + envelope: &MailboxEnvelope, + expected_header: &MailboxEnvelopeHeader, + destination_key: &VerifyingKey, + allowed: &[DeliveryOutcome], ) -> Result { let receipt: SignedDestinationReceipt = exchange(stream, &encode_frame(capability, envelope)?)?; verify_receipt(&receipt, destination_key).map_err(|_| Refusal::Malformed)?; @@ -168,10 +205,7 @@ pub fn send_receipted_frame( || receipt.receipt.message_id != expected_header.message_id || receipt.receipt.address != address || receipt.receipt.execution_epoch != expected_header.execution_epoch - || !matches!( - receipt.receipt.outcome, - DeliveryOutcome::AcceptedDurable | DeliveryOutcome::Duplicate - ) + || !allowed.contains(&receipt.receipt.outcome) { return Err(Refusal::Malformed); } diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 29d8f498c0..6b475c443b 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -196,6 +196,8 @@ pub enum JournalError { Busy, #[error("journal quota exhausted")] Full, + #[error("mailbox expiry settlement needs another bounded pass")] + MaintenancePending, #[error("journal record or chain is invalid")] Corrupt, #[error("journal high-water mark does not match its trusted anchor")] @@ -245,10 +247,6 @@ struct JournalKey { #[derive(Clone, Copy, PartialEq, Eq)] enum AcceptanceTier { Independent, - #[cfg_attr( - not(any(test, feature = "test-support")), - expect(dead_code, reason = "local custody opener is not production-wired yet") - )] Local, } @@ -749,11 +747,11 @@ impl MailboxJournal { ) } - /// Test the software-only local tier against a fsynced same-host anchor. + /// Reopen the software-only local tier against a fsynced same-host anchor. /// - /// Production must obtain the reference from an authenticated restart - /// manifest and verified mailbox scope, not a deserialized route hint. - #[cfg(any(test, feature = "test-support"))] + /// The trusted host must obtain the reference and existing key from an + /// authenticated restart manifest bound to its admitted plan/VM/mailbox. + /// A deserialized reference alone is never authorization to open history. pub fn open_local_with_custody( root: &Path, reference: HistoryKeyRef, @@ -1301,20 +1299,36 @@ impl MailboxJournal { }) } - /// Commit local acceptance only after the ciphertext/dedup record and - /// same-host anchor are fsynced. This cannot detect whole-host rollback. - #[cfg(any(test, feature = "test-support"))] - pub fn admit_local( - &mut self, - request: JournalAdmission<'_>, + /// Verify that an opened local journal still agrees with its held anchor + /// and the destination authority before binding a listener. + pub fn local_ready( + &self, anchor: &FileJournalAnchor, - ) -> Result { + authority: &VerifyingKey, + ) -> Result<(), JournalError> { + if self.poisoned { + return Err(JournalError::Uncertain); + } if self.tier != AcceptanceTier::Local || self.history_key.is_none() { return Err(JournalError::DurableAuthorityUnavailable); } + if &self.authority != authority { + return Err(JournalError::WrongAuthority); + } if anchor.address != self.address || anchor.tip() != self.tip { return Err(JournalError::Rollback); } + Ok(()) + } + + /// Commit local acceptance only after the ciphertext/dedup record and + /// same-host anchor are fsynced. This cannot detect whole-host rollback. + pub fn admit_local( + &mut self, + request: JournalAdmission<'_>, + anchor: &FileJournalAnchor, + ) -> Result { + self.local_ready(anchor, &request.signer.verifying_key())?; self.admit_at_tier(request, AcceptanceTier::Local, |next| anchor.advance(next)) } diff --git a/crates/mvm-hostd/README.md b/crates/mvm-hostd/README.md index c0e4ae34cb..49f55fbef0 100644 --- a/crates/mvm-hostd/README.md +++ b/crates/mvm-hostd/README.md @@ -50,6 +50,13 @@ replacement. An interrupted first provision leaves an orphan and refuses. This is trusted-host local crash recovery only: it is not wired to the production mailbox process, cannot detect whole-host rollback, and does not provide host-blind confidentiality. +The journal-backed service can bind a V2 local receipt to its existing endpoint +and recover an exact retry after restart; on Send and Receive it settles at most +eight expired entries per request, returning unavailable when maintenance +remains rather than reporting an uncommitted terminal state. The guest outbox +still speaks an older untyped success wire. Production mailbox launch remains +refused until the signed receipt handoff and anchored retention/compaction +rules are qualified. ## Main areas diff --git a/crates/mvm-hostd/src/mailbox_service/durable.rs b/crates/mvm-hostd/src/mailbox_service/durable.rs index baa6d078e5..57471123dd 100644 --- a/crates/mvm-hostd/src/mailbox_service/durable.rs +++ b/crates/mvm-hostd/src/mailbox_service/durable.rs @@ -2,8 +2,8 @@ //! //! This listener is separate from the ephemeral mailbox queue. It never //! interprets a successful connection as authority or a signed verdict: -//! authorization, journal fsync, and external anchor publication precede the -//! destination signature being returned to the caller. +//! authorization, journal fsync, and the tier's anchor publication precede +//! the destination signature being returned to the caller. use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; @@ -36,6 +36,7 @@ use super::{GuestSealer, IO_TIMEOUT, MailboxLimits, MailboxPolicy, read_frame}; const ACCEPT_POLL: std::time::Duration = std::time::Duration::from_millis(20); const GUEST_EXCHANGE_TIMEOUT: Duration = Duration::from_secs(12); +const MAX_EXPIRY_RECORDS_PER_REQUEST: usize = 8; struct GuestDeadline<'a> { stream: &'a mut UnixStream, @@ -74,7 +75,8 @@ impl Write for GuestDeadline<'_> { } } -/// A trusted anchor advances outside the journal's rollback domain. +/// Commit delivery-state events; this callback alone cannot authorize a +/// strong `accepted_durable` receipt. pub trait JournalAnchor: Send + Sync { fn commit(&self, tip: JournalTip) -> Result<(), JournalError>; } @@ -104,6 +106,7 @@ struct Shared { state: Mutex, signer: SigningKey, anchor: Box, + local_anchor: Option>, reader: OnceLock, } @@ -117,7 +120,7 @@ pub(crate) struct GuestReaderAuthority { pub(crate) revocations_path: PathBuf, } -/// One destination authority's bounded, journal-backed local admission socket. +/// One destination authority's bounded, journal-backed admission socket. pub struct DurableMailboxService { shared: Arc, path: PathBuf, @@ -135,6 +138,56 @@ impl DurableMailboxService { signer: SigningKey, journal: MailboxJournal, anchor: impl JournalAnchor + 'static, + ) -> std::io::Result { + Self::bind_with( + path, + limits, + policy, + signer, + journal, + Box::new(anchor), + None, + ) + } + + /// Bind the existing mailbox endpoint to a verified local journal. The + /// held anchor and journal must agree before the socket becomes visible. + /// This tier does not claim independent whole-host rollback detection. + pub fn bind_local( + path: &Path, + limits: MailboxLimits, + policy: MailboxPolicy, + signer: SigningKey, + journal: MailboxJournal, + anchor: FileJournalAnchor, + ) -> std::io::Result { + journal + .local_ready(&anchor, &signer.verifying_key()) + .map_err(|error| std::io::Error::other(error.to_string()))?; + let anchor = Arc::new(anchor); + let commit = { + let anchor = Arc::clone(&anchor); + move |tip| anchor.advance(tip) + }; + Self::bind_with( + path, + limits, + policy, + signer, + journal, + Box::new(commit), + Some(anchor), + ) + } + + fn bind_with( + path: &Path, + limits: MailboxLimits, + policy: MailboxPolicy, + signer: SigningKey, + journal: MailboxJournal, + anchor: Box, + local_anchor: Option>, ) -> std::io::Result { if signer.verifying_key() != policy.authority_key { return Err(std::io::Error::new( @@ -155,7 +208,8 @@ impl DurableMailboxService { limits, state: Mutex::new(State { policy, journal }), signer, - anchor: Box::new(anchor), + anchor, + local_anchor, reader: OnceLock::new(), }); let stop = Arc::new(AtomicBool::new(false)); @@ -192,7 +246,9 @@ impl DurableMailboxService { /// Read one guest outbox operation over its dedicated mailbox port. The /// guest sees success only after the canonical ciphertext, dedup identity, - /// and anchor have committed; an idle connection returns `false`. + /// and anchor have committed; an idle connection returns `false`. The + /// guest's older `Result<(), Refusal>` wire does not yet carry a signed + /// receipt, so production may not present it as a receipt-bearing Send. pub fn serve_guest( &self, route: &HostDialRoute, @@ -234,12 +290,13 @@ impl DurableMailboxService { capability: &SignedMailboxCapability, reader: &OpenContext, ) -> Result, JournalError> { - let state = self + let mut state = self .shared .state .lock() .expect("mailbox journal lock poisoned"); let context = reader_context(&state.policy, reader)?; + settle_expired(&self.shared, &mut state.journal, context.now_unix_ms)?; state.journal.receive_next(JournalReceive { capability, context: &context, @@ -263,6 +320,7 @@ impl DurableMailboxService { .expect("mailbox journal lock poisoned"); let context = reader_context(&state.policy, reader)?; let State { policy, journal } = &mut *state; + settle_expired(&self.shared, journal, context.now_unix_ms)?; journal.acknowledge( identity, JournalReceive { @@ -282,7 +340,7 @@ fn serve_reader(shared: &Shared, stream: &mut (impl Read + Write), body: &[u8]) match request { ReaderRequest::Receive => { { - let state = shared.state.lock().expect("mailbox journal lock poisoned"); + let mut state = shared.state.lock().expect("mailbox journal lock poisoned"); let context = guest_context(&state.policy, reader)?; verify_guest_reader_enrollment( &reader.enrollment, @@ -292,6 +350,8 @@ fn serve_reader(shared: &Shared, stream: &mut (impl Read + Write), body: &[u8]) &context.revoked_capabilities, ) .map_err(|_| Refusal::NotAuthorized)?; + settle_expired(shared, &mut state.journal, context.now_unix_ms) + .map_err(reader_refusal)?; } let challenge: [u8; 32] = rand::random(); if challenge == [0; 32] { @@ -312,6 +372,7 @@ fn serve_reader(shared: &Shared, stream: &mut (impl Read + Write), body: &[u8]) let context = guest_context(&state.policy, reader)?; let scope = guest_scope(reader, &context); let State { policy, journal } = &mut *state; + settle_expired(shared, journal, context.now_unix_ms).map_err(reader_refusal)?; let leased = journal .lease_next( JournalGuestLease { @@ -453,6 +514,22 @@ fn reader_refusal(error: JournalError) -> Refusal { } } +fn settle_expired( + shared: &Shared, + journal: &mut MailboxJournal, + now_unix_ms: i64, +) -> Result<(), JournalError> { + for _ in 0..MAX_EXPIRY_RECORDS_PER_REQUEST { + if journal + .expire_next(now_unix_ms, &shared.signer, |tip| shared.anchor.commit(tip))? + .is_none() + { + return Ok(()); + } + } + Err(JournalError::MaintenancePending) +} + fn reader_context( policy: &MailboxPolicy, reader: &OpenContext, @@ -512,28 +589,29 @@ fn admit( let mut state = shared.state.lock().expect("mailbox journal lock poisoned"); state.policy.context.now_unix_ms = now; let State { policy, journal } = &mut *state; - journal - .admit( - JournalAdmission { - envelope, - capability, - context: &policy.context, - encryption_key: &policy.encryption_key, - sender_key: &policy.sender_key, - signer: &shared.signer, - }, - |tip| shared.anchor.commit(tip), - ) - .map_err(|error| match error { - JournalError::Mailbox(MailboxError::WrongRecipient) => Refusal::NotAuthorized, - JournalError::Mailbox(MailboxError::CapabilityPayloadExceeded) => { - Refusal::Oversized(capability.grant.max_payload_bytes as usize) - } - JournalError::Mailbox(error) => Refusal::Mailbox(error.to_string()), - JournalError::Full => Refusal::StorageFull, - JournalError::ConflictingRetry => Refusal::Mailbox(error.to_string()), - _ => Refusal::Unavailable, - }) + settle_expired(shared, journal, now).map_err(reader_refusal)?; + let request = JournalAdmission { + envelope, + capability, + context: &policy.context, + encryption_key: &policy.encryption_key, + sender_key: &policy.sender_key, + signer: &shared.signer, + }; + let accepted = match &shared.local_anchor { + Some(anchor) => journal.admit_local(request, anchor), + None => journal.admit(request, |tip| shared.anchor.commit(tip)), + }; + accepted.map_err(|error| match error { + JournalError::Mailbox(MailboxError::WrongRecipient) => Refusal::NotAuthorized, + JournalError::Mailbox(MailboxError::CapabilityPayloadExceeded) => { + Refusal::Oversized(capability.grant.max_payload_bytes as usize) + } + JournalError::Mailbox(error) => Refusal::Mailbox(error.to_string()), + JournalError::Full => Refusal::StorageFull, + JournalError::ConflictingRetry => Refusal::Mailbox(error.to_string()), + _ => Refusal::Unavailable, + }) } /// Send an immutable, already-sealed delivery to the local destination. diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index 1e0c4f762c..f0a4ad66b0 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1155,6 +1155,167 @@ fn local_file_anchor_cannot_issue_durable_acceptance() { drop(service); } +#[test] +fn local_mailbox_service_returns_exact_signed_v2_receipt_after_restart() { + use mvm_contract::protocol::mailbox::DeliveryOutcome; + use mvm_core::mailbox::frame::{send_local_receipted_frame, send_receipted_frame}; + use mvm_core::mailbox::journal::{ + FileJournalAnchor, HistoryKeyCustody, HistoryKeyRef, JournalError, JournalLimits, + MailboxJournal, + }; + use mvm_hostd::mailbox_service::durable::DurableMailboxService; + use std::os::unix::net::UnixStream; + + struct Custody(HistoryKeyRef); + impl HistoryKeyCustody for Custody { + fn load_existing(&self, reference: &HistoryKeyRef) -> Result { + if reference != &self.0 { + return Err(JournalError::HistoryKeyMismatch); + } + Ok(Key::from_bytes([90; 32])) + } + } + + let (node, channel) = channel_with_one_node(); + let journal_dir = node.home.path().join("journal"); + let anchor_path = node.home.path().join("mailbox-anchor"); + let socket = node.home.path().join("local.sock"); + let signer = channel.authority_key.clone(); + let authority = signer.verifying_key(); + let reference = HistoryKeyRef::new(address(), MbEpoch::new(1).unwrap(), [30; 32], 1) + .expect("versioned history identity"); + let custody = Custody(reference); + let limits = JournalLimits { + max_records: 4, + max_bytes: 64 * 1024, + }; + let mut message_header = header(1, 3); + message_header.expires_at_unix_ms = 4_000_000_000_000; + let mut message_routing = routing(); + message_routing.expires_at_unix_ms = message_header.expires_at_unix_ms; + let mut send_grant = grant(&[MailboxOperation::Send]); + send_grant.expires_at_unix_ms = message_header.expires_at_unix_ms; + let capability = sign_capability(send_grant, &signer).expect("signed Send"); + let envelope = seal_envelope( + SealRequest { + routing: message_routing, + header: message_header.clone(), + plaintext: b"hello", + }, + &channel.encryption_key, + &channel.sender_key, + ) + .expect("canonical encrypted envelope"); + + let start = || { + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [91; 32]).expect("local anchor"); + let journal = MailboxJournal::open_local_with_custody( + &journal_dir, + reference, + authority, + limits, + &custody, + &anchor, + ) + .expect("recover local journal"); + DurableMailboxService::bind_local( + &socket, + LIMITS, + MailboxPolicy { + context: context(1), + encryption_key: Key::from_bytes([23; 32]), + sender_key: channel.sender_key.verifying_key(), + authority_key: authority, + }, + signer.clone(), + journal, + anchor, + ) + .expect("bind verified local endpoint") + }; + let service = start(); + let mut stream = UnixStream::connect(&socket).expect("local Send"); + let accepted = send_local_receipted_frame( + &mut stream, + &capability, + &envelope, + &message_header, + &authority, + ) + .expect("signed local acceptance"); + assert_eq!(accepted.receipt.protocol_version, ProtocolVersion::V2); + assert_eq!( + accepted.receipt.outcome, + DeliveryOutcome::AcceptedLocalDurable + ); + drop(service); + + let service = start(); + let mut stream = UnixStream::connect(&socket).expect("retry after restart"); + let retried = send_local_receipted_frame( + &mut stream, + &capability, + &envelope, + &message_header, + &authority, + ) + .expect("recover exact original receipt"); + assert_eq!(retried, accepted); + let mut strong_stream = UnixStream::connect(&socket).expect("strong-tier client"); + assert_eq!( + send_receipted_frame( + &mut strong_stream, + &capability, + &envelope, + &message_header, + &authority, + ), + Err(Refusal::Malformed), + "a V1 strong-tier client must not upgrade the local receipt" + ); + drop(service); + let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]) + .expect("anchor after service teardown"); + assert_eq!(anchor.tip().cursor, 1, "exact retries must not append"); + drop(anchor); + + let foreign_path = node.home.path().join("foreign.sock"); + let foreign_anchor = FileJournalAnchor::open( + &node.home.path().join("foreign-anchor"), + address(), + [91; 32], + ) + .expect("fresh anchor"); + let foreign_journal = MailboxJournal::open_local_with_custody( + &node.home.path().join("foreign-journal"), + reference, + SigningKey::from_bytes(&[67; 32]).verifying_key(), + limits, + &custody, + &foreign_anchor, + ) + .expect("empty history under a different authority"); + assert!( + DurableMailboxService::bind_local( + &foreign_path, + LIMITS, + MailboxPolicy { + context: context(1), + encryption_key: Key::from_bytes([23; 32]), + sender_key: channel.sender_key.verifying_key(), + authority_key: authority, + }, + signer, + foreign_journal, + foreign_anchor, + ) + .is_err(), + "wrong journal authority must refuse before listener publication" + ); + assert!(!foreign_path.exists()); +} + #[test] fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { use mvm_contract::plan::{Nonce, PlanId}; From 57bbdaba7419161cc48740f6b9c35767c280560f Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 16:29:53 -0800 Subject: [PATCH 32/51] feat(mailbox): require signed local receipt for guest outbox --- .../src/bin/mvm-guest-agent/mailbox.rs | 127 ++++++++++++- crates/mvm-agentd/src/mailbox_outbox.rs | 133 +++++++++++++- crates/mvm-core/src/mailbox/frame.rs | 169 +++++++++++++++--- crates/mvm-hostd/README.md | 11 +- .../mvm-hostd/src/mailbox_service/durable.rs | 21 ++- crates/mvm-hostd/tests/mission_demo.rs | 85 ++++++++- .../057-mailbox-semantics-and-threat-model.md | 21 +++ 7 files changed, 514 insertions(+), 53 deletions(-) diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs index 5157fdf35a..0f3095a24b 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs @@ -12,9 +12,11 @@ use std::path::Path; use std::sync::{OnceLock, atomic::Ordering}; use std::time::{Duration, Instant}; -use ed25519_dalek::SigningKey; +use ed25519_dalek::{SigningKey, VerifyingKey}; use mvm_agentd::flowmux_drive::{MAILBOX_ACK_SIGNING_KEY_FILE, RUN_MVM_DIR}; -use mvm_agentd::mailbox_outbox::{MAILBOX_CMDLINE_TOKEN, OUTBOX_PATH, forward_outbox}; +use mvm_agentd::mailbox_outbox::{ + LOCAL_OUTBOX_PATH, MAILBOX_CMDLINE_TOKEN, OUTBOX_PATH, forward_local_outbox, forward_outbox, +}; use mvm_agentd::telemetry_capture::session::TimedStream; use mvm_contract::protocol::mailbox::MAILBOX_PORT; use mvm_core::mailbox::frame::Refusal; @@ -207,11 +209,11 @@ pub(crate) fn init_mailbox() { mvm_agentd::detached::spawn_named("mailbox-listener", move || { match bind_vsock_listener(MAILBOX_PORT) { Ok(fd) => { - let reader = match (reader_boot, ack_seed) { + let reader = match (reader_boot.clone(), ack_seed) { (Some(binding), Some(seed)) => crate::mailbox_reader::start(binding, seed), _ => None, }; - accept_loop(fd, reader); + accept_loop(fd, reader, reader_boot); } Err(e) => eprintln!( "mvm-guest-agent: mailbox listener bind failed (port {MAILBOX_PORT}): {e}" @@ -224,6 +226,7 @@ pub(crate) fn init_mailbox() { fn accept_loop( listener_fd: RawFd, reader: Option>, + reader_boot: Option, ) { let mut refused = 0u32; while !SHUTDOWN_REQUESTED.load(Ordering::Acquire) { @@ -238,7 +241,13 @@ fn accept_loop( continue; } let mut deadline = MailboxDeadline::new(&mut stream); - let result = wait_for_operation(&mut deadline, reader.as_ref(), Path::new(OUTBOX_PATH)); + let result = wait_for_operation( + &mut deadline, + reader.as_ref(), + reader_boot.as_ref(), + Path::new(LOCAL_OUTBOX_PATH), + Path::new(OUTBOX_PATH), + ); match result { Ok(()) | Err(Refusal::Unavailable) => {} Err(refusal) => { @@ -254,14 +263,34 @@ fn accept_loop( fn wait_for_operation( stream: &mut (impl Read + Write), reader: Option<&std::sync::mpsc::Receiver>, + reader_boot: Option<&MailboxReaderBootBinding>, + local_outbox: &Path, outbox: &Path, ) -> Result<(), Refusal> { for _ in 0..16 { - // A pending Send always gets the existing port first; reader traffic - // cannot starve a workload's outbox. + // A local-durable request cannot be downgraded to the older unit + // verdict even when both requests share the guest mailbox port. + if local_outbox.exists() + && let Some(binding) = reader_boot + { + let authority = + VerifyingKey::from_bytes(&binding.authority_key).map_err(|_| Refusal::Malformed)?; + return forward_local_outbox( + stream, + local_outbox, + &authority, + binding.mailbox, + binding.execution_epoch, + ) + .map(|_| ()); + } + // A pending Send gets the existing port before reader traffic. if outbox.exists() { return forward_outbox(stream, outbox); } + if local_outbox.exists() { + return Err(Refusal::Unavailable); + } if let Some(Ok(pending)) = reader.map(std::sync::mpsc::Receiver::try_recv) { return crate::mailbox_reader::exchange(stream, pending); } @@ -321,11 +350,17 @@ mod tests { use mvm_core::mailbox::frame::OutboxMessage; let dir = tempfile::tempdir().unwrap(); let outbox = dir.path().join("outbox"); + let local_outbox = dir.path().join("outbox-local"); mvm_agentd::mailbox_outbox::write_outbox( &outbox, &OutboxMessage::new(PrincipalId::from_bytes([0; 32]), b"hello".to_vec()), ) .unwrap(); + mvm_agentd::mailbox_outbox::write_outbox( + &local_outbox, + &OutboxMessage::new(PrincipalId::from_bytes([0; 32]), b"local".to_vec()), + ) + .unwrap(); let (mut guest, mut host) = std::os::unix::net::UnixStream::pair().unwrap(); let (_sender, reader) = std::sync::mpsc::sync_channel(1); let peer = std::thread::spawn(move || { @@ -338,9 +373,85 @@ mod tests { host.write_all(&serde_json::to_vec(&Ok::<(), Refusal>(())).unwrap()) .unwrap(); }); - wait_for_operation(&mut guest, Some(&reader), &outbox).unwrap(); + wait_for_operation(&mut guest, Some(&reader), None, &local_outbox, &outbox).unwrap(); peer.join().unwrap(); assert!(!outbox.exists(), "accepted Send is removed as before"); + assert!( + local_outbox.exists(), + "missing boot authority cannot downgrade a local request" + ); + } + + #[test] + fn local_send_requires_boot_authority_and_signed_acceptance() { + use ed25519_dalek::SigningKey; + use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, BootId, DeliveryId, DeliveryOutcome, DestinationReceipt, + ExecutionEpoch, MailboxAddress, MailboxId, PrincipalId, ProtocolVersion, + SignatureAlgorithm, TenantId, + }; + use mvm_core::mailbox::frame::OutboxMessage; + use mvm_core::mailbox::sign_receipt; + + let dir = tempfile::tempdir().expect("outbox root"); + let local = dir.path().join("outbox-local"); + let legacy = dir.path().join("outbox"); + let operation = OutboxMessage::new(PrincipalId::from_bytes([0; 32]), b"report".to_vec()); + mvm_agentd::mailbox_outbox::write_outbox(&local, &operation).expect("pending report"); + let (mut guest, _host) = std::os::unix::net::UnixStream::pair().expect("socket"); + assert_eq!( + wait_for_operation(&mut guest, None, None, &local, &legacy), + Err(Refusal::Unavailable) + ); + assert!(local.exists(), "no boot authority means no acceptance"); + drop(guest); + + let signer = SigningKey::from_bytes(&[51; 32]); + let plan = mvm_core::plan::test_support::PlanFixture::new().build(); + let boot = MailboxReaderBootBinding { + plan_id: plan.plan_id, + plan_nonce: plan.nonce, + mailbox: MailboxId::from_bytes([52; 32]), + authority_key: signer.verifying_key().to_bytes(), + execution_epoch: ExecutionEpoch::new(1).expect("epoch"), + boot: BootId::from_bytes([53; 32]), + guest_identity_key: [54; 32], + ack_public_key: [55; 32], + }; + let (mut guest, mut host) = std::os::unix::net::UnixStream::pair().expect("socket"); + let responder = std::thread::spawn(move || { + let mut length = [0; 4]; + host.read_exact(&mut length).expect("frame length"); + let mut frame = vec![0; u32::from_be_bytes(length) as usize]; + host.read_exact(&mut frame).expect("outbox frame"); + let sent = OutboxMessage::decode(&frame).expect("operation"); + let signed = sign_receipt( + DestinationReceipt { + protocol_version: ProtocolVersion::V2, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: DeliveryId::from_bytes([56; 32]), + message_id: sent.message_id, + address: MailboxAddress { + tenant: TenantId::from_bytes([57; 32]), + authorization_domain: AuthorizationDomainId::from_bytes([58; 32]), + mailbox: MailboxId::from_bytes([52; 32]), + recipient: PrincipalId::from_bytes([59; 32]), + }, + execution_epoch: ExecutionEpoch::new(1).expect("epoch"), + outcome: DeliveryOutcome::AcceptedLocalDurable, + observed_at_unix_ms: 1_500, + ciphertext_commitment: [60; 32], + }, + &signer, + ) + .expect("destination signs"); + host.write_all(&serde_json::to_vec(&Ok::<_, Refusal>(signed)).expect("receipt wire")) + .expect("reply"); + }); + wait_for_operation(&mut guest, None, Some(&boot), &local, &legacy) + .expect("boot-pinned signed acceptance"); + responder.join().expect("host reply"); + assert!(!local.exists(), "verified V2 receipt closes pending send"); } #[test] diff --git a/crates/mvm-agentd/src/mailbox_outbox.rs b/crates/mvm-agentd/src/mailbox_outbox.rs index d2f88f3f7b..7c6ca44a98 100644 --- a/crates/mvm-agentd/src/mailbox_outbox.rs +++ b/crates/mvm-agentd/src/mailbox_outbox.rs @@ -10,18 +10,27 @@ //! [`write_outbox`] is the workload side: it leaves that one message where //! [`forward_outbox`] will find it, using the same [`OutboxMessage`] encoding //! and size limits the host decodes with. +//! `outbox-local` uses that same operation identity but is removed only after +//! [`forward_local_outbox`] verifies a boot-bound signed V2 local receipt. use std::io::{Read, Write}; use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; use std::path::Path; use std::time::Duration; -use mvm_contract::protocol::mailbox::MAX_MAILBOX_PAYLOAD_BYTES; -use mvm_core::mailbox::frame::{OutboxMessage, Refusal, send_outbox_message}; +use ed25519_dalek::VerifyingKey; +use mvm_contract::protocol::mailbox::{ + ExecutionEpoch, MAX_MAILBOX_PAYLOAD_BYTES, MailboxId, SignedDestinationReceipt, +}; +use mvm_core::mailbox::frame::{ + OutboxMessage, Refusal, send_local_outbox_message, send_outbox_message, +}; use mvm_core::util::atomic_io::{atomic_write_new, sync_dir}; /// Where the workload leaves its message and the agent reads it. pub const OUTBOX_PATH: &str = "/run/mvm/mailbox/outbox"; +/// An explicit local-durable request never accepts the older unit-verdict wire. +pub const LOCAL_OUTBOX_PATH: &str = "/run/mvm/mailbox/outbox-local"; /// The launch assertion that grants a guest a mailbox. The agent creates the /// outbox directory and binds the mailbox port only when the kernel cmdline @@ -63,6 +72,27 @@ pub fn wait_and_forward_outbox( /// send leaves it in place for a same-ID retry; only an accepted verdict removes /// it under the writer lock. pub fn forward_outbox(stream: &mut S, outbox: &Path) -> Result<(), Refusal> { + forward_with(stream, outbox, send_outbox_message) +} + +/// Keep a local-durable request pending until a signed V2 receipt is verified. +pub fn forward_local_outbox( + stream: &mut S, + outbox: &Path, + authority: &VerifyingKey, + mailbox: MailboxId, + epoch: ExecutionEpoch, +) -> Result { + forward_with(stream, outbox, |stream, message| { + send_local_outbox_message(stream, message, authority, mailbox, epoch) + }) +} + +fn forward_with( + stream: &mut S, + outbox: &Path, + send: impl FnOnce(&mut S, &OutboxMessage) -> Result, +) -> Result { let dir = outbox.parent().ok_or(Refusal::Malformed)?; let _lock = lock_outbox_dir(dir)?; let mut bytes = Vec::new(); @@ -70,12 +100,18 @@ pub fn forward_outbox(stream: &mut S, outbox: &Path) -> Result< .and_then(|file| file.take(MAX_OUTBOX_BYTES + 1).read_to_end(&mut bytes)) .map_err(|_| Refusal::Malformed)?; let message = OutboxMessage::decode(&bytes)?; - match send_outbox_message(stream, &message) { - Ok(()) => {} + let accepted = match send(stream, &message) { + Ok(accepted) => accepted, Err( - refusal @ (Refusal::RetriesExhausted | Refusal::Oversized(_) | Refusal::NotAuthorized), + refusal @ (Refusal::RetriesExhausted + | Refusal::Oversized(_) + | Refusal::NotAuthorized + | Refusal::Expired), ) => { - let prefix = if matches!(refusal, Refusal::Oversized(_) | Refusal::NotAuthorized) { + let prefix = if matches!( + refusal, + Refusal::Oversized(_) | Refusal::NotAuthorized | Refusal::Expired + ) { "rejected" } else { "uncertain" @@ -84,9 +120,10 @@ pub fn forward_outbox(stream: &mut S, outbox: &Path) -> Result< return Err(refusal); } Err(error) => return Err(error), - } + }; std::fs::remove_file(outbox).map_err(|_| Refusal::Unavailable)?; - sync_dir(dir).map_err(|_| Refusal::Unavailable) + sync_dir(dir).map_err(|_| Refusal::Unavailable)?; + Ok(accepted) } fn archive_outbox(outbox: &Path, message: &OutboxMessage, prefix: &str) -> Result<(), Refusal> { @@ -251,6 +288,86 @@ mod tests { ); } + #[test] + fn local_outbox_is_retained_until_a_signed_v2_receipt_arrives() { + use ed25519_dalek::SigningKey; + use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, DeliveryId, DeliveryOutcome, DestinationReceipt, MailboxAddress, + ProtocolVersion, SignatureAlgorithm, TenantId, + }; + use mvm_core::mailbox::sign_receipt; + + let dir = tempfile::tempdir().expect("private guest outbox"); + let path = dir.path().join("outbox-local"); + let operation = message(b"report"); + write_outbox(&path, &operation).expect("retain original operation"); + let signer = SigningKey::from_bytes(&[10; 32]); + let mailbox = MailboxId::from_bytes([9; 32]); + let epoch = ExecutionEpoch::new(1).expect("epoch"); + let signed = sign_receipt( + DestinationReceipt { + protocol_version: ProtocolVersion::V2, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: DeliveryId::from_bytes([11; 32]), + message_id: operation.message_id, + address: MailboxAddress { + tenant: TenantId::from_bytes([12; 32]), + authorization_domain: AuthorizationDomainId::from_bytes([13; 32]), + mailbox, + recipient: operation.recipient, + }, + execution_epoch: epoch, + outcome: DeliveryOutcome::AcceptedLocalDurable, + observed_at_unix_ms: 1_500, + ciphertext_commitment: [14; 32], + }, + &signer, + ) + .expect("signed local verdict"); + let exchange = |reply: Vec| { + let (mut guest, mut host) = std::os::unix::net::UnixStream::pair().expect("socket"); + let peer = std::thread::spawn(move || { + let mut len = [0; 4]; + host.read_exact(&mut len).expect("outbox length"); + let mut body = vec![0; u32::from_be_bytes(len) as usize]; + host.read_exact(&mut body).expect("outbox operation"); + host.write_all(&reply).expect("host verdict"); + OutboxMessage::decode(&body).expect("original operation") + }); + let result = + forward_local_outbox(&mut guest, &path, &signer.verifying_key(), mailbox, epoch); + assert_eq!(peer.join().expect("host thread"), operation); + result + }; + assert_eq!( + exchange(b"{\"Ok\":null}".to_vec()), + Err(Refusal::Malformed), + "an untyped transport success cannot remove a durable request" + ); + assert!(path.exists(), "uncertain operation retains its stable ID"); + assert_eq!( + exchange( + serde_json::to_vec(&Ok::<_, Refusal>(signed.clone())).expect("V2 receipt wire") + ), + Ok(signed) + ); + assert!( + !path.exists(), + "only verified acceptance removes the outbox" + ); + write_outbox(&path, &operation).expect("another local attempt"); + assert_eq!( + exchange(serde_json::to_vec(&Err::<(), _>(Refusal::Expired)).expect("expired verdict")), + Err(Refusal::Expired) + ); + assert!(!path.exists(), "terminal expiry leaves no pending retry"); + let archived = dir.path().join(format!("rejected-{}", "08".repeat(32))); + assert_eq!( + std::fs::read(archived).expect("terminal operation archived"), + operation.encode() + ); + } + #[test] fn a_connected_guest_delivers_a_message_written_after_the_host_dials() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/mvm-core/src/mailbox/frame.rs b/crates/mvm-core/src/mailbox/frame.rs index ce15587f1f..4a6f4c4e89 100644 --- a/crates/mvm-core/src/mailbox/frame.rs +++ b/crates/mvm-core/src/mailbox/frame.rs @@ -1,9 +1,9 @@ //! The one-frame mailbox exchange both ends of the mailbox port share. //! //! A peer writes a length-prefixed frame (a capability and a sealed envelope), -//! a guest a length-prefixed plaintext [`OutboxMessage`]; either reads one -//! typed verdict, a serialized `Result<(), Refusal>`. Keeping both directions -//! here means the two ends cannot drift. +//! a guest a length-prefixed plaintext [`OutboxMessage`]; a legacy guest reads +//! `Result<(), Refusal>`, while a locally durable sender requires a signed V2 +//! receipt. Keeping both directions here means the two ends cannot drift. //! //! Frame: `u32 BE length`, then `u32 BE capability length`, the capability as //! JSON, and the envelope in its canonical `encode_envelope` form. @@ -12,8 +12,8 @@ use std::io::{Read, Write}; use ed25519_dalek::VerifyingKey; use mvm_contract::protocol::mailbox::{ - DeliveryOutcome, MAX_MAILBOX_PAYLOAD_BYTES, MailboxAddress, MailboxEnvelope, - MailboxEnvelopeHeader, MessageId, PrincipalId, SignedDestinationReceipt, + DeliveryOutcome, ExecutionEpoch, MAX_MAILBOX_PAYLOAD_BYTES, MailboxAddress, MailboxEnvelope, + MailboxEnvelopeHeader, MailboxId, MessageId, PrincipalId, SignedDestinationReceipt, SignedMailboxCapability, }; use serde::{Deserialize, Serialize, de::DeserializeOwned}; @@ -33,6 +33,8 @@ pub enum Refusal { QueueFull(usize), #[error("durable mailbox storage quota is full")] StorageFull, + #[error("mailbox message expired before acceptance")] + Expired, #[error("mailbox retry budget exhausted; reconcile the stable message ID")] RetriesExhausted, #[error("mailbox operation is not authorized")] @@ -217,13 +219,44 @@ pub fn send_outbox_message( stream: &mut S, message: &OutboxMessage, ) -> Result<(), Refusal> { + validate_outbox(message)?; + exchange(stream, &message.encode()) +} + +/// Require a signed V2 local acceptance for one guest-authored operation. +/// +/// The guest cannot independently bind the host-sealed ciphertext commitment +/// to its plaintext. This is a trusted-host receipt, not host-blind delivery. +pub fn send_local_outbox_message( + stream: &mut S, + message: &OutboxMessage, + destination_key: &VerifyingKey, + mailbox: MailboxId, + execution_epoch: ExecutionEpoch, +) -> Result { + validate_outbox(message)?; + let signed: SignedDestinationReceipt = exchange(stream, &message.encode())?; + verify_receipt(&signed, destination_key).map_err(|_| Refusal::Malformed)?; + if signed.receipt.outcome != DeliveryOutcome::AcceptedLocalDurable + || signed.receipt.message_id != message.message_id + || signed.receipt.address.mailbox != mailbox + || signed.receipt.execution_epoch != execution_epoch + || (message.recipient.as_bytes() != &OWN_MAILBOX_RECIPIENT + && signed.receipt.address.recipient != message.recipient) + { + return Err(Refusal::Malformed); + } + Ok(signed) +} + +fn validate_outbox(message: &OutboxMessage) -> Result<(), Refusal> { if message.body.is_empty() { return Err(Refusal::Malformed); } if message.body.len() > MAX_MAILBOX_PAYLOAD_BYTES { return Err(Refusal::Oversized(MAX_MAILBOX_PAYLOAD_BYTES)); } - exchange(stream, &message.encode()) + Ok(()) } fn exchange( @@ -250,6 +283,24 @@ fn exchange( mod tests { use super::*; + struct ReplyPeer(std::io::Cursor>); + + impl Read for ReplyPeer { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.read(buf) + } + } + + impl Write for ReplyPeer { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + #[test] fn an_outbox_message_round_trips_and_malformed_input_is_refused() { let message = OutboxMessage { @@ -280,22 +331,7 @@ mod tests { #[test] fn a_peer_cannot_make_the_verdict_reader_grow_without_bound() { - struct Peer(std::io::Cursor>); - impl Read for Peer { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.0.read(buf) - } - } - impl Write for Peer { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - Ok(buf.len()) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - let mut peer = Peer(std::io::Cursor::new(vec![ + let mut peer = ReplyPeer(std::io::Cursor::new(vec![ b'x'; MAX_VERDICT_BYTES as usize + 2 ])); @@ -329,4 +365,93 @@ mod tests { ); assert!(peer.into_inner().is_empty()); } + + #[test] + fn guest_local_outbox_needs_a_signed_v2_receipt_for_its_own_identity() { + use ed25519_dalek::SigningKey; + use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, DeliveryId, DestinationReceipt, ProtocolVersion, + SignatureAlgorithm, TenantId, + }; + + let signer = SigningKey::from_bytes(&[9; 32]); + let mailbox = MailboxId::from_bytes([3; 32]); + let epoch = ExecutionEpoch::new(7).expect("epoch"); + let message = OutboxMessage { + recipient: PrincipalId::from_bytes(OWN_MAILBOX_RECIPIENT), + message_id: MessageId::from_bytes([4; 32]), + body: b"hello".to_vec(), + }; + let receipt = DestinationReceipt { + protocol_version: ProtocolVersion::V2, + signature_algorithm: SignatureAlgorithm::ED25519, + delivery_id: DeliveryId::from_bytes([5; 32]), + message_id: message.message_id, + address: MailboxAddress { + tenant: TenantId::from_bytes([6; 32]), + authorization_domain: AuthorizationDomainId::from_bytes([7; 32]), + mailbox, + recipient: PrincipalId::from_bytes([8; 32]), + }, + execution_epoch: epoch, + outcome: DeliveryOutcome::AcceptedLocalDurable, + observed_at_unix_ms: 1_500, + ciphertext_commitment: [10; 32], + }; + let signed = super::super::sign_receipt(receipt.clone(), &signer).expect("V2 receipt"); + let wire = serde_json::to_vec(&Ok::<_, Refusal>(signed.clone())).expect("verdict"); + let mut peer = ReplyPeer(std::io::Cursor::new(wire)); + assert_eq!( + send_local_outbox_message(&mut peer, &message, &signer.verifying_key(), mailbox, epoch), + Ok(signed) + ); + + let mut unit = ReplyPeer(std::io::Cursor::new( + serde_json::to_vec(&Ok::<(), Refusal>(())).expect("legacy verdict"), + )); + assert_eq!( + send_local_outbox_message(&mut unit, &message, &signer.verifying_key(), mailbox, epoch), + Err(Refusal::Malformed), + "a transport-level success cannot satisfy local acceptance" + ); + for wrong in [ + DestinationReceipt { + message_id: MessageId::from_bytes([11; 32]), + ..receipt.clone() + }, + DestinationReceipt { + execution_epoch: ExecutionEpoch::new(8).expect("foreign epoch"), + ..receipt.clone() + }, + DestinationReceipt { + outcome: DeliveryOutcome::AcceptedDurable, + protocol_version: ProtocolVersion::V1, + ..receipt.clone() + }, + ] { + let signed = super::super::sign_receipt(wrong.clone(), &signer).expect("signed wrong"); + let mut peer = ReplyPeer(std::io::Cursor::new( + serde_json::to_vec(&Ok::<_, Refusal>(signed)).expect("wrong verdict"), + )); + assert_eq!( + send_local_outbox_message( + &mut peer, + &message, + &signer.verifying_key(), + mailbox, + epoch, + ), + Err(Refusal::Malformed) + ); + } + let foreign = super::super::sign_receipt(receipt, &SigningKey::from_bytes(&[12; 32])) + .expect("foreign authority signs"); + let mut peer = ReplyPeer(std::io::Cursor::new( + serde_json::to_vec(&Ok::<_, Refusal>(foreign)).expect("foreign verdict"), + )); + assert_eq!( + send_local_outbox_message(&mut peer, &message, &signer.verifying_key(), mailbox, epoch), + Err(Refusal::Malformed) + ); + } } diff --git a/crates/mvm-hostd/README.md b/crates/mvm-hostd/README.md index 49f55fbef0..36caa25bb0 100644 --- a/crates/mvm-hostd/README.md +++ b/crates/mvm-hostd/README.md @@ -53,10 +53,13 @@ provide host-blind confidentiality. The journal-backed service can bind a V2 local receipt to its existing endpoint and recover an exact retry after restart; on Send and Receive it settles at most eight expired entries per request, returning unavailable when maintenance -remains rather than reporting an uncommitted terminal state. The guest outbox -still speaks an older untyped success wire. Production mailbox launch remains -refused until the signed receipt handoff and anchored retention/compaction -rules are qualified. +remains rather than reporting an uncommitted terminal state. A separate guest +`outbox-local` path now requires a boot-pinned authority and a verified signed +V2 receipt before removing the pending message; the old `outbox` path retains +its ephemeral unit-verdict behavior. Expired submissions receive a typed +terminal refusal and retain a bounded guest-side rejection record. Production +mailbox launch remains refused until this handoff, the retained custody path, +and anchored retention/compaction are qualified end to end. ## Main areas diff --git a/crates/mvm-hostd/src/mailbox_service/durable.rs b/crates/mvm-hostd/src/mailbox_service/durable.rs index 57471123dd..e2d6296779 100644 --- a/crates/mvm-hostd/src/mailbox_service/durable.rs +++ b/crates/mvm-hostd/src/mailbox_service/durable.rs @@ -246,9 +246,9 @@ impl DurableMailboxService { /// Read one guest outbox operation over its dedicated mailbox port. The /// guest sees success only after the canonical ciphertext, dedup identity, - /// and anchor have committed; an idle connection returns `false`. The - /// guest's older `Result<(), Refusal>` wire does not yet carry a signed - /// receipt, so production may not present it as a receipt-bearing Send. + /// and anchor have committed; an idle connection returns `false`. Local + /// admission returns its signed V2 receipt; the legacy branch retains its + /// original unit-verdict wire. pub fn serve_guest( &self, route: &HostDialRoute, @@ -271,13 +271,16 @@ impl DurableMailboxService { } other => other, }; - let verdict: Result<(), Refusal> = body + let verdict: Result = body .and_then(|body| OutboxMessage::decode(&body)) .and_then(|message| sealer.seal(message)) - .and_then(|(capability, envelope)| { - admit(&self.shared, &capability, &envelope).map(|_| ()) - }); - if let Ok(reply) = serde_json::to_vec(&verdict) { + .and_then(|(capability, envelope)| admit(&self.shared, &capability, &envelope)); + let reply = if self.shared.local_anchor.is_some() { + serde_json::to_vec(&verdict) + } else { + serde_json::to_vec(&verdict.map(|_| ())) + }; + if let Ok(reply) = reply { let _ = stream.write_all(&reply); } Ok(received) @@ -504,6 +507,7 @@ fn guest_scope<'a>( fn reader_refusal(error: JournalError) -> Refusal { match error { JournalError::Full => Refusal::StorageFull, + JournalError::Mailbox(MailboxError::Expired) => Refusal::Expired, JournalError::Mailbox(_) | JournalError::WrongAuthority | JournalError::ConflictingRetry @@ -604,6 +608,7 @@ fn admit( }; accepted.map_err(|error| match error { JournalError::Mailbox(MailboxError::WrongRecipient) => Refusal::NotAuthorized, + JournalError::Mailbox(MailboxError::Expired) => Refusal::Expired, JournalError::Mailbox(MailboxError::CapabilityPayloadExceeded) => { Refusal::Oversized(capability.grant.max_payload_bytes as usize) } diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index f0a4ad66b0..a1dd062a81 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1157,14 +1157,17 @@ fn local_file_anchor_cannot_issue_durable_acceptance() { #[test] fn local_mailbox_service_returns_exact_signed_v2_receipt_after_restart() { + use mvm_agentd::mailbox_outbox::{forward_local_outbox, write_outbox}; use mvm_contract::protocol::mailbox::DeliveryOutcome; - use mvm_core::mailbox::frame::{send_local_receipted_frame, send_receipted_frame}; + use mvm_core::mailbox::frame::{ + OutboxMessage, send_local_receipted_frame, send_receipted_frame, + }; use mvm_core::mailbox::journal::{ FileJournalAnchor, HistoryKeyCustody, HistoryKeyRef, JournalError, JournalLimits, MailboxJournal, }; use mvm_hostd::mailbox_service::durable::DurableMailboxService; - use std::os::unix::net::UnixStream; + use std::os::unix::net::{UnixListener, UnixStream}; struct Custody(HistoryKeyRef); impl HistoryKeyCustody for Custody { @@ -1274,10 +1277,86 @@ fn local_mailbox_service_returns_exact_signed_v2_receipt_after_restart() { Err(Refusal::Malformed), "a V1 strong-tier client must not upgrade the local receipt" ); + + let guest_port = node.home.path().join("guest-port.sock"); + let guest_listener = UnixListener::bind(&guest_port).expect("guest mailbox port"); + let guest_outbox = node.home.path().join("outbox-local"); + let operation = OutboxMessage::new(aircraft_principal(), b"guest".to_vec()); + write_outbox(&guest_outbox, &operation).expect("pending local guest operation"); + let guest = std::thread::spawn(move || { + let (mut stream, _) = guest_listener.accept().expect("host dials guest port"); + forward_local_outbox( + &mut stream, + &guest_outbox, + &authority, + address().mailbox, + MbEpoch::new(1).expect("execution epoch"), + ) + .expect("guest verifies V2 receipt before removing outbox") + }); + let mut guest_sealer = sealer(&channel, &[MailboxOperation::Send]); + guest_sealer.header.expires_at_unix_ms = message_header.expires_at_unix_ms; + guest_sealer.routing.expires_at_unix_ms = message_header.expires_at_unix_ms; + guest_sealer.capability.expires_at_unix_ms = message_header.expires_at_unix_ms; + let intent_dir = node.home.path().join("source-intents"); + guest_sealer.intent_dir = Some(intent_dir.clone()); + assert!( + service + .serve_guest( + &mvm_core::vm_backend::HostDialRoute::PortSocket(guest_port), + &mut guest_sealer, + ) + .expect("host serves guest on existing port") + ); + let guest_receipt = guest.join().expect("guest send thread"); + assert_eq!( + guest_receipt.receipt.outcome, + DeliveryOutcome::AcceptedLocalDurable + ); + assert_eq!(guest_receipt.receipt.message_id, operation.message_id); + drop(service); + + let service = start(); + let retry_port = node.home.path().join("guest-port-retry.sock"); + let listener = UnixListener::bind(&retry_port).expect("restarted guest port"); + let retry_outbox = node.home.path().join("outbox-local"); + write_outbox(&retry_outbox, &operation).expect("same-ID source retry"); + let retry_guest = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("host redials guest"); + forward_local_outbox( + &mut stream, + &retry_outbox, + &authority, + address().mailbox, + MbEpoch::new(1).expect("execution epoch"), + ) + .expect("original signed local verdict") + }); + let mut retry_sealer = sealer(&channel, &[MailboxOperation::Send]); + retry_sealer.header.expires_at_unix_ms = message_header.expires_at_unix_ms; + retry_sealer.routing.expires_at_unix_ms = message_header.expires_at_unix_ms; + retry_sealer.capability.expires_at_unix_ms = message_header.expires_at_unix_ms; + retry_sealer.intent_dir = Some(intent_dir); + assert!( + service + .serve_guest( + &mvm_core::vm_backend::HostDialRoute::PortSocket(retry_port), + &mut retry_sealer, + ) + .expect("host redials after helper restart") + ); + assert_eq!( + retry_guest.join().expect("guest retry thread"), + guest_receipt + ); drop(service); let anchor = FileJournalAnchor::open(&anchor_path, address(), [91; 32]) .expect("anchor after service teardown"); - assert_eq!(anchor.tip().cursor, 1, "exact retries must not append"); + assert_eq!( + anchor.tip().cursor, + 2, + "exact retries append nothing; guest Send commits once" + ); drop(anchor); let foreign_path = node.home.path().join("foreign.sock"); diff --git a/specs/adrs/057-mailbox-semantics-and-threat-model.md b/specs/adrs/057-mailbox-semantics-and-threat-model.md index 22508312e9..df13e53724 100644 --- a/specs/adrs/057-mailbox-semantics-and-threat-model.md +++ b/specs/adrs/057-mailbox-semantics-and-threat-model.md @@ -281,6 +281,16 @@ must reproduce the original bytes; an unprovable acceptance is unknown, not a new version-one `duplicate` that loses the acceptance tier. Different bytes under the same ID conflict. Version-one `duplicate` behavior is unchanged. +The guest's local-durable outbox request is sender-owned pending work, not an +acceptance receipt. Its host retains the canonical sealed bytes and records a +bounded dispatch attempt before destination I/O. The guest removes the +pending request only after checking a version-two destination signature bound +to its boot-pinned mailbox authority, execution epoch, and message ID. An +older unit success cannot satisfy this path; wrong-key, stale-epoch, or +unverifiable replies keep the operation pending or yield a typed terminal +refusal. An already-dispatched envelope is not rewritten for a replacement +epoch, and this rule does not authorize transfer of an accepted message. + `accepted_ephemeral` means the message is owned by the named live mailbox epoch. If that epoch terminates, the signed epoch-terminal observation closes the volatile queue; senders that need recovery use stable IDs and retry into a @@ -308,6 +318,17 @@ corruption, wrong key, and quota exhaustion fail closed. Crash recovery is an idempotent reconciliation of durable state, not a sleep-polled foreground path. +For the standalone local tier, a signed plan must declare a finite retention +window before admission; an absent policy does not imply an infinite default. +Compaction first durably publishes an encrypted, destination-authenticated +snapshot of still-live delivery state and dedup tombstones through each +message's expiry plus that window, then advances the local chain head, and +only then prunes superseded records. Recovery verifies the appropriate old +prefix or committed snapshot after interruption; it never guesses a new +genesis from a gap. If the snapshot or retained tombstones cannot fit the +finite quota, new acceptance refuses rather than discarding replay evidence. +This host-local sequence does not detect rollback of the entire host. + ### 8. Selective receive is bounded and fair Selective receive may match only authenticated inner-header fields declared by From 385619f4a394704492c6595a919066b107b87f8d Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 16:29:53 -0800 Subject: [PATCH 33/51] feat(mailbox): bind retained purpose keys to launch template --- crates/mvm-hostd/src/mailbox_custody.rs | 133 ++++++++++++++++++++++++ crates/mvm-hostd/src/mailbox_launch.rs | 38 +++++++ crates/mvm-hostd/src/mailbox_process.rs | 15 +-- 3 files changed, 179 insertions(+), 7 deletions(-) diff --git a/crates/mvm-hostd/src/mailbox_custody.rs b/crates/mvm-hostd/src/mailbox_custody.rs index 0cd6acf830..29856215ef 100644 --- a/crates/mvm-hostd/src/mailbox_custody.rs +++ b/crates/mvm-hostd/src/mailbox_custody.rs @@ -279,6 +279,18 @@ impl RetainedMailboxKeys { self.binding.history } + pub(crate) fn address(&self) -> MailboxAddress { + self.binding.address + } + + pub(crate) fn epoch(&self) -> ExecutionEpoch { + self.binding.epoch + } + + pub(crate) fn authority_key(&self) -> [u8; 32] { + self.binding.authority_key + } + pub(crate) fn payload_key(&self) -> Key { Key::from_bytes( self.material[32..64] @@ -426,6 +438,127 @@ mod tests { ); } + #[test] + fn retained_keys_reopen_the_same_encrypted_envelope_and_local_acceptance() { + use mvm_contract::protocol::mailbox::DeliveryOutcome; + use mvm_core::mailbox::frame::OutboxMessage; + use mvm_core::mailbox::journal::{ + FileJournalAnchor, JournalAdmission, JournalLimits, MailboxJournal, + }; + + let (dir, plan, address) = setup(); + let vm = plan.image.name.as_str(); + let posture = MailboxLaunchPosture::for_test(&plan, Variant::Dev); + let source_state = dir.path().join("source-vm"); + fs::create_dir(&source_state).expect("source VM state"); + let keys = RetainedMailboxKeys::provision(dir.path(), &plan, vm, &posture, address) + .expect("verified first provision"); + let derived = derive_launch_keys(dir.path(), &plan.mailbox.expect("grant")) + .expect("mailbox signing material") + .with_retained(&keys) + .expect("bind retained purpose keys to the canonical template"); + let (signer, journal_key, anchor_key) = derived.durable_material(); + assert_eq!( + journal_key, + *keys + .load_existing(&keys.history_reference()) + .expect("history key") + .expose_bytes() + ); + assert_eq!(anchor_key, keys.anchor_key()); + let (policy, mut sealer) = derived.trust(); + let payload_key = keys.payload_key(); + assert_eq!( + *policy.encryption_key.expose_bytes(), + *payload_key.expose_bytes() + ); + assert_eq!(sealer.header.encryption.key_id, keys.payload_key_id()); + let (capability, envelope) = sealer + .seal(OutboxMessage::new( + policy.context.expected_recipient, + b"hello".to_vec(), + )) + .expect("seal under retained payload key"); + let journal_root = dir.path().join("recipient-journal"); + let anchor_path = dir.path().join("recipient-anchor"); + let limits = JournalLimits { + max_records: 4, + max_bytes: 64 * 1024, + }; + let anchor = FileJournalAnchor::open(&anchor_path, address, keys.anchor_key()) + .expect("local anchor genesis"); + let mut journal = MailboxJournal::open_local_with_custody( + &journal_root, + keys.history_reference(), + signer.verifying_key(), + limits, + &keys, + &anchor, + ) + .expect("empty recipient journal"); + let accepted = journal + .admit_local( + JournalAdmission { + envelope: &envelope, + capability: &capability, + context: &policy.context, + encryption_key: &payload_key, + sender_key: &policy.sender_key, + signer: &signer, + }, + &anchor, + ) + .expect("local encrypted acceptance"); + assert_eq!( + accepted.receipt.outcome, + DeliveryOutcome::AcceptedLocalDurable + ); + assert_eq!(journal.tip(), anchor.tip()); + drop(journal); + drop(anchor); + drop(keys); + fs::remove_dir(&source_state).expect("source teardown"); + + let keys = RetainedMailboxKeys::open_existing( + dir.path(), + &plan, + vm, + &posture.handoff(None), + None, + address, + ) + .expect("authenticated restart binding"); + let anchor = FileJournalAnchor::open(&anchor_path, address, keys.anchor_key()) + .expect("retained local anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &journal_root, + keys.history_reference(), + signer.verifying_key(), + limits, + &keys, + &anchor, + ) + .expect("decrypt and authenticate retained envelope"); + let retry = journal + .admit_local( + JournalAdmission { + envelope: &envelope, + capability: &capability, + context: &policy.context, + encryption_key: &keys.payload_key(), + sender_key: &policy.sender_key, + signer: &signer, + }, + &anchor, + ) + .expect("recover original signed receipt"); + assert_eq!( + serde_jcs::to_vec(&retry).expect("retry bytes"), + serde_jcs::to_vec(&accepted).expect("original bytes") + ); + assert_eq!(anchor.tip().cursor, 1, "a retry cannot append"); + } + #[test] fn recovery_refuses_missing_or_changed_keys_and_signed_scope() { let (dir, plan, address) = setup(); diff --git a/crates/mvm-hostd/src/mailbox_launch.rs b/crates/mvm-hostd/src/mailbox_launch.rs index ddb5e24bb3..58f2e80b60 100644 --- a/crates/mvm-hostd/src/mailbox_launch.rs +++ b/crates/mvm-hostd/src/mailbox_launch.rs @@ -165,6 +165,44 @@ impl DerivedMailboxKeys { ) } + /// Bind the retained purpose keys to this host-derived mailbox identity. + #[cfg(test)] + pub(crate) fn with_retained( + mut self, + retained: &crate::mailbox_custody::RetainedMailboxKeys, + ) -> Result { + use mvm_core::mailbox::journal::HistoryKeyCustody as _; + + let header = &self.template.header; + let address = MailboxAddress { + tenant: header.tenant, + authorization_domain: header.authorization_domain, + mailbox: header.mailbox, + recipient: header.recipient, + }; + if address != retained.address() + || self.template.capability.address != address + || header.execution_epoch != retained.epoch() + || SigningKey::from_bytes(&self.authority) + .verifying_key() + .to_bytes() + != retained.authority_key() + { + return Err(Error::new( + ErrorKind::PermissionDenied, + "retained mailbox purpose keys do not match the admitted template", + )); + } + self.encryption = *retained.payload_key().expose_bytes(); + self.template.header.encryption.key_id = retained.payload_key_id(); + self.journal_mac = *retained + .load_existing(&retained.history_reference()) + .map_err(Error::other)? + .expose_bytes(); + self.anchor_mac = retained.anchor_key(); + Ok(self) + } + /// The host trust and sealer these keys stand for. Pure: no file is read, /// so it can be rebuilt on every dial attempt. pub fn trust(&self) -> (MailboxPolicy, GuestSealer) { diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index fd713bd193..67498b41d9 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -19,10 +19,11 @@ //! declares the capability. A grant on one that does not refuses the launch //! rather than boot a guest that asserts a mailbox nobody serves. //! -//! Until an independent checkpoint and persistent history-key provider exist, -//! only admitted Dev launches serve the ephemeral Send queue. Production -//! mailbox launches refuse before boot, and reader frames fail closed. -//! The local file anchor never authorizes a signed `AcceptedDurable` claim. +//! Until the separate local receipt, retained custody, retention, and guest +//! handoff are qualified end to end, only admitted Dev launches serve the +//! ephemeral Send queue. Production mailbox launches refuse before boot, and +//! reader frames fail closed. The local anchor never authorizes a signed +//! `AcceptedDurable` claim. use std::io::{Error, ErrorKind, Result, Write as _}; use std::os::unix::process::CommandExt as _; @@ -160,7 +161,7 @@ fn serve_ephemeral(config: MailboxProcessConfig) -> Result<()> { if posture.variant() != Variant::Dev || plan.mailbox != Some(grant) { return Err(Error::new( ErrorKind::PermissionDenied, - "production mailbox requires an independent checkpoint and key custody", + "production mailbox local-durable path is not yet qualified", )); } let (policy, sealer) = keys.trust(); @@ -325,7 +326,7 @@ pub fn require_servable( if posture.variant() != Variant::Dev { return Err(Error::new( ErrorKind::PermissionDenied, - "production mailbox needs an independent checkpoint and persistent key custody", + "production mailbox local-durable path is not yet qualified", ) .into()); } @@ -398,7 +399,7 @@ pub fn spawn_for_launch( if posture.variant() != Variant::Dev { return Err(Error::new( ErrorKind::PermissionDenied, - "production mailbox needs independent durability", + "production mailbox local-durable path is not yet qualified", )); } require_wired(plan, backend)?; From c656ac70a6d0e4e87ade591f3c1dd152eb933cba Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 18:24:40 -0800 Subject: [PATCH 34/51] feat(mailbox): bind signed local retention to custody --- .../src/bin/mvm-guest-agent/mailbox_reader.rs | 1 + crates/mvm-cli/src/exec/transient.rs | 1 + .../mvm-client/src/admission/mailbox_grant.rs | 1 + crates/mvm-contract/src/protocol/mailbox.rs | 73 +++++++++++++++ crates/mvm-contract/tests/mailbox_delivery.rs | 1 + .../mvm-contract/tests/mailbox_retention.rs | 91 +++++++++++++++++++ crates/mvm-core/src/mailbox/journal.rs | 1 + .../mvm-core/src/mailbox/transport/tests.rs | 1 + crates/mvm-core/src/plan/signing.rs | 1 + crates/mvm-core/src/plan/synthesis.rs | 1 + crates/mvm-core/tests/mailbox_reader.rs | 1 + crates/mvm-hostd/src/mailbox_custody.rs | 86 +++++++++++++++++- crates/mvm-hostd/src/mailbox_launch.rs | 1 + crates/mvm-hostd/src/mailbox_process.rs | 1 + .../mvm-hostd/src/mailbox_process/reader.rs | 1 + crates/mvm-hostd/tests/mission_demo.rs | 3 + .../src/workload_runner/runner/sockets.rs | 1 + .../src/workload_runner/runner/spawner.rs | 1 + crates/mvm-vmm/src/host/cmdline.rs | 1 + crates/mvm-vmm/src/host/flowmux_identity.rs | 2 + .../057-mailbox-semantics-and-threat-model.md | 3 + 21 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 crates/mvm-contract/tests/mailbox_retention.rs diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs index e5909f88aa..df93b7c4ca 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox_reader.rs @@ -188,6 +188,7 @@ fn perform_exchange( authority_key: boot.authority_key, epoch: boot.execution_epoch, report: None, + retention: None, }; let authority = VerifyingKey::from_bytes(&boot.authority_key) .map_err(|_| Refusal::NotAuthorized)?; diff --git a/crates/mvm-cli/src/exec/transient.rs b/crates/mvm-cli/src/exec/transient.rs index ecb0daf716..8653a7e92f 100644 --- a/crates/mvm-cli/src/exec/transient.rs +++ b/crates/mvm-cli/src/exec/transient.rs @@ -721,6 +721,7 @@ mod tests { authority_key: [2; 32], epoch: ExecutionEpoch::new(1).unwrap(), report: None, + retention: None, }; let error = boot(Some(granted)).err().expect("the grant is refused"); assert!(format!("{error:#}").contains("mailbox"), "{error:#}"); diff --git a/crates/mvm-client/src/admission/mailbox_grant.rs b/crates/mvm-client/src/admission/mailbox_grant.rs index 2d076516b7..36aa6efebf 100644 --- a/crates/mvm-client/src/admission/mailbox_grant.rs +++ b/crates/mvm-client/src/admission/mailbox_grant.rs @@ -27,6 +27,7 @@ pub fn grant_for(mailbox: MailboxId, authority: &VerifyingKey) -> MailboxGrant { authority_key: authority.to_bytes(), epoch: ExecutionEpoch::new(1).expect("epoch 1 is nonzero"), report: None, + retention: None, } } diff --git a/crates/mvm-contract/src/protocol/mailbox.rs b/crates/mvm-contract/src/protocol/mailbox.rs index 60fa3403e0..42fb3cae8f 100644 --- a/crates/mvm-contract/src/protocol/mailbox.rs +++ b/crates/mvm-contract/src/protocol/mailbox.rs @@ -242,6 +242,69 @@ impl From for MailboxReportSendPolicyWire { } } +/// Signed, versioned retention window for a mailbox's dedup identity after expiry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde( + try_from = "MailboxRetentionPolicyWire", + into = "MailboxRetentionPolicyWire" +)] +pub struct MailboxRetentionPolicy { + retention_ms: u64, +} + +impl MailboxRetentionPolicy { + /// Refuse zero or a duration that cannot be checked against signed `i64` expiry. + pub fn new(retention_ms: u64) -> Result { + if retention_ms == 0 || retention_ms > i64::MAX as u64 { + return Err(MailboxContractError::InvalidRetentionPolicyBounds); + } + Ok(Self { retention_ms }) + } + + #[must_use] + pub const fn retention_ms(self) -> u64 { + self.retention_ms + } + + /// The last millisecond for which this signed grant retains a dedup identity. + /// Refuse negative expiry or overflow instead of wrapping the deadline. + pub fn retained_until(self, expires_at_unix_ms: i64) -> Result { + if expires_at_unix_ms < 0 { + return Err(MailboxContractError::InvalidRetentionPolicyBounds); + } + expires_at_unix_ms + .checked_add(self.retention_ms as i64) + .ok_or(MailboxContractError::InvalidRetentionPolicyBounds) + } +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct MailboxRetentionPolicyWire { + version: u16, + retention_ms: u64, +} + +impl TryFrom for MailboxRetentionPolicy { + type Error = MailboxContractError; + + fn try_from(wire: MailboxRetentionPolicyWire) -> Result { + if wire.version != 1 { + return Err(MailboxContractError::UnsupportedRetentionPolicyVersion); + } + Self::new(wire.retention_ms) + } +} + +impl From for MailboxRetentionPolicyWire { + fn from(policy: MailboxRetentionPolicy) -> Self { + Self { + version: 1, + retention_ms: policy.retention_ms, + } + } +} + /// What a signed `ExecutionPlan` admits for one workload's mailbox: which /// mailbox, the key whose capabilities are honored, and the execution epoch. /// Presence grants that workload its own mailbox, including the ability for @@ -261,6 +324,10 @@ pub struct MailboxGrant { /// on older signed plans so their canonical bytes remain unchanged. #[serde(default, skip_serializing_if = "Option::is_none")] pub report: Option, + /// Required for local-durable retention; absent legacy grants remain + /// unchanged and do not authorize this stronger acceptance tier. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retention: Option, } impl MailboxGrant { @@ -1316,6 +1383,10 @@ pub enum MailboxContractError { UnsupportedReportPolicyVersion, #[error("workload report policy bounds are invalid")] InvalidReportPolicyBounds, + #[error("mailbox retention policy version is unsupported")] + UnsupportedRetentionPolicyVersion, + #[error("mailbox retention policy bounds are invalid")] + InvalidRetentionPolicyBounds, #[error("workload report is not granted by the signed plan")] ReportNotGranted, #[error("workload report must be a cast without a reply route")] @@ -1350,6 +1421,8 @@ impl MailboxContractError { Self::InvalidEpochRange => "invalid_epoch_range", Self::UnsupportedReportPolicyVersion => "unsupported_report_policy_version", Self::InvalidReportPolicyBounds => "invalid_report_policy_bounds", + Self::UnsupportedRetentionPolicyVersion => "unsupported_retention_policy_version", + Self::InvalidRetentionPolicyBounds => "invalid_retention_policy_bounds", Self::ReportNotGranted => "report_not_granted", Self::InvalidReportMessageKind => "invalid_report_message_kind", Self::InvalidReportContentType => "invalid_report_content_type", diff --git a/crates/mvm-contract/tests/mailbox_delivery.rs b/crates/mvm-contract/tests/mailbox_delivery.rs index 00e88a4c31..2f6a996e9f 100644 --- a/crates/mvm-contract/tests/mailbox_delivery.rs +++ b/crates/mvm-contract/tests/mailbox_delivery.rs @@ -24,6 +24,7 @@ fn plan_grant() -> MailboxGrant { authority_key: [5; 32], epoch: ExecutionEpoch::new(7).expect("epoch"), report: None, + retention: None, } } diff --git a/crates/mvm-contract/tests/mailbox_retention.rs b/crates/mvm-contract/tests/mailbox_retention.rs new file mode 100644 index 0000000000..5fa2293d77 --- /dev/null +++ b/crates/mvm-contract/tests/mailbox_retention.rs @@ -0,0 +1,91 @@ +use mvm_contract::protocol::mailbox::{ + ExecutionEpoch, MailboxContractError, MailboxGrant, MailboxId, MailboxRetentionPolicy, +}; +use serde::Serialize; + +#[derive(Serialize)] +struct LegacyGrant { + mailbox: MailboxId, + authority_key: [u8; 32], + epoch: ExecutionEpoch, +} + +fn grant() -> MailboxGrant { + MailboxGrant { + mailbox: MailboxId::from_bytes([1; 32]), + authority_key: [2; 32], + epoch: ExecutionEpoch::new(3).expect("epoch"), + report: None, + retention: None, + } +} + +#[test] +fn absent_retention_preserves_legacy_signed_grant_bytes() { + let grant = grant(); + let old = LegacyGrant { + mailbox: grant.mailbox, + authority_key: grant.authority_key, + epoch: grant.epoch, + }; + assert_eq!( + serde_json::to_vec(&grant).expect("grant"), + serde_json::to_vec(&old).expect("legacy grant") + ); + let decoded: MailboxGrant = + serde_json::from_slice(&serde_json::to_vec(&old).expect("old bytes")) + .expect("old grant decodes"); + assert_eq!(decoded.retention, None); +} + +#[test] +fn a_finite_signed_window_roundtrips_without_becoming_a_default() { + let mut declared = grant(); + declared.retention = Some(MailboxRetentionPolicy::new(10_000).expect("finite window")); + let encoded = serde_json::to_value(declared).expect("encode"); + assert_eq!(encoded["retention"]["version"], 1); + assert_eq!(encoded["retention"]["retention_ms"], 10_000); + assert_eq!( + serde_json::from_value::(encoded).expect("decode"), + declared + ); + assert_eq!( + declared + .retention + .expect("retention") + .retained_until(50_000), + Ok(60_000) + ); + assert_eq!(grant().retention, None); +} + +#[test] +fn malformed_or_unbounded_retention_cannot_enter_a_signed_grant() { + assert_eq!( + MailboxRetentionPolicy::new(0), + Err(MailboxContractError::InvalidRetentionPolicyBounds) + ); + assert_eq!( + MailboxRetentionPolicy::new(u64::MAX), + Err(MailboxContractError::InvalidRetentionPolicyBounds) + ); + let window = MailboxRetentionPolicy::new(1).expect("one millisecond"); + assert_eq!( + window.retained_until(i64::MAX), + Err(MailboxContractError::InvalidRetentionPolicyBounds) + ); + assert_eq!( + window.retained_until(-1), + Err(MailboxContractError::InvalidRetentionPolicyBounds) + ); + for raw in [ + r#"{"version":2,"retention_ms":1000}"#, + r#"{"version":1,"retention_ms":0}"#, + r#"{"version":1,"retention_ms":1000,"unexpected":true}"#, + ] { + assert!( + serde_json::from_str::(raw).is_err(), + "{raw}" + ); + } +} diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 6b475c443b..81351ea79a 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -617,6 +617,7 @@ fn verify_recovered_lease( authority_key: authority.to_bytes(), epoch: lease.lease.execution_epoch, report: None, + retention: None, }; let scope = MailboxGuestReaderScope { plan: &plan, diff --git a/crates/mvm-core/src/mailbox/transport/tests.rs b/crates/mvm-core/src/mailbox/transport/tests.rs index 8f5b957d6e..7985f38c20 100644 --- a/crates/mvm-core/src/mailbox/transport/tests.rs +++ b/crates/mvm-core/src/mailbox/transport/tests.rs @@ -508,6 +508,7 @@ fn guest_lease_and_signed_ack_are_durable_replay_fenced_and_boot_bound() { authority_key: signer.verifying_key().to_bytes(), epoch: epoch(), report: None, + retention: None, }; let plan_id = PlanId(format!("sha256:{}", "a".repeat(64))); let plan_nonce = Nonce::from_bytes(id(52)); diff --git a/crates/mvm-core/src/plan/signing.rs b/crates/mvm-core/src/plan/signing.rs index 19ca9071b3..829e9f5d18 100644 --- a/crates/mvm-core/src/plan/signing.rs +++ b/crates/mvm-core/src/plan/signing.rs @@ -504,6 +504,7 @@ mod tests { authority_key: [9; 32], epoch: ExecutionEpoch::new(3).unwrap(), report: None, + retention: None, } } diff --git a/crates/mvm-core/src/plan/synthesis.rs b/crates/mvm-core/src/plan/synthesis.rs index b394974431..2dca81b573 100644 --- a/crates/mvm-core/src/plan/synthesis.rs +++ b/crates/mvm-core/src/plan/synthesis.rs @@ -1627,6 +1627,7 @@ mod tests { authority_key: [9; 32], epoch: ExecutionEpoch::new(1).unwrap(), report: None, + retention: None, }; let mut inp = input("vm"); inp.mailbox = Some(grant); diff --git a/crates/mvm-core/tests/mailbox_reader.rs b/crates/mvm-core/tests/mailbox_reader.rs index 9a77d3a823..dd1c6f38be 100644 --- a/crates/mvm-core/tests/mailbox_reader.rs +++ b/crates/mvm-core/tests/mailbox_reader.rs @@ -43,6 +43,7 @@ fn fixture() -> ( authority_key: authority.verifying_key().to_bytes(), epoch, report: None, + retention: None, }; let plan_id = PlanId(format!("sha256:{}", "a".repeat(64))); let plan_nonce = Nonce::from_bytes([7; 16]); diff --git a/crates/mvm-hostd/src/mailbox_custody.rs b/crates/mvm-hostd/src/mailbox_custody.rs index 29856215ef..d8b7c36df2 100644 --- a/crates/mvm-hostd/src/mailbox_custody.rs +++ b/crates/mvm-hostd/src/mailbox_custody.rs @@ -40,6 +40,7 @@ struct Binding { vm_name: String, address: MailboxAddress, epoch: ExecutionEpoch, + retention_ms: u64, authority_key: [u8; 32], host_signer: [u8; 32], key_version: u32, @@ -62,6 +63,7 @@ struct Identity<'a> { vm_name: &'a str, address: MailboxAddress, epoch: ExecutionEpoch, + retention_ms: u64, } /// Three independent purpose keys in one no-clobber, fsynced private bundle. @@ -129,6 +131,10 @@ fn identity<'a>( let grant = plan .mailbox .context("admitted plan does not grant a mailbox")?; + let retention_ms = grant + .retention + .context("admitted plan does not declare local mailbox retention")? + .retention_ms(); ensure!( address.mailbox == grant.mailbox && plan.image.name == vm_name, "mailbox custody identity does not match the admitted VM" @@ -150,6 +156,7 @@ fn identity<'a>( vm_name, address, epoch: grant.epoch, + retention_ms, }) } @@ -189,6 +196,7 @@ impl RetainedMailboxKeys { vm_name: vm_name.to_owned(), address, epoch: identity.epoch, + retention_ms: identity.retention_ms, authority_key: plan.mailbox.context("mailbox grant")?.authority_key, host_signer: signer.verifying_key().to_bytes(), key_version: KEY_VERSION, @@ -247,6 +255,7 @@ impl RetainedMailboxKeys { && binding.vm_name == identity.vm_name && binding.address == address && binding.epoch == identity.epoch + && binding.retention_ms == identity.retention_ms && binding.authority_key == plan.mailbox.context("mailbox grant")?.authority_key && binding.host_signer == signer.verifying_key().to_bytes() && binding.key_version == KEY_VERSION @@ -326,7 +335,9 @@ impl HistoryKeyCustody for RetainedMailboxKeys { #[cfg(test)] mod tests { use super::*; - use mvm_contract::protocol::mailbox::{AuthorizationDomainId, MailboxGrant, MailboxId}; + use mvm_contract::protocol::mailbox::{ + AuthorizationDomainId, MailboxGrant, MailboxId, MailboxRetentionPolicy, + }; use mvm_core::plan::{Variant, compute_plan_id, test_support::PlanFixture}; use std::fs; @@ -342,6 +353,7 @@ mod tests { .to_bytes(), epoch: ExecutionEpoch::new(1).expect("epoch"), report: None, + retention: Some(MailboxRetentionPolicy::new(10_000).expect("signed retention")), }); plan.plan_id = compute_plan_id(&plan); let (policy, _) = derive_launch_keys(dir.path(), &plan.mailbox.expect("grant")) @@ -356,6 +368,78 @@ mod tests { (dir, plan, address) } + #[test] + fn local_custody_requires_and_binds_the_signed_retention_window() { + let (dir, plan, address) = setup(); + let vm = plan.image.name.as_str(); + let posture = MailboxLaunchPosture::for_test(&plan, Variant::Dev); + let mut absent = plan.clone(); + absent.mailbox.as_mut().expect("grant").retention = None; + absent.plan_id = compute_plan_id(&absent); + let absent_posture = MailboxLaunchPosture::for_test(&absent, Variant::Dev); + assert!( + RetainedMailboxKeys::provision(dir.path(), &absent, vm, &absent_posture, address,) + .is_err(), + "legacy signed plans cannot provision local custody" + ); + let retained = RetainedMailboxKeys::provision(dir.path(), &plan, vm, &posture, address) + .expect("declared retention"); + assert_eq!(retained.binding.retention_ms, 10_000); + assert!( + RetainedMailboxKeys::open_existing( + dir.path(), + &absent, + vm, + &absent_posture.handoff(None), + None, + address, + ) + .is_err(), + "legacy signed plans cannot recover local custody either" + ); + + let mut changed = plan.clone(); + changed.mailbox.as_mut().expect("grant").retention = + Some(MailboxRetentionPolicy::new(20_000).expect("changed window")); + changed.plan_id = compute_plan_id(&changed); + let changed_posture = MailboxLaunchPosture::for_test(&changed, Variant::Dev); + assert!( + RetainedMailboxKeys::open_existing( + dir.path(), + &changed, + vm, + &changed_posture.handoff(None), + None, + address, + ) + .is_err(), + "a different signed retention policy cannot inherit prior custody" + ); + let identity = identity(dir.path(), &plan, vm, address).expect("trusted identity"); + let (_, manifest) = paths(dir.path(), &identity).expect("scope path"); + let mut forged: serde_json::Value = + serde_json::from_slice(&fs::read(&manifest).expect("signed manifest")) + .expect("manifest JSON"); + forged["binding"]["retention_ms"] = serde_json::json!(20_000); + fs::write( + &manifest, + serde_json::to_vec(&forged).expect("forged manifest"), + ) + .expect("alter manifest"); + assert!( + RetainedMailboxKeys::open_existing( + dir.path(), + &plan, + vm, + &posture.handoff(None), + None, + address, + ) + .is_err(), + "a modified manifest window must fail signature verification" + ); + } + #[test] fn purpose_keys_and_signed_manifest_survive_restart_without_launch_secret() { let (dir, plan, address) = setup(); diff --git a/crates/mvm-hostd/src/mailbox_launch.rs b/crates/mvm-hostd/src/mailbox_launch.rs index 58f2e80b60..a6f5184905 100644 --- a/crates/mvm-hostd/src/mailbox_launch.rs +++ b/crates/mvm-hostd/src/mailbox_launch.rs @@ -493,6 +493,7 @@ mod tests { authority_key: authority_key_for(dir, &mailbox).unwrap().to_bytes(), epoch: ExecutionEpoch::new(1).unwrap(), report: None, + retention: None, } } diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index 67498b41d9..001da50ec3 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -493,6 +493,7 @@ mod tests { .to_bytes(), epoch: ExecutionEpoch::new(1).unwrap(), report: None, + retention: None, } } diff --git a/crates/mvm-hostd/src/mailbox_process/reader.rs b/crates/mvm-hostd/src/mailbox_process/reader.rs index 3e9901a848..afb8fef974 100644 --- a/crates/mvm-hostd/src/mailbox_process/reader.rs +++ b/crates/mvm-hostd/src/mailbox_process/reader.rs @@ -218,6 +218,7 @@ mod tests { .to_bytes(), epoch: ExecutionEpoch::new(1).unwrap(), report: None, + retention: None, }; let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); plan.mailbox = Some(grant); diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index a1dd062a81..254c802392 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -774,6 +774,7 @@ fn plan_and_trust(channel: &Channel, granted: bool) -> (ExecutionPlan, MailboxPo authority_key: channel.authority_key.verifying_key().to_bytes(), epoch: MbEpoch::new(1).unwrap(), report: None, + retention: None, }); } let trust = MailboxPolicy { @@ -914,6 +915,7 @@ fn granted_plan(authority_key: [u8; 32]) -> ExecutionPlan { authority_key, epoch: MbEpoch::new(1).unwrap(), report: None, + retention: None, }); plan } @@ -1550,6 +1552,7 @@ fn signed_guest_lease_and_exact_ack_survive_mock_anchored_restart() { authority_key: authority.to_bytes(), epoch: MbEpoch::new(1).unwrap(), report: None, + retention: None, }; let mut reader_grant = grant(&[MailboxOperation::Receive]); reader_grant.subject = aircraft_principal(); diff --git a/crates/mvm-runtime/src/workload_runner/runner/sockets.rs b/crates/mvm-runtime/src/workload_runner/runner/sockets.rs index f528927cf1..b536522b73 100644 --- a/crates/mvm-runtime/src/workload_runner/runner/sockets.rs +++ b/crates/mvm-runtime/src/workload_runner/runner/sockets.rs @@ -137,6 +137,7 @@ mod tests { authority_key: [0; 32], epoch: mvm_contract::protocol::mailbox::ExecutionEpoch::new(1).unwrap(), report: None, + retention: None, }); let config = VmStartConfig { plan_json: Some(serde_json::to_string(&plan).unwrap()), diff --git a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs index e7a691b721..49f24af058 100644 --- a/crates/mvm-runtime/src/workload_runner/runner/spawner.rs +++ b/crates/mvm-runtime/src/workload_runner/runner/spawner.rs @@ -770,6 +770,7 @@ mod tests { authority_key: host.verifying_key().to_bytes(), epoch: ExecutionEpoch::new(3).unwrap(), report: None, + retention: None, }); plan.plan_id = mvm_core::plan::compute_plan_id(&plan); let redaction = RedactionPolicy::default(); diff --git a/crates/mvm-vmm/src/host/cmdline.rs b/crates/mvm-vmm/src/host/cmdline.rs index bf44c9dcbc..3baa4aacb4 100644 --- a/crates/mvm-vmm/src/host/cmdline.rs +++ b/crates/mvm-vmm/src/host/cmdline.rs @@ -552,6 +552,7 @@ mod tests { authority_key: [2; 32], epoch: ExecutionEpoch::new(1).unwrap(), report: None, + retention: None, })); let absent = build(None); let tokens = |c: &str| c.split_whitespace().map(str::to_owned).collect::>(); diff --git a/crates/mvm-vmm/src/host/flowmux_identity.rs b/crates/mvm-vmm/src/host/flowmux_identity.rs index 3101660fa3..5174992142 100644 --- a/crates/mvm-vmm/src/host/flowmux_identity.rs +++ b/crates/mvm-vmm/src/host/flowmux_identity.rs @@ -545,6 +545,7 @@ mod tests { authority_key: anchor.to_bytes(), epoch: ExecutionEpoch::new(3).unwrap(), report: None, + retention: None, }); plan.plan_id = mvm_core::plan::compute_plan_id(&plan); let binding = MailboxReaderBootBinding::mint( @@ -597,6 +598,7 @@ mod tests { authority_key: host_key().verifying_key().to_bytes(), epoch: ExecutionEpoch::new(3).unwrap(), report: None, + retention: None, }); plan.plan_id = mvm_core::plan::compute_plan_id(&plan); let dir = tempfile::tempdir().unwrap(); diff --git a/specs/adrs/057-mailbox-semantics-and-threat-model.md b/specs/adrs/057-mailbox-semantics-and-threat-model.md index df13e53724..addfac7b14 100644 --- a/specs/adrs/057-mailbox-semantics-and-threat-model.md +++ b/specs/adrs/057-mailbox-semantics-and-threat-model.md @@ -320,6 +320,9 @@ path. For the standalone local tier, a signed plan must declare a finite retention window before admission; an absent policy does not imply an infinite default. +The versioned optional `MailboxGrant.retention` preserves old signed-plan +bytes when absent; a present duration is an explicit request bounded by host +policy, and expiry-plus-retention arithmetic fails closed on overflow. Compaction first durably publishes an encrypted, destination-authenticated snapshot of still-live delivery state and dedup tombstones through each message's expiry plus that window, then advances the local chain head, and From af9ac5c4cf0c8ac04b78ecc06badae64972608c1 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 18:29:06 -0800 Subject: [PATCH 35/51] fix(mailbox): keep local journal opener test-scoped until handoff --- crates/mvm-core/src/mailbox/journal.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 81351ea79a..bca048fb52 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -748,11 +748,12 @@ impl MailboxJournal { ) } - /// Reopen the software-only local tier against a fsynced same-host anchor. + /// Reopen a test-scoped software-only local tier against a fsynced same-host anchor. /// - /// The trusted host must obtain the reference and existing key from an - /// authenticated restart manifest bound to its admitted plan/VM/mailbox. - /// A deserialized reference alone is never authorization to open history. + /// Production must first supply an authenticated restart binding to the + /// admitted plan/VM/mailbox and the signed retention policy. A deserialized + /// reference alone is never authorization to open history. + #[cfg(any(test, feature = "test-support"))] pub fn open_local_with_custody( root: &Path, reference: HistoryKeyRef, From bf8972b653f1a4cb26c874667a9f76d1498c080b Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 18:52:00 -0800 Subject: [PATCH 36/51] feat(mailbox): enforce local retention window at admission --- crates/mvm-core/src/mailbox/journal.rs | 77 +++++++++++++++++++++++-- crates/mvm-core/tests/mailbox_crypto.rs | 4 ++ crates/mvm-hostd/src/mailbox_custody.rs | 9 ++- crates/mvm-hostd/tests/mission_demo.rs | 4 ++ 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index bca048fb52..b56d0e1a2e 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -18,9 +18,10 @@ use hmac::{Hmac, Mac, digest::KeyInit}; use mvm_contract::protocol::mailbox::{ ContentType, DELIVERY_ACK_SIGNATURE_DOMAIN, DeliveryLeaseNonce, DeliveryOutcome, DestinationReceipt, EncryptionAlgorithm, ExecutionEpoch, MailboxAddress, MailboxDeliveryLease, - MailboxEnvelope, MailboxGrant, MailboxGuestReaderScope, MailboxOperation, ProtocolVersion, - SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, - SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, + MailboxEnvelope, MailboxGrant, MailboxGuestReaderScope, MailboxOperation, + MailboxRetentionPolicy, ProtocolVersion, SignatureAlgorithm, SignedDestinationReceipt, + SignedMailboxCapability, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, + SignedMailboxGuestReaderEnrollment, }; use serde::{Deserialize, Serialize}; use sha2::Sha256; @@ -242,6 +243,7 @@ struct JournalKey { bytes: Zeroizing<[u8; 32]>, reference: Option, tier: AcceptanceTier, + retention: Option, } #[derive(Clone, Copy, PartialEq, Eq)] @@ -256,6 +258,7 @@ impl JournalKey { bytes: Zeroizing::new(bytes), reference: None, tier: AcceptanceTier::Independent, + retention: None, } } } @@ -331,6 +334,7 @@ pub struct MailboxJournal { key: Zeroizing<[u8; 32]>, history_key: Option, tier: AcceptanceTier, + retention: Option, limits: JournalLimits, tip: JournalTip, bytes: u64, @@ -742,6 +746,7 @@ impl MailboxJournal { bytes: Zeroizing::new(*key.expose_bytes()), reference: Some(reference), tier: AcceptanceTier::Independent, + retention: None, }, limits, anchor, @@ -761,6 +766,7 @@ impl MailboxJournal { limits: JournalLimits, custody: &impl HistoryKeyCustody, anchor: &FileJournalAnchor, + retention: MailboxRetentionPolicy, ) -> Result { reference.validate()?; if reference.address != anchor.address { @@ -775,6 +781,7 @@ impl MailboxJournal { bytes: Zeroizing::new(*key.expose_bytes()), reference: Some(reference), tier: AcceptanceTier::Local, + retention: Some(retention), }, limits, anchor.tip(), @@ -831,6 +838,7 @@ impl MailboxJournal { key: key.bytes, history_key: key.reference, tier: key.tier, + retention: key.retention, limits, tip: JournalTip::genesis(), bytes: 0, @@ -937,6 +945,11 @@ impl MailboxJournal { frame::decode_frame(&raw).map_err(|_| JournalError::Corrupt)?; if receipt.delivery_id != envelope.routing.delivery_id || receipt.ciphertext_commitment != envelope.ciphertext_commitment + || journal.retention.is_some_and(|policy| { + policy + .retained_until(envelope.routing.expires_at_unix_ms) + .is_err() + }) || journal .seen .insert( @@ -1184,6 +1197,12 @@ impl MailboxJournal { }) { return Err(JournalError::HistoryKeyMismatch); } + if tier == AcceptanceTier::Local { + self.retention + .ok_or(JournalError::DurableAuthorityUnavailable)? + .retained_until(envelope.routing.expires_at_unix_ms) + .map_err(|_| JournalError::InvalidLimits)?; + } let mut probe = Probe::default(); let opened = open_authenticated_envelope( envelope, @@ -1939,6 +1958,10 @@ mod anchor_tests { } } + fn local_retention() -> MailboxRetentionPolicy { + MailboxRetentionPolicy::new(10_000).expect("finite test retention") + } + fn versioned_custody() -> (HistoryKeyRef, TestCustody) { let reference = HistoryKeyRef::new( address(), @@ -2149,6 +2172,7 @@ mod anchor_tests { limits(), &custody, &anchor, + local_retention(), ) .expect("empty local journal"); let accepted = journal @@ -2184,6 +2208,7 @@ mod anchor_tests { limits(), &custody, &anchor, + local_retention(), ) .expect("recover retained ciphertext and dedup"); context.now_unix_ms += 1; @@ -2228,12 +2253,52 @@ mod anchor_tests { signer.verifying_key(), limits(), &custody, - &anchor + &anchor, + local_retention(), ), Err(JournalError::Rollback) )); } + #[test] + fn local_acceptance_refuses_retention_deadline_overflow_without_a_record() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let anchor_path = dir.path().join("anchor").join("tip.json"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let (reference, custody) = versioned_custody(); + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("local anchor"); + let policy = mvm_contract::protocol::mailbox::MailboxRetentionPolicy::new(i64::MAX as u64) + .expect("finite but overflowing for this envelope"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + policy, + ) + .expect("empty journal"); + assert!(matches!( + journal.admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ), + Err(JournalError::InvalidLimits) + )); + assert_eq!(journal.tip(), JournalTip::genesis()); + assert_eq!(anchor.tip(), JournalTip::genesis()); + } + #[test] fn interrupted_terminal_expiry_reconciles_without_an_early_receipt() { let dir = tempfile::tempdir().expect("journal root"); @@ -2250,6 +2315,7 @@ mod anchor_tests { limits(), &custody, &anchor, + local_retention(), ) .expect("empty journal"); journal @@ -2298,6 +2364,7 @@ mod anchor_tests { limits(), &custody, &anchor, + local_retention(), ) .expect("verify and reconcile terminal suffix"); assert_eq!(anchor.tip().cursor, 2); @@ -2360,6 +2427,7 @@ mod anchor_tests { limits(), &custody, &anchor, + local_retention(), ) .expect("empty journal"); journal @@ -2421,6 +2489,7 @@ mod anchor_tests { limits(), &custody, &anchor, + local_retention(), ), Err(JournalError::Corrupt) )); diff --git a/crates/mvm-core/tests/mailbox_crypto.rs b/crates/mvm-core/tests/mailbox_crypto.rs index 0912e0d222..9570ebb308 100644 --- a/crates/mvm-core/tests/mailbox_crypto.rs +++ b/crates/mvm-core/tests/mailbox_crypto.rs @@ -445,6 +445,8 @@ fn local_journal_expiry_recovers_after_interrupted_anchor_commit() { limits, &custody, &anchor, + mvm_contract::protocol::mailbox::MailboxRetentionPolicy::new(10_000) + .expect("test retention"), ) .expect("empty local journal"); let accepted = journal @@ -484,6 +486,8 @@ fn local_journal_expiry_recovers_after_interrupted_anchor_commit() { limits, &custody, &anchor, + mvm_contract::protocol::mailbox::MailboxRetentionPolicy::new(10_000) + .expect("test retention"), ) .expect("reconcile verified terminal state"); let expired = journal diff --git a/crates/mvm-hostd/src/mailbox_custody.rs b/crates/mvm-hostd/src/mailbox_custody.rs index d8b7c36df2..68c662c5dd 100644 --- a/crates/mvm-hostd/src/mailbox_custody.rs +++ b/crates/mvm-hostd/src/mailbox_custody.rs @@ -12,7 +12,7 @@ use anyhow::{Context as _, Result, ensure}; use ed25519_dalek::{Signature, Signer as _, SigningKey, Verifier as _}; use mvm_contract::plan::{Nonce, PlanId}; use mvm_contract::protocol::mailbox::{ - BootId, EncryptionAlgorithm, ExecutionEpoch, MailboxAddress, + BootId, EncryptionAlgorithm, ExecutionEpoch, MailboxAddress, MailboxRetentionPolicy, }; use mvm_core::crypto::aead::Key; use mvm_core::mailbox::journal::{HistoryKeyCustody, HistoryKeyRef, JournalError}; @@ -296,6 +296,11 @@ impl RetainedMailboxKeys { self.binding.epoch } + pub(crate) fn retention(&self) -> Result { + MailboxRetentionPolicy::new(self.binding.retention_ms) + .context("signed mailbox retention is invalid") + } + pub(crate) fn authority_key(&self) -> [u8; 32] { self.binding.authority_key } @@ -578,6 +583,7 @@ mod tests { limits, &keys, &anchor, + keys.retention().expect("signed window"), ) .expect("empty recipient journal"); let accepted = journal @@ -621,6 +627,7 @@ mod tests { limits, &keys, &anchor, + keys.retention().expect("signed window"), ) .expect("decrypt and authenticate retained envelope"); let retry = journal diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index 254c802392..6dc4cb3ac0 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -1222,6 +1222,8 @@ fn local_mailbox_service_returns_exact_signed_v2_receipt_after_restart() { limits, &custody, &anchor, + mvm_contract::protocol::mailbox::MailboxRetentionPolicy::new(10_000) + .expect("test retention"), ) .expect("recover local journal"); DurableMailboxService::bind_local( @@ -1375,6 +1377,8 @@ fn local_mailbox_service_returns_exact_signed_v2_receipt_after_restart() { limits, &custody, &foreign_anchor, + mvm_contract::protocol::mailbox::MailboxRetentionPolicy::new(10_000) + .expect("test retention"), ) .expect("empty history under a different authority"); assert!( From e189e1f711f7115c4731205487940eaf3b9d657f Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 19:12:00 -0800 Subject: [PATCH 37/51] docs(mailbox): require configured local retention ceiling --- specs/adrs/057-mailbox-semantics-and-threat-model.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specs/adrs/057-mailbox-semantics-and-threat-model.md b/specs/adrs/057-mailbox-semantics-and-threat-model.md index addfac7b14..316239832c 100644 --- a/specs/adrs/057-mailbox-semantics-and-threat-model.md +++ b/specs/adrs/057-mailbox-semantics-and-threat-model.md @@ -323,6 +323,12 @@ window before admission; an absent policy does not imply an infinite default. The versioned optional `MailboxGrant.retention` preserves old signed-plan bytes when absent; a present duration is an explicit request bounded by host policy, and expiry-plus-retention arithmetic fails closed on overflow. +The workload configures the requested window before the plan is signed; the +host operator independently configures a maximum. Neither has a permissive +default: without both, local-durable admission refuses, and a request above +the host maximum refuses rather than being silently clamped. Changing the +maximum affects new acceptance, not the recovery or retention deadline of +messages already accepted under a verified signed plan and custody binding. Compaction first durably publishes an encrypted, destination-authenticated snapshot of still-live delivery state and dedup tombstones through each message's expiry plus that window, then advances the local chain head, and From 9612cb2deaa15a966252b205d87115ed5c9b080c Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 19:46:56 -0800 Subject: [PATCH 38/51] feat(mailbox): configure signed retention and host ceiling --- .../mvm-cli/src/commands/machine/runtime.rs | 1 + crates/mvm-cli/src/commands/tests.rs | 53 +++++++++++ crates/mvm-cli/src/commands/vm/exec.rs | 8 ++ .../src/commands/vm/exec/run_args_default.rs | 1 + .../mvm-client/src/admission/mailbox_grant.rs | 75 +++++++++++++++- crates/mvm-core/src/user_config.rs | 28 +++++- crates/mvm-hostd/src/plan_admission.rs | 90 +++++++++++++++++++ .../content/docs/reference/cli-commands.md | 9 ++ 8 files changed, 259 insertions(+), 6 deletions(-) diff --git a/crates/mvm-cli/src/commands/machine/runtime.rs b/crates/mvm-cli/src/commands/machine/runtime.rs index cdc4f63602..8e530af43a 100644 --- a/crates/mvm-cli/src/commands/machine/runtime.rs +++ b/crates/mvm-cli/src/commands/machine/runtime.rs @@ -339,6 +339,7 @@ pub(crate) fn start_args_with_grant(args: &MachineRunArgs, name: &str) -> Result let mut start = start_args_for_run(args, name); start.mailbox = mvm_client::admission::mailbox_grant::resolve( args.run.mailbox.as_deref(), + args.run.mailbox_retention_ms, args.run.from_workload_ir.as_deref(), )?; Ok(start) diff --git a/crates/mvm-cli/src/commands/tests.rs b/crates/mvm-cli/src/commands/tests.rs index 07e54d867e..4d81f2e0ed 100644 --- a/crates/mvm-cli/src/commands/tests.rs +++ b/crates/mvm-cli/src/commands/tests.rs @@ -4336,6 +4336,59 @@ fn run_timeout_parses_to_some() { } } +#[test] +fn mailbox_retention_flag_is_explicit_on_machine_runs() { + let mailbox = "ab".repeat(32); + let cli = Cli::try_parse_from([ + "mvmctl", + "machine", + "run", + "--image", + "alpine", + "--mailbox", + &mailbox, + "--mailbox-retention-ms", + "10000", + "--", + "true", + ]) + .expect("parse explicit mailbox window"); + let Commands::Machine(group) = cli.command else { + panic!("expected machine command"); + }; + let machine::MachineAction::Run(args) = group.action else { + panic!("expected machine run"); + }; + assert_eq!(args.run.mailbox_retention_ms, Some(10_000)); + let direct = Cli::try_parse_from([ + "mvmctl", + "run", + "--mailbox", + &mailbox, + "--mailbox-retention-ms", + "10000", + "--", + "true", + ]) + .expect("parse direct run retention"); + let Commands::Run(run) = direct.command else { + panic!("expected direct run"); + }; + assert_eq!(run.machine.run.mailbox_retention_ms, Some(10_000)); + assert!( + Cli::try_parse_from([ + "mvmctl", + "machine", + "run", + "--mailbox-retention-ms", + "0", + "--", + "true", + ]) + .is_err() + ); +} + #[test] fn machine_run_interactive_image_shell_dx_parses() { let args = parse_machine_run(&["--net", "-it", "--image", "alpine", "--", "/bin/sh"]) diff --git a/crates/mvm-cli/src/commands/vm/exec.rs b/crates/mvm-cli/src/commands/vm/exec.rs index 6667b37fa1..743aafb74b 100644 --- a/crates/mvm-cli/src/commands/vm/exec.rs +++ b/crates/mvm-cli/src/commands/vm/exec.rs @@ -246,6 +246,9 @@ pub(in crate::commands) struct RunArgs { /// Grant a mailbox, named by a 64-character hex id. #[arg(long, value_name = "ID")] pub mailbox: Option, + /// Retain local mailbox dedup identities this many milliseconds after expiry. + #[arg(long, value_name = "MILLISECONDS", value_parser = clap::value_parser!(u64).range(1..))] + pub mailbox_retention_ms: Option, /// Set memory (for example, 512M or 1G). #[arg(long, default_value = "512M")] pub memory: String, @@ -677,6 +680,7 @@ pub(in crate::commands) fn run_secure_started( let admit_outputs = outputs.grants(); let admit_mailbox = mvm_client::admission::mailbox_grant::resolve( args.mailbox.as_deref(), + args.mailbox_retention_ms, args.from_workload_ir.as_deref(), )?; let host_config = mvm_core::user_config::load(None); @@ -1937,6 +1941,10 @@ mod tests { parsed.machine.run.cpu_limit, expected.machine.run.cpu_limit, "--cpu-limit default" ); + assert_eq!( + parsed.machine.run.mailbox_retention_ms, expected.machine.run.mailbox_retention_ms, + "--mailbox-retention-ms default" + ); assert_eq!( parsed.machine.run.allow_host, expected.machine.run.allow_host, "--allow-host default" diff --git a/crates/mvm-cli/src/commands/vm/exec/run_args_default.rs b/crates/mvm-cli/src/commands/vm/exec/run_args_default.rs index 6e20488ef7..d823003a1d 100644 --- a/crates/mvm-cli/src/commands/vm/exec/run_args_default.rs +++ b/crates/mvm-cli/src/commands/vm/exec/run_args_default.rs @@ -48,6 +48,7 @@ impl Default for RunArgs { cpu_limit: None, grants_file: None, mailbox: None, + mailbox_retention_ms: None, memory: "512M".to_string(), explicit_resources: Default::default(), profile: RunProfile::Dev, diff --git a/crates/mvm-client/src/admission/mailbox_grant.rs b/crates/mvm-client/src/admission/mailbox_grant.rs index 36aa6efebf..ff68650eef 100644 --- a/crates/mvm-client/src/admission/mailbox_grant.rs +++ b/crates/mvm-client/src/admission/mailbox_grant.rs @@ -5,7 +5,9 @@ use anyhow::{Context, Result, ensure}; use ed25519_dalek::VerifyingKey; -use mvm_contract::protocol::mailbox::{ExecutionEpoch, MailboxGrant, MailboxId}; +use mvm_contract::protocol::mailbox::{ + ExecutionEpoch, MailboxGrant, MailboxId, MailboxRetentionPolicy, +}; /// Parse a 64-character hex mailbox id. pub fn parse_mailbox_id(raw: &str) -> Result { @@ -63,10 +65,27 @@ pub fn combine(flag: Option, declared: Option) -> Result, + mailbox: Option, +) -> Result> { + let Some(requested_ms) = requested_ms else { + return Ok(None); + }; + ensure!( + mailbox.is_some(), + "--mailbox-retention-ms requires a mailbox grant" + ); + MailboxRetentionPolicy::new(requested_ms) + .context("--mailbox-retention-ms must be a finite, nonzero duration") + .map(Some) +} + /// Resolve both sources against the host signer. Neither means no grant and no /// key is loaded. pub fn resolve( flag: Option<&str>, + retention_ms: Option, workload_ir: Option<&std::path::Path>, ) -> Result> { let flag = flag.map(parse_mailbox_id).transpose()?; @@ -75,7 +94,9 @@ pub fn resolve( .map(declared_in) .transpose()? .flatten(); - let Some(id) = combine(flag, declared)? else { + let id = combine(flag, declared)?; + let retention = retention_request(retention_ms, id)?; + let Some(id) = id else { return Ok(None); }; let keys_dir = mvm_core::config::mvm_keys_dir(); @@ -84,7 +105,9 @@ pub fn resolve( .context("loading the host signer as the mailbox authority")?; let authority = mvm_hostd::mailbox_launch::authority_key_for(&keys_dir, &id) .context("deriving the mailbox authority")?; - Ok(Some(grant_for(id, &authority))) + let mut grant = grant_for(id, &authority); + grant.retention = retention; + Ok(Some(grant)) } #[cfg(test)] @@ -109,7 +132,7 @@ mod tests { #[test] fn no_flag_grants_nothing() { - assert!(resolve(None, None).unwrap().is_none()); + assert!(resolve(None, None, None).unwrap().is_none()); } fn workload_with(ext: serde_json::Value) -> mvm_contract::ir::Workload { @@ -144,4 +167,48 @@ mod tests { assert!(combine(Some(a), Some(b)).is_err()); assert_eq!(combine(None, None).unwrap(), None); } + + #[test] + fn requested_retention_needs_a_mailbox_and_is_signed_into_its_grant() { + let id = MailboxId::from_bytes([1; 32]); + assert!(retention_request(Some(10_000), None).is_err()); + assert!(retention_request(Some(0), Some(id)).is_err()); + assert!(retention_request(Some(u64::MAX), Some(id)).is_err()); + assert_eq!(retention_request(None, Some(id)).unwrap(), None); + let requested = retention_request(Some(10_000), Some(id)) + .unwrap() + .expect("finite window"); + let mut grant = grant_for( + id, + &ed25519_dalek::SigningKey::from_bytes(&[5; 32]).verifying_key(), + ); + grant.retention = Some(requested); + assert_eq!( + serde_json::to_value(grant).unwrap()["retention"]["retention_ms"], + 10_000 + ); + + let home = tempfile::tempdir().expect("isolated host home"); + let mut env = mvm_core::util::test_env::TestEnv::new(); + env.isolate_mvm_home(home.path()); + let flag = "ab".repeat(32); + assert!(resolve(Some(&flag), Some(0), None).is_err()); + assert!( + !mvm_core::config::mvm_keys_dir() + .join("host-signer.ed25519") + .exists(), + "invalid retention must not mint the host signer" + ); + let resolved = resolve(Some(&flag), Some(10_000), None) + .expect("host-derived grant") + .expect("mailbox grant"); + assert_eq!(resolved.retention, Some(requested)); + let legacy = resolve(Some(&flag), None, None) + .expect("legacy grant") + .expect("mailbox grant"); + assert_eq!( + legacy.retention, None, + "an absent flag cannot mint a window" + ); + } } diff --git a/crates/mvm-core/src/user_config.rs b/crates/mvm-core/src/user_config.rs index 23ab2eb5cd..0073e436fb 100644 --- a/crates/mvm-core/src/user_config.rs +++ b/crates/mvm-core/src/user_config.rs @@ -8,8 +8,8 @@ pub const DEFAULT_STANDBY_MEMORY_MIB: u64 = 2048; /// Persistent operator configuration stored at `~/.mvm/config/config.toml`. /// -/// CLI flags always take precedence over these values. This config is -/// `mvmctl`-specific; `mvmd` maintains its own separate config. +/// CLI flags take precedence over defaults, but never over host ceilings. +/// This config is `mvmctl`-specific; `mvmd` maintains its own separate config. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct MvmConfig { @@ -53,6 +53,9 @@ pub struct MvmConfig { /// value here refuses an explicitly unbounded grant rather than clamping /// it, so a caller learns the host forbids what it asked for. pub max_wall_clock_secs: Option, + /// Maximum local mailbox retention after envelope expiry, in milliseconds. + /// `None` refuses a requested local-durable window; it does not mean unbounded. + pub max_mailbox_retention_ms: Option, /// Default CPU share, in thousandths of one host core, for a workload no /// higher surface capped. `None` = uncapped. /// @@ -266,6 +269,7 @@ impl Default for MvmConfig { max_cpu_millicores: None, max_memory_mib: None, max_wall_clock_secs: None, + max_mailbox_retention_ms: None, // Also unset: a default invented here would cap every local run // with a number nobody chose. default_cpu_millicores: None, @@ -470,6 +474,26 @@ mod tests { use super::*; use crate::util::test_env::TestEnv; + #[test] + fn mailbox_retention_ceiling_is_optional_and_roundtrips() { + assert_eq!(MvmConfig::default().max_mailbox_retention_ms, None); + assert!( + !toml::to_string(&MvmConfig::default()) + .expect("default operator config") + .contains("max_mailbox_retention_ms"), + "the default config must not suggest local durability is enabled" + ); + let mut config = MvmConfig::default(); + config.max_mailbox_retention_ms = Some(10_000); + let encoded = toml::to_string(&config).expect("operator config"); + assert_eq!( + toml::from_str::(&encoded) + .expect("operator config roundtrip") + .max_mailbox_retention_ms, + Some(10_000) + ); + } + #[test] fn config_parse_errors_do_not_retain_sensitive_source() { let sensitive = format!("private-{}", std::process::id()); diff --git a/crates/mvm-hostd/src/plan_admission.rs b/crates/mvm-hostd/src/plan_admission.rs index e5723e3624..37d7961ed7 100644 --- a/crates/mvm-hostd/src/plan_admission.rs +++ b/crates/mvm-hostd/src/plan_admission.rs @@ -65,6 +65,7 @@ use mvm_build::guest_libc::GuestLibc; use mvm_contract::grants::ceiling::GrantCeiling; use mvm_contract::grants::{CpuGrant, Grants}; use mvm_contract::protocol::capability_negotiation::negotiate_grants; +use mvm_contract::protocol::mailbox::{MailboxGrant, MailboxRetentionPolicy}; use mvm_contract::protocol::resource_controls::{EnforcedGrants, ResourceControls}; use mvm_contract::provenance::{ ActorRef, AttestationBinding, DecisionActorRole, DecisionCategory, DecisionOutcome, @@ -481,6 +482,10 @@ pub fn admit_plan_for_run( // signed admitted one to anything downstream that only checks the // signature. admit_grants(plan, &host_grant_ceiling(), posture)?; + enforce_mailbox_retention( + plan.mailbox.as_ref(), + mvm_core::user_config::load(None).max_mailbox_retention_ms, + )?; // The ceiling above bounds this one workload; the budget bounds the sum of // every live one. Both sit in the pre-keystore window for the same reason, @@ -565,6 +570,10 @@ pub fn admit_signed_plan_for_run( let extensions = verify_extension_packs(&verified, None)?; verify_registry_pack_assets(&verified)?; admit_grants(&verified, &host_grant_ceiling(), posture)?; + enforce_mailbox_retention( + verified.mailbox.as_ref(), + mvm_core::user_config::load(None).max_mailbox_retention_ms, + )?; admit_within_host_budget(&verified)?; admit_host_interface(&verified)?; @@ -1010,6 +1019,23 @@ fn admit_grants(plan: &ExecutionPlan, ceiling: &GrantCeiling, posture: RunPostur enforceability_gate(grants, plan, posture) } +fn enforce_mailbox_retention(mailbox: Option<&MailboxGrant>, max_ms: Option) -> Result<()> { + let Some(requested) = mailbox.and_then(|grant| grant.retention) else { + return Ok(()); + }; + let max_ms = max_ms + .context("local mailbox retention requires max_mailbox_retention_ms in host config.toml")?; + let ceiling = MailboxRetentionPolicy::new(max_ms) + .context("host max_mailbox_retention_ms must be a finite, nonzero duration")?; + anyhow::ensure!( + requested.retention_ms() <= ceiling.retention_ms(), + "local mailbox retention {} ms exceeds this host's {} ms ceiling", + requested.retention_ms(), + ceiling.retention_ms(), + ); + Ok(()) +} + /// Refuse a plan whose two encodings of one wall-clock bound disagree. /// /// `resources.timeouts.exec_secs` is the projection of `grants.wall_clock`, and @@ -2458,11 +2484,33 @@ mod admitted_environment_tests { mod tests { use super::*; use chrono::{DateTime, TimeZone, Utc}; + use mvm_contract::protocol::mailbox::{ + ExecutionEpoch, MailboxGrant, MailboxId, MailboxRetentionPolicy, + }; use mvm_core::plan::{PlanSeccompTier, SecretReleasePolicy}; use rand::Rng; const FIXTURE_SHA: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + #[test] + fn local_retention_requires_a_configured_host_ceiling() { + let mut grant = MailboxGrant { + mailbox: MailboxId::from_bytes([1; 32]), + authority_key: [2; 32], + epoch: ExecutionEpoch::new(1).expect("epoch"), + report: None, + retention: None, + }; + assert!(enforce_mailbox_retention(Some(&grant), None).is_ok()); + grant.retention = Some(MailboxRetentionPolicy::new(10_000).expect("window")); + assert!(enforce_mailbox_retention(Some(&grant), None).is_err()); + assert!(enforce_mailbox_retention(Some(&grant), Some(0)).is_err()); + assert!(enforce_mailbox_retention(Some(&grant), Some(u64::MAX)).is_err()); + assert!(enforce_mailbox_retention(Some(&grant), Some(9_999)).is_err()); + assert!(enforce_mailbox_retention(Some(&grant), Some(10_000)).is_ok()); + assert!(enforce_mailbox_retention(Some(&grant), Some(20_000)).is_ok()); + } + #[cfg(feature = "manifest-verify")] const PYTHON_MANIFEST: &[u8] = include_bytes!("../../mvm-cli/tests/fixtures/signed-registry-python/manifest.json"); @@ -4009,6 +4057,48 @@ mod tests { .expect("a grant inside the ceiling admits"); } + #[test] + fn mailbox_retention_above_or_without_host_ceiling_refuses_before_signing() { + let (_env, _home) = host_with_ceiling(Default::default()); + let keys = tempfile::tempdir().expect("keystore"); + let mut input = fixture_input("vm-retention"); + input.mailbox = Some(MailboxGrant { + mailbox: MailboxId::from_bytes([1; 32]), + authority_key: [2; 32], + epoch: ExecutionEpoch::new(1).expect("epoch"), + report: None, + retention: Some(MailboxRetentionPolicy::new(10_000).expect("window")), + }); + let attempt = || { + admit_for_run( + &input, + &SystemClock, + &InMemoryNonceLedger::new(), + Some(keys.path()), + None, + RunPosture::without_backend(Variant::Dev), + ) + }; + let missing = attempt().expect_err("missing operator ceiling must refuse"); + assert!( + missing.to_string().contains("max_mailbox_retention_ms"), + "{missing:#}" + ); + assert!( + !keys.path().join("host-signer.ed25519").exists(), + "a refused grant must not reach the keystore" + ); + let mut config = mvm_core::user_config::load(None); + config.max_mailbox_retention_ms = Some(9_999); + mvm_core::user_config::save(&config, None).expect("set host ceiling"); + let over = attempt().expect_err("request above host ceiling must refuse"); + assert!(over.to_string().contains("ceiling"), "{over:#}"); + assert!( + !keys.path().join("host-signer.ed25519").exists(), + "an over-ceiling grant must not be signed" + ); + } + #[test] fn admission_refuses_before_signing() { // Not "an error came back" — no *signature* may exist for a run we diff --git a/public/src/content/docs/reference/cli-commands.md b/public/src/content/docs/reference/cli-commands.md index 1df1842413..9404621c3e 100644 --- a/public/src/content/docs/reference/cli-commands.md +++ b/public/src/content/docs/reference/cli-commands.md @@ -1569,6 +1569,15 @@ default of read-only mounts unless `:rw` is explicit. `--name` is optional: when omitted, `machine create` auto-generates a name and prints it (mirroring `machine run -d`). +`mvmctl run` and `mvmctl machine run` accept `--mailbox <64-hex-id>` and an +optional `--mailbox-retention-ms `. The requested retention +window is signed into the mailbox grant and requires a mailbox. The operator +sets `max_mailbox_retention_ms` in `$MVM_HOME/config/config.toml`; a missing, +invalid, or smaller ceiling refuses a retention request before plan signing, +without silently clamping it. Both values are unset by default. This does not +enable local-durable production service: its launch remains gated until +recoverable compaction and the service/guest handoff are qualified. + A host can also admit plans it did not synthesize. By default every plan booted locally is synthesized and self-signed under the host key at admission time; a fleet control plane can instead sign an `ExecutionPlan` itself and have a From af9b4d3178159bf1f343c9468ac6a1478eb0600e Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 19:51:50 -0800 Subject: [PATCH 39/51] test(mailbox): initialize retention ceiling fixture directly --- crates/mvm-core/src/user_config.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/mvm-core/src/user_config.rs b/crates/mvm-core/src/user_config.rs index 0073e436fb..6d964cb33f 100644 --- a/crates/mvm-core/src/user_config.rs +++ b/crates/mvm-core/src/user_config.rs @@ -483,8 +483,10 @@ mod tests { .contains("max_mailbox_retention_ms"), "the default config must not suggest local durability is enabled" ); - let mut config = MvmConfig::default(); - config.max_mailbox_retention_ms = Some(10_000); + let config = MvmConfig { + max_mailbox_retention_ms: Some(10_000), + ..MvmConfig::default() + }; let encoded = toml::to_string(&config).expect("operator config"); assert_eq!( toml::from_str::(&encoded) From a389daaf10eb960bbe20a12a29b951a3aa2bb2dc Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 21:46:34 -0800 Subject: [PATCH 40/51] feat(mailbox): checkpoint local journal with retained tombstones --- crates/mvm-core/src/mailbox/journal.rs | 670 +++++++++++++++++- .../057-mailbox-semantics-and-threat-model.md | 19 + 2 files changed, 674 insertions(+), 15 deletions(-) diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index b56d0e1a2e..041ab32df9 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -45,6 +45,7 @@ const ANCHOR_DOMAIN: &[u8] = b"mvm.mailbox.anchor.v1\0"; const ACCEPTANCE_INTENT_DOMAIN_V2: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v2\0"; const ACCEPTANCE_INTENT_DOMAIN_V3: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v3\0"; const EXPIRY_INTENT_DOMAIN: &[u8] = b"mvm.mailbox.journal-expiry.v1\0"; +const CHECKPOINT_DOMAIN: &[u8] = b"mvm.mailbox.local-checkpoint.v1\0"; const MAX_ANCHOR_BYTES: u64 = 1024; const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; const MAX_RECORDS: usize = 100_000; @@ -296,8 +297,14 @@ enum JournalEvent { expired_at_unix_ms: i64, authorization_signature: Vec, }, + Checkpoint { + state: Box, + authorization_signature: Vec, + }, } +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct LeaseState { retry_count: u8, signed: SignedMailboxDeliveryLease, @@ -305,6 +312,8 @@ struct LeaseState { receive_capability: SignedMailboxCapability, } +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] struct Seen { envelope: MailboxEnvelope, capability: SignedMailboxCapability, @@ -315,6 +324,25 @@ struct Seen { signed_ack: Option, } +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CheckpointEntry { + identity: ReplayIdentity, + seen: Seen, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CheckpointState { + source: JournalTip, + address: MailboxAddress, + history_key: HistoryKeyRef, + retention: MailboxRetentionPolicy, + created_at_unix_ms: i64, + entries: Vec, + used_challenges: Vec<([u8; 32], ReplayIdentity)>, +} + #[derive(Default)] struct Probe(Option); @@ -338,9 +366,10 @@ pub struct MailboxJournal { limits: JournalLimits, tip: JournalTip, bytes: u64, + records: usize, seen: BTreeMap, order: Vec, - used_challenges: BTreeSet<[u8; 32]>, + used_challenges: BTreeMap<[u8; 32], ReplayIdentity>, poisoned: bool, _lock: FileLock, } @@ -842,25 +871,68 @@ impl MailboxJournal { limits, tip: JournalTip::genesis(), bytes: 0, + records: 0, seen: BTreeMap::new(), order: Vec::new(), - used_challenges: BTreeSet::new(), + used_challenges: BTreeMap::new(), poisoned: false, _lock: lock, }; let mut paths = Vec::new(); + let mut checkpoints = Vec::new(); for entry in fs::read_dir(root)? { let entry = entry?; - if entry.file_name() != "journal.lock" { - paths.push(entry.path()); - if paths.len() > limits.max_records { - return Err(JournalError::Full); + if entry.file_name() == "journal.lock" { + continue; + } + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_str().ok_or(JournalError::Corrupt)?; + if let Some(raw) = name.strip_prefix("checkpoint-") { + let cursor = raw + .strip_suffix(".bin") + .and_then(|raw| raw.parse::().ok()) + .ok_or(JournalError::Corrupt)?; + if path != checkpoint_path(root, cursor) { + return Err(JournalError::Corrupt); } + checkpoints.push((cursor, path)); + } else if let Some(raw) = name.strip_prefix("entry-") { + let cursor = raw + .strip_suffix(".bin") + .and_then(|raw| raw.parse::().ok()) + .ok_or(JournalError::Corrupt)?; + if path != record_path(root, cursor) { + return Err(JournalError::Corrupt); + } + paths.push((cursor, path)); + } else { + return Err(JournalError::Corrupt); + } + if paths.len() > limits.max_records || checkpoints.len() > 2 { + return Err(JournalError::Full); } } + checkpoints.sort(); + if !checkpoints.is_empty() && journal.tier != AcceptanceTier::Local { + return Err(JournalError::Corrupt); + } + if let Some((cursor, path)) = checkpoints + .iter() + .rev() + .find(|(cursor, _)| *cursor <= expected.cursor) + { + journal.restore_checkpoint(path, *cursor)?; + if *cursor == expected.cursor && journal.tip != expected { + return Err(JournalError::Rollback); + } + } + let start = journal.tip.cursor; + let (mut paths, obsolete): (Vec<_>, Vec<_>) = + paths.into_iter().partition(|(cursor, _)| *cursor > start); paths.sort(); let mut suffix_synced = false; - for path in paths { + for (_, path) in paths { let cursor = journal .tip .cursor @@ -875,6 +947,7 @@ impl MailboxJournal { .checked_add(bytes.len() as u64) .filter(|n| *n <= limits.max_bytes) .ok_or(JournalError::Full)?; + journal.records += 1; let (tag, ciphertext) = bytes.split_at_checked(32).ok_or(JournalError::Corrupt)?; let expected_tag = journal.mac(ciphertext); if tag != expected_tag { @@ -1028,7 +1101,11 @@ impl MailboxJournal { enrollment: *enrollment, receive_capability: *receive_capability, }); - if !journal.used_challenges.insert(challenge) { + if journal + .used_challenges + .insert(challenge, identity) + .is_some() + { return Err(JournalError::Corrupt); } } @@ -1047,7 +1124,12 @@ impl MailboxJournal { .checked_add(1) .filter(|count| *count <= MAX_LEASE_RETRIES) .ok_or(JournalError::Corrupt)?; - if challenge == [0; 32] || !journal.used_challenges.insert(challenge) { + if challenge == [0; 32] + || journal + .used_challenges + .insert(challenge, identity) + .is_some() + { return Err(JournalError::Corrupt); } } @@ -1115,6 +1197,7 @@ impl MailboxJournal { .map_err(|_| JournalError::Corrupt)?; entry.expired_at_unix_ms = Some(expired_at_unix_ms); } + JournalEvent::Checkpoint { .. } => return Err(JournalError::Corrupt), } let next = JournalTip { cursor, @@ -1139,9 +1222,172 @@ impl MailboxJournal { if journal.tip.cursor < expected.cursor { return Err(JournalError::Rollback); } + if start != 0 { + for (_, path) in obsolete { + fs::remove_file(path)?; + } + } + for (cursor, path) in checkpoints { + if cursor > expected.cursor || cursor < start { + // Pre-anchor or superseded generations are removed only after + // the committed prefix has been verified against the anchor. + fs::remove_file(&path)?; + } + } + sync_dir(root).map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; Ok(journal) } + fn restore_checkpoint(&mut self, path: &Path, cursor: u64) -> Result<(), JournalError> { + let bytes = read_record(path)?; + if bytes.len() as u64 > self.limits.max_bytes { + return Err(JournalError::Full); + } + let (previous, sealed) = bytes.split_at_checked(32).ok_or(JournalError::Corrupt)?; + let (tag, ciphertext) = sealed.split_at_checked(32).ok_or(JournalError::Corrupt)?; + let previous: [u8; 32] = previous.try_into().map_err(|_| JournalError::Corrupt)?; + let expected_tag = self.mac_after(previous, ciphertext); + if self.tier != AcceptanceTier::Local || tag != expected_tag { + return Err(JournalError::Corrupt); + } + let plaintext = Zeroizing::new( + aead::open( + &self.record_key(), + ciphertext, + &[DOMAIN, &previous].concat(), + ) + .map_err(|_| JournalError::Corrupt)?, + ); + let record: Record = + serde_json::from_slice(&plaintext).map_err(|_| JournalError::Corrupt)?; + if record.version != 3 + || record.cursor != cursor + || record.previous != previous + || record.address != self.address + || record.history_key != self.history_key + { + return Err(JournalError::Corrupt); + } + let JournalEvent::Checkpoint { + state, + authorization_signature, + } = record.event + else { + return Err(JournalError::Corrupt); + }; + if state.source.cursor.checked_add(1) != Some(cursor) + || state.source.head != previous + || state.address != self.address + || Some(state.history_key) != self.history_key + || Some(state.retention) != self.retention + || state.created_at_unix_ms < 0 + { + return Err(JournalError::Corrupt); + } + let signature: [u8; 64] = authorization_signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + self.authority + .verify( + &canonical_bytes(CHECKPOINT_DOMAIN, &state).map_err(|_| JournalError::Corrupt)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; + let mut seen = BTreeMap::new(); + let mut order = Vec::new(); + for CheckpointEntry { + identity, + seen: entry, + } in state.entries + { + let receipt = &entry.accepted_receipt; + if receipt.address != self.address + || receipt.outcome != DeliveryOutcome::AcceptedLocalDurable + || receipt.message_id != identity.message_id + || receipt.execution_epoch != identity.execution_epoch + || receipt.delivery_id != entry.envelope.routing.delivery_id + || receipt.ciphertext_commitment != entry.envelope.ciphertext_commitment + || receipt.validate_versioned_outcome().is_err() + || state + .retention + .retained_until(entry.envelope.routing.expires_at_unix_ms) + .is_err() + || entry + .expired_at_unix_ms + .is_some_and(|time| time < entry.envelope.routing.expires_at_unix_ms) + || (entry.acknowledged && entry.expired_at_unix_ms.is_some()) + || (entry.signed_ack.is_some() && !entry.acknowledged) + { + return Err(JournalError::Corrupt); + } + if let Some(lease) = &entry.lease { + if lease.retry_count > MAX_LEASE_RETRIES { + return Err(JournalError::Corrupt); + } + let at = entry + .signed_ack + .as_ref() + .map_or(lease.signed.lease.issued_at_unix_ms, |ack| { + ack.ack.acknowledged_at_unix_ms + }); + let guest_key = verify_recovered_lease( + &entry, + identity, + &lease.signed, + &lease.enrollment, + &lease.receive_capability, + &self.authority, + at, + )?; + if let Some(ack) = &entry.signed_ack { + if ack.ack.lease != lease.signed || ack.encode().is_err() { + return Err(JournalError::Corrupt); + } + let signature: [u8; 64] = ack + .signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + guest_key + .verify( + &canonical_bytes(DELIVERY_ACK_SIGNATURE_DOMAIN, &ack.ack) + .map_err(|_| JournalError::Corrupt)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; + } else if entry.acknowledged { + return Err(JournalError::Corrupt); + } + } else if entry.signed_ack.is_some() { + return Err(JournalError::Corrupt); + } + if seen.insert(identity, entry).is_some() { + return Err(JournalError::Corrupt); + } + order.push(identity); + } + let mut used_challenges = BTreeMap::new(); + for (challenge, identity) in state.used_challenges { + if challenge == [0; 32] + || !seen.contains_key(&identity) + || used_challenges.insert(challenge, identity).is_some() + { + return Err(JournalError::Corrupt); + } + } + self.seen = seen; + self.order = order; + self.used_challenges = used_challenges; + self.tip = JournalTip { + cursor, + head: expected_tag, + }; + self.bytes = bytes.len() as u64; + self.records = 1; + Ok(()) + } + /// The legacy local callback cannot prove an independent anchor or /// recoverable key custody, so it cannot issue durable acceptance. /// @@ -1353,6 +1599,136 @@ impl MailboxJournal { self.admit_at_tier(request, AcceptanceTier::Local, |next| anchor.advance(next)) } + /// Commit a signed local snapshot before pruning the superseded prefix. + /// + /// A quota refusal leaves the old history untouched; uncertain I/O poisons + /// this handle until recovery has verified the committed anchor. The caller + /// must provide trusted UNIX time for terminal-state retention. + pub fn compact_local( + &mut self, + now_unix_ms: i64, + signer: &SigningKey, + anchor: &FileJournalAnchor, + ) -> Result { + self.local_ready(anchor, &signer.verifying_key())?; + let retention = self.retention.ok_or(JournalError::InvalidLimits)?; + let history_key = self + .history_key + .ok_or(JournalError::HistoryKeyUnavailable)?; + if now_unix_ms < 0 { + return Err(JournalError::InvalidLimits); + } + let mut retired = BTreeSet::new(); + for (identity, entry) in &self.seen { + if (entry.acknowledged || entry.expired_at_unix_ms.is_some()) + && now_unix_ms + > retention + .retained_until(entry.envelope.routing.expires_at_unix_ms) + .map_err(|_| JournalError::Corrupt)? + { + retired.insert(*identity); + } + } + let state = CheckpointState { + source: self.tip, + address: self.address, + history_key, + retention, + created_at_unix_ms: now_unix_ms, + entries: self + .order + .iter() + .filter(|identity| !retired.contains(identity)) + .map(|identity| { + self.seen + .get(identity) + .cloned() + .map(|seen| CheckpointEntry { + identity: *identity, + seen, + }) + }) + .collect::>>() + .ok_or(JournalError::Corrupt)?, + used_challenges: self + .used_challenges + .iter() + .filter(|(_, identity)| !retired.contains(identity)) + .map(|(challenge, identity)| (*challenge, *identity)) + .collect(), + }; + let signature = signer + .sign(&canonical_bytes(CHECKPOINT_DOMAIN, &state)?) + .to_bytes() + .to_vec(); + let cursor = self.tip.cursor.checked_add(1).ok_or(JournalError::Full)?; + let record = Record { + version: 3, + cursor, + previous: self.tip.head, + address: self.address, + history_key: self.history_key, + event: JournalEvent::Checkpoint { + state: Box::new(state), + authorization_signature: signature, + }, + }; + let body = Zeroizing::new(serde_json::to_vec(&record).map_err(|_| JournalError::Corrupt)?); + if body.len() + 64 + aead::NONCE_SIZE + aead::TAG_SIZE > MAX_RECORD_BYTES { + return Err(JournalError::Full); + } + let ciphertext = aead::seal( + &self.record_key(), + &body, + &[DOMAIN, &self.tip.head].concat(), + ); + let size = (ciphertext.len() + 64) as u64; + if self + .bytes + .checked_add(size) + .is_none_or(|n| n > self.limits.max_bytes) + { + return Err(JournalError::Full); + } + let tag = self.mac(&ciphertext); + let mut bytes = self.tip.head.to_vec(); + bytes.extend_from_slice(&tag); + bytes.extend_from_slice(&ciphertext); + let path = checkpoint_path(&self.root, cursor); + match fs::symlink_metadata(&path) { + Ok(_) => return Err(JournalError::Corrupt), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + if let Err(error) = atomic_write_new(&path, &bytes).and_then(|()| sync_dir(&self.root)) { + self.poisoned = true; + return Err(JournalError::Storage(std::io::Error::other(error))); + } + let next = JournalTip { cursor, head: tag }; + if let Err(error) = anchor.advance(next) { + self.poisoned = true; + return Err(error); + } + self.tip = next; + self.poisoned = true; + for entry in fs::read_dir(&self.root)? { + let old = entry?.path(); + if old != path && old != self.root.join("journal.lock") { + fs::remove_file(old)?; + } + } + sync_dir(&self.root) + .map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; + self.order.retain(|identity| !retired.contains(identity)); + self.seen.retain(|identity, _| !retired.contains(identity)); + self.used_challenges + .retain(|_, identity| !retired.contains(identity)); + self.bytes = size; + self.records = 1; + self.poisoned = false; + Ok(next) + } + fn append( &mut self, event: JournalEvent, @@ -1362,7 +1738,7 @@ impl MailboxJournal { return Err(JournalError::Uncertain); } let cursor = self.tip.cursor.checked_add(1).ok_or(JournalError::Full)?; - if cursor > self.limits.max_records as u64 { + if self.records >= self.limits.max_records { return Err(JournalError::Full); } let record = Record { @@ -1402,6 +1778,7 @@ impl MailboxJournal { } self.tip = tip; self.bytes += bytes.len() as u64; + self.records += 1; if let Err(error) = anchor(tip) { self.poisoned = true; return Err(error); @@ -1567,7 +1944,7 @@ impl MailboxJournal { { return Err(JournalError::Busy); } - if self.used_challenges.contains(request.challenge) { + if self.used_challenges.contains_key(request.challenge) { return Err(JournalError::ConflictingRetry); } if active.retry_count >= MAX_LEASE_RETRIES { @@ -1584,7 +1961,7 @@ impl MailboxJournal { }, anchor, )?; - self.used_challenges.insert(*request.challenge); + self.used_challenges.insert(*request.challenge, identity); self.seen .get_mut(&identity) .and_then(|entry| entry.lease.as_mut()) @@ -1595,7 +1972,7 @@ impl MailboxJournal { lease: signed, })); } - if self.used_challenges.contains(request.challenge) { + if self.used_challenges.contains_key(request.challenge) { return Err(JournalError::ConflictingRetry); } let header = message.opened.header(); @@ -1652,7 +2029,7 @@ impl MailboxJournal { }, anchor, )?; - self.used_challenges.insert(*request.challenge); + self.used_challenges.insert(*request.challenge, identity); self.seen .get_mut(&identity) .ok_or(JournalError::Corrupt)? @@ -1838,10 +2215,14 @@ impl MailboxJournal { } fn mac(&self, bytes: &[u8]) -> [u8; 32] { + self.mac_after(self.tip.head, bytes) + } + + fn mac_after(&self, previous: [u8; 32], bytes: &[u8]) -> [u8; 32] { let mut mac = as KeyInit>::new_from_slice(&self.key[..]) .expect("a 32-byte HMAC key is valid"); mac.update(DOMAIN); - mac.update(&self.tip.head); + mac.update(&previous); mac.update(bytes); mac.finalize().into_bytes().into() } @@ -1858,6 +2239,10 @@ fn record_path(root: &Path, cursor: u64) -> PathBuf { root.join(format!("entry-{cursor:020}.bin")) } +fn checkpoint_path(root: &Path, cursor: u64) -> PathBuf { + root.join(format!("checkpoint-{cursor:020}.bin")) +} + fn read_record(path: &Path) -> Result, JournalError> { let meta = fs::symlink_metadata(path)?; if !meta.file_type().is_file() || meta.len() > MAX_RECORD_BYTES as u64 { @@ -2299,6 +2684,261 @@ mod anchor_tests { assert_eq!(anchor.tip(), JournalTip::genesis()); } + #[test] + fn local_checkpoint_preserves_the_original_receipt_after_pruning() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let anchor_path = dir.path().join("anchor").join("tip.json"); + let (envelope, capability, mut context, payload_key, sender_key, signer) = message(); + let (reference, custody) = versioned_custody(); + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("local anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ) + .expect("empty journal"); + let original = journal + .admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("signed local receipt"); + let old_record = fs::read(record_path(&root, 1)).expect("pre-checkpoint prefix"); + journal.limits.max_bytes = journal.bytes + 1; + assert!(matches!( + journal.compact_local(context.now_unix_ms, &signer, &anchor), + Err(JournalError::Full) + )); + assert_eq!(journal.tip(), anchor.tip(), "quota refusal cannot advance"); + assert!(record_path(&root, 1).exists(), "quota keeps old prefix"); + journal.limits = limits(); + journal + .compact_local(context.now_unix_ms, &signer, &anchor) + .expect("signed checkpoint and anchor"); + assert!( + !record_path(&root, 1).exists(), + "old prefix pruned after anchor" + ); + assert!( + !fs::read(checkpoint_path(&root, 2)) + .expect("encrypted checkpoint") + .windows(b"hello".len()) + .any(|window| window == b"hello"), + "checkpoint cannot expose guest payload" + ); + assert_eq!(journal.tip(), anchor.tip()); + fs::write(record_path(&root, 1), old_record).expect("simulated post-anchor prune crash"); + drop(journal); + drop(anchor); + + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("retained anchor"); + let mut recovered = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ) + .expect("reopen from signed checkpoint without old prefix"); + assert!( + !record_path(&root, 1).exists(), + "recovery finishes the interrupted prune" + ); + context.now_unix_ms += 1; + let duplicate = recovered + .admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("same immutable acceptance after restart"); + assert_eq!( + serde_jcs::to_vec(&duplicate).expect("duplicate bytes"), + serde_jcs::to_vec(&original).expect("original bytes") + ); + assert_eq!(recovered.tip().cursor, 2, "duplicate cannot append"); + let first_checkpoint = checkpoint_path(&root, 2); + recovered + .compact_local(context.now_unix_ms, &signer, &anchor) + .expect("subsequent generation can replace the first"); + assert!(!first_checkpoint.exists(), "superseded checkpoint pruned"); + assert_eq!(recovered.tip().cursor, 3); + drop(recovered); + drop(anchor); + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("later anchor"); + let recovered = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ) + .expect("later checkpoint reopens"); + assert_eq!(recovered.tip(), anchor.tip()); + } + + #[test] + fn anchored_checkpoint_cannot_fall_back_to_a_shorter_prefix() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let anchor_path = dir.path().join("anchor").join("tip.json"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let (reference, custody) = versioned_custody(); + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("local anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ) + .expect("empty journal"); + journal + .admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("accepted"); + journal + .compact_local(context.now_unix_ms, &signer, &anchor) + .expect("checkpoint anchored"); + drop(journal); + drop(anchor); + let path = checkpoint_path(&root, 2); + let original = fs::read(&path).expect("checkpoint bytes"); + let mut tampered = original.clone(); + let last = tampered.last_mut().expect("nonempty checkpoint"); + *last ^= 1; + fs::write(&path, tampered).expect("corrupt stored ciphertext"); + let anchor = + FileJournalAnchor::open(&anchor_path, address(), [22; 32]).expect("retained anchor"); + assert!(matches!( + MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ), + Err(JournalError::Corrupt) + )); + fs::remove_file(&path).expect("delete checkpoint"); + assert!(matches!( + MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ), + Err(JournalError::Rollback) + )); + } + + #[test] + fn checkpoint_keeps_tombstone_through_window_then_retires_terminal_state() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + let (envelope, capability, context, payload_key, sender_key, signer) = message(); + let (reference, custody) = versioned_custody(); + let anchor = FileJournalAnchor::open( + &dir.path().join("anchor").join("tip.json"), + address(), + [22; 32], + ) + .expect("local anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ) + .expect("empty journal"); + journal + .admit_local( + admission( + &envelope, + &capability, + &context, + &payload_key, + &sender_key, + &signer, + ), + &anchor, + ) + .expect("accepted"); + let expiry = envelope.routing.expires_at_unix_ms; + journal + .expire_next(expiry, &signer, |next| anchor.advance(next)) + .expect("terminal settlement"); + let horizon = local_retention() + .retained_until(expiry) + .expect("finite horizon"); + journal + .compact_local(horizon, &signer, &anchor) + .expect("retain through the horizon"); + assert_eq!(journal.seen.len(), 1); + journal + .compact_local(horizon + 1, &signer, &anchor) + .expect("retire after the horizon"); + assert!(journal.seen.is_empty()); + drop(journal); + let recovered = MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ) + .expect("empty retired generation reopens"); + assert!(recovered.seen.is_empty()); + assert_eq!(recovered.tip(), anchor.tip()); + } + #[test] fn interrupted_terminal_expiry_reconciles_without_an_early_receipt() { let dir = tempfile::tempdir().expect("journal root"); diff --git a/specs/adrs/057-mailbox-semantics-and-threat-model.md b/specs/adrs/057-mailbox-semantics-and-threat-model.md index 316239832c..4361b489c4 100644 --- a/specs/adrs/057-mailbox-semantics-and-threat-model.md +++ b/specs/adrs/057-mailbox-semantics-and-threat-model.md @@ -338,6 +338,25 @@ genesis from a gap. If the snapshot or retained tombstones cannot fit the finite quota, new acceptance refuses rather than discarding replay evidence. This host-local sequence does not detect rollback of the entire host. +The checkpoint is an immutable journal generation, not a second acceptance +authority. It binds the previous committed tip, the next monotonic cursor, +the history-key version, the exact signed retention window, and a content +commitment over the retained ciphertext, receipt bodies, leases, signed guest +ACKs, terminal dispositions, and dedup state. The destination signs this +commitment before encrypting and fsyncing the checkpoint. The local anchor +then advances to the checkpoint tip before any earlier record is removed. +Only terminal messages strictly past expiry plus their signed retention +window, and challenges owned by those messages, may leave that snapshot; +pending messages and their replay evidence remain until terminal settlement. +Recovery of a pre-anchor interruption requires the complete old prefix; +recovery of a post-anchor interruption requires the checkpoint and its +matching tip even when old files are still present or partly pruned. A +missing, divergent, oversized, or unverifiable checkpoint refuses rather +than falling back to a shorter history. Snapshotting cannot turn a leased +message into an unsigned host ACK or reset the original signed acceptance +time. The same bounded writer lock excludes concurrent admission, lease, +ACK, and compaction while this transition is in flight. + ### 8. Selective receive is bounded and fair Selective receive may match only authenticated inner-header fields declared by From 367eccd5ca6076ae184cc9655ab81263d050b216 Mon Sep 17 00:00:00 2001 From: Ari Date: Wed, 11 Nov 2026 23:49:04 -0800 Subject: [PATCH 41/51] feat(mailbox): serve retained local durable production mailbox --- crates/mvm-agentd/README.md | 9 +- .../tests/steps/mission_demo.rs | 35 + crates/mvm-core/README.md | 33 +- crates/mvm-core/src/mailbox/journal.rs | 898 ++---------------- crates/mvm-core/src/mailbox/journal/anchor.rs | 134 +++ .../src/mailbox/journal/checkpoint.rs | 354 +++++++ .../mvm-core/src/mailbox/journal/recovery.rs | 429 +++++++++ crates/mvm-hostd/README.md | 16 +- crates/mvm-hostd/src/lib.rs | 1 - crates/mvm-hostd/src/mailbox_custody.rs | 61 ++ crates/mvm-hostd/src/mailbox_launch.rs | 2 - crates/mvm-hostd/src/mailbox_process.rs | 622 +++++++++++- .../mvm-hostd/src/mailbox_service/durable.rs | 44 + crates/mvm-hostd/src/plan_admission.rs | 144 +-- .../src/plan_admission/grant_validation.rs | 39 + .../src/plan_admission/mailbox_posture.rs | 98 ++ .../mvm-runtime/src/workload_runner/runner.rs | 10 +- .../workload_runner/runner/console_boot.rs | 6 + .../s40_mission_demo/mission_demo.feature | 6 + .../content/docs/reference/cli-commands.md | 10 +- 20 files changed, 1920 insertions(+), 1031 deletions(-) create mode 100644 crates/mvm-core/src/mailbox/journal/anchor.rs create mode 100644 crates/mvm-core/src/mailbox/journal/checkpoint.rs create mode 100644 crates/mvm-core/src/mailbox/journal/recovery.rs create mode 100644 crates/mvm-hostd/src/plan_admission/grant_validation.rs create mode 100644 crates/mvm-hostd/src/plan_admission/mailbox_posture.rs diff --git a/crates/mvm-agentd/README.md b/crates/mvm-agentd/README.md index ba44a8332c..df9075415e 100644 --- a/crates/mvm-agentd/README.md +++ b/crates/mvm-agentd/README.md @@ -42,11 +42,12 @@ ACK signing key inside the guest agent, not in the workload. Only a signed plan granting this workload's own mailbox can enable Receive. The agent binds `/run/mvm/mailbox/receive.sock` when its separate boot ACK key -was provisioned, but the current Dev host helper refuses reader frames until -independent journal authority and key custody are deployed. The local protocol +was provisioned. Dev's ephemeral host helper refuses reader frames; a production +mailbox with signed retention uses a locally anchored journal and boot-bound +Receive authority. The local protocol is versioned, length-prefixed, and bounded; use `ApplicationReaderRequest` and -`ReaderResponse` from `mvm_core::mailbox::reader_wire`. Once backed by an -independent destination, `Receive` returns a signed lease, +`ReaderResponse` from `mvm_core::mailbox::reader_wire`. Once backed by that +local destination, `Receive` returns a signed lease, content type, and payload. Process the payload **before** sending `Ack { lease }` on a new connection. An empty inbox and a refusal are not successful delivery; reading never automatically ACKs. diff --git a/crates/mvm-conformance/tests/steps/mission_demo.rs b/crates/mvm-conformance/tests/steps/mission_demo.rs index 9c8862f52b..78e28f0334 100644 --- a/crates/mvm-conformance/tests/steps/mission_demo.rs +++ b/crates/mvm-conformance/tests/steps/mission_demo.rs @@ -21,6 +21,41 @@ fn run_demo(world: &mut CliWorld) { world.last_run = Some(output); } +#[when("the production mailbox helper witness runs")] +fn run_production_mailbox_helper(world: &mut CliWorld) { + let output = Command::new("cargo") + .args([ + "test", + "-p", + "mvm-hostd", + "--lib", + "mailbox_process::tests::production_helper_requires_retained_custody_and_returns_signed_local_receipt", + "--", + "--exact", + ]) + .current_dir(crate::steps::cli::workspace_root()) + .env_remove("RUSTFLAGS") + .output() + .expect("spawn the production mailbox helper test"); + world.last_run = Some(output); +} + +#[then("the production mailbox helper witness passes")] +fn production_mailbox_helper_passes(world: &mut CliWorld) { + let run = world.last_run.as_ref().expect("the witness ran"); + let stdout = String::from_utf8_lossy(&run.stdout); + assert!( + run.status.success() + && stdout.contains( + "production_helper_requires_retained_custody_and_returns_signed_local_receipt" + ) + && stdout.contains("1 passed; 0 failed"), + "production mailbox helper witness failed:\n{}\n{}", + stdout, + String::from_utf8_lossy(&run.stderr) + ); +} + fn stdout(world: &CliWorld) -> String { let run = world.last_run.as_ref().expect("the demo ran"); String::from_utf8_lossy(&run.stdout).into_owned() diff --git a/crates/mvm-core/README.md b/crates/mvm-core/README.md index 423e66519d..39cda4021b 100644 --- a/crates/mvm-core/README.md +++ b/crates/mvm-core/README.md @@ -64,7 +64,7 @@ canonical encrypted mailbox frame and its deduplication identity. Its the current `(cursor, head)` and atomically compare-and-advance it after a journal write. The journal persists unsigned receipt state with a destination-signed, domain-separated admission intent binding the verified -identity, canonical frame, and receipt state. The version-3 test path also +identity, canonical frame, and receipt state. Version-3 local history also binds a `HistoryKeyRef` (mailbox address, epoch, AES-256-GCM key ID and nonzero version) to the encrypted record and signed intent. Its recovery obtains the exact key through an existing-only `HistoryKeyCustody` lookup and refuses @@ -87,15 +87,18 @@ anchor have been fsynced. An exact local retry re-verifies current admission and the committed local tip before re-signing the original body/time, yielding the same signed bytes after restart. Recovery refuses a local V2 receipt in an independently anchored journal or a V1 strong receipt in a local journal. -The hostd service can bind that local journal to its existing endpoint, but -the production launch path still refuses until verified custody, retention, -and guest receipt handoff are integrated end to end. +The production hostd helper binds the local journal to its existing endpoint +only after verifying signed-plan custody, retained purpose keys, the journal +anchor, the finite retention window, and the guest boot identity. Dev retains +the separate ephemeral Send queue and does not serve Receive. `expire_next` records an authority-signed terminal expiry only after the message's absolute deadline; a lost response can be reconstructed after recovery. Expired messages are not disclosed, and an uncertain expiry write -cannot mint a terminal receipt. This does not yet implement an anchored -retention/compaction boundary: exhausted record or byte quotas still refuse -new work instead of deleting dedup history. +cannot mint a terminal receipt. Local compaction fsyncs a destination-signed, +encrypted checkpoint and advances the host-local anchor before pruning old +entries. It preserves pending delivery and dedup state through expiry plus +the signed window, retires terminal state only after that horizon, and +refuses when a checkpoint cannot fit the bounded quota. Retained guest-reader controls are checked against the historical signed Receive capability, enrollment, lease and guest ACK before recovery accepts their delivery state. An exact ACK retry still needs current boot, session, @@ -105,24 +108,20 @@ ACK, including during recovery; it requires that guest's signed ACK. `FileJournalAnchor` is a host-local anchor, **not** an independent authority: restoring both the journal and its host-local anchor can hide -rollback. The hostd test-scoped custody prototype stores distinct retained -purpose keys under a signed restart binding, but it is not connected to the -production service. Until verified custody and the existing service's -admission handoff are integrated and tested, local -journal writes must not be presented as `accepted_durable`. The trusted-host +rollback. Hostd retains distinct purpose keys under a signed restart binding +and uses them for the production local-durable service. Local journal writes +must not be presented as the stronger `accepted_durable`. The trusted-host profile can access plaintext; host-blind confidentiality requires a different attested key-release and protected-memory architecture. -The planned standalone `accepted_local_durable` tier has a distinct versioned -receipt contract and signer. Its history and payload-purpose keys belong in the existing +The standalone `accepted_local_durable` tier has a distinct versioned +receipt contract and signer. Its history and payload-purpose keys live in the existing protected host keys directory, scoped to tenant/domain/mailbox/epoch and key version, with an authority signer pinned for retained history. Recovery must load the existing key version; it must not mint a new signer or use the erased per-launch secret as its only opening key. Rotation and deletion need explicit retention and refusal rules. A host-local chain can witness ordinary crash and -process restart, **not** rollback of the entire host. The local journal test -path is not production activation: its custody is synthetic, with no retained -key provider, authenticated restart manifest, or service adapter. The stronger +process restart, **not** rollback of the entire host. The stronger `accepted_durable` outcome continues to require a separately trusted compare-and-advance checkpoint; no such provider is wired. diff --git a/crates/mvm-core/src/mailbox/journal.rs b/crates/mvm-core/src/mailbox/journal.rs index 041ab32df9..d5ce843f36 100644 --- a/crates/mvm-core/src/mailbox/journal.rs +++ b/crates/mvm-core/src/mailbox/journal.rs @@ -6,22 +6,21 @@ //! acceptance only after the journal and independent anchor commit. A hash //! chain alone cannot detect deletion of its suffix. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::fs; use std::io::Read; +use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; use std::path::{Path, PathBuf}; -use std::sync::Mutex; use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; -use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; use hmac::{Hmac, Mac, digest::KeyInit}; use mvm_contract::protocol::mailbox::{ - ContentType, DELIVERY_ACK_SIGNATURE_DOMAIN, DeliveryLeaseNonce, DeliveryOutcome, - DestinationReceipt, EncryptionAlgorithm, ExecutionEpoch, MailboxAddress, MailboxDeliveryLease, - MailboxEnvelope, MailboxGrant, MailboxGuestReaderScope, MailboxOperation, - MailboxRetentionPolicy, ProtocolVersion, SignatureAlgorithm, SignedDestinationReceipt, - SignedMailboxCapability, SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, - SignedMailboxGuestReaderEnrollment, + ContentType, DeliveryLeaseNonce, DeliveryOutcome, DestinationReceipt, EncryptionAlgorithm, + ExecutionEpoch, MailboxAddress, MailboxDeliveryLease, MailboxEnvelope, MailboxGrant, + MailboxGuestReaderScope, MailboxOperation, MailboxRetentionPolicy, ProtocolVersion, + SignatureAlgorithm, SignedDestinationReceipt, SignedMailboxCapability, + SignedMailboxDeliveryAck, SignedMailboxDeliveryLease, SignedMailboxGuestReaderEnrollment, }; use serde::{Deserialize, Serialize}; use sha2::Sha256; @@ -36,22 +35,24 @@ use super::{ frame, open_authenticated_envelope, sign_receipt, }; use crate::crypto::aead::{self, Key}; -use crate::private_fs; -use crate::util::atomic_io::{FileLock, atomic_write_durable, atomic_write_new, sync_dir}; +use crate::util::atomic_io::{FileLock, atomic_write_new, sync_dir}; const DOMAIN: &[u8] = b"mvm.mailbox.journal.v1\0"; const ENCRYPTION_DOMAIN: &[u8] = b"mvm.mailbox.journal.encryption.v1\0"; -const ANCHOR_DOMAIN: &[u8] = b"mvm.mailbox.anchor.v1\0"; const ACCEPTANCE_INTENT_DOMAIN_V2: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v2\0"; const ACCEPTANCE_INTENT_DOMAIN_V3: &[u8] = b"mvm.mailbox.journal-acceptance-intent.v3\0"; const EXPIRY_INTENT_DOMAIN: &[u8] = b"mvm.mailbox.journal-expiry.v1\0"; -const CHECKPOINT_DOMAIN: &[u8] = b"mvm.mailbox.local-checkpoint.v1\0"; -const MAX_ANCHOR_BYTES: u64 = 1024; const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024; const MAX_RECORDS: usize = 100_000; const MAX_JOURNAL_BYTES: u64 = 1024 * 1024 * 1024; const MAX_LEASE_RETRIES: u8 = 3; +mod anchor; +mod checkpoint; +mod recovery; +pub use anchor::FileJournalAnchor; +use checkpoint::{CheckpointState, checkpoint_path}; + /// Limits are checked both before admission and while reconstructing state. #[derive(Debug, Clone, Copy)] pub struct JournalLimits { @@ -324,25 +325,6 @@ struct Seen { signed_ack: Option, } -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct CheckpointEntry { - identity: ReplayIdentity, - seen: Seen, -} - -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct CheckpointState { - source: JournalTip, - address: MailboxAddress, - history_key: HistoryKeyRef, - retention: MailboxRetentionPolicy, - created_at_unix_ms: i64, - entries: Vec, - used_challenges: Vec<([u8; 32], ReplayIdentity)>, -} - #[derive(Default)] struct Probe(Option); @@ -467,113 +449,6 @@ impl ReceivedMailboxMessage { } } -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct AnchorBody { - version: u8, - address: MailboxAddress, - tip: JournalTip, -} - -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct AnchorRecord { - body: AnchorBody, - tag: [u8; 32], -} - -/// A software-first, locally fsynced anchor. Keep its file outside the -/// journal directory. It detects journal-only rollback, not simultaneous -/// rollback of the entire trusted host and its anchor storage. -pub struct FileJournalAnchor { - path: PathBuf, - address: MailboxAddress, - key: Zeroizing<[u8; 32]>, - tip: Mutex, - _lock: FileLock, -} - -impl FileJournalAnchor { - /// Open a private anchor file, or genesis if none exists. - pub fn open(path: &Path, address: MailboxAddress, key: [u8; 32]) -> Result { - let parent = path.parent().ok_or(JournalError::Corrupt)?; - private_fs::ensure_private_dir(parent)?; - let lock = FileLock::try_acquire(path) - .map_err(|e| std::io::Error::other(e.to_string()))? - .ok_or(JournalError::Busy)?; - let tip = match fs::symlink_metadata(path) { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => JournalTip::genesis(), - Err(e) => return Err(e.into()), - Ok(meta) if !meta.file_type().is_file() || meta.len() > MAX_ANCHOR_BYTES => { - return Err(JournalError::Corrupt); - } - Ok(_) => { - let mut bytes = Vec::new(); - fs::File::open(path)? - .take(MAX_ANCHOR_BYTES + 1) - .read_to_end(&mut bytes)?; - if bytes.len() as u64 > MAX_ANCHOR_BYTES { - return Err(JournalError::Corrupt); - } - let record: AnchorRecord = - serde_json::from_slice(&bytes).map_err(|_| JournalError::Corrupt)?; - let body = serde_json::to_vec(&record.body).map_err(|_| JournalError::Corrupt)?; - if record.body.version != 1 - || record.body.address != address - || anchor_mac(&key, &body) != record.tag - { - return Err(JournalError::Corrupt); - } - record.body.tip - } - }; - Ok(Self { - path: path.to_owned(), - address, - key: Zeroizing::new(key), - tip: Mutex::new(tip), - _lock: lock, - }) - } - - #[must_use] - pub fn tip(&self) -> JournalTip { - *self.tip.lock().expect("mailbox anchor lock poisoned") - } - - /// Advance by one committed journal entry, fsyncing the file and its - /// containing directory before returning. - pub fn advance(&self, tip: JournalTip) -> Result<(), JournalError> { - let mut current = self.tip.lock().expect("mailbox anchor lock poisoned"); - if current.cursor.checked_add(1) != Some(tip.cursor) { - return Err(JournalError::Rollback); - } - let body = AnchorBody { - version: 1, - address: self.address, - tip, - }; - let tag = anchor_mac( - &self.key[..], - &serde_json::to_vec(&body).map_err(|_| JournalError::Corrupt)?, - ); - let bytes = - serde_json::to_vec(&AnchorRecord { body, tag }).map_err(|_| JournalError::Corrupt)?; - atomic_write_durable(&self.path, &bytes) - .map_err(|e| JournalError::Storage(std::io::Error::other(e.to_string())))?; - *current = tip; - Ok(()) - } -} - -fn anchor_mac(key: &[u8], body: &[u8]) -> [u8; 32] { - let mut mac = - as KeyInit>::new_from_slice(key).expect("a 32-byte HMAC key is valid"); - mac.update(ANCHOR_DOMAIN); - mac.update(body); - mac.finalize().into_bytes().into() -} - #[derive(Serialize)] struct AcceptanceIntent<'a> { identity: ReplayIdentity, @@ -782,12 +657,11 @@ impl MailboxJournal { ) } - /// Reopen a test-scoped software-only local tier against a fsynced same-host anchor. + /// Reopen the software-only local tier against a fsynced same-host anchor. /// - /// Production must first supply an authenticated restart binding to the - /// admitted plan/VM/mailbox and the signed retention policy. A deserialized + /// The caller must first verify the admitted plan, VM, mailbox, signed + /// retention policy and authenticated custody manifest. A deserialized /// reference alone is never authorization to open history. - #[cfg(any(test, feature = "test-support"))] pub fn open_local_with_custody( root: &Path, reference: HistoryKeyRef, @@ -835,559 +709,6 @@ impl MailboxJournal { }) } - fn open_with_recovery( - root: &Path, - address: MailboxAddress, - authority: VerifyingKey, - key: JournalKey, - limits: JournalLimits, - expected: JournalTip, - mut reconcile: impl FnMut(JournalTip) -> Result<(), JournalError>, - ) -> Result { - if limits.max_records == 0 - || limits.max_records > MAX_RECORDS - || limits.max_bytes == 0 - || limits.max_bytes > MAX_JOURNAL_BYTES - { - return Err(JournalError::InvalidLimits); - } - if expected.cursor == 0 && expected != JournalTip::genesis() { - return Err(JournalError::Rollback); - } - private_fs::ensure_private_dir(root)?; - sync_dir(root.parent().ok_or(JournalError::Corrupt)?) - .map_err(|e| JournalError::Storage(std::io::Error::other(e.to_string())))?; - let lock = FileLock::try_acquire(&root.join("journal")) - .map_err(|e| std::io::Error::other(e.to_string()))? - .ok_or(JournalError::Busy)?; - let mut journal = Self { - root: root.to_owned(), - address, - authority, - key: key.bytes, - history_key: key.reference, - tier: key.tier, - retention: key.retention, - limits, - tip: JournalTip::genesis(), - bytes: 0, - records: 0, - seen: BTreeMap::new(), - order: Vec::new(), - used_challenges: BTreeMap::new(), - poisoned: false, - _lock: lock, - }; - let mut paths = Vec::new(); - let mut checkpoints = Vec::new(); - for entry in fs::read_dir(root)? { - let entry = entry?; - if entry.file_name() == "journal.lock" { - continue; - } - let path = entry.path(); - let name = entry.file_name(); - let name = name.to_str().ok_or(JournalError::Corrupt)?; - if let Some(raw) = name.strip_prefix("checkpoint-") { - let cursor = raw - .strip_suffix(".bin") - .and_then(|raw| raw.parse::().ok()) - .ok_or(JournalError::Corrupt)?; - if path != checkpoint_path(root, cursor) { - return Err(JournalError::Corrupt); - } - checkpoints.push((cursor, path)); - } else if let Some(raw) = name.strip_prefix("entry-") { - let cursor = raw - .strip_suffix(".bin") - .and_then(|raw| raw.parse::().ok()) - .ok_or(JournalError::Corrupt)?; - if path != record_path(root, cursor) { - return Err(JournalError::Corrupt); - } - paths.push((cursor, path)); - } else { - return Err(JournalError::Corrupt); - } - if paths.len() > limits.max_records || checkpoints.len() > 2 { - return Err(JournalError::Full); - } - } - checkpoints.sort(); - if !checkpoints.is_empty() && journal.tier != AcceptanceTier::Local { - return Err(JournalError::Corrupt); - } - if let Some((cursor, path)) = checkpoints - .iter() - .rev() - .find(|(cursor, _)| *cursor <= expected.cursor) - { - journal.restore_checkpoint(path, *cursor)?; - if *cursor == expected.cursor && journal.tip != expected { - return Err(JournalError::Rollback); - } - } - let start = journal.tip.cursor; - let (mut paths, obsolete): (Vec<_>, Vec<_>) = - paths.into_iter().partition(|(cursor, _)| *cursor > start); - paths.sort(); - let mut suffix_synced = false; - for (_, path) in paths { - let cursor = journal - .tip - .cursor - .checked_add(1) - .ok_or(JournalError::Full)?; - if path != record_path(root, cursor) { - return Err(JournalError::Corrupt); - } - let bytes = read_record(&path)?; - journal.bytes = journal - .bytes - .checked_add(bytes.len() as u64) - .filter(|n| *n <= limits.max_bytes) - .ok_or(JournalError::Full)?; - journal.records += 1; - let (tag, ciphertext) = bytes.split_at_checked(32).ok_or(JournalError::Corrupt)?; - let expected_tag = journal.mac(ciphertext); - if tag != expected_tag { - return Err(JournalError::Corrupt); - } - let plaintext = Zeroizing::new( - aead::open( - &journal.record_key(), - ciphertext, - &[DOMAIN, &journal.tip.head].concat(), - ) - .map_err(|_| JournalError::Corrupt)?, - ); - let record: Record = - serde_json::from_slice(&plaintext).map_err(|_| JournalError::Corrupt)?; - let expected_version = if journal.history_key.is_some() { 3 } else { 2 }; - if record.version != expected_version - || record.cursor != cursor - || record.previous != journal.tip.head - || record.address != address - { - return Err(JournalError::Corrupt); - } - if let Some(reference) = &record.history_key { - reference.validate().map_err(|_| JournalError::Corrupt)?; - } - if record.history_key != journal.history_key { - return Err(JournalError::HistoryKeyMismatch); - } - match record.event { - JournalEvent::Accepted { - identity, - frame: encoded_frame, - receipt, - authorization_signature, - } => { - let expected_outcome = match journal.tier { - AcceptanceTier::Independent => DeliveryOutcome::AcceptedDurable, - AcceptanceTier::Local => DeliveryOutcome::AcceptedLocalDurable, - }; - if receipt.address != address - || receipt.outcome != expected_outcome - || receipt.message_id != identity.message_id - || receipt.execution_epoch != identity.execution_epoch - || receipt.validate_versioned_outcome().is_err() - { - return Err(JournalError::Corrupt); - } - let signature: [u8; 64] = authorization_signature - .as_slice() - .try_into() - .map_err(|_| JournalError::Corrupt)?; - authority - .verify( - &acceptance_intent_bytes( - identity, - &encoded_frame, - &receipt, - journal.history_key.as_ref(), - )?, - &Signature::from_bytes(&signature), - ) - .map_err(|_| JournalError::Corrupt)?; - let raw = STANDARD_NO_PAD - .decode(&encoded_frame) - .map_err(|_| JournalError::Corrupt)?; - let (capability, envelope) = - frame::decode_frame(&raw).map_err(|_| JournalError::Corrupt)?; - if receipt.delivery_id != envelope.routing.delivery_id - || receipt.ciphertext_commitment != envelope.ciphertext_commitment - || journal.retention.is_some_and(|policy| { - policy - .retained_until(envelope.routing.expires_at_unix_ms) - .is_err() - }) - || journal - .seen - .insert( - identity, - Seen { - envelope, - capability, - accepted_receipt: *receipt, - acknowledged: false, - expired_at_unix_ms: None, - lease: None, - signed_ack: None, - }, - ) - .is_some() - { - return Err(JournalError::Corrupt); - } - journal.order.push(identity); - } - JournalEvent::Acknowledged { - identity, - acknowledged_at_unix_ms, - } => { - let Some(entry) = journal.seen.get_mut(&identity) else { - return Err(JournalError::Corrupt); - }; - if entry.acknowledged - || entry.expired_at_unix_ms.is_some() - || entry.lease.is_some() - || acknowledged_at_unix_ms < 0 - { - return Err(JournalError::Corrupt); - } - entry.acknowledged = true; - } - JournalEvent::Leased { - identity, - challenge, - lease, - enrollment, - receive_capability, - } => { - let entry = journal - .seen - .get_mut(&identity) - .ok_or(JournalError::Corrupt)?; - if entry.acknowledged - || entry.expired_at_unix_ms.is_some() - || lease.encode().is_err() - || enrollment.encode().is_err() - || lease.lease.address != address - || lease.lease.delivery_id != entry.envelope.routing.delivery_id - || lease.lease.message_id != identity.message_id - || lease.lease.execution_epoch != identity.execution_epoch - || lease.lease.ciphertext_commitment != entry.envelope.ciphertext_commitment - || lease.lease.consumer_session != enrollment.enrollment.consumer_session - || lease.lease.receive_capability_id - != receive_capability.grant.capability_id - || challenge == [0; 32] - { - return Err(JournalError::Corrupt); - } - verify_recovered_lease( - entry, - identity, - &lease, - &enrollment, - &receive_capability, - &authority, - lease.lease.issued_at_unix_ms, - )?; - entry.lease = Some(LeaseState { - retry_count: 0, - signed: *lease, - enrollment: *enrollment, - receive_capability: *receive_capability, - }); - if journal - .used_challenges - .insert(challenge, identity) - .is_some() - { - return Err(JournalError::Corrupt); - } - } - JournalEvent::LeasedRetry { - identity, - challenge, - } => { - let active = journal - .seen - .get_mut(&identity) - .filter(|entry| !entry.acknowledged && entry.expired_at_unix_ms.is_none()) - .and_then(|entry| entry.lease.as_mut()) - .ok_or(JournalError::Corrupt)?; - active.retry_count = active - .retry_count - .checked_add(1) - .filter(|count| *count <= MAX_LEASE_RETRIES) - .ok_or(JournalError::Corrupt)?; - if challenge == [0; 32] - || journal - .used_challenges - .insert(challenge, identity) - .is_some() - { - return Err(JournalError::Corrupt); - } - } - JournalEvent::AcknowledgedSigned { identity, ack } => { - let entry = journal - .seen - .get_mut(&identity) - .ok_or(JournalError::Corrupt)?; - let active = entry.lease.as_ref().ok_or(JournalError::Corrupt)?; - if entry.acknowledged - || entry.expired_at_unix_ms.is_some() - || ack.encode().is_err() - || ack.ack.lease != active.signed - { - return Err(JournalError::Corrupt); - } - let guest_key = verify_recovered_lease( - entry, - identity, - &active.signed, - &active.enrollment, - &active.receive_capability, - &authority, - ack.ack.acknowledged_at_unix_ms, - )?; - let signature: [u8; 64] = ack - .signature - .as_slice() - .try_into() - .map_err(|_| JournalError::Corrupt)?; - guest_key - .verify( - &canonical_bytes(DELIVERY_ACK_SIGNATURE_DOMAIN, &ack.ack) - .map_err(|_| JournalError::Corrupt)?, - &Signature::from_bytes(&signature), - ) - .map_err(|_| JournalError::Corrupt)?; - entry.acknowledged = true; - entry.signed_ack = Some(*ack); - } - JournalEvent::Expired { - identity, - expired_at_unix_ms, - authorization_signature, - } => { - let entry = journal - .seen - .get_mut(&identity) - .ok_or(JournalError::Corrupt)?; - if entry.acknowledged - || entry.expired_at_unix_ms.is_some() - || expired_at_unix_ms < entry.envelope.routing.expires_at_unix_ms - { - return Err(JournalError::Corrupt); - } - let signature: [u8; 64] = authorization_signature - .as_slice() - .try_into() - .map_err(|_| JournalError::Corrupt)?; - authority - .verify( - &expiry_intent_bytes(address, identity, entry, expired_at_unix_ms)?, - &Signature::from_bytes(&signature), - ) - .map_err(|_| JournalError::Corrupt)?; - entry.expired_at_unix_ms = Some(expired_at_unix_ms); - } - JournalEvent::Checkpoint { .. } => return Err(JournalError::Corrupt), - } - let next = JournalTip { - cursor, - head: expected_tag, - }; - if cursor == expected.cursor && next != expected { - return Err(JournalError::Rollback); - } - if cursor > expected.cursor { - if !suffix_synced { - // A predecessor could crash after publishing a record but - // before syncing its directory entry. Recovery must not - // advance an independent anchor ahead of that entry. - sync_dir(root) - .map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; - suffix_synced = true; - } - reconcile(next)?; - } - journal.tip = next; - } - if journal.tip.cursor < expected.cursor { - return Err(JournalError::Rollback); - } - if start != 0 { - for (_, path) in obsolete { - fs::remove_file(path)?; - } - } - for (cursor, path) in checkpoints { - if cursor > expected.cursor || cursor < start { - // Pre-anchor or superseded generations are removed only after - // the committed prefix has been verified against the anchor. - fs::remove_file(&path)?; - } - } - sync_dir(root).map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; - Ok(journal) - } - - fn restore_checkpoint(&mut self, path: &Path, cursor: u64) -> Result<(), JournalError> { - let bytes = read_record(path)?; - if bytes.len() as u64 > self.limits.max_bytes { - return Err(JournalError::Full); - } - let (previous, sealed) = bytes.split_at_checked(32).ok_or(JournalError::Corrupt)?; - let (tag, ciphertext) = sealed.split_at_checked(32).ok_or(JournalError::Corrupt)?; - let previous: [u8; 32] = previous.try_into().map_err(|_| JournalError::Corrupt)?; - let expected_tag = self.mac_after(previous, ciphertext); - if self.tier != AcceptanceTier::Local || tag != expected_tag { - return Err(JournalError::Corrupt); - } - let plaintext = Zeroizing::new( - aead::open( - &self.record_key(), - ciphertext, - &[DOMAIN, &previous].concat(), - ) - .map_err(|_| JournalError::Corrupt)?, - ); - let record: Record = - serde_json::from_slice(&plaintext).map_err(|_| JournalError::Corrupt)?; - if record.version != 3 - || record.cursor != cursor - || record.previous != previous - || record.address != self.address - || record.history_key != self.history_key - { - return Err(JournalError::Corrupt); - } - let JournalEvent::Checkpoint { - state, - authorization_signature, - } = record.event - else { - return Err(JournalError::Corrupt); - }; - if state.source.cursor.checked_add(1) != Some(cursor) - || state.source.head != previous - || state.address != self.address - || Some(state.history_key) != self.history_key - || Some(state.retention) != self.retention - || state.created_at_unix_ms < 0 - { - return Err(JournalError::Corrupt); - } - let signature: [u8; 64] = authorization_signature - .as_slice() - .try_into() - .map_err(|_| JournalError::Corrupt)?; - self.authority - .verify( - &canonical_bytes(CHECKPOINT_DOMAIN, &state).map_err(|_| JournalError::Corrupt)?, - &Signature::from_bytes(&signature), - ) - .map_err(|_| JournalError::Corrupt)?; - let mut seen = BTreeMap::new(); - let mut order = Vec::new(); - for CheckpointEntry { - identity, - seen: entry, - } in state.entries - { - let receipt = &entry.accepted_receipt; - if receipt.address != self.address - || receipt.outcome != DeliveryOutcome::AcceptedLocalDurable - || receipt.message_id != identity.message_id - || receipt.execution_epoch != identity.execution_epoch - || receipt.delivery_id != entry.envelope.routing.delivery_id - || receipt.ciphertext_commitment != entry.envelope.ciphertext_commitment - || receipt.validate_versioned_outcome().is_err() - || state - .retention - .retained_until(entry.envelope.routing.expires_at_unix_ms) - .is_err() - || entry - .expired_at_unix_ms - .is_some_and(|time| time < entry.envelope.routing.expires_at_unix_ms) - || (entry.acknowledged && entry.expired_at_unix_ms.is_some()) - || (entry.signed_ack.is_some() && !entry.acknowledged) - { - return Err(JournalError::Corrupt); - } - if let Some(lease) = &entry.lease { - if lease.retry_count > MAX_LEASE_RETRIES { - return Err(JournalError::Corrupt); - } - let at = entry - .signed_ack - .as_ref() - .map_or(lease.signed.lease.issued_at_unix_ms, |ack| { - ack.ack.acknowledged_at_unix_ms - }); - let guest_key = verify_recovered_lease( - &entry, - identity, - &lease.signed, - &lease.enrollment, - &lease.receive_capability, - &self.authority, - at, - )?; - if let Some(ack) = &entry.signed_ack { - if ack.ack.lease != lease.signed || ack.encode().is_err() { - return Err(JournalError::Corrupt); - } - let signature: [u8; 64] = ack - .signature - .as_slice() - .try_into() - .map_err(|_| JournalError::Corrupt)?; - guest_key - .verify( - &canonical_bytes(DELIVERY_ACK_SIGNATURE_DOMAIN, &ack.ack) - .map_err(|_| JournalError::Corrupt)?, - &Signature::from_bytes(&signature), - ) - .map_err(|_| JournalError::Corrupt)?; - } else if entry.acknowledged { - return Err(JournalError::Corrupt); - } - } else if entry.signed_ack.is_some() { - return Err(JournalError::Corrupt); - } - if seen.insert(identity, entry).is_some() { - return Err(JournalError::Corrupt); - } - order.push(identity); - } - let mut used_challenges = BTreeMap::new(); - for (challenge, identity) in state.used_challenges { - if challenge == [0; 32] - || !seen.contains_key(&identity) - || used_challenges.insert(challenge, identity).is_some() - { - return Err(JournalError::Corrupt); - } - } - self.seen = seen; - self.order = order; - self.used_challenges = used_challenges; - self.tip = JournalTip { - cursor, - head: expected_tag, - }; - self.bytes = bytes.len() as u64; - self.records = 1; - Ok(()) - } - /// The legacy local callback cannot prove an independent anchor or /// recoverable key custody, so it cannot issue durable acceptance. /// @@ -1599,136 +920,6 @@ impl MailboxJournal { self.admit_at_tier(request, AcceptanceTier::Local, |next| anchor.advance(next)) } - /// Commit a signed local snapshot before pruning the superseded prefix. - /// - /// A quota refusal leaves the old history untouched; uncertain I/O poisons - /// this handle until recovery has verified the committed anchor. The caller - /// must provide trusted UNIX time for terminal-state retention. - pub fn compact_local( - &mut self, - now_unix_ms: i64, - signer: &SigningKey, - anchor: &FileJournalAnchor, - ) -> Result { - self.local_ready(anchor, &signer.verifying_key())?; - let retention = self.retention.ok_or(JournalError::InvalidLimits)?; - let history_key = self - .history_key - .ok_or(JournalError::HistoryKeyUnavailable)?; - if now_unix_ms < 0 { - return Err(JournalError::InvalidLimits); - } - let mut retired = BTreeSet::new(); - for (identity, entry) in &self.seen { - if (entry.acknowledged || entry.expired_at_unix_ms.is_some()) - && now_unix_ms - > retention - .retained_until(entry.envelope.routing.expires_at_unix_ms) - .map_err(|_| JournalError::Corrupt)? - { - retired.insert(*identity); - } - } - let state = CheckpointState { - source: self.tip, - address: self.address, - history_key, - retention, - created_at_unix_ms: now_unix_ms, - entries: self - .order - .iter() - .filter(|identity| !retired.contains(identity)) - .map(|identity| { - self.seen - .get(identity) - .cloned() - .map(|seen| CheckpointEntry { - identity: *identity, - seen, - }) - }) - .collect::>>() - .ok_or(JournalError::Corrupt)?, - used_challenges: self - .used_challenges - .iter() - .filter(|(_, identity)| !retired.contains(identity)) - .map(|(challenge, identity)| (*challenge, *identity)) - .collect(), - }; - let signature = signer - .sign(&canonical_bytes(CHECKPOINT_DOMAIN, &state)?) - .to_bytes() - .to_vec(); - let cursor = self.tip.cursor.checked_add(1).ok_or(JournalError::Full)?; - let record = Record { - version: 3, - cursor, - previous: self.tip.head, - address: self.address, - history_key: self.history_key, - event: JournalEvent::Checkpoint { - state: Box::new(state), - authorization_signature: signature, - }, - }; - let body = Zeroizing::new(serde_json::to_vec(&record).map_err(|_| JournalError::Corrupt)?); - if body.len() + 64 + aead::NONCE_SIZE + aead::TAG_SIZE > MAX_RECORD_BYTES { - return Err(JournalError::Full); - } - let ciphertext = aead::seal( - &self.record_key(), - &body, - &[DOMAIN, &self.tip.head].concat(), - ); - let size = (ciphertext.len() + 64) as u64; - if self - .bytes - .checked_add(size) - .is_none_or(|n| n > self.limits.max_bytes) - { - return Err(JournalError::Full); - } - let tag = self.mac(&ciphertext); - let mut bytes = self.tip.head.to_vec(); - bytes.extend_from_slice(&tag); - bytes.extend_from_slice(&ciphertext); - let path = checkpoint_path(&self.root, cursor); - match fs::symlink_metadata(&path) { - Ok(_) => return Err(JournalError::Corrupt), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - if let Err(error) = atomic_write_new(&path, &bytes).and_then(|()| sync_dir(&self.root)) { - self.poisoned = true; - return Err(JournalError::Storage(std::io::Error::other(error))); - } - let next = JournalTip { cursor, head: tag }; - if let Err(error) = anchor.advance(next) { - self.poisoned = true; - return Err(error); - } - self.tip = next; - self.poisoned = true; - for entry in fs::read_dir(&self.root)? { - let old = entry?.path(); - if old != path && old != self.root.join("journal.lock") { - fs::remove_file(old)?; - } - } - sync_dir(&self.root) - .map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; - self.order.retain(|identity| !retired.contains(identity)); - self.seen.retain(|identity, _| !retired.contains(identity)); - self.used_challenges - .retain(|_, identity| !retired.contains(identity)); - self.bytes = size; - self.records = 1; - self.poisoned = false; - Ok(next) - } - fn append( &mut self, event: JournalEvent, @@ -2239,18 +1430,17 @@ fn record_path(root: &Path, cursor: u64) -> PathBuf { root.join(format!("entry-{cursor:020}.bin")) } -fn checkpoint_path(root: &Path, cursor: u64) -> PathBuf { - root.join(format!("checkpoint-{cursor:020}.bin")) -} - fn read_record(path: &Path) -> Result, JournalError> { - let meta = fs::symlink_metadata(path)?; - if !meta.file_type().is_file() || meta.len() > MAX_RECORD_BYTES as u64 { + let file = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path)?; + let meta = file.metadata()?; + if !meta.file_type().is_file() || meta.nlink() != 1 || meta.len() > MAX_RECORD_BYTES as u64 { return Err(JournalError::Corrupt); } let mut bytes = Vec::new(); - fs::File::open(path)? - .take(MAX_RECORD_BYTES as u64 + 1) + file.take(MAX_RECORD_BYTES as u64 + 1) .read_to_end(&mut bytes)?; if bytes.len() > MAX_RECORD_BYTES { return Err(JournalError::Corrupt); @@ -2261,6 +1451,7 @@ fn read_record(path: &Path) -> Result, JournalError> { #[cfg(test)] mod anchor_tests { use super::*; + use ed25519_dalek::{Signature, Verifier}; use mvm_contract::protocol::mailbox::{ AuthorizationDomainId, AuthorizationDomainRouteLabel, BootId, CapabilityId, CapabilityOperations, CausationId, CiphertextPadding, ContentType, CorrelationId, @@ -2269,6 +1460,7 @@ mod anchor_tests { MailboxRoutingFrame, MessageId, OrderingScope, PrincipalId, ReplayNonce, RevocationId, SequenceNumber, TenantId, TransportBindingId, }; + use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; struct IndependentAnchor { @@ -2874,6 +2066,42 @@ mod anchor_tests { )); } + #[test] + fn recovery_never_follows_a_replaced_record_outside_its_private_directory() { + let dir = tempfile::tempdir().expect("journal root"); + let root = dir.path().join("journal"); + crate::private_fs::ensure_private_dir(&root).expect("private journal"); + let external = dir.path().join("outside"); + fs::write(&external, b"outside must not be read").expect("external file"); + std::os::unix::fs::symlink(&external, record_path(&root, 1)) + .expect("replace record with symlink"); + let signer = SigningKey::from_bytes(&[6; 32]); + let (reference, custody) = versioned_custody(); + let anchor = FileJournalAnchor::open( + &dir.path().join("anchor").join("tip.json"), + address(), + [22; 32], + ) + .expect("local anchor"); + assert!( + MailboxJournal::open_local_with_custody( + &root, + reference, + signer.verifying_key(), + limits(), + &custody, + &anchor, + local_retention(), + ) + .is_err() + ); + assert_eq!( + fs::read(&external).expect("external file retained"), + b"outside must not be read" + ); + assert_eq!(anchor.tip(), JournalTip::genesis()); + } + #[test] fn checkpoint_keeps_tombstone_through_window_then_retires_terminal_state() { let dir = tempfile::tempdir().expect("journal root"); diff --git a/crates/mvm-core/src/mailbox/journal/anchor.rs b/crates/mvm-core/src/mailbox/journal/anchor.rs new file mode 100644 index 0000000000..323f992bd8 --- /dev/null +++ b/crates/mvm-core/src/mailbox/journal/anchor.rs @@ -0,0 +1,134 @@ +//! The same-host, fsynced high-water mark for local durability. + +use std::fs; +use std::io::Read; +use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use hmac::{Hmac, Mac, digest::KeyInit}; +use mvm_contract::protocol::mailbox::MailboxAddress; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use zeroize::Zeroizing; + +use super::{JournalError, JournalTip}; +use crate::private_fs; +use crate::util::atomic_io::{FileLock, atomic_write_durable}; + +const ANCHOR_DOMAIN: &[u8] = b"mvm.mailbox.anchor.v1\0"; +const MAX_ANCHOR_BYTES: u64 = 1024; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AnchorBody { + version: u8, + address: MailboxAddress, + tip: JournalTip, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AnchorRecord { + body: AnchorBody, + tag: [u8; 32], +} + +/// A software-first, locally fsynced anchor. Keep its file outside the +/// journal directory. It detects journal-only rollback, not simultaneous +/// rollback of the entire trusted host and its anchor storage. +pub struct FileJournalAnchor { + path: PathBuf, + pub(super) address: MailboxAddress, + key: Zeroizing<[u8; 32]>, + tip: Mutex, + _lock: FileLock, +} + +impl FileJournalAnchor { + /// Open a private anchor file, or genesis if none exists. + pub fn open(path: &Path, address: MailboxAddress, key: [u8; 32]) -> Result { + let parent = path.parent().ok_or(JournalError::Corrupt)?; + private_fs::ensure_private_dir(parent)?; + let lock = FileLock::try_acquire(path) + .map_err(|e| std::io::Error::other(e.to_string()))? + .ok_or(JournalError::Busy)?; + let tip = match fs::symlink_metadata(path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => JournalTip::genesis(), + Err(e) => return Err(e.into()), + Ok(meta) if !meta.file_type().is_file() || meta.len() > MAX_ANCHOR_BYTES => { + return Err(JournalError::Corrupt); + } + Ok(_) => { + let file = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path)?; + let meta = file.metadata()?; + if !meta.file_type().is_file() || meta.nlink() != 1 || meta.len() > MAX_ANCHOR_BYTES + { + return Err(JournalError::Corrupt); + } + let mut bytes = Vec::new(); + file.take(MAX_ANCHOR_BYTES + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_ANCHOR_BYTES { + return Err(JournalError::Corrupt); + } + let record: AnchorRecord = + serde_json::from_slice(&bytes).map_err(|_| JournalError::Corrupt)?; + let body = serde_json::to_vec(&record.body).map_err(|_| JournalError::Corrupt)?; + if record.body.version != 1 + || record.body.address != address + || anchor_mac(&key, &body) != record.tag + { + return Err(JournalError::Corrupt); + } + record.body.tip + } + }; + Ok(Self { + path: path.to_owned(), + address, + key: Zeroizing::new(key), + tip: Mutex::new(tip), + _lock: lock, + }) + } + + #[must_use] + pub fn tip(&self) -> JournalTip { + *self.tip.lock().expect("mailbox anchor lock poisoned") + } + + /// Advance by one committed journal entry, fsyncing the file and its + /// containing directory before returning. + pub fn advance(&self, tip: JournalTip) -> Result<(), JournalError> { + let mut current = self.tip.lock().expect("mailbox anchor lock poisoned"); + if current.cursor.checked_add(1) != Some(tip.cursor) { + return Err(JournalError::Rollback); + } + let body = AnchorBody { + version: 1, + address: self.address, + tip, + }; + let tag = anchor_mac( + &self.key[..], + &serde_json::to_vec(&body).map_err(|_| JournalError::Corrupt)?, + ); + let bytes = + serde_json::to_vec(&AnchorRecord { body, tag }).map_err(|_| JournalError::Corrupt)?; + atomic_write_durable(&self.path, &bytes) + .map_err(|e| JournalError::Storage(std::io::Error::other(e.to_string())))?; + *current = tip; + Ok(()) + } +} + +fn anchor_mac(key: &[u8], body: &[u8]) -> [u8; 32] { + let mut mac = + as KeyInit>::new_from_slice(key).expect("a 32-byte HMAC key is valid"); + mac.update(ANCHOR_DOMAIN); + mac.update(body); + mac.finalize().into_bytes().into() +} diff --git a/crates/mvm-core/src/mailbox/journal/checkpoint.rs b/crates/mvm-core/src/mailbox/journal/checkpoint.rs new file mode 100644 index 0000000000..25e3663f57 --- /dev/null +++ b/crates/mvm-core/src/mailbox/journal/checkpoint.rs @@ -0,0 +1,354 @@ +//! Signed local generations and crash-safe prefix replacement. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier}; +use mvm_contract::protocol::mailbox::{ + DELIVERY_ACK_SIGNATURE_DOMAIN, DeliveryOutcome, MailboxAddress, MailboxRetentionPolicy, +}; +use serde::{Deserialize, Serialize}; +use zeroize::Zeroizing; + +use super::{ + AcceptanceTier, DOMAIN, FileJournalAnchor, HistoryKeyRef, JournalError, JournalEvent, + JournalTip, MAX_LEASE_RETRIES, MAX_RECORD_BYTES, MailboxJournal, Record, ReplayIdentity, Seen, + aead, atomic_write_new, canonical_bytes, read_record, sync_dir, verify_recovered_lease, +}; + +const CHECKPOINT_DOMAIN: &[u8] = b"mvm.mailbox.local-checkpoint.v1\0"; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CheckpointEntry { + identity: ReplayIdentity, + seen: Seen, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct CheckpointState { + source: JournalTip, + address: MailboxAddress, + history_key: HistoryKeyRef, + retention: MailboxRetentionPolicy, + created_at_unix_ms: i64, + entries: Vec, + used_challenges: Vec<([u8; 32], ReplayIdentity)>, +} + +pub(super) fn checkpoint_path(root: &Path, cursor: u64) -> PathBuf { + root.join(format!("checkpoint-{cursor:020}.bin")) +} + +impl MailboxJournal { + pub(super) fn restore_checkpoint( + &mut self, + path: &Path, + cursor: u64, + ) -> Result<(), JournalError> { + let bytes = read_record(path)?; + if bytes.len() as u64 > self.limits.max_bytes { + return Err(JournalError::Full); + } + let (previous, sealed) = bytes.split_at_checked(32).ok_or(JournalError::Corrupt)?; + let (tag, ciphertext) = sealed.split_at_checked(32).ok_or(JournalError::Corrupt)?; + let previous: [u8; 32] = previous.try_into().map_err(|_| JournalError::Corrupt)?; + let expected_tag = self.mac_after(previous, ciphertext); + if self.tier != AcceptanceTier::Local || tag != expected_tag { + return Err(JournalError::Corrupt); + } + let plaintext = Zeroizing::new( + aead::open( + &self.record_key(), + ciphertext, + &[DOMAIN, &previous].concat(), + ) + .map_err(|_| JournalError::Corrupt)?, + ); + let record: Record = + serde_json::from_slice(&plaintext).map_err(|_| JournalError::Corrupt)?; + if record.version != 3 + || record.cursor != cursor + || record.previous != previous + || record.address != self.address + || record.history_key != self.history_key + { + return Err(JournalError::Corrupt); + } + let JournalEvent::Checkpoint { + state, + authorization_signature, + } = record.event + else { + return Err(JournalError::Corrupt); + }; + if state.source.cursor.checked_add(1) != Some(cursor) + || state.source.head != previous + || state.address != self.address + || Some(state.history_key) != self.history_key + || Some(state.retention) != self.retention + || state.created_at_unix_ms < 0 + { + return Err(JournalError::Corrupt); + } + let signature: [u8; 64] = authorization_signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + self.authority + .verify( + &canonical_bytes(CHECKPOINT_DOMAIN, &state).map_err(|_| JournalError::Corrupt)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; + let mut seen = BTreeMap::new(); + let mut order = Vec::new(); + for CheckpointEntry { + identity, + seen: entry, + } in state.entries + { + let receipt = &entry.accepted_receipt; + if receipt.address != self.address + || receipt.outcome != DeliveryOutcome::AcceptedLocalDurable + || receipt.message_id != identity.message_id + || receipt.execution_epoch != identity.execution_epoch + || receipt.delivery_id != entry.envelope.routing.delivery_id + || receipt.ciphertext_commitment != entry.envelope.ciphertext_commitment + || receipt.validate_versioned_outcome().is_err() + || state + .retention + .retained_until(entry.envelope.routing.expires_at_unix_ms) + .is_err() + || entry + .expired_at_unix_ms + .is_some_and(|time| time < entry.envelope.routing.expires_at_unix_ms) + || (entry.acknowledged && entry.expired_at_unix_ms.is_some()) + || (entry.signed_ack.is_some() && !entry.acknowledged) + { + return Err(JournalError::Corrupt); + } + if let Some(lease) = &entry.lease { + if lease.retry_count > MAX_LEASE_RETRIES { + return Err(JournalError::Corrupt); + } + let at = entry + .signed_ack + .as_ref() + .map_or(lease.signed.lease.issued_at_unix_ms, |ack| { + ack.ack.acknowledged_at_unix_ms + }); + let guest_key = verify_recovered_lease( + &entry, + identity, + &lease.signed, + &lease.enrollment, + &lease.receive_capability, + &self.authority, + at, + )?; + if let Some(ack) = &entry.signed_ack { + if ack.ack.lease != lease.signed || ack.encode().is_err() { + return Err(JournalError::Corrupt); + } + let signature: [u8; 64] = ack + .signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + guest_key + .verify( + &canonical_bytes(DELIVERY_ACK_SIGNATURE_DOMAIN, &ack.ack) + .map_err(|_| JournalError::Corrupt)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; + } else if entry.acknowledged { + return Err(JournalError::Corrupt); + } + } else if entry.signed_ack.is_some() { + return Err(JournalError::Corrupt); + } + if seen.insert(identity, entry).is_some() { + return Err(JournalError::Corrupt); + } + order.push(identity); + } + let mut used_challenges = BTreeMap::new(); + for (challenge, identity) in state.used_challenges { + if challenge == [0; 32] + || !seen.contains_key(&identity) + || used_challenges.insert(challenge, identity).is_some() + { + return Err(JournalError::Corrupt); + } + } + self.seen = seen; + self.order = order; + self.used_challenges = used_challenges; + self.tip = JournalTip { + cursor, + head: expected_tag, + }; + self.bytes = bytes.len() as u64; + self.records = 1; + Ok(()) + } + + /// Whether local maintenance should checkpoint before quota exhaustion or + /// retire terminal replay evidence past its signed retention horizon. + pub fn local_compaction_due(&self, now_unix_ms: i64) -> Result { + let retention = self.retention.ok_or(JournalError::InvalidLimits)?; + if now_unix_ms < 0 || self.tier != AcceptanceTier::Local { + return Err(JournalError::InvalidLimits); + } + if self.records >= self.limits.max_records.saturating_sub(8).max(1) + || (self.records > 1 && self.bytes >= self.limits.max_bytes / 2) + { + return Ok(true); + } + for entry in self.seen.values() { + if (entry.acknowledged || entry.expired_at_unix_ms.is_some()) + && now_unix_ms + > retention + .retained_until(entry.envelope.routing.expires_at_unix_ms) + .map_err(|_| JournalError::Corrupt)? + { + return Ok(true); + } + } + Ok(false) + } + + /// Commit a signed local snapshot before pruning the superseded prefix. + /// + /// A quota refusal leaves the old history untouched; uncertain I/O poisons + /// this handle until recovery has verified the committed anchor. The caller + /// must provide trusted UNIX time for terminal-state retention. + pub fn compact_local( + &mut self, + now_unix_ms: i64, + signer: &SigningKey, + anchor: &FileJournalAnchor, + ) -> Result { + self.local_ready(anchor, &signer.verifying_key())?; + let retention = self.retention.ok_or(JournalError::InvalidLimits)?; + let history_key = self + .history_key + .ok_or(JournalError::HistoryKeyUnavailable)?; + if now_unix_ms < 0 { + return Err(JournalError::InvalidLimits); + } + let mut retired = BTreeSet::new(); + for (identity, entry) in &self.seen { + if (entry.acknowledged || entry.expired_at_unix_ms.is_some()) + && now_unix_ms + > retention + .retained_until(entry.envelope.routing.expires_at_unix_ms) + .map_err(|_| JournalError::Corrupt)? + { + retired.insert(*identity); + } + } + let state = CheckpointState { + source: self.tip, + address: self.address, + history_key, + retention, + created_at_unix_ms: now_unix_ms, + entries: self + .order + .iter() + .filter(|identity| !retired.contains(identity)) + .map(|identity| { + self.seen + .get(identity) + .cloned() + .map(|seen| CheckpointEntry { + identity: *identity, + seen, + }) + }) + .collect::>>() + .ok_or(JournalError::Corrupt)?, + used_challenges: self + .used_challenges + .iter() + .filter(|(_, identity)| !retired.contains(identity)) + .map(|(challenge, identity)| (*challenge, *identity)) + .collect(), + }; + let signature = signer + .sign(&canonical_bytes(CHECKPOINT_DOMAIN, &state)?) + .to_bytes() + .to_vec(); + let cursor = self.tip.cursor.checked_add(1).ok_or(JournalError::Full)?; + let record = Record { + version: 3, + cursor, + previous: self.tip.head, + address: self.address, + history_key: self.history_key, + event: JournalEvent::Checkpoint { + state: Box::new(state), + authorization_signature: signature, + }, + }; + let body = Zeroizing::new(serde_json::to_vec(&record).map_err(|_| JournalError::Corrupt)?); + if body.len() + 64 + aead::NONCE_SIZE + aead::TAG_SIZE > MAX_RECORD_BYTES { + return Err(JournalError::Full); + } + let ciphertext = aead::seal( + &self.record_key(), + &body, + &[DOMAIN, &self.tip.head].concat(), + ); + let size = (ciphertext.len() + 64) as u64; + if self + .bytes + .checked_add(size) + .is_none_or(|n| n > self.limits.max_bytes) + { + return Err(JournalError::Full); + } + let tag = self.mac(&ciphertext); + let mut bytes = self.tip.head.to_vec(); + bytes.extend_from_slice(&tag); + bytes.extend_from_slice(&ciphertext); + let path = checkpoint_path(&self.root, cursor); + match fs::symlink_metadata(&path) { + Ok(_) => return Err(JournalError::Corrupt), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + if let Err(error) = atomic_write_new(&path, &bytes).and_then(|()| sync_dir(&self.root)) { + self.poisoned = true; + return Err(JournalError::Storage(std::io::Error::other(error))); + } + let next = JournalTip { cursor, head: tag }; + if let Err(error) = anchor.advance(next) { + self.poisoned = true; + return Err(error); + } + self.tip = next; + self.poisoned = true; + for entry in fs::read_dir(&self.root)? { + let old = entry?.path(); + if old != path && old != self.root.join("journal.lock") { + fs::remove_file(old)?; + } + } + sync_dir(&self.root) + .map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; + self.order.retain(|identity| !retired.contains(identity)); + self.seen.retain(|identity, _| !retired.contains(identity)); + self.used_challenges + .retain(|_, identity| !retired.contains(identity)); + self.bytes = size; + self.records = 1; + self.poisoned = false; + Ok(next) + } +} diff --git a/crates/mvm-core/src/mailbox/journal/recovery.rs b/crates/mvm-core/src/mailbox/journal/recovery.rs new file mode 100644 index 0000000000..1422b6b6c5 --- /dev/null +++ b/crates/mvm-core/src/mailbox/journal/recovery.rs @@ -0,0 +1,429 @@ +//! Reconstruct the authenticated journal, reconcile only intact suffixes, and +//! never infer a new genesis from a missing anchored generation. + +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; + +use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; +use ed25519_dalek::{Signature, Verifier, VerifyingKey}; +use mvm_contract::protocol::mailbox::{ + DELIVERY_ACK_SIGNATURE_DOMAIN, DeliveryOutcome, MailboxAddress, +}; +use zeroize::Zeroizing; + +use super::super::{canonical_bytes, frame}; +use super::{ + AcceptanceTier, DOMAIN, JournalError, JournalEvent, JournalKey, JournalLimits, JournalTip, + LeaseState, MAX_JOURNAL_BYTES, MAX_LEASE_RETRIES, MAX_RECORDS, MailboxJournal, Record, Seen, + acceptance_intent_bytes, checkpoint_path, expiry_intent_bytes, read_record, record_path, + verify_recovered_lease, +}; +use crate::crypto::aead; +use crate::private_fs; +use crate::util::atomic_io::{FileLock, sync_dir}; + +impl MailboxJournal { + pub(super) fn open_with_recovery( + root: &Path, + address: MailboxAddress, + authority: VerifyingKey, + key: JournalKey, + limits: JournalLimits, + expected: JournalTip, + mut reconcile: impl FnMut(JournalTip) -> Result<(), JournalError>, + ) -> Result { + if limits.max_records == 0 + || limits.max_records > MAX_RECORDS + || limits.max_bytes == 0 + || limits.max_bytes > MAX_JOURNAL_BYTES + { + return Err(JournalError::InvalidLimits); + } + if expected.cursor == 0 && expected != JournalTip::genesis() { + return Err(JournalError::Rollback); + } + private_fs::ensure_private_dir(root)?; + sync_dir(root.parent().ok_or(JournalError::Corrupt)?) + .map_err(|e| JournalError::Storage(std::io::Error::other(e.to_string())))?; + let lock = FileLock::try_acquire(&root.join("journal")) + .map_err(|e| std::io::Error::other(e.to_string()))? + .ok_or(JournalError::Busy)?; + let mut journal = Self { + root: root.to_owned(), + address, + authority, + key: key.bytes, + history_key: key.reference, + tier: key.tier, + retention: key.retention, + limits, + tip: JournalTip::genesis(), + bytes: 0, + records: 0, + seen: BTreeMap::new(), + order: Vec::new(), + used_challenges: BTreeMap::new(), + poisoned: false, + _lock: lock, + }; + let mut paths = Vec::new(); + let mut checkpoints = Vec::new(); + for entry in fs::read_dir(root)? { + let entry = entry?; + if entry.file_name() == "journal.lock" { + continue; + } + let path = entry.path(); + let name = entry.file_name(); + let name = name.to_str().ok_or(JournalError::Corrupt)?; + if let Some(raw) = name.strip_prefix("checkpoint-") { + let cursor = raw + .strip_suffix(".bin") + .and_then(|raw| raw.parse::().ok()) + .ok_or(JournalError::Corrupt)?; + if path != checkpoint_path(root, cursor) { + return Err(JournalError::Corrupt); + } + checkpoints.push((cursor, path)); + } else if let Some(raw) = name.strip_prefix("entry-") { + let cursor = raw + .strip_suffix(".bin") + .and_then(|raw| raw.parse::().ok()) + .ok_or(JournalError::Corrupt)?; + if path != record_path(root, cursor) { + return Err(JournalError::Corrupt); + } + paths.push((cursor, path)); + } else { + return Err(JournalError::Corrupt); + } + if paths.len() > limits.max_records || checkpoints.len() > 2 { + return Err(JournalError::Full); + } + } + checkpoints.sort(); + if !checkpoints.is_empty() && journal.tier != AcceptanceTier::Local { + return Err(JournalError::Corrupt); + } + if let Some((cursor, path)) = checkpoints + .iter() + .rev() + .find(|(cursor, _)| *cursor <= expected.cursor) + { + journal.restore_checkpoint(path, *cursor)?; + if *cursor == expected.cursor && journal.tip != expected { + return Err(JournalError::Rollback); + } + } + let start = journal.tip.cursor; + let (mut paths, obsolete): (Vec<_>, Vec<_>) = + paths.into_iter().partition(|(cursor, _)| *cursor > start); + paths.sort(); + let mut suffix_synced = false; + for (_, path) in paths { + let cursor = journal + .tip + .cursor + .checked_add(1) + .ok_or(JournalError::Full)?; + if path != record_path(root, cursor) { + return Err(JournalError::Corrupt); + } + let bytes = read_record(&path)?; + journal.bytes = journal + .bytes + .checked_add(bytes.len() as u64) + .filter(|n| *n <= limits.max_bytes) + .ok_or(JournalError::Full)?; + journal.records += 1; + let (tag, ciphertext) = bytes.split_at_checked(32).ok_or(JournalError::Corrupt)?; + let expected_tag = journal.mac(ciphertext); + if tag != expected_tag { + return Err(JournalError::Corrupt); + } + let plaintext = Zeroizing::new( + aead::open( + &journal.record_key(), + ciphertext, + &[DOMAIN, &journal.tip.head].concat(), + ) + .map_err(|_| JournalError::Corrupt)?, + ); + let record: Record = + serde_json::from_slice(&plaintext).map_err(|_| JournalError::Corrupt)?; + let expected_version = if journal.history_key.is_some() { 3 } else { 2 }; + if record.version != expected_version + || record.cursor != cursor + || record.previous != journal.tip.head + || record.address != address + { + return Err(JournalError::Corrupt); + } + if let Some(reference) = &record.history_key { + reference.validate().map_err(|_| JournalError::Corrupt)?; + } + if record.history_key != journal.history_key { + return Err(JournalError::HistoryKeyMismatch); + } + match record.event { + JournalEvent::Accepted { + identity, + frame: encoded_frame, + receipt, + authorization_signature, + } => { + let expected_outcome = match journal.tier { + AcceptanceTier::Independent => DeliveryOutcome::AcceptedDurable, + AcceptanceTier::Local => DeliveryOutcome::AcceptedLocalDurable, + }; + if receipt.address != address + || receipt.outcome != expected_outcome + || receipt.message_id != identity.message_id + || receipt.execution_epoch != identity.execution_epoch + || receipt.validate_versioned_outcome().is_err() + { + return Err(JournalError::Corrupt); + } + let signature: [u8; 64] = authorization_signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + authority + .verify( + &acceptance_intent_bytes( + identity, + &encoded_frame, + &receipt, + journal.history_key.as_ref(), + )?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; + let raw = STANDARD_NO_PAD + .decode(&encoded_frame) + .map_err(|_| JournalError::Corrupt)?; + let (capability, envelope) = + frame::decode_frame(&raw).map_err(|_| JournalError::Corrupt)?; + if receipt.delivery_id != envelope.routing.delivery_id + || receipt.ciphertext_commitment != envelope.ciphertext_commitment + || journal.retention.is_some_and(|policy| { + policy + .retained_until(envelope.routing.expires_at_unix_ms) + .is_err() + }) + || journal + .seen + .insert( + identity, + Seen { + envelope, + capability, + accepted_receipt: *receipt, + acknowledged: false, + expired_at_unix_ms: None, + lease: None, + signed_ack: None, + }, + ) + .is_some() + { + return Err(JournalError::Corrupt); + } + journal.order.push(identity); + } + JournalEvent::Acknowledged { + identity, + acknowledged_at_unix_ms, + } => { + let Some(entry) = journal.seen.get_mut(&identity) else { + return Err(JournalError::Corrupt); + }; + if entry.acknowledged + || entry.expired_at_unix_ms.is_some() + || entry.lease.is_some() + || acknowledged_at_unix_ms < 0 + { + return Err(JournalError::Corrupt); + } + entry.acknowledged = true; + } + JournalEvent::Leased { + identity, + challenge, + lease, + enrollment, + receive_capability, + } => { + let entry = journal + .seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)?; + if entry.acknowledged + || entry.expired_at_unix_ms.is_some() + || lease.encode().is_err() + || enrollment.encode().is_err() + || lease.lease.address != address + || lease.lease.delivery_id != entry.envelope.routing.delivery_id + || lease.lease.message_id != identity.message_id + || lease.lease.execution_epoch != identity.execution_epoch + || lease.lease.ciphertext_commitment != entry.envelope.ciphertext_commitment + || lease.lease.consumer_session != enrollment.enrollment.consumer_session + || lease.lease.receive_capability_id + != receive_capability.grant.capability_id + || challenge == [0; 32] + { + return Err(JournalError::Corrupt); + } + verify_recovered_lease( + entry, + identity, + &lease, + &enrollment, + &receive_capability, + &authority, + lease.lease.issued_at_unix_ms, + )?; + entry.lease = Some(LeaseState { + retry_count: 0, + signed: *lease, + enrollment: *enrollment, + receive_capability: *receive_capability, + }); + if journal + .used_challenges + .insert(challenge, identity) + .is_some() + { + return Err(JournalError::Corrupt); + } + } + JournalEvent::LeasedRetry { + identity, + challenge, + } => { + let active = journal + .seen + .get_mut(&identity) + .filter(|entry| !entry.acknowledged && entry.expired_at_unix_ms.is_none()) + .and_then(|entry| entry.lease.as_mut()) + .ok_or(JournalError::Corrupt)?; + active.retry_count = active + .retry_count + .checked_add(1) + .filter(|count| *count <= MAX_LEASE_RETRIES) + .ok_or(JournalError::Corrupt)?; + if challenge == [0; 32] + || journal + .used_challenges + .insert(challenge, identity) + .is_some() + { + return Err(JournalError::Corrupt); + } + } + JournalEvent::AcknowledgedSigned { identity, ack } => { + let entry = journal + .seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)?; + let active = entry.lease.as_ref().ok_or(JournalError::Corrupt)?; + if entry.acknowledged + || entry.expired_at_unix_ms.is_some() + || ack.encode().is_err() + || ack.ack.lease != active.signed + { + return Err(JournalError::Corrupt); + } + let guest_key = verify_recovered_lease( + entry, + identity, + &active.signed, + &active.enrollment, + &active.receive_capability, + &authority, + ack.ack.acknowledged_at_unix_ms, + )?; + let signature: [u8; 64] = ack + .signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + guest_key + .verify( + &canonical_bytes(DELIVERY_ACK_SIGNATURE_DOMAIN, &ack.ack) + .map_err(|_| JournalError::Corrupt)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; + entry.acknowledged = true; + entry.signed_ack = Some(*ack); + } + JournalEvent::Expired { + identity, + expired_at_unix_ms, + authorization_signature, + } => { + let entry = journal + .seen + .get_mut(&identity) + .ok_or(JournalError::Corrupt)?; + if entry.acknowledged + || entry.expired_at_unix_ms.is_some() + || expired_at_unix_ms < entry.envelope.routing.expires_at_unix_ms + { + return Err(JournalError::Corrupt); + } + let signature: [u8; 64] = authorization_signature + .as_slice() + .try_into() + .map_err(|_| JournalError::Corrupt)?; + authority + .verify( + &expiry_intent_bytes(address, identity, entry, expired_at_unix_ms)?, + &Signature::from_bytes(&signature), + ) + .map_err(|_| JournalError::Corrupt)?; + entry.expired_at_unix_ms = Some(expired_at_unix_ms); + } + JournalEvent::Checkpoint { .. } => return Err(JournalError::Corrupt), + } + let next = JournalTip { + cursor, + head: expected_tag, + }; + if cursor == expected.cursor && next != expected { + return Err(JournalError::Rollback); + } + if cursor > expected.cursor { + if !suffix_synced { + // A predecessor could crash after publishing a record but + // before syncing its directory entry. Recovery must not + // advance an independent anchor ahead of that entry. + sync_dir(root) + .map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; + suffix_synced = true; + } + reconcile(next)?; + } + journal.tip = next; + } + if journal.tip.cursor < expected.cursor { + return Err(JournalError::Rollback); + } + if start != 0 { + for (_, path) in obsolete { + fs::remove_file(path)?; + } + } + for (cursor, path) in checkpoints { + if cursor > expected.cursor || cursor < start { + // Pre-anchor or superseded generations are removed only after + // the committed prefix has been verified against the anchor. + fs::remove_file(&path)?; + } + } + sync_dir(root).map_err(|error| JournalError::Storage(std::io::Error::other(error)))?; + Ok(journal) + } +} diff --git a/crates/mvm-hostd/README.md b/crates/mvm-hostd/README.md index 36caa25bb0..ad0802f25c 100644 --- a/crates/mvm-hostd/README.md +++ b/crates/mvm-hostd/README.md @@ -41,15 +41,15 @@ signing, and audit emission each pass through their dedicated policy gate. Key material remains inside the smallest role that needs it; callers receive only the result. -The test-scoped `mailbox_custody` prototype retains three distinct random +`mailbox_custody` retains three distinct random purpose keys (journal history, recipient payload opening, local anchor) in a private bundle under the existing host keys directory. A host-signed manifest binds their IDs and version to the admitted plan, VM, address, epoch and authority; recovery loads the existing signer and exact bundle, never mints a replacement. An interrupted first provision leaves an orphan and refuses. -This is trusted-host local crash recovery only: it is not wired to the -production mailbox process, cannot detect whole-host rollback, and does not -provide host-blind confidentiality. +The production helper verifies this custody and the guest boot binding before +binding its listener. This is trusted-host local crash recovery only: it +cannot detect whole-host rollback or provide host-blind confidentiality. The journal-backed service can bind a V2 local receipt to its existing endpoint and recover an exact retry after restart; on Send and Receive it settles at most eight expired entries per request, returning unavailable when maintenance @@ -58,8 +58,12 @@ remains rather than reporting an uncommitted terminal state. A separate guest V2 receipt before removing the pending message; the old `outbox` path retains its ephemeral unit-verdict behavior. Expired submissions receive a typed terminal refusal and retain a bounded guest-side rejection record. Production -mailbox launch remains refused until this handoff, the retained custody path, -and anchored retention/compaction are qualified end to end. +mailboxes require a signed retention window under the host's configured ceiling; +the local helper returns only `accepted_local_durable` after anchoring and +retains signed guest ACK and dedup state across checkpoint and restart. +The source retry spool holds at most 64 immutable intents per admitted mailbox +scope; saturation refuses further sends rather than discarding uncertain +operations or inventing fresh envelopes for their IDs. ## Main areas diff --git a/crates/mvm-hostd/src/lib.rs b/crates/mvm-hostd/src/lib.rs index e775dd1ca2..e8e92208f7 100644 --- a/crates/mvm-hostd/src/lib.rs +++ b/crates/mvm-hostd/src/lib.rs @@ -64,7 +64,6 @@ pub mod jailer; /// [`keyholder::SecretResolver`] trait + the single-host /// [`keyholder::LocalResolver`]. pub mod keyholder; -#[cfg(test)] mod mailbox_custody; pub mod mailbox_launch; pub mod mailbox_process; diff --git a/crates/mvm-hostd/src/mailbox_custody.rs b/crates/mvm-hostd/src/mailbox_custody.rs index 68c662c5dd..0d23ece99c 100644 --- a/crates/mvm-hostd/src/mailbox_custody.rs +++ b/crates/mvm-hostd/src/mailbox_custody.rs @@ -75,6 +75,12 @@ pub(crate) struct RetainedMailboxKeys { material: Zeroizing<[u8; KEY_BYTES]>, } +pub(crate) struct MailboxStoragePaths { + pub(crate) journal: PathBuf, + pub(crate) anchor: PathBuf, + pub(crate) source_intents: PathBuf, +} + fn paths(keys_dir: &Path, identity: &Identity<'_>) -> Result<(PathBuf, PathBuf)> { let digest = Sha256::digest(serde_jcs::to_vec(identity)?); let name = hex::encode(digest); @@ -161,6 +167,61 @@ fn identity<'a>( } impl RetainedMailboxKeys { + /// First launch provisions once; subsequent launches require the exact + /// existing signed manifest. A partial first write cannot be replaced. + pub(crate) fn provision_or_open( + keys_dir: &Path, + plan: &ExecutionPlan, + vm_name: &str, + posture: &MailboxLaunchPosture, + current_boot: Option, + address: MailboxAddress, + ) -> Result { + posture.verify(plan, vm_name)?; + let identity = identity(keys_dir, plan, vm_name, address)?; + let (bundle, manifest) = paths(keys_dir, &identity)?; + let mut existing = false; + for path in [&bundle, &manifest] { + match std::fs::symlink_metadata(path) { + Ok(_) => existing = true, + Err(error) if error.kind() == ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + } + if existing { + Self::open_existing( + keys_dir, + plan, + vm_name, + &posture.handoff(current_boot), + current_boot, + address, + ) + } else { + Self::provision(keys_dir, plan, vm_name, posture, address) + } + } + + /// Host-local storage remains outside the per-VM state directory so a VM + /// teardown cannot erase accepted history or its high-water mark. + pub(crate) fn storage_paths(&self, keys_dir: &Path) -> Result { + let binding = &self.binding; + let identity = Identity { + plan_id: &binding.plan_id, + plan_nonce: &binding.plan_nonce, + vm_name: &binding.vm_name, + address: binding.address, + epoch: binding.epoch, + retention_ms: binding.retention_ms, + }; + let (bundle, _) = paths(keys_dir, &identity)?; + Ok(MailboxStoragePaths { + journal: bundle.with_extension("journal"), + anchor: bundle.with_extension("anchor"), + source_intents: bundle.with_extension("intents"), + }) + } + /// Provision once under a verified launch posture; an interrupted first /// provision leaves an orphan bundle and refuses instead of replacing it. pub(crate) fn provision( diff --git a/crates/mvm-hostd/src/mailbox_launch.rs b/crates/mvm-hostd/src/mailbox_launch.rs index a6f5184905..9e8fbcf6da 100644 --- a/crates/mvm-hostd/src/mailbox_launch.rs +++ b/crates/mvm-hostd/src/mailbox_launch.rs @@ -156,7 +156,6 @@ impl Drop for DerivedMailboxKeys { impl DerivedMailboxKeys { /// Mailbox-scoped destination signer and separate journal/anchor MAC keys. - #[cfg(test)] pub(crate) fn durable_material(&self) -> (SigningKey, [u8; 32], [u8; 32]) { ( SigningKey::from_bytes(&self.authority), @@ -166,7 +165,6 @@ impl DerivedMailboxKeys { } /// Bind the retained purpose keys to this host-derived mailbox identity. - #[cfg(test)] pub(crate) fn with_retained( mut self, retained: &crate::mailbox_custody::RetainedMailboxKeys, diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index 001da50ec3..f13840a3ec 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -19,11 +19,11 @@ //! declares the capability. A grant on one that does not refuses the launch //! rather than boot a guest that asserts a mailbox nobody serves. //! -//! Until the separate local receipt, retained custody, retention, and guest -//! handoff are qualified end to end, only admitted Dev launches serve the -//! ephemeral Send queue. Production mailbox launches refuse before boot, and -//! reader frames fail closed. The local anchor never authorizes a signed -//! `AcceptedDurable` claim. +//! Admitted Dev launches retain the ephemeral Send queue. Production requires +//! an explicit signed retention window, verified guest boot, and retained +//! custody before binding a receipt-bearing local journal and guest reader. +//! The same-host anchor authorizes `AcceptedLocalDurable`, never the stronger +//! whole-host rollback-resistant `AcceptedDurable` claim. use std::io::{Error, ErrorKind, Result, Write as _}; use std::os::unix::process::CommandExt as _; @@ -33,18 +33,20 @@ use std::sync::mpsc::sync_channel; use std::time::Duration; use mvm_contract::protocol::mailbox::{MAILBOX_PORT, MailboxGrant}; +use mvm_core::mailbox::journal::{FileJournalAnchor, JournalError, JournalLimits, MailboxJournal}; use mvm_core::plan::{ExecutionPlan, Variant}; use mvm_core::vm_backend::{HostDialRoute, VmId, VmStartConfig}; use mvm_runtime::backend::AnyBackend; use mvm_vmm::host::aux_bin::AuxBin; use mvm_vmm::host::flowmux_identity::{MAILBOX_READER_BOOT_FILE, MailboxReaderBootBinding}; +use crate::mailbox_custody::RetainedMailboxKeys; use crate::mailbox_launch::{DerivedMailboxKeys, LAUNCH_LIMITS, derive_launch_keys}; use crate::mailbox_service::MailboxService; +use crate::mailbox_service::durable::DurableMailboxService; use crate::plan_admission::{MailboxLaunchPosture, MailboxPostureHandoff}; use crate::vm_lifetime::{endpoint_needed, launcher_is_alive}; -#[cfg(test)] mod reader; /// The helper binary this module spawns. @@ -57,6 +59,10 @@ const POLL_INTERVAL: Duration = Duration::from_millis(500); const READY_TIMEOUT: Duration = Duration::from_secs(10); const READY_MARKER: &[u8; 6] = b"READY\n"; const MAX_HELPER_PLAN_BYTES: u64 = 1024 * 1024; +const LOCAL_JOURNAL_LIMITS: JournalLimits = JournalLimits { + max_records: 10_000, + max_bytes: 64 * 1024 * 1024, +}; /// What the process reads on stdin. `deny_unknown_fields`, so a field a newer /// launcher adds fails closed here. @@ -65,6 +71,8 @@ const MAX_HELPER_PLAN_BYTES: u64 = 1024 * 1024; pub struct MailboxProcessConfig { pub grant: MailboxGrant, pub keys: DerivedMailboxKeys, + /// Absolute host key directory from the launcher; helper environment is empty. + pub keys_dir: PathBuf, pub socket: PathBuf, pub guest_route: HostDialRoute, /// Absent on an old or unauthenticated restart: fail closed, never default Dev. @@ -113,7 +121,7 @@ fn load_admitted_plan(state_dir: &Path) -> Result { /// Serve until neither the launcher nor the VM is left; the exit code. pub fn serve(config: MailboxProcessConfig) -> i32 { - match serve_ephemeral(config) { + match serve_verified(config) { Ok(()) => 0, Err(error) => { tracing::error!(%error, "mailbox service failed closed"); @@ -122,12 +130,13 @@ pub fn serve(config: MailboxProcessConfig) -> i32 { } } -fn serve_ephemeral(config: MailboxProcessConfig) -> Result<()> { +fn serve_verified(config: MailboxProcessConfig) -> Result<()> { // SAFETY: `getppid` is async-signal-safe and infallible. let launcher = unsafe { libc::getppid() }; let MailboxProcessConfig { grant, keys, + keys_dir, socket, guest_route, posture, @@ -158,10 +167,32 @@ fn serve_ephemeral(config: MailboxProcessConfig) -> Result<()> { reader_boot.as_ref().map(|binding| binding.boot), ) .map_err(Error::other)?; - if posture.variant() != Variant::Dev || plan.mailbox != Some(grant) { + if plan.mailbox != Some(grant) { + return Err(Error::new( + ErrorKind::PermissionDenied, + "mailbox helper grant does not match its admitted plan", + )); + } + if posture.variant() == Variant::Prod { + return serve_local( + launcher, + &plan, + MailboxProcessConfig { + grant, + keys, + keys_dir, + socket, + guest_route, + posture: Some(posture), + reader_boot, + vm_state_dir, + }, + ); + } + if posture.variant() != Variant::Dev { return Err(Error::new( ErrorKind::PermissionDenied, - "production mailbox local-durable path is not yet qualified", + "invalid mailbox posture", )); } let (policy, sealer) = keys.trust(); @@ -182,6 +213,121 @@ fn serve_ephemeral(config: MailboxProcessConfig) -> Result<()> { Ok(()) } +fn serve_local( + launcher: libc::pid_t, + plan: &ExecutionPlan, + config: MailboxProcessConfig, +) -> Result<()> { + let MailboxProcessConfig { + grant, + keys, + keys_dir, + socket, + guest_route, + posture, + reader_boot, + vm_state_dir, + } = config; + let posture = posture.ok_or_else(|| { + Error::new( + ErrorKind::PermissionDenied, + "production mailbox has no admitted posture", + ) + })?; + let vm_name = vm_state_dir + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| Error::new(ErrorKind::InvalidData, "mailbox VM identity is invalid"))?; + if !keys_dir.is_absolute() { + return Err(Error::new( + ErrorKind::PermissionDenied, + "mailbox custody directory must be absolute", + )); + } + let binding = reader_boot.as_ref().ok_or_else(|| { + Error::new( + ErrorKind::PermissionDenied, + "production mailbox requires a verified guest reader boot", + ) + })?; + let retention = grant.retention.ok_or_else(|| { + Error::new( + ErrorKind::PermissionDenied, + "production mailbox has no signed retention", + ) + })?; + let (initial_policy, _) = keys.trust(); + let address = mvm_contract::protocol::mailbox::MailboxAddress { + tenant: initial_policy.context.expected_tenant, + authorization_domain: initial_policy.context.expected_authorization_domain, + mailbox: grant.mailbox, + recipient: initial_policy.context.expected_recipient, + }; + let retained = RetainedMailboxKeys::open_existing( + &keys_dir, + plan, + vm_name, + &posture, + Some(binding.boot), + address, + ) + .map_err(Error::other)?; + if retained.retention().map_err(Error::other)? != retention { + return Err(Error::new( + ErrorKind::PermissionDenied, + "local mailbox retention differs from signed custody", + )); + } + let paths = retained.storage_paths(&keys_dir).map_err(Error::other)?; + let derived = keys.with_retained(&retained)?; + let (signer, _, _) = derived.durable_material(); + let (policy, sealer) = derived.trust(); + let mut sealer = sealer.with_grant(&grant)?; + mvm_core::private_fs::ensure_private_dir(&paths.source_intents).map_err(Error::other)?; + sealer.intent_dir = Some(paths.source_intents); + let anchor = FileJournalAnchor::open(&paths.anchor, address, retained.anchor_key()) + .map_err(Error::other)?; + let journal = MailboxJournal::open_local_with_custody( + &paths.journal, + retained.history_reference(), + signer.verifying_key(), + LOCAL_JOURNAL_LIMITS, + &retained, + &anchor, + retention, + ) + .map_err(Error::other)?; + let reader = reader::load_or_enroll( + &vm_state_dir, + &grant, + binding.clone(), + &sealer.capability, + &signer, + )?; + let service = DurableMailboxService::bind_local( + &socket, + LAUNCH_LIMITS, + policy.with_grant(&grant)?, + signer, + journal, + anchor, + )?; + service.install_guest_reader(reader)?; + std::io::stdout().write_all(READY_MARKER)?; + while vm_state_dir.exists() + && endpoint_needed(launcher_is_alive(launcher), vm_is_alive(&vm_state_dir)) + { + if service.serve_guest(&guest_route, &mut sealer).is_err() { + std::thread::sleep(POLL_INTERVAL); + } + match service.maintain_local() { + Ok(()) | Err(JournalError::MaintenancePending) => {} + Err(error) => return Err(Error::other(error)), + } + } + Ok(()) +} + fn vm_is_alive(state_dir: &Path) -> bool { mvm_vmm::host::process_liveness::state_dir_has_live_process(state_dir) } @@ -197,6 +343,7 @@ pub fn build_config( Ok(MailboxProcessConfig { grant: *grant, keys: derive_launch_keys(keys_dir, grant)?, + keys_dir: keys_dir.to_path_buf(), socket: mvm_core::config::vm_mailbox_socket_at(state_dir), guest_route: backend .host_dial_route(state_dir, MAILBOX_PORT) @@ -323,10 +470,12 @@ pub fn require_servable( ) })?; posture.verify(&plan, &config.name)?; - if posture.variant() != Variant::Dev { + if posture.variant() == Variant::Prod + && plan.mailbox.is_some_and(|grant| grant.retention.is_none()) + { return Err(Error::new( ErrorKind::PermissionDenied, - "production mailbox local-durable path is not yet qualified", + "production mailbox requires signed retention", ) .into()); } @@ -396,10 +545,10 @@ pub fn spawn_for_launch( ) })?; posture.verify(plan, vm_name).map_err(Error::other)?; - if posture.variant() != Variant::Dev { + if posture.variant() == Variant::Prod && grant.retention.is_none() { return Err(Error::new( ErrorKind::PermissionDenied, - "production mailbox local-durable path is not yet qualified", + "production mailbox requires signed retention", )); } require_wired(plan, backend)?; @@ -411,6 +560,30 @@ pub fn spawn_for_launch( backend, )?; config.reader_boot = load_reader_boot_for_plan(&state_dir, plan)?; + if posture.variant() == Variant::Prod { + let binding = config.reader_boot.as_ref().ok_or_else(|| { + Error::new( + ErrorKind::PermissionDenied, + "production mailbox requires guest boot binding", + ) + })?; + let (policy, _) = config.keys.trust(); + let address = mvm_contract::protocol::mailbox::MailboxAddress { + tenant: policy.context.expected_tenant, + authorization_domain: policy.context.expected_authorization_domain, + mailbox: grant.mailbox, + recipient: policy.context.expected_recipient, + }; + RetainedMailboxKeys::provision_or_open( + &config.keys_dir, + plan, + vm_name, + posture, + Some(binding.boot), + address, + ) + .map_err(Error::other)?; + } config.posture = Some(posture.handoff(config.reader_boot.as_ref().map(|binding| binding.boot))); let bin = mvm_vmm::host::broker_services_spawn::resolve_subprocess_bin_to_spawn(&AuxBin::new( MAILBOX_BIN, @@ -484,6 +657,31 @@ mod tests { dir } + fn accept_guest(listener: &std::os::unix::net::UnixListener) -> std::os::unix::net::UnixStream { + use std::os::fd::AsRawFd as _; + let mut ready = libc::pollfd { + fd: listener.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + // SAFETY: `ready` is a live pollfd for this owned listener, valid for one entry. + let result = unsafe { libc::poll(&mut ready, 1, 10_000) }; + assert_eq!(result, 1, "helper did not dial before the guest deadline"); + assert_ne!( + ready.revents & libc::POLLIN, + 0, + "guest port was not readable" + ); + let (stream, _) = listener.accept().expect("helper dialed guest port"); + stream + .set_nonblocking(false) + .expect("blocking guest exchange"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("bounded guest read"); + stream + } + fn grant(dir: &Path) -> MailboxGrant { let mailbox = MailboxId::from_bytes([4; 32]); MailboxGrant { @@ -751,6 +949,7 @@ mod tests { .unwrap(); let config = MailboxProcessConfig { keys: derive_launch_keys(dir.path(), &grant).unwrap(), + keys_dir: dir.path().to_path_buf(), grant, socket: dir.path().join("mailbox.sock"), guest_route: HostDialRoute::PortSocket(dir.path().join("absent-port")), @@ -811,6 +1010,7 @@ mod tests { let socket = keys.path().join("mailbox.sock"); let config = MailboxProcessConfig { keys: derive_launch_keys(keys.path(), &grant).unwrap(), + keys_dir: keys.path().to_path_buf(), grant, socket: socket.clone(), guest_route: HostDialRoute::PortSocket(guest_port), @@ -866,6 +1066,345 @@ mod tests { assert_eq!(helper.join().unwrap(), 0); } + #[test] + fn production_helper_requires_retained_custody_and_returns_signed_local_receipt() { + use ed25519_dalek::SigningKey; + use mvm_contract::protocol::mailbox::{DeliveryOutcome, PrincipalId}; + use mvm_core::mailbox::frame::{OWN_MAILBOX_RECIPIENT, OutboxMessage}; + + let home = tempfile::tempdir_in("/tmp").expect("isolated host home"); + let mut env = mvm_core::util::test_env::TestEnv::new(); + env.isolate_mvm_home(home.path()); + let keys_dir = mvm_core::config::mvm_keys_dir(); + crate::audit::host_keypair::load_or_init_at(&keys_dir).expect("host signer"); + let state = tempfile::tempdir_in("/tmp").expect("VM state"); + let mut grant = grant(&keys_dir); + grant.retention = Some( + mvm_contract::protocol::mailbox::MailboxRetentionPolicy::new(10_000) + .expect("signed retention"), + ); + let mut plan = mvm_core::plan::test_support::PlanFixture::new().build(); + plan.mailbox = Some(grant); + plan.image.name = state + .path() + .file_name() + .expect("VM name") + .to_string_lossy() + .into_owned(); + plan.plan_id = mvm_core::plan::compute_plan_id(&plan); + std::fs::write( + state.path().join("plan.json"), + serde_json::to_vec(&plan).expect("plan"), + ) + .expect("admitted plan"); + let binding = MailboxReaderBootBinding::mint( + &plan, + SigningKey::from_bytes(&[45; 32]).verifying_key(), + SigningKey::from_bytes(&[44; 32]).verifying_key().to_bytes(), + ) + .expect("cold boot identity"); + binding.persist(state.path()).expect("persist guest boot"); + { + use base64::Engine as _; + mvm_vmm::host::flowmux_identity::InheritableIdentity { + session_id: "production-guest".into(), + guest_verifying_key_base64: base64::engine::general_purpose::STANDARD + .encode(SigningKey::from_bytes(&[45; 32]).verifying_key().as_bytes()), + } + .persist(state.path()) + .expect("persist pinned guest identity"); + } + let posture = MailboxLaunchPosture::for_test(&plan, Variant::Prod); + let keys = derive_launch_keys(&keys_dir, &grant).expect("derived mailbox keys"); + let (policy, _) = keys.trust(); + let address = mvm_contract::protocol::mailbox::MailboxAddress { + tenant: policy.context.expected_tenant, + authorization_domain: policy.context.expected_authorization_domain, + mailbox: grant.mailbox, + recipient: policy.context.expected_recipient, + }; + let retained = RetainedMailboxKeys::provision_or_open( + &keys_dir, + &plan, + &plan.image.name, + &posture, + Some(binding.boot), + address, + ) + .expect("signed custody"); + let paths = retained.storage_paths(&keys_dir).expect("persistent paths"); + let guest_port = home.path().join("guest-port.sock"); + let listener = std::sync::Arc::new( + std::os::unix::net::UnixListener::bind(&guest_port).expect("guest port"), + ); + listener + .set_nonblocking(true) + .expect("bounded guest accept"); + let socket = home.path().join("mailbox.sock"); + let restart_keys = keys.clone(); + let restart_binding = binding.clone(); + let wrong_keys = MailboxProcessConfig { + keys: keys.clone(), + keys_dir: home.path().join("missing-keys"), + grant, + socket: socket.clone(), + guest_route: HostDialRoute::PortSocket(guest_port.clone()), + posture: Some(posture.handoff(Some(binding.boot))), + reader_boot: Some(binding.clone()), + vm_state_dir: state.path().to_path_buf(), + }; + assert!( + serve_verified(wrong_keys).is_err(), + "the helper must load exact retained custody, not use its launch keys alone" + ); + assert!(!socket.exists(), "failed custody cannot bind a listener"); + let config = MailboxProcessConfig { + keys, + keys_dir: keys_dir.clone(), + grant, + socket: socket.clone(), + guest_route: HostDialRoute::PortSocket(guest_port.clone()), + posture: Some(posture.handoff(Some(binding.boot))), + reader_boot: Some(binding), + vm_state_dir: state.path().to_path_buf(), + }; + let (stopped_tx, stopped_rx) = sync_channel(1); + let helper = std::thread::spawn(move || { + let outcome = serve_verified(config); + let _ = stopped_tx.send(()); + outcome + }); + let outbox = home.path().join("outbox"); + let message = OutboxMessage::new( + PrincipalId::from_bytes(OWN_MAILBOX_RECIPIENT), + b"local report".to_vec(), + ); + mvm_agentd::mailbox_outbox::write_outbox(&outbox, &message).expect("outbox"); + let retry_outbox = outbox.clone(); + let send_listener = std::sync::Arc::clone(&listener); + let guest = std::thread::spawn(move || { + let mut stream = accept_guest(&send_listener); + mvm_agentd::mailbox_outbox::forward_local_outbox( + &mut stream, + &outbox, + &ed25519_dalek::VerifyingKey::from_bytes(&grant.authority_key).expect("authority"), + grant.mailbox, + grant.epoch, + ) + .expect("verified signed local receipt") + }); + let accepted = match guest.join() { + Ok(accepted) => accepted, + Err(panic) => { + if helper.is_finished() { + panic!( + "production helper failed: {:?}", + helper.join().expect("helper task") + ); + } + std::panic::resume_unwind(panic); + } + }; + assert_eq!( + accepted.receipt.outcome, + DeliveryOutcome::AcceptedLocalDurable + ); + assert!(paths.journal.is_dir(), "history survives outside VM state"); + assert!(paths.anchor.is_file(), "anchor committed before receipt"); + + use mvm_contract::protocol::mailbox::{ + DeliveryAckNonce, MailboxDeliveryAck, ProtocolVersion, SignatureAlgorithm, + }; + use mvm_core::mailbox::reader::{sign_delivery_ack, sign_reader_possession}; + use mvm_core::mailbox::reader_wire::{ + ReaderRequest, ReaderResponse, read_response, write_request, + }; + let ack_key = SigningKey::from_bytes(&[44; 32]); + let mut receiving = accept_guest(&listener); + write_request(&mut receiving, &ReaderRequest::Receive).expect("request Receive"); + let ReaderResponse::Challenge { + enrollment, + challenge, + .. + } = read_response(&mut receiving).expect("reader challenge") + else { + panic!("guest did not receive a boot-bound challenge"); + }; + let signature = sign_reader_possession(&enrollment, &challenge, &ack_key) + .expect("guest possession proof"); + write_request( + &mut receiving, + &ReaderRequest::Possession { + signature: signature.to_vec(), + }, + ) + .expect("answer challenge"); + let ReaderResponse::Delivery { lease, payload, .. } = + read_response(&mut receiving).expect("leased delivery") + else { + panic!("guest did not receive a durable lease"); + }; + assert_eq!(payload, b"local report"); + drop(receiving); + + let now: i64 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_millis() + .try_into() + .expect("bounded time"); + let signed_ack = sign_delivery_ack( + MailboxDeliveryAck { + protocol_version: ProtocolVersion::V1, + signature_algorithm: SignatureAlgorithm::ED25519, + lease: *lease, + ack_nonce: DeliveryAckNonce::from_bytes([66; 16]), + acknowledged_at_unix_ms: now, + }, + &ack_key, + ) + .expect("signed guest ACK"); + let mut forged = signed_ack.clone(); + forged.signature[0] ^= 1; + let mut rejecting = accept_guest(&listener); + write_request( + &mut rejecting, + &ReaderRequest::Ack { + signed: Box::new(forged), + }, + ) + .expect("forged guest ACK request"); + assert_eq!( + read_response(&mut rejecting), + Ok(ReaderResponse::Refused { + reason: mvm_core::mailbox::frame::Refusal::NotAuthorized + }), + "a forged guest signature must not complete the lease" + ); + drop(rejecting); + let mut acknowledging = accept_guest(&listener); + write_request( + &mut acknowledging, + &ReaderRequest::Ack { + signed: Box::new(signed_ack.clone()), + }, + ) + .expect("guest ACK"); + assert_eq!( + read_response(&mut acknowledging), + Ok(ReaderResponse::Acknowledged { + ack: Box::new(signed_ack) + }), + "only the guest's exact signed ACK may complete the lease" + ); + drop(acknowledging); + drop(listener); + let state_path = state.path().to_path_buf(); + let parked = home.path().join("paused-vm-state"); + std::fs::rename(&state_path, &parked).expect("stop helper without losing VM state"); + stopped_rx + .recv_timeout(Duration::from_secs(20)) + .expect("helper stopped after VM state moved"); + helper.join().expect("helper task").expect("clean stop"); + std::fs::rename(&parked, &state_path).expect("restore same VM identity"); + let anchor = FileJournalAnchor::open(&paths.anchor, address, retained.anchor_key()) + .expect("held local anchor"); + let mut journal = MailboxJournal::open_local_with_custody( + &paths.journal, + retained.history_reference(), + ed25519_dalek::VerifyingKey::from_bytes(&grant.authority_key).expect("authority"), + LOCAL_JOURNAL_LIMITS, + &retained, + &anchor, + grant.retention.expect("signed retention"), + ) + .expect("recover signed guest ACK"); + journal + .compact_local(now, &restart_keys.durable_material().0, &anchor) + .expect("checkpoint ACK before helper restart"); + drop(journal); + drop(anchor); + + std::fs::remove_file(&guest_port).expect("release old guest socket"); + let listener = std::sync::Arc::new( + std::os::unix::net::UnixListener::bind(&guest_port).expect("restart guest port"), + ); + listener + .set_nonblocking(true) + .expect("bounded guest accept"); + let config = MailboxProcessConfig { + keys: restart_keys, + keys_dir: keys_dir.clone(), + grant, + socket, + guest_route: HostDialRoute::PortSocket(guest_port), + posture: Some(posture.handoff(Some(restart_binding.boot))), + reader_boot: Some(restart_binding), + vm_state_dir: state_path, + }; + let (restarted_tx, restarted_rx) = sync_channel(1); + let restarted = std::thread::spawn(move || { + let outcome = serve_verified(config); + let _ = restarted_tx.send(()); + outcome + }); + mvm_agentd::mailbox_outbox::write_outbox(&retry_outbox, &message) + .expect("replay same operation"); + let retry_listener = std::sync::Arc::clone(&listener); + let retry = std::thread::spawn(move || { + let mut stream = accept_guest(&retry_listener); + mvm_agentd::mailbox_outbox::forward_local_outbox( + &mut stream, + &retry_outbox, + &ed25519_dalek::VerifyingKey::from_bytes(&grant.authority_key).expect("authority"), + grant.mailbox, + grant.epoch, + ) + .expect("recover exact signed receipt") + }) + .join() + .expect("retry guest"); + assert_eq!(retry, accepted, "checkpoint retained exact acceptance"); + let mut receiving = accept_guest(&listener); + write_request(&mut receiving, &ReaderRequest::Receive).expect("Receive after restart"); + let ReaderResponse::Challenge { + enrollment, + challenge, + .. + } = read_response(&mut receiving).expect("restart challenge") + else { + panic!("guest reader enrollment was not restored"); + }; + let signature = sign_reader_possession(&enrollment, &challenge, &ack_key) + .expect("same guest boot possession"); + write_request( + &mut receiving, + &ReaderRequest::Possession { + signature: signature.to_vec(), + }, + ) + .expect("answer restart challenge"); + assert_eq!( + read_response(&mut receiving), + Ok(ReaderResponse::Empty), + "signed ACK must remain terminal after checkpoint and restart" + ); + drop(receiving); + drop(listener); + drop(state); + restarted_rx + .recv_timeout(Duration::from_secs(20)) + .expect("restarted helper stopped after VM teardown"); + restarted + .join() + .expect("restarted helper task") + .expect("clean stop"); + assert!( + paths.journal.is_dir(), + "VM teardown cannot erase accepted history" + ); + } + #[test] fn a_grant_that_cannot_be_served_fails_the_launch() { let dir = keys_dir(); @@ -917,6 +1456,23 @@ mod tests { let dev = posture_for(&config, Variant::Dev); assert!(require_servable(&config, &backend, dev.as_ref()).is_ok()); + let mut retained = grant(dir.path()); + retained.retention = Some( + mvm_contract::protocol::mailbox::MailboxRetentionPolicy::new(10_000) + .expect("finite signed window"), + ); + let prod_config = config_for(Some(retained)); + let prod = posture_for(&prod_config, Variant::Prod); + assert!( + require_servable(&prod_config, &backend, prod.as_ref()).is_ok(), + "a verified production grant with explicit retention may reach the helper" + ); + let unwired = backend_named("wasm"); + assert!( + require_servable(&prod_config, &unwired, prod.as_ref()).is_err(), + "a signed window cannot make an unwired backend serve" + ); + let mut forged = serde_json::to_value(dev.unwrap().handoff(None)).unwrap(); forged["plan_id"] = serde_json::json!("sha256:wrong"); let forged: MailboxPostureHandoff = serde_json::from_value(forged).unwrap(); @@ -949,6 +1505,7 @@ mod tests { .unwrap(); let config = MailboxProcessConfig { keys: derive_launch_keys(dir.path(), &grant).unwrap(), + keys_dir: dir.path().to_path_buf(), grant, socket: dir.path().join("mailbox.sock"), guest_route: HostDialRoute::PortSocket(dir.path().join("absent-port")), @@ -960,6 +1517,43 @@ mod tests { assert!(!dir.path().join("mailbox.sock").exists()); } + #[test] + fn production_helper_without_guest_boot_refuses_before_custody_or_binding() { + let keys = keys_dir(); + let mut grant = grant(keys.path()); + grant.retention = Some( + mvm_contract::protocol::mailbox::MailboxRetentionPolicy::new(10_000) + .expect("signed window"), + ); + let start = config_for(Some(grant)); + let plan = granted_plan(&start).expect("admitted plan"); + let state = tempfile::tempdir_in("/tmp").expect("VM state root"); + let vm_state_dir = state.path().join("vm-under-test"); + std::fs::create_dir(&vm_state_dir).expect("VM state"); + std::fs::write( + vm_state_dir.join("plan.json"), + start.plan_json.expect("plan JSON"), + ) + .expect("persist admitted plan"); + let socket = state.path().join("mailbox.sock"); + let config = MailboxProcessConfig { + grant, + keys: derive_launch_keys(keys.path(), &grant).expect("mailbox keys"), + keys_dir: keys.path().to_path_buf(), + socket: socket.clone(), + guest_route: HostDialRoute::PortSocket(state.path().join("guest-port.sock")), + posture: Some(MailboxLaunchPosture::for_test(&plan, Variant::Prod).handoff(None)), + reader_boot: None, + vm_state_dir, + }; + let error = serve_verified(config).expect_err("no signed guest boot"); + assert_eq!(error.kind(), ErrorKind::PermissionDenied); + assert!( + !socket.exists(), + "a refused helper cannot publish a listener" + ); + } + #[test] fn helper_refuses_oversized_or_symlinked_plan_state() { let state = tempfile::tempdir().unwrap(); diff --git a/crates/mvm-hostd/src/mailbox_service/durable.rs b/crates/mvm-hostd/src/mailbox_service/durable.rs index e2d6296779..f66558fe98 100644 --- a/crates/mvm-hostd/src/mailbox_service/durable.rs +++ b/crates/mvm-hostd/src/mailbox_service/durable.rs @@ -129,6 +129,50 @@ pub struct DurableMailboxService { } impl DurableMailboxService { + /// Install the verified, boot-bound Receive authority before advertising + /// readiness to the guest. It cannot be replaced within this helper. + pub(crate) fn install_guest_reader(&self, reader: GuestReaderAuthority) -> std::io::Result<()> { + self.shared.reader.set(reader).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "mailbox guest reader was already installed", + ) + }) + } + + /// Reclaim the bounded local prefix and settle expiry even when no guest + /// connects. An incomplete expiry pass resumes on the helper's next loop. + pub(crate) fn maintain_local(&self) -> Result<(), JournalError> { + let anchor = self + .shared + .local_anchor + .as_ref() + .ok_or(JournalError::DurableAuthorityUnavailable)?; + let now_unix_ms: i64 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| JournalError::Uncertain)? + .as_millis() + .try_into() + .map_err(|_| JournalError::Uncertain)?; + let mut state = self + .shared + .state + .lock() + .expect("mailbox journal lock poisoned"); + if state.journal.local_compaction_due(now_unix_ms)? { + state + .journal + .compact_local(now_unix_ms, &self.shared.signer, anchor)?; + } + settle_expired(&self.shared, &mut state.journal, now_unix_ms)?; + if state.journal.local_compaction_due(now_unix_ms)? { + state + .journal + .compact_local(now_unix_ms, &self.shared.signer, anchor)?; + } + Ok(()) + } + /// Bind a live destination. Caller provides an already-recovered journal /// and an anchor backed by a different rollback domain. pub fn bind( diff --git a/crates/mvm-hostd/src/plan_admission.rs b/crates/mvm-hostd/src/plan_admission.rs index 37d7961ed7..34242348df 100644 --- a/crates/mvm-hostd/src/plan_admission.rs +++ b/crates/mvm-hostd/src/plan_admission.rs @@ -65,7 +65,6 @@ use mvm_build::guest_libc::GuestLibc; use mvm_contract::grants::ceiling::GrantCeiling; use mvm_contract::grants::{CpuGrant, Grants}; use mvm_contract::protocol::capability_negotiation::negotiate_grants; -use mvm_contract::protocol::mailbox::{MailboxGrant, MailboxRetentionPolicy}; use mvm_contract::protocol::resource_controls::{EnforcedGrants, ResourceControls}; use mvm_contract::provenance::{ ActorRef, AttestationBinding, DecisionActorRole, DecisionCategory, DecisionOutcome, @@ -97,13 +96,17 @@ mod canonical_kernel; mod composition; mod display_grants; mod final_gates; +mod grant_validation; mod host_gates; +mod mailbox_posture; mod nonce_ledger; mod registry_pack; mod verb_grant_sidecar; use assurance_runtime::{AssuranceRuntimeBinding, verify_selected_runtime}; +use grant_validation::{enforce_mailbox_retention, enforce_wall_clock_projection}; use host_gates::{admit_host_interface, admit_within_host_budget}; +pub use mailbox_posture::{MailboxLaunchPosture, MailboxPostureHandoff}; pub use nonce_ledger::InMemoryNonceLedger; use registry_pack::verify_registry_pack_assets; use verb_grant_sidecar::mint_verb_grant_sidecar; @@ -168,103 +171,6 @@ pub struct AdmittedPlan { dispatched: bool, } -/// Launch-only posture retained from host admission, never inferred from plan labels. -#[derive(Debug, Clone)] -pub struct MailboxLaunchPosture { - variant: Variant, - plan_id: PlanId, - plan_nonce: mvm_core::plan::Nonce, - vm_name: String, - grant: mvm_contract::protocol::mailbox::MailboxGrant, -} - -/// The private-pipe projection; unlike the launch token, parsing this is not -/// admission authority. Only the launcher holding a verified token sends it. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MailboxPostureHandoff { - variant: Variant, - plan_id: PlanId, - plan_nonce: mvm_core::plan::Nonce, - vm_name: String, - grant: mvm_contract::protocol::mailbox::MailboxGrant, - reader_boot: Option, -} - -impl MailboxPostureHandoff { - pub(crate) fn verify( - &self, - plan: &ExecutionPlan, - vm_name: &str, - reader_boot: Option, - ) -> Result<()> { - MailboxLaunchPosture { - variant: self.variant, - plan_id: self.plan_id.clone(), - plan_nonce: self.plan_nonce.clone(), - vm_name: self.vm_name.clone(), - grant: self.grant, - } - .verify(plan, vm_name)?; - anyhow::ensure!( - self.reader_boot == reader_boot, - "mailbox helper posture belongs to another guest boot" - ); - Ok(()) - } - - pub(crate) const fn variant(&self) -> Variant { - self.variant - } -} - -impl MailboxLaunchPosture { - #[cfg(test)] - pub(crate) fn for_test(plan: &ExecutionPlan, variant: Variant) -> Self { - Self { - variant, - plan_id: plan.plan_id.clone(), - plan_nonce: plan.nonce.clone(), - vm_name: plan.image.name.clone(), - grant: plan.mailbox.expect("mailbox posture fixture has a grant"), - } - } - - /// Check the private helper handoff against this VM's admitted plan. - pub fn verify(&self, plan: &ExecutionPlan, vm_name: &str) -> Result<()> { - anyhow::ensure!( - plan.plan_id == mvm_core::plan::compute_plan_id(plan) - && self.plan_id == plan.plan_id - && self.plan_nonce == plan.nonce - && self.vm_name == vm_name - && plan.image.name == vm_name - && plan.mailbox == Some(self.grant), - "mailbox helper posture does not match the admitted VM and plan" - ); - Ok(()) - } - - /// The posture admission selected; not a hint from the image or guest. - #[must_use] - pub const fn variant(&self) -> Variant { - self.variant - } - - pub(crate) fn handoff( - &self, - reader_boot: Option, - ) -> MailboxPostureHandoff { - MailboxPostureHandoff { - variant: self.variant, - plan_id: self.plan_id.clone(), - plan_nonce: self.plan_nonce.clone(), - vm_name: self.vm_name.clone(), - grant: self.grant, - reader_boot, - } - } -} - /// Re-verified host artifact behind one extension binding in the signed plan. #[derive(Debug, Clone)] pub struct AdmittedExtension { @@ -1019,48 +925,6 @@ fn admit_grants(plan: &ExecutionPlan, ceiling: &GrantCeiling, posture: RunPostur enforceability_gate(grants, plan, posture) } -fn enforce_mailbox_retention(mailbox: Option<&MailboxGrant>, max_ms: Option) -> Result<()> { - let Some(requested) = mailbox.and_then(|grant| grant.retention) else { - return Ok(()); - }; - let max_ms = max_ms - .context("local mailbox retention requires max_mailbox_retention_ms in host config.toml")?; - let ceiling = MailboxRetentionPolicy::new(max_ms) - .context("host max_mailbox_retention_ms must be a finite, nonzero duration")?; - anyhow::ensure!( - requested.retention_ms() <= ceiling.retention_ms(), - "local mailbox retention {} ms exceeds this host's {} ms ceiling", - requested.retention_ms(), - ceiling.retention_ms(), - ); - Ok(()) -} - -/// Refuse a plan whose two encodings of one wall-clock bound disagree. -/// -/// `resources.timeouts.exec_secs` is the projection of `grants.wall_clock`, and -/// synthesis is the only thing that writes it. A signed plan reaching admission -/// with the two out of step was not produced by that path, and there is no -/// right answer to pick between them: enforcing the grant ignores a signed -/// field, enforcing the field ignores what the ceiling was applied to. Refusing -/// is the only choice that cannot silently run a workload under a bound nobody -/// authored. -/// -/// The comparison runs against the projection, never against the grant -/// directly, because the projection is lossy: an absent grant and an explicit -/// `Unbounded` both encode as `0`, and only the ceiling distinguishes them. -fn enforce_wall_clock_projection(plan: &ExecutionPlan, grants: &Grants) -> Result<()> { - let projected = mvm_contract::grants::projection::exec_secs_from_grants(grants); - let carried = plan.resources.timeouts.exec_secs; - if projected != carried { - anyhow::bail!( - "plan carries exec_secs {carried} but its wall-clock grant projects to \ - {projected}; refusing to choose between two encodings of one bound" - ); - } - Ok(()) -} - /// Refuse (prod) or warn (dev) when the backend this run boots on has no /// mechanism behind a declared grant. fn enforceability_gate(grants: &Grants, plan: &ExecutionPlan, posture: RunPosture) -> Result<()> { diff --git a/crates/mvm-hostd/src/plan_admission/grant_validation.rs b/crates/mvm-hostd/src/plan_admission/grant_validation.rs new file mode 100644 index 0000000000..a86ca99808 --- /dev/null +++ b/crates/mvm-hostd/src/plan_admission/grant_validation.rs @@ -0,0 +1,39 @@ +//! Cross-field and host-ceiling checks before signing or dispatching a plan. + +use anyhow::{Context as _, Result}; +use mvm_contract::grants::Grants; +use mvm_contract::protocol::mailbox::{MailboxGrant, MailboxRetentionPolicy}; +use mvm_core::plan::ExecutionPlan; + +pub(super) fn enforce_mailbox_retention( + mailbox: Option<&MailboxGrant>, + max_ms: Option, +) -> Result<()> { + let Some(requested) = mailbox.and_then(|grant| grant.retention) else { + return Ok(()); + }; + let max_ms = max_ms + .context("local mailbox retention requires max_mailbox_retention_ms in host config.toml")?; + let ceiling = MailboxRetentionPolicy::new(max_ms) + .context("host max_mailbox_retention_ms must be a finite, nonzero duration")?; + anyhow::ensure!( + requested.retention_ms() <= ceiling.retention_ms(), + "local mailbox retention {} ms exceeds this host's {} ms ceiling", + requested.retention_ms(), + ceiling.retention_ms(), + ); + Ok(()) +} + +/// A grant cannot carry a wall-clock bound that contradicts its projection. +pub(super) fn enforce_wall_clock_projection(plan: &ExecutionPlan, grants: &Grants) -> Result<()> { + let projected = mvm_contract::grants::projection::exec_secs_from_grants(grants); + let carried = plan.resources.timeouts.exec_secs; + if projected != carried { + anyhow::bail!( + "plan carries exec_secs {carried} but its wall-clock grant projects to \ + {projected}; refusing to choose between two encodings of one bound" + ); + } + Ok(()) +} diff --git a/crates/mvm-hostd/src/plan_admission/mailbox_posture.rs b/crates/mvm-hostd/src/plan_admission/mailbox_posture.rs new file mode 100644 index 0000000000..fc0f6300c6 --- /dev/null +++ b/crates/mvm-hostd/src/plan_admission/mailbox_posture.rs @@ -0,0 +1,98 @@ +//! A verified admission's one-VM mailbox authority and private helper handoff. + +use anyhow::Result; +use mvm_contract::protocol::mailbox::{BootId, MailboxGrant}; +use mvm_core::plan::{ExecutionPlan, Nonce, PlanId, Variant}; + +/// Launch-only posture retained from host admission, never inferred from plan labels. +#[derive(Debug, Clone)] +pub struct MailboxLaunchPosture { + pub(super) variant: Variant, + pub(super) plan_id: PlanId, + pub(super) plan_nonce: Nonce, + pub(super) vm_name: String, + pub(super) grant: MailboxGrant, +} + +/// The private-pipe projection is not itself admission authority. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MailboxPostureHandoff { + variant: Variant, + plan_id: PlanId, + plan_nonce: Nonce, + vm_name: String, + grant: MailboxGrant, + reader_boot: Option, +} + +impl MailboxPostureHandoff { + pub(crate) fn verify( + &self, + plan: &ExecutionPlan, + vm_name: &str, + reader_boot: Option, + ) -> Result<()> { + MailboxLaunchPosture { + variant: self.variant, + plan_id: self.plan_id.clone(), + plan_nonce: self.plan_nonce.clone(), + vm_name: self.vm_name.clone(), + grant: self.grant, + } + .verify(plan, vm_name)?; + anyhow::ensure!( + self.reader_boot == reader_boot, + "mailbox helper posture belongs to another guest boot" + ); + Ok(()) + } + + pub(crate) const fn variant(&self) -> Variant { + self.variant + } +} + +impl MailboxLaunchPosture { + #[cfg(test)] + pub(crate) fn for_test(plan: &ExecutionPlan, variant: Variant) -> Self { + Self { + variant, + plan_id: plan.plan_id.clone(), + plan_nonce: plan.nonce.clone(), + vm_name: plan.image.name.clone(), + grant: plan.mailbox.expect("mailbox posture fixture has a grant"), + } + } + + /// Check the private helper handoff against this VM's admitted plan. + pub fn verify(&self, plan: &ExecutionPlan, vm_name: &str) -> Result<()> { + anyhow::ensure!( + plan.plan_id == mvm_core::plan::compute_plan_id(plan) + && self.plan_id == plan.plan_id + && self.plan_nonce == plan.nonce + && self.vm_name == vm_name + && plan.image.name == vm_name + && plan.mailbox == Some(self.grant), + "mailbox helper posture does not match the admitted VM and plan" + ); + Ok(()) + } + + /// The posture admission selected; not a hint from the image or guest. + #[must_use] + pub const fn variant(&self) -> Variant { + self.variant + } + + pub(crate) fn handoff(&self, reader_boot: Option) -> MailboxPostureHandoff { + MailboxPostureHandoff { + variant: self.variant, + plan_id: self.plan_id.clone(), + plan_nonce: self.plan_nonce.clone(), + vm_name: self.vm_name.clone(), + grant: self.grant, + reader_boot, + } + } +} diff --git a/crates/mvm-runtime/src/workload_runner/runner.rs b/crates/mvm-runtime/src/workload_runner/runner.rs index b47b73e552..12c6ac0109 100644 --- a/crates/mvm-runtime/src/workload_runner/runner.rs +++ b/crates/mvm-runtime/src/workload_runner/runner.rs @@ -79,6 +79,7 @@ struct PreparedWorkloadBoot { use admission::{admitted_ingress, admitted_network_limits}; pub use broker::RealBrokerRegistrar; use claim_lease::WarmClaimLease; +use console_boot::supervisor_owns_console; use parent_verify::{reserve_and_verify_parent, schedule_resident_checkpoint_reclaim}; use refusal::{ admitted_seccomp_tier, refuse, require_admitted_seccomp_tier, require_fresh_child_identity, @@ -1496,15 +1497,6 @@ fn child_start_config(claim: &StandbyClaim, child: &VmId, rootfs_dir: &Path) -> cfg } -/// Whether the backend's own supervisor holds the console capture for the -/// VM's whole life, so the launcher must not start a competing writer. -/// -/// Only the HVF supervisor owns capture today. Every other backend leaves it -/// to the launcher's follower. -fn supervisor_owns_console(kind: mvm_core::vm_backend::BackendKind) -> bool { - kind == mvm_core::vm_backend::BackendKind::Hvf -} - #[cfg(test)] mod tests { #[test] diff --git a/crates/mvm-runtime/src/workload_runner/runner/console_boot.rs b/crates/mvm-runtime/src/workload_runner/runner/console_boot.rs index 7b88e35672..c81a185389 100644 --- a/crates/mvm-runtime/src/workload_runner/runner/console_boot.rs +++ b/crates/mvm-runtime/src/workload_runner/runner/console_boot.rs @@ -1,5 +1,11 @@ use anyhow::Result; +/// Only the HVF supervisor owns console capture for the VM's whole life; +/// other backends need the launcher's follower, not a competing writer. +pub(super) fn supervisor_owns_console(kind: mvm_core::vm_backend::BackendKind) -> bool { + kind == mvm_core::vm_backend::BackendKind::Hvf +} + /// Boot the VMM while the best-effort console follower is being installed. pub(super) fn boot_while_starting_console( boot: impl FnOnce() -> Result, diff --git a/features/suites/s40_mission_demo/mission_demo.feature b/features/suites/s40_mission_demo/mission_demo.feature index 75737c4d4a..d67c0c7c4e 100644 --- a/features/suites/s40_mission_demo/mission_demo.feature +++ b/features/suites/s40_mission_demo/mission_demo.feature @@ -12,6 +12,8 @@ Feature: The three-node mission demo contains, terminates and rebuilds host (macOS 26+ Apple Silicon; one message per VM, a re-witness is pending); neither is a CI lane, and the "Demo status" section of specs/contracts/host-interface-distributed-execution.md records both. + The production helper scenario below exercises the real local journal and + guest wire over a loopback socket; it is not a booted-VM or host-blind witness. Scenario: The compromised workload is contained and replaced in the audit timeline When the scripted mission demo runs @@ -27,3 +29,7 @@ Feature: The three-node mission demo contains, terminates and rebuilds When the scripted mission demo runs Then the demo passes And the cross-node delivery test ran and passed + + Scenario: A production helper persists a local receipt and guest ACK + When the production mailbox helper witness runs + Then the production mailbox helper witness passes diff --git a/public/src/content/docs/reference/cli-commands.md b/public/src/content/docs/reference/cli-commands.md index 9404621c3e..bcfceb599c 100644 --- a/public/src/content/docs/reference/cli-commands.md +++ b/public/src/content/docs/reference/cli-commands.md @@ -1574,9 +1574,13 @@ optional `--mailbox-retention-ms `. The requested retention window is signed into the mailbox grant and requires a mailbox. The operator sets `max_mailbox_retention_ms` in `$MVM_HOME/config/config.toml`; a missing, invalid, or smaller ceiling refuses a retention request before plan signing, -without silently clamping it. Both values are unset by default. This does not -enable local-durable production service: its launch remains gated until -recoverable compaction and the service/guest handoff are qualified. +without silently clamping it. Both values are unset by default. In production, +a mailbox also requires the signed retention window and a verified guest boot +binding. The helper reopens retained keys and the anchored journal before it +reports readiness; sends return a signed `accepted_local_durable` receipt, and +Receive requires a boot-bound guest lease and signed ACK. This same-host tier +does not detect rollback of the entire host and does not issue the stronger +`accepted_durable` receipt. Developer launches retain the ephemeral Send queue. A host can also admit plans it did not synthesize. By default every plan booted locally is synthesized and self-signed under the host key at admission time; From 406a97b161a7a4074c95e3a3f65e198dd89a3edb Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 12 Nov 2026 00:02:02 -0800 Subject: [PATCH 42/51] fix(cli): keep mailbox retention help within summary limit --- crates/mvm-cli/src/commands/vm/exec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/mvm-cli/src/commands/vm/exec.rs b/crates/mvm-cli/src/commands/vm/exec.rs index 743aafb74b..7c88424595 100644 --- a/crates/mvm-cli/src/commands/vm/exec.rs +++ b/crates/mvm-cli/src/commands/vm/exec.rs @@ -246,7 +246,7 @@ pub(in crate::commands) struct RunArgs { /// Grant a mailbox, named by a 64-character hex id. #[arg(long, value_name = "ID")] pub mailbox: Option, - /// Retain local mailbox dedup identities this many milliseconds after expiry. + /// Retain local mailbox dedup IDs after expiry (milliseconds). #[arg(long, value_name = "MILLISECONDS", value_parser = clap::value_parser!(u64).range(1..))] pub mailbox_retention_ms: Option, /// Set memory (for example, 512M or 1G). From f5d906b2dbc21a68b2a4fa6d5c84624c4f1d6c6f Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 12 Nov 2026 02:35:02 -0800 Subject: [PATCH 43/51] fix(mailbox): use live admission clock and distinct spool identities --- crates/mvm-hostd/src/mailbox_service.rs | 26 ++++++++++++++++++------- crates/mvm-hostd/tests/mission_demo.rs | 9 +++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/crates/mvm-hostd/src/mailbox_service.rs b/crates/mvm-hostd/src/mailbox_service.rs index 5b06b19656..b1f7e9b651 100644 --- a/crates/mvm-hostd/src/mailbox_service.rs +++ b/crates/mvm-hostd/src/mailbox_service.rs @@ -341,8 +341,8 @@ pub fn serve_grant( } /// Bind the service for `grant` without dialing the guest, and hand back the -/// grant-bound sealer to serve it with, so a caller can serve the same queue -/// and replay state over many connections. +/// grant-bound sealer to serve it with. Live admission uses the current wall +/// clock on every request, not the time the helper first bound the listener. pub fn bind_grant( grant: &MailboxGrant, trust: MailboxPolicy, @@ -351,7 +351,7 @@ pub fn bind_grant( socket: &Path, ) -> std::io::Result<(MailboxService, GuestSealer)> { let sealer = sealer.with_grant(grant)?; - let service = MailboxService::bind(socket, limits, trust.with_grant(grant)?)?; + let service = MailboxService::bind_live(socket, limits, trust.with_grant(grant)?)?; Ok((service, sealer)) } @@ -1009,14 +1009,26 @@ mod tests { } } - /// What the real launch path builds for one epoch: the service and the - /// sealer, both from the host key directory through the grant. + // These historical-epoch fixtures use their fixed context time; production + // bind_grant instead consults the current wall clock on every admission. + fn bind_fixture_grant( + grant: &MailboxGrant, + trust: MailboxPolicy, + sealer: GuestSealer, + limits: MailboxLimits, + socket: &Path, + ) -> std::io::Result<(MailboxService, GuestSealer)> { + let sealer = sealer.with_grant(grant)?; + let service = MailboxService::bind(socket, limits, trust.with_grant(grant)?)?; + Ok((service, sealer)) + } + fn launch(keys: &Path, socket: &Path, epoch: u64) -> (MailboxService, GuestSealer) { let grant = grant_at(keys, epoch); let (policy, sealer) = derive_launch_keys(keys, &grant, &test_secret()) .unwrap() .trust(); - bind_grant(&grant, policy, sealer, LAUNCH_LIMITS, socket).unwrap() + bind_fixture_grant(&grant, policy, sealer, LAUNCH_LIMITS, socket).unwrap() } #[test] @@ -1080,7 +1092,7 @@ mod tests { let grant = grant_at(keys, 1); let (policy, sealer) = derive_launch_keys(keys, &grant, secret).unwrap().trust(); let socket = keys.join(name); - let _bound = bind_grant(&grant, policy, sealer, LAUNCH_LIMITS, &socket).unwrap(); + let _bound = bind_fixture_grant(&grant, policy, sealer, LAUNCH_LIMITS, &socket).unwrap(); send(&socket, &sealed.0, &sealed.1) } diff --git a/crates/mvm-hostd/tests/mission_demo.rs b/crates/mvm-hostd/tests/mission_demo.rs index 1d7305b335..a7fcf37867 100644 --- a/crates/mvm-hostd/tests/mission_demo.rs +++ b/crates/mvm-hostd/tests/mission_demo.rs @@ -664,7 +664,9 @@ fn a_full_host_queue_delays_the_guest_outbox_instead_of_dropping_it() { let outbox = node.home.path().join("outbox"); std::fs::create_dir(&outbox).unwrap(); for i in 0..total { + let identity_byte: u8 = (i + 1).try_into().expect("bounded spool fixture"); let message = OutboxMessage { + message_id: MessageId::from_bytes([identity_byte; 32]), body: format!("m{i}").into_bytes(), ..outbox_message() }; @@ -1117,11 +1119,15 @@ fn a_running_guest_submits_again_after_its_first_message_was_consumed() { let outbox = dir.path().join("outbox"); mvm_agentd::mailbox_outbox::write_outbox(&outbox, &outbox_message()).unwrap(); let guest_outbox = outbox.clone(); + let (forwarded_tx, forwarded_rx) = std::sync::mpsc::sync_channel(1); let guest = std::thread::spawn(move || { for _ in 0..2 { let (mut stream, _) = listener.accept().unwrap(); mvm_agentd::mailbox_outbox::wait_and_forward_outbox(&mut stream, &guest_outbox) .expect("guest submits pending operation"); + forwarded_tx + .send(()) + .expect("forwarding completion observer"); } }); let launch = serve_derived( @@ -1145,6 +1151,9 @@ fn a_running_guest_submits_again_after_its_first_message_was_consumed() { .expect("the live guest message arrives") }; assert_eq!(receive(), b"hello"); + forwarded_rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("first guest operation was removed"); let next = OutboxMessage::new(aircraft_principal(), b"after-start".to_vec()); mvm_agentd::mailbox_outbox::write_outbox(&outbox, &next).unwrap(); assert_eq!(receive(), b"after-start"); From de0f06afba21b685a86c1bc420e4c08cfd38c228 Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 12 Nov 2026 10:42:28 -0800 Subject: [PATCH 44/51] fix(lab): run KVM witness without root and check legacy refusal --- .../persistent_machine_live.feature | 2 +- scripts/run-gcp-kvm-test-remote.sh | 37 ++++++++++++++++--- scripts/run-gcp-kvm-test.test.sh | 8 ++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/features/suites/s8_readme_contract/persistent_machine_live.feature b/features/suites/s8_readme_contract/persistent_machine_live.feature index ef244cf845..68b087c791 100644 --- a/features/suites/s8_readme_contract/persistent_machine_live.feature +++ b/features/suites/s8_readme_contract/persistent_machine_live.feature @@ -57,7 +57,7 @@ Feature: README persistent machine lifecycle works end to end And the image "python:3.12" is prepared in the live home When I run mvmctl in an isolated live home with "machine create bdd-tool-command --image python:3.12 --policy features/suites/s8_readme_contract/fixtures/legacy-tool-policy.toml" Then the command exits with code 1 - And the output contains "tool-specific policy is no longer supported" + And the error output contains "`tools`: unknown field `tools`" @live @firecracker @ci_live Scenario: a workload process uses admitted egress and a destination-bound secret diff --git a/scripts/run-gcp-kvm-test-remote.sh b/scripts/run-gcp-kvm-test-remote.sh index 58dc34fc39..810d29684e 100755 --- a/scripts/run-gcp-kvm-test-remote.sh +++ b/scripts/run-gcp-kvm-test-remote.sh @@ -109,13 +109,21 @@ if ! command -v firecracker >/dev/null 2>&1 \ /usr/local/bin/jailer fi +cd "$repo" +# Keep toolchains and build state in the tree so the unprivileged test user can +# own them without granting access to root's home. +# shellcheck source=/dev/null +source scripts/dev-env.sh +export RUSTUP_HOME="$repo/.mvm-test/rustup" +export PATH="$CARGO_HOME/bin:$PATH" + if ! command -v rustup >/dev/null 2>&1; then curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ | sh -s -- -y --profile minimal --default-toolchain stable fi # The file is installed by rustup immediately above. # shellcheck source=/dev/null -. /root/.cargo/env +. "$CARGO_HOME/env" guest_rust="$(awk '/^\[workspace\.metadata\.mvm\.toolchain\]/{p=1; next} /^\[/{p=0} p && $1 == "rust" {gsub(/"/, "", $3); print $3}' "$repo/Cargo.toml")" @@ -151,15 +159,32 @@ if ((${#command_argv[@]} == 0)); then exit 1 fi -cd "$repo" -# Repository-owned worktree isolation helper. -# shellcheck source=/dev/null -source scripts/dev-env.sh +# Firecracker refuses a root caller even when nested KVM is available. Give the +# test user only the device's group, not root or a privileged host service. +kvm_group="$(stat -c %G /dev/kvm)" +if [[ "$kvm_group" == UNKNOWN ]]; then + echo "error: /dev/kvm has no named access group" >&2 + exit 1 +fi +useradd -m -s /bin/bash -G "$kvm_group" mvm-test +chown -R mvm-test:mvm-test "$repo" "$results" +if ! runuser -u mvm-test -- test -r /dev/kvm -a -w /dev/kvm; then + echo "error: unprivileged test user cannot access /dev/kvm" >&2 + exit 1 +fi printf '>> running:' printf ' %q' "${command_argv[@]}" printf '\n' set +e -"${command_argv[@]}" 2>&1 | tee "$results/command.log" +runuser -u mvm-test -- env \ + HOME=/home/mvm-test \ + PATH="$CARGO_HOME/bin:/usr/local/bin:/usr/bin:/bin" \ + RUSTUP_HOME="$RUSTUP_HOME" \ + CARGO_HOME="$CARGO_HOME" \ + CARGO_TARGET_DIR="$CARGO_TARGET_DIR" \ + MVM_HOME="$MVM_HOME" \ + MVM_GCP_KVM_RESULTS_DIR="$results" \ + "${command_argv[@]}" 2>&1 | tee "$results/command.log" run_status=${PIPESTATUS[0]} set -e exit "$run_status" diff --git a/scripts/run-gcp-kvm-test.test.sh b/scripts/run-gcp-kvm-test.test.sh index fc3e003d10..92e6d5b0ab 100755 --- a/scripts/run-gcp-kvm-test.test.sh +++ b/scripts/run-gcp-kvm-test.test.sh @@ -98,6 +98,14 @@ grep -Fq 'just_sha256=4a5cc2f53e6f0f8c59092a6cc38291eb729d46a7dd95d3ae582008881b grep -Fq '[[ "$(just --version)" == "just $just_version" ]]' "$remote_runner" grep -Fq 'firecracker_sha256=06094a1108ae9e82aa4c23a775aa92758f53f1175d422270d9d6162cb9ade558' "$remote_runner" grep -Fq '| sha256sum -c -' "$remote_runner" +# The bootstrap is privileged, but Firecracker and the requested witness are not. +# These are literal source fragments, not expressions for this test shell. +# shellcheck disable=SC2016 +grep -Fq 'useradd -m -s /bin/bash -G "$kvm_group" mvm-test' "$remote_runner" +grep -Fq 'runuser -u mvm-test -- test -r /dev/kvm -a -w /dev/kvm' "$remote_runner" +grep -Fq 'runuser -u mvm-test -- env' "$remote_runner" +# shellcheck disable=SC2016 +grep -Fq 'RUSTUP_HOME="$repo/.mvm-test/rustup"' "$remote_runner" # A tracked-only upload has no .git directory. Recover the real, reachable # commit without changing any uploaded bytes, and reject malformed or From c8b755f5d8434f6d74be503c0faf0e70f44c76fc Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 12 Nov 2026 14:13:18 -0800 Subject: [PATCH 45/51] fix(mailbox): keep Unix journal portable and enable KVM user session --- crates/mvm-core/src/mailbox.rs | 1 + crates/mvm-core/src/mailbox/reader.rs | 1 + crates/mvm-core/src/mailbox/transport.rs | 2 +- crates/mvm-core/tests/mailbox_crypto.rs | 2 +- crates/mvm-hostd/src/mailbox_process.rs | 23 +++++++++---------- scripts/run-gcp-kvm-test-remote.sh | 16 +++++++++++++ scripts/run-gcp-kvm-test.sh | 5 +++- scripts/run-gcp-kvm-test.test.sh | 7 +++++- .../057-mailbox-semantics-and-threat-model.md | 2 +- 9 files changed, 42 insertions(+), 17 deletions(-) diff --git a/crates/mvm-core/src/mailbox.rs b/crates/mvm-core/src/mailbox.rs index 89992c7e76..c61e6a0540 100644 --- a/crates/mvm-core/src/mailbox.rs +++ b/crates/mvm-core/src/mailbox.rs @@ -30,6 +30,7 @@ const ENVELOPE_AEAD_DOMAIN: &[u8] = b"mvm.mailbox.payload.v1\0"; pub mod frame; /// Bounded destination-side ciphertext journal and durable acceptance. +#[cfg(unix)] pub mod journal; /// Cryptographic guest reader enrollment, separate from transport admission. diff --git a/crates/mvm-core/src/mailbox/reader.rs b/crates/mvm-core/src/mailbox/reader.rs index 8d7dbb0cd2..63081c41ed 100644 --- a/crates/mvm-core/src/mailbox/reader.rs +++ b/crates/mvm-core/src/mailbox/reader.rs @@ -84,6 +84,7 @@ impl MailboxReaderBootBinding { } /// Reopen a bounded host record without following a symlink. + #[cfg(unix)] pub fn load(state_dir: &Path) -> AnyResult { use std::io::Read as _; use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; diff --git a/crates/mvm-core/src/mailbox/transport.rs b/crates/mvm-core/src/mailbox/transport.rs index 649944c444..7237addbf1 100644 --- a/crates/mvm-core/src/mailbox/transport.rs +++ b/crates/mvm-core/src/mailbox/transport.rs @@ -867,5 +867,5 @@ pub fn connect_with_fallback( mod limits; pub use limits::{ResourceLedger, ResourceLimitError, ResourceLimits, ResourceLimitsBuilder}; -#[cfg(test)] +#[cfg(all(test, unix))] mod tests; diff --git a/crates/mvm-core/tests/mailbox_crypto.rs b/crates/mvm-core/tests/mailbox_crypto.rs index 9570ebb308..34735798ef 100644 --- a/crates/mvm-core/tests/mailbox_crypto.rs +++ b/crates/mvm-core/tests/mailbox_crypto.rs @@ -393,7 +393,7 @@ fn identity_scope_epoch_expiry_and_content_type_are_checked() { } } -#[cfg(feature = "test-support")] +#[cfg(all(feature = "test-support", unix))] #[test] fn local_journal_expiry_recovers_after_interrupted_anchor_commit() { use mvm_core::mailbox::journal::{ diff --git a/crates/mvm-hostd/src/mailbox_process.rs b/crates/mvm-hostd/src/mailbox_process.rs index f46efd198f..a7a40474c2 100644 --- a/crates/mvm-hostd/src/mailbox_process.rs +++ b/crates/mvm-hostd/src/mailbox_process.rs @@ -1500,18 +1500,17 @@ mod tests { "{name}" ); } - for name in ["wasm"] { - let backend = backend_named(name); - assert!(!backend.capabilities().guest_mailbox, "{name}"); - assert!( - require_servable(&granted, &backend, posture.as_ref()).is_err(), - "{name}" - ); - assert!( - require_servable(&config_for(None), &backend, None).is_ok(), - "{name}" - ); - } + let name = "wasm"; + let backend = backend_named(name); + assert!(!backend.capabilities().guest_mailbox, "{name}"); + assert!( + require_servable(&granted, &backend, posture.as_ref()).is_err(), + "{name}" + ); + assert!( + require_servable(&config_for(None), &backend, None).is_ok(), + "{name}" + ); } #[test] diff --git a/scripts/run-gcp-kvm-test-remote.sh b/scripts/run-gcp-kvm-test-remote.sh index 810d29684e..5a9e5fe585 100755 --- a/scripts/run-gcp-kvm-test-remote.sh +++ b/scripts/run-gcp-kvm-test-remote.sh @@ -172,12 +172,28 @@ if ! runuser -u mvm-test -- test -r /dev/kvm -a -w /dev/kvm; then echo "error: unprivileged test user cannot access /dev/kvm" >&2 exit 1 fi +# Firecracker's resource limits require a real user-manager bus and delegated +# systemd scope. A bare runuser process has neither, even with /dev/kvm access. +loginctl enable-linger mvm-test +test_uid="$(id -u mvm-test)" +systemctl start "user@${test_uid}.service" +user_runtime_dir="/run/user/$test_uid" +if [[ ! -S "$user_runtime_dir/bus" ]] \ + || ! runuser -u mvm-test -- env \ + XDG_RUNTIME_DIR="$user_runtime_dir" \ + DBUS_SESSION_BUS_ADDRESS="unix:path=$user_runtime_dir/bus" \ + systemctl --user is-active --quiet default.target; then + echo "error: unprivileged test user has no active systemd session bus" >&2 + exit 1 +fi printf '>> running:' printf ' %q' "${command_argv[@]}" printf '\n' set +e runuser -u mvm-test -- env \ HOME=/home/mvm-test \ + XDG_RUNTIME_DIR="$user_runtime_dir" \ + DBUS_SESSION_BUS_ADDRESS="unix:path=$user_runtime_dir/bus" \ PATH="$CARGO_HOME/bin:/usr/local/bin:/usr/bin:/bin" \ RUSTUP_HOME="$RUSTUP_HOME" \ CARGO_HOME="$CARGO_HOME" \ diff --git a/scripts/run-gcp-kvm-test.sh b/scripts/run-gcp-kvm-test.sh index 67e3c3f20e..57d8e432a3 100755 --- a/scripts/run-gcp-kvm-test.sh +++ b/scripts/run-gcp-kvm-test.sh @@ -208,8 +208,11 @@ trap 'exit 130' INT trap 'exit 143' TERM echo ">> creating the Spot instance" -gcloud "${create_args[@]}" +# The API may create the instance before gcloud returns an error (for example, +# a client-side updater crash). Treat even an uncertain create as ours to +# delete; otherwise the EXIT trap could leave a billable VM behind. instance_created=1 +gcloud "${create_args[@]}" # Compute RUNNING does not mean the independently owned guest sshd is ready. # Google exposes no sshd-ready event, so reconcile it at most five times with diff --git a/scripts/run-gcp-kvm-test.test.sh b/scripts/run-gcp-kvm-test.test.sh index 92e6d5b0ab..9f162fd87e 100755 --- a/scripts/run-gcp-kvm-test.test.sh +++ b/scripts/run-gcp-kvm-test.test.sh @@ -53,7 +53,7 @@ run_case() { } run_case success 0 0 0 0 0 1 -run_case create-failure 13 13 0 0 0 0 +run_case create-failure 13 13 0 0 0 1 run_case upload-failure 17 0 17 0 0 1 run_case remote-failure 23 0 0 23 0 1 run_case results-failure 1 0 0 0 29 1 @@ -106,6 +106,11 @@ grep -Fq 'runuser -u mvm-test -- test -r /dev/kvm -a -w /dev/kvm' "$remote_runne grep -Fq 'runuser -u mvm-test -- env' "$remote_runner" # shellcheck disable=SC2016 grep -Fq 'RUSTUP_HOME="$repo/.mvm-test/rustup"' "$remote_runner" +grep -Fq 'loginctl enable-linger mvm-test' "$remote_runner" +# shellcheck disable=SC2016 +grep -Fq 'systemctl start "user@${test_uid}.service"' "$remote_runner" +# shellcheck disable=SC2016 +grep -Fq 'DBUS_SESSION_BUS_ADDRESS="unix:path=$user_runtime_dir/bus"' "$remote_runner" # A tracked-only upload has no .git directory. Recover the real, reachable # commit without changing any uploaded bytes, and reject malformed or diff --git a/specs/adrs/057-mailbox-semantics-and-threat-model.md b/specs/adrs/057-mailbox-semantics-and-threat-model.md index 4361b489c4..a460add974 100644 --- a/specs/adrs/057-mailbox-semantics-and-threat-model.md +++ b/specs/adrs/057-mailbox-semantics-and-threat-model.md @@ -198,7 +198,7 @@ per-boot guest-held mailbox ACK signing key, not the FlowMux egress identity or the host's mailbox sealing key. The destination authority enrolls its public key against the signed plan ID/nonce, recipient, current authenticated guest boot and reader session, execution/policy epoch, capability ID, and validity -window. The guest proves possession over a fresh challenge before delivery; +window. The guest must demonstrate possession over a fresh challenge before delivery; the destination signs a lease binding that session, delivery, ciphertext commitment, and deadline. ACKs are signed by the enrolled guest key over the exact lease and are replay-checked and journaled before a terminal disposition. From 72715fb5f45145bd332cff6d8ffef0c05df5b7ca Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 12 Nov 2026 16:51:47 -0800 Subject: [PATCH 46/51] fix(lab): pin Firecracker witness and account for uncertain creation --- crates/mvm-agentd/src/mailbox_outbox.rs | 25 ++++++++++++++++++----- crates/mvm-conformance/tests/steps/cli.rs | 4 ++++ scripts/run-cve-2026-80521-gcp.test.sh | 3 ++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/crates/mvm-agentd/src/mailbox_outbox.rs b/crates/mvm-agentd/src/mailbox_outbox.rs index 8eb6f51cb3..d17eec8957 100644 --- a/crates/mvm-agentd/src/mailbox_outbox.rs +++ b/crates/mvm-agentd/src/mailbox_outbox.rs @@ -179,8 +179,14 @@ fn lock_outbox_dir(dir: &Path) -> Result { .read(true) .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) .open(dir) - .map_err(|_| Refusal::Unavailable)?; - file.try_lock().map_err(|_| Refusal::Unavailable)?; + .map_err(|error| { + eprintln!("mvm-guest-agent: outbox directory open failed: {error}"); + Refusal::Unavailable + })?; + file.try_lock().map_err(|error| { + eprintln!("mvm-guest-agent: outbox directory lock failed: {error}"); + Refusal::Unavailable + })?; Ok(file) } @@ -399,10 +405,19 @@ fn write_single_outbox(outbox: &Path, message: &OutboxMessage) -> Result<(), Ref Ok(_) if existing == bytes => return Ok(()), Ok(_) => return Err(Refusal::QueueFull(1)), Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(_) => return Err(Refusal::Unavailable), + Err(error) => { + eprintln!("mvm-guest-agent: outbox existing request read failed: {error}"); + return Err(Refusal::Unavailable); + } } - atomic_write_new(outbox, &bytes).map_err(|_| Refusal::Unavailable)?; - sync_dir(dir).map_err(|_| Refusal::Unavailable) + atomic_write_new(outbox, &bytes).map_err(|error| { + eprintln!("mvm-guest-agent: outbox atomic write failed: {error:#}"); + Refusal::Unavailable + })?; + sync_dir(dir).map_err(|error| { + eprintln!("mvm-guest-agent: outbox directory sync failed: {error:#}"); + Refusal::Unavailable + }) } fn write_spooled(spool: &Path, message: &OutboxMessage) -> Result<(), Refusal> { diff --git a/crates/mvm-conformance/tests/steps/cli.rs b/crates/mvm-conformance/tests/steps/cli.rs index dcb4e89406..da96908c6b 100644 --- a/crates/mvm-conformance/tests/steps/cli.rs +++ b/crates/mvm-conformance/tests/steps/cli.rs @@ -665,9 +665,13 @@ fn run_workload_origin_network_secret_live_witness(world: &mut CliWorld) { } fn workload_origin_network_secret_argv() -> Vec { + // This @firecracker witness uses the released Firecracker kernel, which + // cannot boot under QEMU's PVH loader; never inherit a backend override. [ "machine", "run", + "--hypervisor", + "firecracker", "--image", "python:3.12", "--policy", diff --git a/scripts/run-cve-2026-80521-gcp.test.sh b/scripts/run-cve-2026-80521-gcp.test.sh index 7896aef5da..2603919a70 100755 --- a/scripts/run-cve-2026-80521-gcp.test.sh +++ b/scripts/run-cve-2026-80521-gcp.test.sh @@ -51,7 +51,8 @@ run_case() { } run_case success 0 0 0 0 0 1 -run_case create-failure 13 13 0 0 0 0 +# A client may report failure after the API created a billable instance. +run_case create-failure 13 13 0 0 0 1 run_case upload-failure 17 0 17 0 0 1 run_case remote-failure 23 0 0 23 0 1 run_case evidence-failure 1 0 0 0 29 1 From 2999b082f2726c08403897542c58aa6959c9230e Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 12 Nov 2026 18:01:13 -0800 Subject: [PATCH 47/51] fix(lab): pin Firecracker for isolated GCP homes --- scripts/run-gcp-kvm-test-remote.sh | 12 ++++++++++++ scripts/run-gcp-kvm-test.test.sh | 2 ++ 2 files changed, 14 insertions(+) diff --git a/scripts/run-gcp-kvm-test-remote.sh b/scripts/run-gcp-kvm-test-remote.sh index 5a9e5fe585..f0df902a09 100755 --- a/scripts/run-gcp-kvm-test-remote.sh +++ b/scripts/run-gcp-kvm-test-remote.sh @@ -108,6 +108,17 @@ if ! command -v firecracker >/dev/null 2>&1 \ "/tmp/$firecracker_dir/jailer-v${firecracker_version}-x86_64" \ /usr/local/bin/jailer fi +# Isolated live homes have no Firecracker cache yet. The capability probe does +# not search PATH, so pass the installed root-owned binary explicitly, but only +# after verifying its bytes against this checkout's release pin. +firecracker_binary_sha256="$(awk '/^\[x86_64\]/{p=1; next} + /^\[/{p=0} p && $1 == "binary_sha256" {gsub(/"/, "", $3); print $3}' \ + "$repo/crates/mvm-core/firecracker.lock")" +if [[ ! "$firecracker_binary_sha256" =~ ^[0-9a-f]{64}$ ]]; then + echo "error: missing x86_64 Firecracker binary pin" >&2 + exit 1 +fi +printf '%s %s\n' "$firecracker_binary_sha256" /usr/local/bin/firecracker | sha256sum -c - cd "$repo" # Keep toolchains and build state in the tree so the unprivileged test user can @@ -199,6 +210,7 @@ runuser -u mvm-test -- env \ CARGO_HOME="$CARGO_HOME" \ CARGO_TARGET_DIR="$CARGO_TARGET_DIR" \ MVM_HOME="$MVM_HOME" \ + MVM_FIRECRACKER_BIN=/usr/local/bin/firecracker \ MVM_GCP_KVM_RESULTS_DIR="$results" \ "${command_argv[@]}" 2>&1 | tee "$results/command.log" run_status=${PIPESTATUS[0]} diff --git a/scripts/run-gcp-kvm-test.test.sh b/scripts/run-gcp-kvm-test.test.sh index 9f162fd87e..ac3b4742db 100755 --- a/scripts/run-gcp-kvm-test.test.sh +++ b/scripts/run-gcp-kvm-test.test.sh @@ -98,6 +98,8 @@ grep -Fq 'just_sha256=4a5cc2f53e6f0f8c59092a6cc38291eb729d46a7dd95d3ae582008881b grep -Fq '[[ "$(just --version)" == "just $just_version" ]]' "$remote_runner" grep -Fq 'firecracker_sha256=06094a1108ae9e82aa4c23a775aa92758f53f1175d422270d9d6162cb9ade558' "$remote_runner" grep -Fq '| sha256sum -c -' "$remote_runner" +grep -Fq 'crates/mvm-core/firecracker.lock' "$remote_runner" +grep -Fq 'MVM_FIRECRACKER_BIN=/usr/local/bin/firecracker' "$remote_runner" # The bootstrap is privileged, but Firecracker and the requested witness are not. # These are literal source fragments, not expressions for this test shell. # shellcheck disable=SC2016 From 94caaf6667ee0a245f9d9beeb2e8782cfbed6b27 Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 12 Nov 2026 20:05:42 -0800 Subject: [PATCH 48/51] docs(xtask): record merged mailbox footprint measurement --- xtask/src/check_rust_footprint.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xtask/src/check_rust_footprint.rs b/xtask/src/check_rust_footprint.rs index 5b3ef6691d..20d3e26691 100644 --- a/xtask/src/check_rust_footprint.rs +++ b/xtask/src/check_rust_footprint.rs @@ -35,9 +35,9 @@ use std::process::Command; use crate::rust_source::{CommentSyntax, blank_comments}; // The integrated mailbox journal and newer main CI witnesses add bounded -// recovery, timeout, overclaim, and refusal coverage. The pre-merge mailbox -// tree measured 981,855 SLOC; keep a small margin rather than delete security -// checks to fit main's 966,160 SLOC ratchet. +// recovery, timeout, overclaim, and refusal coverage. The merged tree measures +// 982,225 SLOC and 1,248,875 physical lines; keep a small margin rather than +// delete security checks to fit main's 966,160 SLOC ratchet. /// Ceiling on total SLOC, all categories. const MAX_TOTAL_SLOC: usize = 984_000; /// Ceiling on total physical lines, all categories. From e90577f420e8d04ff5bac02290c461e15d2aed8f Mon Sep 17 00:00:00 2001 From: Ari Date: Thu, 12 Nov 2026 23:36:29 -0800 Subject: [PATCH 49/51] fix(agentd): place safety rationale on Linux test block --- crates/mvm-agentd/src/guest_mount.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/mvm-agentd/src/guest_mount.rs b/crates/mvm-agentd/src/guest_mount.rs index 5d78cfd0bc..62fa3392cb 100644 --- a/crates/mvm-agentd/src/guest_mount.rs +++ b/crates/mvm-agentd/src/guest_mount.rs @@ -2404,9 +2404,9 @@ mod privilege_tests { )), "the agent must retain only the workload group; got:\n{status}" ); - // SAFETY: PR_GET_DUMPABLE has no pointer arguments and reads only - // the current process's dumpability. assert_eq!( + // SAFETY: PR_GET_DUMPABLE has no pointer arguments and reads only + // the current process's dumpability. unsafe { libc::prctl(libc::PR_GET_DUMPABLE, 0, 0, 0, 0) }, 0, "the workload UID must not be able to inspect the agent's keys" From 2e5ec98d50cc70f058a87fe1b7b19d743c2861d8 Mon Sep 17 00:00:00 2001 From: Ari Date: Fri, 13 Nov 2026 00:11:45 -0800 Subject: [PATCH 50/51] test(agentd): make mailbox fixture and ACK failure deterministic --- crates/mvm-agentd/src/bin/mvm-guest-agent.rs | 32 +++++++++++++------ .../src/bin/mvm-guest-agent/mailbox.rs | 25 +++++++++------ 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent.rs index 4e2dc7bbdc..182030b443 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent.rs @@ -223,7 +223,7 @@ impl ColdContinuation { } fn complete_cold_response( - file: &mut std::fs::File, + file: &mut impl Write, session: &mut AuthenticatedSession, next: Option, response: &GuestResponse, @@ -1658,6 +1658,17 @@ mod tests { #[test] fn cold_ack_write_failure_cannot_enable_continuation() { + struct RejectWrite; + impl Write for RejectWrite { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let (mut host, mut guest) = UnixStream::pair().unwrap(); host.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); guest @@ -1674,18 +1685,19 @@ mod tests { let (mut session, _) = AuthenticatedSession::host_retaining_peer(&mut host, "ack-failure", host_key).unwrap(); peer.join().unwrap(); - let descriptor: std::os::fd::OwnedFd = host.into(); - let mut file = std::fs::File::from(descriptor); // This tests the real protected write seam, not activation or native // ownership. A candidate ACK alone cannot create a continuation. + let Err(error) = complete_cold_response( + &mut RejectWrite, + &mut session, + Some(ColdContinuation::Exec), + &GuestResponse::ActivateEnvironmentAck, + ) else { + panic!("the injected writer must fail before continuation"); + }; assert!( - complete_cold_response( - &mut file, - &mut session, - Some(ColdContinuation::Exec), - &GuestResponse::ActivateEnvironmentAck, - ) - .is_err() + format!("{error:#}").contains("control frame write failed"), + "the injected writer must fail before continuation" ); } diff --git a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs index 9db911d5da..e585843de8 100644 --- a/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs +++ b/crates/mvm-agentd/src/bin/mvm-guest-agent/mailbox.rs @@ -323,6 +323,17 @@ fn wait_for_operation( #[cfg(test)] mod tests { use super::*; + use mvm_core::mailbox::frame::OutboxMessage; + + fn stage_pending_outbox(path: &Path, message: &OutboxMessage) { + // These tests exercise the guest port, not a concurrent workload + // writer. Stage immutable bytes without competing for the writer's + // directory lock during the forwarding assertion. + mvm_core::util::atomic_io::atomic_write_new(path, &message.encode()) + .expect("stage pending mailbox request"); + mvm_core::util::atomic_io::sync_dir(path.parent().expect("outbox parent")) + .expect("sync pending mailbox request"); + } #[test] fn an_idle_guest_logs_a_bounded_number_of_refusals() { @@ -373,20 +384,17 @@ mod tests { #[test] fn a_pending_send_still_uses_the_existing_port_when_receive_is_enabled() { use mvm_contract::protocol::mailbox::PrincipalId; - use mvm_core::mailbox::frame::OutboxMessage; let dir = tempfile::tempdir().unwrap(); let outbox = dir.path().join("outbox"); let local_outbox = dir.path().join("outbox-local"); - mvm_agentd::mailbox_outbox::write_outbox( + stage_pending_outbox( &outbox, &OutboxMessage::new(PrincipalId::from_bytes([0; 32]), b"hello".to_vec()), - ) - .unwrap(); - mvm_agentd::mailbox_outbox::write_outbox( + ); + stage_pending_outbox( &local_outbox, &OutboxMessage::new(PrincipalId::from_bytes([0; 32]), b"local".to_vec()), - ) - .unwrap(); + ); let (mut guest, mut host) = std::os::unix::net::UnixStream::pair().unwrap(); let (_sender, reader) = std::sync::mpsc::sync_channel(1); let peer = std::thread::spawn(move || { @@ -416,14 +424,13 @@ mod tests { ExecutionEpoch, MailboxAddress, MailboxId, PrincipalId, ProtocolVersion, SignatureAlgorithm, TenantId, }; - use mvm_core::mailbox::frame::OutboxMessage; use mvm_core::mailbox::sign_receipt; let dir = tempfile::tempdir().expect("outbox root"); let local = dir.path().join("outbox-local"); let legacy = dir.path().join("outbox"); let operation = OutboxMessage::new(PrincipalId::from_bytes([0; 32]), b"report".to_vec()); - mvm_agentd::mailbox_outbox::write_outbox(&local, &operation).expect("pending report"); + stage_pending_outbox(&local, &operation); let (mut guest, _host) = std::os::unix::net::UnixStream::pair().expect("socket"); assert_eq!( wait_for_operation(&mut guest, None, None, &local, &legacy), From 7f06f100315b519abc90fbea532d5f0295ff01fa Mon Sep 17 00:00:00 2001 From: Ari Date: Fri, 13 Nov 2026 01:31:55 -0800 Subject: [PATCH 51/51] chore: measure merged Rust footprint --- xtask/src/check_rust_footprint.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xtask/src/check_rust_footprint.rs b/xtask/src/check_rust_footprint.rs index 43ff5e2951..b31f5e6591 100644 --- a/xtask/src/check_rust_footprint.rs +++ b/xtask/src/check_rust_footprint.rs @@ -38,9 +38,9 @@ use crate::rust_source::{CommentSyntax, blank_comments}; // tests remain measured; the library health/signing path adds regressions. // Leave room for main changes; physical lines have operator-approved headroom. /// Ceiling on total SLOC, all categories. -const MAX_TOTAL_SLOC: usize = 985_000; +const MAX_TOTAL_SLOC: usize = 991_000; /// Ceiling on total physical lines, all categories. -const MAX_TOTAL_PHYSICAL: usize = 1_252_000; +const MAX_TOTAL_PHYSICAL: usize = 1_259_000; /// How far under a ceiling the tree may sit before the ceiling must drop. const SLACK: usize = 10_000;