vfio-pci: mmap the BARs into the guest

crates/hw-vfio/src/lib.rs

@@ -5,8 +5,9 @@
 //! So far this is `vfio-pci` on Linux hosts, QEMU's hw/vfio/pci.c over the legacy container
 //! of hw/vfio/container-legacy.c: the config space, BARs and ROM of a host function bound to
 //! `vfio-pci` are given to the guest, with the parts QEMU emulates emulated the same way, and the
-//! guest's RAM is mapped for the device's DMA, and INTx, MSI and MSI-X reach the guest. DMA
-//! through a vIOMMU, iommufd and the device quirks come later. The plan for the rest of this
+//! guest's RAM is mapped for the device's DMA, and INTx, MSI and MSI-X reach the guest. The BARs
+//! are mapped into the guest, and DMA can go through a vIOMMU. iommufd and the device quirks
+//! come later. The plan for the rest of this
 //! crate is in `spec/24-workspace-layout.md`.
 
 mod host;
@@ -46,6 +47,11 @@ pub struct VfioPciConf {
     pub multifunction: bool,
     /// `skip-vsc-check`.
     pub skip_vsc_check: bool,
+    /// `x-no-mmap`: every BAR access is trapped, none goes to a mapping of the BAR.
+    pub no_mmap: bool,
+    /// `x-intx-mmap-timeout-ms`: how long after an INTx the BAR mappings stay off, 0 for
+    /// until INTx is turned off.
+    pub intx_mmap_timeout_ms: u32,
 }
 
 impl Default for VfioPciConf {
@@ -65,6 +71,8 @@ impl Default for VfioPciConf {
             romfile: None,
             multifunction: false,
             skip_vsc_check: true,
+            no_mmap: false,
+            intx_mmap_timeout_ms: 1100,
         }
     }
 }

crates/mem/src/address_space.rs

@@ -100,7 +100,7 @@ impl AddressSpace {
         // of it, so an outl to 0xcf8 on q35 reaches CONFIG_ADDRESS although the reset
         // control register sits on top of 0xcf9.
         let ram = match range.target.ty {
-            RegionType::Ram => true,
+            RegionType::Ram | RegionType::RamDevice => true,
             RegionType::RomDevice => range.romd_mode,
             _ => false,
         };
@@ -252,7 +252,9 @@ fn slow_path(range: Option<&FlatRange>) -> (&dyn MmioOps, &str) {
     match range {
         None => (&UNASSIGNED, UNASSIGNED_NAME),
         Some(r) => match &r.target.kind {
-            TargetKind::Mmio(ops) | TargetKind::RomDevice(_, ops) => (ops.as_ref(), r.name()),
+            TargetKind::Mmio(ops)
+            | TargetKind::RomDevice(_, ops)
+            | TargetKind::RamDevice(_, ops) => (ops.as_ref(), r.name()),
             _ => (&UNASSIGNED, r.name()),
         },
     }
@@ -263,7 +265,7 @@ fn access_allowed(range: Option<&FlatRange>, offset: u64, len: u64, attrs: MemTx
     if !attrs.memory() {
         return true;
     }
-    if range.is_some_and(|r| r.target.ty == RegionType::Ram) {
+    if range.is_some_and(|r| matches!(r.target.ty, RegionType::Ram | RegionType::RamDevice)) {
         return true;
     }
     let name = range.map_or(UNASSIGNED_NAME, |r| r.name());

crates/mem/src/flatview.rs

@@ -176,7 +176,9 @@ pub(crate) struct RenderFlags {
 pub(crate) fn dirty_log_mask(n: &Node, flags: RenderFlags) -> DirtyMask {
     let mut mask = n.dirty_log_mask;
     let ram = n.target.ram_block().is_some();
-    if flags.global_dirty_log && (ram || n.ty() == RegionType::Iommu) {
+    // A RAM device block is never migratable, so migration does not log it.
+    let migratable = ram && n.ty() != RegionType::RamDevice;
+    if flags.global_dirty_log && (migratable || n.ty() == RegionType::Iommu) {
         mask = mask.with(DirtyClient::Migration);
     }
     if flags.code_dirty_log && ram {

crates/mem/src/region.rs

@@ -33,6 +33,10 @@ pub enum RegionType {
     /// A RAM block read directly in romd mode and a device otherwise, and always a device for
     /// writes.
     RomDevice,
+    /// Host memory of a device, `memory_region_init_ram_device_ptr()`: a mapped BAR of a
+    /// passthrough device. A memory listener sees it as RAM, but the CPU reaches it through
+    /// sized callbacks so that each access is one load or store of its width.
+    RamDevice,
     /// A device.
     Mmio,
     /// A window onto another region.
@@ -59,6 +63,7 @@ pub(crate) struct RegionTarget {
 pub(crate) enum TargetKind {
     Ram(Arc<RamBlock>),
     RomDevice(Arc<RamBlock>, Arc<dyn MmioOps>),
+    RamDevice(Arc<RamBlock>, Arc<dyn MmioOps>),
     Mmio(Arc<dyn MmioOps>),
     Iommu(Arc<dyn IommuOps>),
     /// Containers and aliases, which never appear in a FlatView.
@@ -68,7 +73,9 @@ pub(crate) enum TargetKind {
 impl RegionTarget {
     pub(crate) fn ram_block(&self) -> Option<&Arc<RamBlock>> {
         match &self.kind {
-            TargetKind::Ram(b) | TargetKind::RomDevice(b, _) => Some(b),
+            TargetKind::Ram(b) | TargetKind::RomDevice(b, _) | TargetKind::RamDevice(b, _) => {
+                Some(b)
+            }
             _ => None,
         }
     }

crates/mem/src/system.rs

@@ -12,15 +12,17 @@ use std::fmt;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, MutexGuard};
 
-use crate::access::MmioOps;
+use crate::access::{AccessConstraints, AccessSize, Endian, MmioOps};
 use crate::address_space::AddressSpace;
+use crate::attrs::{AccessCtx, MemResult, MemTxResult};
 use crate::dirty::{DirtyClient, DirtySnapshot};
 use crate::error::MemError;
 use crate::flatview::{FlatView, RenderFlags, SPACE_END, dirty_log_mask, render};
 use crate::iommu::IommuOps;
 use crate::listener::{ListenerId, MemoryListener};
 use crate::ram::RamBlock;
 use crate::region::{Arena, Node, RegionId, RegionInfo, RegionTarget, RegionType, TargetKind};
+use ruvm_sys::{HostMemory, hostatomic};
 
 /// `GLOBAL_DIRTY_MIGRATION`: global dirty tracking for live migration.
 pub const GLOBAL_DIRTY_MIGRATION: u32 = 1 << 0;
@@ -304,6 +306,21 @@ impl MemorySystem {
         )
     }
 
+    /// Host memory of a device, `memory_region_init_ram_device_ptr()`. The region takes `mem`,
+    /// which is usually a mapped BAR, and unmaps it when the region goes. CPU accesses go
+    /// through [`RamDeviceOps`] so the device sees each one at its width.
+    pub fn new_ram_device_ptr(&self, name: &str, mem: HostMemory) -> Result<RegionId, MemError> {
+        let size = mem.len() as u64;
+        let block = Self::track(RamBlock::from_memory(name, mem, self.config.page_bits));
+        let ops: Arc<dyn MmioOps> = Arc::new(RamDeviceOps(Arc::clone(&block)));
+        self.create(
+            name,
+            u128::from(size),
+            RegionType::RamDevice,
+            TargetKind::RamDevice(block, ops),
+        )
+    }
+
     /// A device, `memory_region_init_io()`.
     pub fn new_io(
         &self,
@@ -953,3 +970,56 @@ fn topology_pass(
         }
     }
 }
+
+/// `ram_device_mem_ops`: each access is one host load or store of its width, in host byte
+/// order, so a device register sees the size the guest used. Unaligned accesses are allowed
+/// and copied a byte at a time, as `ldl_he_p()` would.
+pub(crate) struct RamDeviceOps(Arc<RamBlock>);
+
+impl RamDeviceOps {
+    fn bytes(&self, offset: u64, size: AccessSize) -> MemResult<&[std::sync::atomic::AtomicU8]> {
+        let start = usize::try_from(offset).map_err(|_| MemTxResult::ERROR)?;
+        let end = start.checked_add(size.bytes() as usize).ok_or(MemTxResult::ERROR)?;
+        self.0.atomic_bytes().get(start..end).ok_or(MemTxResult::ERROR)
+    }
+}
+
+impl MmioOps for RamDeviceOps {
+    fn read(&self, _cx: &AccessCtx, offset: u64, size: AccessSize) -> MemResult<u64> {
+        let mem = self.bytes(offset, size)?;
+        if let Some(v) = hostatomic::load(mem) {
+            return Ok(v);
+        }
+        let mut b = [0u8; 8];
+        let n = mem.len();
+        self.0.read(offset, &mut b[..n]).map_err(|_| MemTxResult::ERROR)?;
+        Ok(if cfg!(target_endian = "big") {
+            u64::from_be_bytes(b) >> (64 - 8 * n)
+        } else {
+            u64::from_le_bytes(b)
+        })
+    }
+
+    fn write(&self, _cx: &AccessCtx, offset: u64, size: AccessSize, value: u64) -> MemResult<()> {
+        let mem = self.bytes(offset, size)?;
+        if hostatomic::store(mem, value) {
+            return Ok(());
+        }
+        let n = mem.len();
+        let b = value.to_ne_bytes();
+        let b = if cfg!(target_endian = "big") { &b[8 - n..] } else { &b[..n] };
+        self.0.write(offset, b).map_err(|_| MemTxResult::ERROR)
+    }
+
+    fn valid(&self) -> AccessConstraints {
+        AccessConstraints::any_size(1, 8).allow_unaligned()
+    }
+
+    fn impl_constraints(&self) -> AccessConstraints {
+        AccessConstraints::any_size(1, 8).allow_unaligned()
+    }
+
+    fn endianness(&self) -> Endian {
+        Endian::HOST
+    }
+}

crates/mem/tests/api.rs

@@ -247,6 +247,33 @@ fn rom_device_switches_between_ram_and_callbacks() {
     assert!(matches!(sys.set_romd(root, false), Err(MemError::WrongKind(_))));
 }
 
+#[test]
+fn ram_device_is_ram_for_listeners_and_sized_for_the_cpu() {
+    let (sys, root, space) = setup();
+    let bar = sys.new_ram_device_ptr("bar", ruvm_sys::HostMemory::new(0x1000).unwrap()).unwrap();
+    sys.add_subregion(root, 0x4000, bar).unwrap();
+    assert_eq!(sys.region(bar).unwrap().ty, ruvm_mem::RegionType::RamDevice);
+    assert_eq!(space.write_u32(0x4010, U, 0x1122_3344), MemTxResult::OK);
+    let mut b = [0; 4];
+    sys.ram_block(bar).unwrap().read(0x10, &mut b).unwrap();
+    assert_eq!(b, 0x1122_3344u32.to_ne_bytes());
+    assert_eq!(space.load(0x4010, 8, Endian::HOST, U), (0x1122_3344, MemTxResult::OK));
+    // Unaligned accesses are allowed and copied a byte at a time.
+    assert_eq!(space.store(0x4021, 4, 0xa1b2_c3d4, Endian::Little, U), MemTxResult::OK);
+    assert_eq!(space.load(0x4021, 4, Endian::Little, U), (0xa1b2_c3d4, MemTxResult::OK));
+    assert_eq!(space.load(0x4022, 2, Endian::Little, U), (0xb2c3, MemTxResult::OK));
+    // The memory attribute lets it through, like RAM.
+    let mem = MemTxAttrs::new().with_memory(true);
+    assert_eq!(space.read_u32(0x4010, mem), (0x1122_3344, MemTxResult::OK));
+    // A mapped BAR is never migrated, so migration does not log it.
+    sys.global_dirty_log_start(GLOBAL_DIRTY_MIGRATION).unwrap();
+    let fv = space.flatview();
+    let r = fv.ranges().iter().find(|r| r.name() == "bar").unwrap();
+    assert!(!r.dirty_log_mask().contains(DirtyClient::Migration));
+    assert!(r.ram_block().is_some());
+    sys.global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
+}
+
 struct Iommu {
     target: Arc<AddressSpace>,
 }

crates/sys/Cargo.toml

@@ -11,7 +11,7 @@ authors.workspace = true
 publish = false
 
 [package.metadata.ruvm]
-unsafe-budget = 33
+unsafe-budget = 37
 
 [dependencies]
 

crates/sys/src/hostmem.rs

@@ -141,6 +141,99 @@ impl HostMemory {
         Ok(HostMemory { ptr, len, fd })
     }
 
+    /// Maps the `areas` of a device region, each an offset and a size in the region, the way
+    /// `vfio_region_mmap()` does: the region of `size` bytes at `fd_offset` of `fd` gets an
+    /// address aligned to its size rounded up to a power of two, capped at 1 GiB, so the kernel
+    /// can use huge mappings, and each area is mapped shared at its place in it. The areas must
+    /// be sorted, apart and inside the region. Each one comes back as memory of its own, which
+    /// unmaps it when it goes. If one fails, none stay mapped.
+    #[cfg(any(target_os = "linux", target_os = "android"))]
+    pub fn map_device(
+        fd: std::os::fd::BorrowedFd<'_>,
+        fd_offset: u64,
+        size: u64,
+        areas: &[(u64, u64)],
+        read: bool,
+        write: bool,
+    ) -> std::io::Result<Vec<HostMemory>> {
+        use std::os::fd::AsRawFd;
+
+        let invalid = || std::io::Error::from(std::io::ErrorKind::InvalidInput);
+        let size = usize::try_from(size).map_err(|_| invalid())?;
+        let mut end = 0;
+        let mut spans = Vec::with_capacity(areas.len());
+        for &(off, len) in areas {
+            let off = usize::try_from(off).map_err(|_| invalid())?;
+            let len = usize::try_from(len).map_err(|_| invalid())?;
+            if off < end || len == 0 || off.checked_add(len).is_none_or(|e| e > size) {
+                return Err(invalid());
+            }
+            let file_off = fd_offset.checked_add(off as u64).ok_or_else(invalid)?;
+            let file_off = libc::off_t::try_from(file_off).map_err(|_| invalid())?;
+            spans.push((off, len, file_off));
+            end = off + len;
+        }
+        let align = size.checked_next_power_of_two().unwrap_or(usize::MAX).min(1 << 30);
+        let total = size.checked_add(align).ok_or_else(invalid)?;
+        // SAFETY: a new inaccessible mapping at an address the kernel picks aliases nothing in
+        // this process.
+        let base = unsafe {
+            libc::mmap(
+                std::ptr::null_mut(),
+                total,
+                libc::PROT_NONE,
+                libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_NORESERVE,
+                -1,
+                0,
+            )
+        };
+        if base == libc::MAP_FAILED {
+            return Err(std::io::Error::last_os_error());
+        }
+        let base = base as usize;
+        let start = base.next_multiple_of(align);
+        unmap_reserved(base, start - base);
+        unmap_reserved(start + size, align - (start - base));
+
+        let mut prot = 0;
+        if read {
+            prot |= libc::PROT_READ;
+        }
+        if write {
+            prot |= libc::PROT_WRITE;
+        }
+        let mut out = Vec::with_capacity(spans.len());
+        let mut done = 0;
+        for (off, len, file_off) in spans {
+            unmap_reserved(start + done, off - done);
+            // SAFETY: the target is the part of the reservation above for this area, which
+            // nothing else in the process maps, and the device memory is only ever reached
+            // through the `AtomicU8` view.
+            let p = unsafe {
+                libc::mmap(
+                    (start + off) as *mut libc::c_void,
+                    len,
+                    prot,
+                    libc::MAP_SHARED | libc::MAP_FIXED,
+                    fd.as_raw_fd(),
+                    file_off,
+                )
+            };
+            if p == libc::MAP_FAILED {
+                let err = std::io::Error::last_os_error();
+                // The areas mapped so far go with `out`.
+                unmap_reserved(start + off, size - off);
+                return Err(err);
+            }
+            let ptr =
+                std::ptr::NonNull::new(p.cast::<AtomicU8>()).expect("mmap never returns null");
+            out.push(HostMemory { ptr, len, fd: None });
+            done = off + len;
+        }
+        unmap_reserved(start + done, size - done);
+        Ok(out)
+    }
+
     /// The descriptor of shared memory, which another process can map.
     #[cfg(unix)]
     pub fn fd(&self) -> Option<std::os::fd::BorrowedFd<'_>> {
@@ -392,6 +485,16 @@ impl HostMemory {
     }
 }
 
+/// Gives back `len` bytes at `addr` of a reservation [`HostMemory::map_device`] made and no
+/// [`HostMemory`] owns.
+#[cfg(any(target_os = "linux", target_os = "android"))]
+fn unmap_reserved(addr: usize, len: usize) {
+    if len != 0 {
+        // SAFETY: the range is part of an inaccessible reservation nothing points into.
+        unsafe { libc::munmap(addr as *mut libc::c_void, len) };
+    }
+}
+
 #[cfg(unix)]
 impl Drop for HostMemory {
     fn drop(&mut self) {
@@ -408,6 +511,34 @@ mod tests {
     use super::*;
     use std::sync::atomic::Ordering;
 
+    #[cfg(any(target_os = "linux", target_os = "android"))]
+    #[test]
+    fn device_areas_are_aligned_and_shared() {
+        use std::os::fd::AsFd;
+
+        let file = HostMemory::shared("dev", 0x8000).unwrap();
+        file.as_slice()[0x5008].store(0x5a, Ordering::Relaxed);
+        let fd = file.fd().unwrap();
+        let areas = [(0, 0x2000), (0x5000, 0x1000)];
+        let maps = HostMemory::map_device(fd.as_fd(), 0, 0x8000, &areas, true, true).unwrap();
+        assert_eq!(maps.len(), 2);
+        assert_eq!(maps[0].host_addr() % 0x8000, 0);
+        assert_eq!(maps[1].host_addr(), maps[0].host_addr() + 0x5000);
+        assert_eq!(maps[1].len(), 0x1000);
+        assert_eq!(maps[1].as_slice()[8].load(Ordering::Relaxed), 0x5a);
+        maps[0].as_slice()[1].store(3, Ordering::Relaxed);
+        assert_eq!(file.as_slice()[1].load(Ordering::Relaxed), 3);
+        // Out of order, overlapping and empty areas are refused.
+        for bad in [
+            [(0x5000, 0x1000), (0, 0x1000)],
+            [(0, 0x2000), (0x1000, 0x1000)],
+            [(0, 0), (0x1000, 0x1000)],
+        ] {
+            assert!(HostMemory::map_device(fd.as_fd(), 0, 0x8000, &bad, true, true).is_err());
+        }
+        assert!(HostMemory::map_device(fd.as_fd(), 0, 0x1000, &[(0, 0x2000)], true, true).is_err());
+    }
+
     #[test]
     fn zeroed_and_writable() {
         let m = HostMemory::new(1 << 20).unwrap();