From 248b4b330f69989d0d2ca70a4ea223e6e004f33a Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:21:06 +0700 Subject: [PATCH 1/2] hw-vfio: map the BARs of vfio-pci into the guest --- crates/hw-vfio/src/lib.rs | 12 +- crates/hw-vfio/src/pci.rs | 398 ++++++++++++++++++++++++++++++-- crates/mem/src/address_space.rs | 8 +- crates/mem/src/flatview.rs | 4 +- crates/mem/src/region.rs | 9 +- crates/mem/src/system.rs | 72 +++++- crates/mem/tests/api.rs | 27 +++ crates/sys/Cargo.toml | 2 +- crates/sys/src/hostmem.rs | 131 +++++++++++ crates/sys/src/vfio.rs | 169 +++++++++++++- crates/system/src/vfio.rs | 10 +- crates/system/src/x86.rs | 10 +- 12 files changed, 816 insertions(+), 36 deletions(-) diff --git a/crates/hw-vfio/src/lib.rs b/crates/hw-vfio/src/lib.rs index 36de4faa..659e6f3c 100644 --- a/crates/hw-vfio/src/lib.rs +++ b/crates/hw-vfio/src/lib.rs @@ -5,8 +5,9 @@ //! So far this is `vfio-pci` on Linux hosts, QEMU's hw/vfio/pci.c over the legacy container //! of hw/vfio/container-legacy.c: the config space, BARs and ROM of a host function bound to //! `vfio-pci` are given to the guest, with the parts QEMU emulates emulated the same way, and the -//! guest's RAM is mapped for the device's DMA, and INTx, MSI and MSI-X reach the guest. DMA -//! through a vIOMMU, iommufd and the device quirks come later. The plan for the rest of this +//! guest's RAM is mapped for the device's DMA, and INTx, MSI and MSI-X reach the guest. The BARs +//! are mapped into the guest, and DMA can go through a vIOMMU. iommufd and the device quirks +//! come later. The plan for the rest of this //! crate is in `spec/24-workspace-layout.md`. mod host; @@ -46,6 +47,11 @@ pub struct VfioPciConf { pub multifunction: bool, /// `skip-vsc-check`. pub skip_vsc_check: bool, + /// `x-no-mmap`: every BAR access is trapped, none goes to a mapping of the BAR. + pub no_mmap: bool, + /// `x-intx-mmap-timeout-ms`: how long after an INTx the BAR mappings stay off, 0 for + /// until INTx is turned off. + pub intx_mmap_timeout_ms: u32, } impl Default for VfioPciConf { @@ -65,6 +71,8 @@ impl Default for VfioPciConf { romfile: None, multifunction: false, skip_vsc_check: true, + no_mmap: false, + intx_mmap_timeout_ms: 1100, } } } diff --git a/crates/hw-vfio/src/pci.rs b/crates/hw-vfio/src/pci.rs index c367102a..8c27279e 100644 --- a/crates/hw-vfio/src/pci.rs +++ b/crates/hw-vfio/src/pci.rs @@ -15,8 +15,12 @@ //! vector gets its eventfd when the guest first unmasks it, and one that fires while masked //! goes into the emulated PBA. Behind a vIOMMU the device's address space holds an IOMMU region, //! and the container follows it with a notifier the way `vfio_iommu_map_notify()` does, mapping -//! each IOVA the guest maps and unmapping it again when the guest invalidates it. mmap of the -//! BARs, hot reset and the device quirks are not here yet. +//! each IOVA the guest maps and unmapping it again when the guest invalidates it. The parts of +//! a BAR the kernel lets be mapped are mapped into the process and laid over the trapped BAR as +//! RAM device regions, as `vfio_region_mmap()` does, leaving the MSI-X table trapped. While INTx +//! is pending they are turned off, so that the guest's next BAR access traps and serves as the +//! EOI, and a timer turns them on again once INTx has gone quiet. Hot reset and the device +//! quirks are not here yet. use std::fmt; use std::io; @@ -30,8 +34,9 @@ use std::thread::{self, JoinHandle}; use rustix::event::{EventfdFlags, PollFd, PollFlags, eventfd, poll}; use ruvm_base::error::{Error, Result, strerror}; -use ruvm_base::report::{error_report, report_error, warn_report}; +use ruvm_base::report::{error_report, format_message, report_error, warn_report}; use ruvm_hw_core::IrqLine; +use ruvm_hw_core::timer::{Clock, Timer}; use ruvm_hw_pci::pcie::{ PCI_EXP_DEVCAP, PCI_EXP_FLAGS_TYPE, PCI_EXP_LNKCAP, PCI_EXP_LNKCAP2, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL2, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA2, PCI_EXP_TYPE_ENDPOINT, PCI_EXP_TYPE_LEG_END, @@ -58,12 +63,14 @@ use ruvm_mem::{ MemTxAttrs, MemoryListener, MemorySystem, MmioOps, RegionId, RegionType, register_iommu_notifier, unregister_iommu_notifier, }; +use ruvm_sys::HostMemory; use ruvm_sys::vfio::{ API_VERSION, DEVICE_FLAGS_PCI, DEVICE_FLAGS_RESET, DeviceInfo, GROUP_FLAGS_VIABLE, IRQ_INFO_NORESIZE, IRQ_SET_ACTION_TRIGGER, IRQ_SET_ACTION_UNMASK, PCI_CONFIG_REGION_INDEX, PCI_ERR_IRQ_INDEX, PCI_INTX_IRQ_INDEX, PCI_MSI_IRQ_INDEX, PCI_MSIX_IRQ_INDEX, - PCI_ROM_REGION_INDEX, RegionInfo, TYPE1_IOMMU, TYPE1V2_IOMMU, UNMAP_ALL, VfioContainer, - VfioDevice, VfioGroup, + PCI_ROM_REGION_INDEX, REGION_INFO_CAP_MSIX_MAPPABLE, REGION_INFO_FLAG_MMAP, + REGION_INFO_FLAG_READ, REGION_INFO_FLAG_WRITE, RegionCaps, RegionInfo, TYPE1_IOMMU, + TYPE1V2_IOMMU, UNMAP_ALL, VfioContainer, VfioDevice, VfioGroup, }; use crate::{PCI_ANY_ID, VfioPciConf}; @@ -87,6 +94,8 @@ const PCI_VENDOR_ID_CHELSIO: u16 = 0x1425; const PCI_VENDOR_ID_BAIDU: u16 = 0x1d22; const PCI_DEVICE_ID_KUNLUN_VF: u16 = 0x3685; const PCI_EXP_LNKCAP_X1_2_5GT: u32 = 0x11; +/// `PCI_MSIX_ENTRY_SIZE`. +const PCI_MSIX_ENTRY_SIZE: u64 = 16; /// Bars 0 to 5, `ARRAY_SIZE(vdev->bars)`. const NUM_BARS: usize = PCI_ROM_SLOT; /// What `read()` gives back for a short config read, `EFAULT`. @@ -178,9 +187,11 @@ impl DmaListener { /// `vfio_listener_valid_section()` and `vfio_get_section_iova_range()`. fn section(&self, fr: &FlatRange) -> Option<(u64, u128)> { // vfio_listener_skipped_section(): ROM devices are not RAM to QEMU either, and an - // address with bit 63 set is a 64-bit BAR being sized. + // address with bit 63 set is a 64-bit BAR being sized. RAM devices, the mapped BARs of + // assigned devices, are RAM. let kind = fr.region_type(); - if (kind != RegionType::Ram && kind != RegionType::Iommu) || fr.addr() & (1 << 63) != 0 { + let ram = matches!(kind, RegionType::Ram | RegionType::RamDevice); + if (!ram && kind != RegionType::Iommu) || fr.addr() & (1 << 63) != 0 { return None; } let off = self.page_size - 1; @@ -242,7 +253,7 @@ impl DmaListener { let view = system.flatview(); let fr = view.lookup(addr); let Some((fr, block)) = fr - .filter(|fr| fr.region_type() == RegionType::Ram) + .filter(|fr| matches!(fr.region_type(), RegionType::Ram | RegionType::RamDevice)) .and_then(|fr| fr.ram_block().map(|b| (fr, b))) else { let xlat = fr.map_or(addr, |fr| fr.offset_in_region() + (addr - fr.addr())); @@ -365,6 +376,8 @@ impl MemoryListener for DmaListener { }; let size = size as u64; // memory_region_get_ram_ptr() + offset_within_region + (iova - offset_within_address_space). + // A RAM device is only mapped in whole IOMMU pages, which section() already cut it to, + // since the container takes the host page as its smallest one. let vaddr = block.host_addr() + (fr.offset_in_region() + (iova - fr.addr())) as usize; if let Err(e) = self.map(iova, size, vaddr, fr.readonly()) { let ret = -e.raw_os_error().unwrap_or(0); @@ -372,6 +385,18 @@ impl MemoryListener for DmaListener { "vfio_container_dma_map({self:p}, {iova:#x}, {size:#x}, {vaddr:#x}) = {ret} ({})", strerror(&e) ); + if fr.region_type() == RegionType::RamDevice { + // A BAR that cannot be mapped for DMA only breaks peer to peer DMA, so it is + // not fatal. vfio_device_error_append() names the device that owns the BAR, + // whose name is the start of every mmap region's name. + let owner = fr.name().rsplit_once(" BAR ").map_or(fr.name(), |(n, _)| n); + static WARNED: AtomicBool = AtomicBool::new(false); + let e = Error::generic(msg).hint(format_args!( + "{owner}: PCI peer-to-peer transactions on BARs are not supported.\n" + )); + warn_err_once(&WARNED, &e); + return; + } self.fail(fr.name(), Error::generic(msg)); } } @@ -603,6 +628,17 @@ fn lock(m: &Mutex) -> MutexGuard<'_, T> { m.lock().unwrap_or_else(PoisonError::into_inner) } +/// `warn_report_err_once()`: the warning and its hint, the first time `done` is seen clear. +fn warn_err_once(done: &AtomicBool, e: &Error) { + if !done.swap(true, Ordering::Relaxed) { + let mut out = format_message("warning: ", e.message()); + if let Some(h) = e.hint_text() { + out.push_str(h); + } + eprint!("{out}"); + } +} + /// One eventfd slot per interrupt, filled when the interrupt is set up. type Slots = Arc>>>>; @@ -769,6 +805,62 @@ impl Pba { } } +/// The mapped parts of the BARs, the `mmaps` of each `VFIORegion`, and what turns them off and +/// on around INTx. +#[derive(Debug)] +struct Mmaps { + memory: Weak, + /// Their regions. The lock is held over each switch and over what decides it, so that the + /// INTx thread and the timer do not undo each other. + state: Mutex, + /// The dma-bufs of the BARs, which live as long as the device. + dma_bufs: Mutex>, + /// `intx.mmap_timer`, made when the device has an INTx pin. + timer: OnceLock, + /// `intx.mmap_timeout` in milliseconds, 0 to leave them off once INTx fired. + timeout_ms: u32, +} + +#[derive(Debug)] +struct MmapState { + regions: Vec, + enabled: bool, +} + +impl Mmaps { + fn new(memory: &Arc, timeout_ms: u32) -> Self { + Mmaps { + memory: Arc::downgrade(memory), + state: Mutex::new(MmapState { regions: Vec::new(), enabled: true }), + dma_bufs: Mutex::new(Vec::new()), + timer: OnceLock::new(), + timeout_ms, + } + } + + /// `vfio_mmap_set_enabled()`, with `state` locked. Setting the state the regions are + /// already in changes nothing. + fn set_enabled(&self, state: &mut MmapState, enabled: bool) { + if state.enabled == enabled { + return; + } + state.enabled = enabled; + if let Some(memory) = self.memory.upgrade() { + for &r in &state.regions { + let _ = memory.set_enabled(r, enabled); + } + } + } + + /// `timer_mod()` of the mmap timer to the timeout from now on the virtual clock. + fn arm(&self) { + let Some(t) = self.timer.get() else { return }; + if let Some(clock) = t.clock() { + t.modify((clock.get_ms() + i64::from(self.timeout_ms)) * 1_000_000); + } + } +} + /// What the BAR, ROM and config callbacks share: the device, where its config space is and the /// state of its interrupts. #[derive(Debug)] @@ -777,6 +869,7 @@ struct Shared { interrupt: Mutex, device: VfioDevice, config_offset: u64, + mmaps: Arc, _attachment: Attachment, } @@ -910,10 +1003,31 @@ impl Shared { else { return; }; + if let Some(t) = self.mmaps.timer.get() { + t.del(); + } // vfio_device_irq_disable(). let _ = self.device.set_irqs(PCI_INTX_IRQ_INDEX, 0, 0, IRQ_SET_ACTION_TRIGGER); *lock(&intx.pending) = false; intx.irq.lower(); + // The thread has to be gone before the mmaps go back on, or it could turn them off. + drop(intx); + self.mmaps.set_enabled(&mut lock(&self.mmaps.state), true); + } + + /// `vfio_intx_mmap_enable()`: the mmap timer fired. The mmaps go back on unless INTx is + /// still pending, in which case the timer waits another timeout. + fn intx_mmap_enable(&self) { + let mut state = lock(&self.mmaps.state); + let pending = match &*lock(&self.interrupt) { + Interrupt::Intx(i) => *lock(&i.pending), + _ => false, + }; + if pending { + self.mmaps.arm(); + } else { + self.mmaps.set_enabled(&mut state, true); + } } /// The part of `vfio_intx_enable()` after the pin is known: an eventfd for the kernel to @@ -922,12 +1036,24 @@ impl Shared { let pending = Arc::new(Mutex::new(false)); let notifier = { let (pending, irq) = (Arc::clone(&pending), irq.clone()); - // vfio_intx_interrupt(). The lock is held over the raise so that an EOI cannot - // lower the pin in between. + let mmaps = Arc::clone(&self.mmaps); + // vfio_intx_interrupt(). The mmaps go off before the pin goes up, so that the + // guest's first BAR access after it traps and is taken as the EOI, and they stay + // locked until the pin is pending, so the timer cannot turn them on in between. + // The pending lock is held over the raise so that an EOI cannot lower the pin in + // between. Notifiers::new("intx-interrupt", 1, move |_| { - let mut p = lock(&pending); - *p = true; - irq.raise(); + let mut state = lock(&mmaps.state); + mmaps.set_enabled(&mut state, false); + { + let mut p = lock(&pending); + *p = true; + irq.raise(); + } + drop(state); + if mmaps.timeout_ms != 0 { + mmaps.arm(); + } })? }; // vfio_device_irq_set_signaling(). @@ -1421,6 +1547,10 @@ struct MsixInfo { struct Populated { shared: Arc, bars: [RegionInfo; NUM_BARS], + /// The capability chain of each BAR's info. + caps: [RegionCaps; NUM_BARS], + /// `VFIORegion::mmaps` of each BAR, offsets and sizes in it. + mmaps: [Vec<(u64, u64)>; NUM_BARS], express: bool, reset_works: bool, config: Vec, @@ -1430,7 +1560,9 @@ struct Populated { /// free slot. `express_bus` says whether `bus` is PCI Express, `pci_bus_is_express()`, and /// `dma_as` is the address space the device's DMA goes to, `pci_device_iommu_address_space()`. /// `system` is the system memory address space, the only one an IOMMU in `dma_as` may map to, -/// and `viommu` is the vIOMMU of the bus, if it wants to know about assigned devices. +/// and `viommu` is the vIOMMU of the bus, if it wants to know about assigned devices. `clock` is +/// the virtual clock, which times how long the BAR mmaps stay off after INTx. +#[allow(clippy::too_many_arguments)] pub fn realize( bus: &Arc, devfn: Option, @@ -1438,6 +1570,7 @@ pub fn realize( dma_as: &Arc, system: &Arc, viommu: Option<&dyn HostIommu>, + clock: &Arc, conf: &VfioPciConf, ) -> Result> { let sysfsdev = match (&conf.sysfsdev, conf.host) { @@ -1467,7 +1600,7 @@ pub fn realize( "x-balloon-allowed only potentially compatible with mdev devices", ))); } - let p = populate(&sysfsdev, &name, bus.memory(), dma_as, system).map_err(prefix)?; + let p = populate(&sysfsdev, &name, bus.memory(), dma_as, system, conf).map_err(prefix)?; let info = PciDeviceInfo { name: "vfio-pci".to_string(), id: conf.id.clone(), @@ -1493,7 +1626,17 @@ pub fn realize( let ops = Arc::new(ops); let _ = ops.dev.set(Arc::downgrade(&dev)); dev.set_ops(Arc::clone(&ops) as Arc); - // vfio_pci_interrupt_setup(). + // vfio_pci_interrupt_setup(): a device with an INTx pin gets the timer that maps + // its BARs again once INTx goes quiet. + if ops.config_read(&dev, PCI_INTERRUPT_PIN as u32, 1) != 0 { + let shared = Arc::downgrade(&ops.shared); + let timer = clock.new_timer(move || { + if let Some(s) = shared.upgrade() { + s.intx_mmap_enable(); + } + }); + let _ = ops.shared.mmaps.timer.set(timer); + } if let Err(e) = ops.intx_enable(&dev) { unset(s.idev_set); bus.unregister_device(&dev); @@ -1531,6 +1674,7 @@ fn populate( memory: &Arc, space: &Arc, system: &Arc, + conf: &VfioPciConf, ) -> Result { let (attachment, device, info) = attach(sysfsdev, name, memory, space, system)?; @@ -1546,11 +1690,15 @@ fn populate( if info.num_irqs < PCI_MSIX_IRQ_INDEX + 1 { return Err(Error::generic(format!("unexpected number of irqs {}", info.num_irqs))); } + // vfio_region_setup() for each BAR. let mut bars = [RegionInfo::default(); NUM_BARS]; - for (i, bar) in bars.iter_mut().enumerate() { - *bar = device - .region_info(i as u32) + let mut caps: [RegionCaps; NUM_BARS] = Default::default(); + let mut mmaps: [Vec<(u64, u64)>; NUM_BARS] = Default::default(); + for i in 0..NUM_BARS { + (bars[i], caps[i]) = device + .region_info_caps(i as u32) .map_err(|e| Error::from_io(format_args!("failed to get region {i} info"), e))?; + mmaps[i] = region_mmaps(i, &bars[i], &caps[i], conf.no_mmap)?; } let config_info = device .region_info(PCI_CONFIG_REGION_INDEX) @@ -1580,11 +1728,14 @@ fn populate( interrupt: Mutex::new(Interrupt::None), device, config_offset: config_info.offset, + mmaps: Arc::new(Mmaps::new(memory, conf.intx_mmap_timeout_ms)), _attachment: attachment, }); Ok(Populated { shared, bars, + caps, + mmaps, express, reset_works: info.flags & DEVICE_FLAGS_RESET != 0, config, @@ -1597,6 +1748,8 @@ struct Setup<'a> { bus: &'a PciBus, shared: Arc, bars: [RegionInfo; NUM_BARS], + caps: [RegionCaps; NUM_BARS], + mmaps: [Vec<(u64, u64)>; NUM_BARS], reset_works: bool, express_bus: bool, conf: &'a VfioPciConf, @@ -1629,6 +1782,8 @@ impl<'a> Setup<'a> { bus, shared: p.shared, bars: p.bars, + caps: p.caps, + mmaps: p.mmaps, reset_works: p.reset_works, express_bus, conf, @@ -1859,12 +2014,79 @@ impl<'a> Setup<'a> { let region = mem.new_io(&format!("{name} BAR {nr}"), size, Arc::new(ops)).map_err(mem_error)?; mem.add_subregion(bar, 0, region).map_err(mem_error)?; + if let Err(e) = self.bar_mmap(nr, region) { + if let Some(e) = e { + report_error(&e); + } + error_report(&format!("Failed to mmap {name} BAR {nr}. Performance may be slow")); + } self.dev.register_bar(nr, type_, bar); self.containers[nr] = Some(bar); } Ok(()) } + /// `vfio_region_mmap()` for BAR `nr`, whose trapped region is `region`: each of its mmaps + /// is mapped and laid over it, then a dma-buf of them is asked for. An error that QEMU + /// only traces is `None`. + fn bar_mmap(&self, nr: usize, region: RegionId) -> std::result::Result<(), Option> { + let areas = &self.mmaps[nr]; + if areas.is_empty() { + return Ok(()); + } + let info = &self.bars[nr]; + let maps = HostMemory::map_device( + self.shared.device.as_fd(), + info.offset, + info.size, + areas, + info.flags & REGION_INFO_FLAG_READ != 0, + info.flags & REGION_INFO_FLAG_WRITE != 0, + ) + .map_err(|_| None)?; + let mem = self.bus.memory(); + let name = &self.shared.name; + let mut regions = Vec::with_capacity(maps.len()); + for (i, m) in maps.into_iter().enumerate() { + let r = mem + .new_ram_device_ptr(&format!("{name} BAR {nr} mmaps[{i}]"), m) + .map_err(|e| Some(mem_error(e)))?; + regions.push(r); + } + for (&r, &(offset, _)) in regions.iter().zip(areas) { + mem.add_subregion(region, offset, r).map_err(|e| Some(mem_error(e)))?; + } + lock(&self.shared.mmaps.state).regions.extend(regions); + + // vfio_region_create_dma_buf(). The legacy container's device takes the ioctl, so it + // is always asked. + let open = rustix::fs::OFlags::RDWR.bits(); + match self.shared.device.dma_buf(nr as u32, open, areas) { + Ok(fd) => lock(&self.shared.mmaps.dma_bufs).push(fd), + Err(e) if e.raw_os_error() == Some(rustix::io::Errno::NOTTY.raw_os_error()) => { + static WARNED: AtomicBool = AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + warn_report( + "VFIO dma-buf not supported in kernel, using mmap fallback, P2P DMA \ + will not work", + ); + } + } + Err(e) => { + static WARNED: AtomicBool = AtomicBool::new(false); + let e = Error::from_io( + format_args!( + "{name} BAR {nr}: dma-buf unavailable, using mmap fallback, P2P DMA \ + will not work" + ), + e, + ); + warn_err_once(&WARNED, &e); + } + } + Ok(()) + } + /// `vfio_msix_early_setup()`: where the MSI-X table and PBA are, from the device. fn msix_early_setup(&mut self) -> Result<()> { let pos = usize::from(self.dev.find_capability(PCI_CAP_ID_MSIX)); @@ -1915,6 +2137,17 @@ impl<'a> Setup<'a> { } } self.msix = Some(msix); + // vfio_pci_fixup_msix_region(): unless the kernel lets the table be mapped, its host + // pages stay trapped. + if !self.caps[msix.table_bar].has(REGION_INFO_CAP_MSIX_MAPPABLE) { + fixup_msix_mmaps( + &mut self.mmaps[msix.table_bar], + self.bars[msix.table_bar].size, + msix.table_offset, + msix.entries, + rustix::param::page_size() as u64, + ); + } Ok(()) } @@ -2225,10 +2458,137 @@ fn ext_cap_max_size(config: &[u8], pos: u16) -> u16 { next - pos } +/// The mmaps `vfio_region_setup()` gives BAR `nr`, each an offset and a size in it, sorted. +/// Without a sparse mmap capability the whole BAR is one. +fn region_mmaps( + nr: usize, + info: &RegionInfo, + caps: &RegionCaps, + no_mmap: bool, +) -> Result> { + if info.size == 0 || no_mmap || info.flags & REGION_INFO_FLAG_MMAP == 0 { + return Ok(Vec::new()); + } + let Some(areas) = caps.sparse_mmap() else { + return Ok(vec![(0, info.size)]); + }; + // vfio_setup_region_sparse_mmaps(). + let mut areas: Vec<(u64, u64)> = areas.into_iter().filter(|a| a.1 != 0).collect(); + areas.sort_by_key(|a| a.0); + for w in areas.windows(2) { + let prev_end = w[0].0.wrapping_add(w[0].1); + if prev_end > w[1].0 { + return Err(Error::generic(format!( + "vfio_setup_region_sparse_mmaps: overlapping sparse mmap regions detected in \ + region {nr}: [0x{:x}-0x{:x}] overlaps with [0x{:x}-0x{:x}]", + w[0].0, + prev_end.wrapping_sub(1), + w[1].0, + w[1].0.wrapping_add(w[1].1).wrapping_sub(1) + ))); + } + } + Ok(areas) +} + +/// `vfio_pci_fixup_msix_region()`: when one mmap covers the whole BAR of the MSI-X table, the +/// host pages of the table are cut out of it, which leaves the part before them, the part after +/// them, both, or nothing. Any other layout is left as the kernel gave it. +fn fixup_msix_mmaps( + mmaps: &mut Vec<(u64, u64)>, + size: u64, + table_offset: u32, + entries: u32, + page_size: u64, +) { + if mmaps.len() != 1 || mmaps[0] != (0, size) { + return; + } + let start = u64::from(table_offset) & !(page_size - 1); + let end = (u64::from(table_offset) + u64::from(entries) * PCI_MSIX_ENTRY_SIZE) + .next_multiple_of(page_size); + if start == 0 { + if end >= size { + mmaps.clear(); + } else { + mmaps[0] = (end, size - end); + } + } else if end >= size { + mmaps[0].1 = start; + } else { + *mmaps = vec![(0, start), (end, size - end)]; + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn region_mmaps_like_vfio_region_setup() { + let mut info = RegionInfo::default(); + info.size = 0x4000; + info.flags = REGION_INFO_FLAG_READ | REGION_INFO_FLAG_WRITE | REGION_INFO_FLAG_MMAP; + // A sparse mmap capability at 32 with the areas given. + let sparse = |areas: &[(u64, u64)]| { + let mut b = vec![0u8; 32]; + b.extend(1u16.to_ne_bytes()); + b.extend(1u16.to_ne_bytes()); + b.extend(0u32.to_ne_bytes()); + b.extend((areas.len() as u32).to_ne_bytes()); + b.extend(0u32.to_ne_bytes()); + for &(o, s) in areas { + b.extend(o.to_ne_bytes()); + b.extend(s.to_ne_bytes()); + } + RegionCaps::new(b, 32) + }; + let none = RegionCaps::default(); + assert_eq!(region_mmaps(0, &info, &none, false).unwrap(), vec![(0, 0x4000)]); + assert!(region_mmaps(0, &info, &none, true).unwrap().is_empty()); + // Empty areas go, and the rest are sorted. + let caps = sparse(&[(0x3000, 0x1000), (0x2000, 0), (0, 0x1000)]); + assert_eq!( + region_mmaps(0, &info, &caps, false).unwrap(), + vec![(0, 0x1000), (0x3000, 0x1000)] + ); + assert!(region_mmaps(0, &info, &sparse(&[]), false).unwrap().is_empty()); + let caps = sparse(&[(0x1000, 0x2000), (0, 0x1800)]); + assert_eq!( + region_mmaps(2, &info, &caps, false).unwrap_err().message(), + "vfio_setup_region_sparse_mmaps: overlapping sparse mmap regions detected in region \ + 2: [0x0-0x17ff] overlaps with [0x1000-0x2fff]" + ); + info.flags &= !REGION_INFO_FLAG_MMAP; + assert!(region_mmaps(0, &info, &none, false).unwrap().is_empty()); + info.flags |= REGION_INFO_FLAG_MMAP; + info.size = 0; + assert!(region_mmaps(0, &info, &none, false).unwrap().is_empty()); + } + + #[test] + fn msix_table_pages_are_cut_out_of_the_mmap() { + let fix = |size, offset, entries| { + let mut m = vec![(0, size)]; + fixup_msix_mmaps(&mut m, size, offset, entries, 0x1000); + m + }; + // e1000e: a 16k BAR 3 with the table at 0 and the PBA at 0x2000. + assert_eq!(fix(0x4000, 0, 5), vec![(0x1000, 0x3000)]); + assert_eq!(fix(0x1000, 0, 5), vec![]); + assert_eq!(fix(0x4000, 0x3000, 5), vec![(0, 0x3000)]); + assert_eq!(fix(0x4000, 0x2008, 64), vec![(0, 0x2000), (0x3000, 0x1000)]); + // A table that spills into the next page takes that page too. + assert_eq!(fix(0x8000, 0x1ff0, 2), vec![(0, 0x1000), (0x3000, 0x5000)]); + // Layouts that are not one mmap of the whole BAR stay as they are. + let mut m = vec![(0x1000, 0x3000)]; + fixup_msix_mmaps(&mut m, 0x4000, 0, 5, 0x1000); + assert_eq!(m, vec![(0x1000, 0x3000)]); + let mut m = vec![]; + fixup_msix_mmaps(&mut m, 0x4000, 0, 5, 0x1000); + assert!(m.is_empty()); + } + #[test] fn basename_like_glib() { assert_eq!(basename("/sys/bus/pci/devices/0000:00:04.0"), "0000:00:04.0"); diff --git a/crates/mem/src/address_space.rs b/crates/mem/src/address_space.rs index 5f8f0dbd..b8d86846 100644 --- a/crates/mem/src/address_space.rs +++ b/crates/mem/src/address_space.rs @@ -100,7 +100,7 @@ impl AddressSpace { // of it, so an outl to 0xcf8 on q35 reaches CONFIG_ADDRESS although the reset // control register sits on top of 0xcf9. let ram = match range.target.ty { - RegionType::Ram => true, + RegionType::Ram | RegionType::RamDevice => true, RegionType::RomDevice => range.romd_mode, _ => false, }; @@ -252,7 +252,9 @@ fn slow_path(range: Option<&FlatRange>) -> (&dyn MmioOps, &str) { match range { None => (&UNASSIGNED, UNASSIGNED_NAME), Some(r) => match &r.target.kind { - TargetKind::Mmio(ops) | TargetKind::RomDevice(_, ops) => (ops.as_ref(), r.name()), + TargetKind::Mmio(ops) + | TargetKind::RomDevice(_, ops) + | TargetKind::RamDevice(_, ops) => (ops.as_ref(), r.name()), _ => (&UNASSIGNED, r.name()), }, } @@ -263,7 +265,7 @@ fn access_allowed(range: Option<&FlatRange>, offset: u64, len: u64, attrs: MemTx if !attrs.memory() { return true; } - if range.is_some_and(|r| r.target.ty == RegionType::Ram) { + if range.is_some_and(|r| matches!(r.target.ty, RegionType::Ram | RegionType::RamDevice)) { return true; } let name = range.map_or(UNASSIGNED_NAME, |r| r.name()); diff --git a/crates/mem/src/flatview.rs b/crates/mem/src/flatview.rs index 11f554db..b7d32d2e 100644 --- a/crates/mem/src/flatview.rs +++ b/crates/mem/src/flatview.rs @@ -176,7 +176,9 @@ pub(crate) struct RenderFlags { pub(crate) fn dirty_log_mask(n: &Node, flags: RenderFlags) -> DirtyMask { let mut mask = n.dirty_log_mask; let ram = n.target.ram_block().is_some(); - if flags.global_dirty_log && (ram || n.ty() == RegionType::Iommu) { + // A RAM device block is never migratable, so migration does not log it. + let migratable = ram && n.ty() != RegionType::RamDevice; + if flags.global_dirty_log && (migratable || n.ty() == RegionType::Iommu) { mask = mask.with(DirtyClient::Migration); } if flags.code_dirty_log && ram { diff --git a/crates/mem/src/region.rs b/crates/mem/src/region.rs index f9c6ea90..b98a1af5 100644 --- a/crates/mem/src/region.rs +++ b/crates/mem/src/region.rs @@ -33,6 +33,10 @@ pub enum RegionType { /// A RAM block read directly in romd mode and a device otherwise, and always a device for /// writes. RomDevice, + /// Host memory of a device, `memory_region_init_ram_device_ptr()`: a mapped BAR of a + /// passthrough device. A memory listener sees it as RAM, but the CPU reaches it through + /// sized callbacks so that each access is one load or store of its width. + RamDevice, /// A device. Mmio, /// A window onto another region. @@ -59,6 +63,7 @@ pub(crate) struct RegionTarget { pub(crate) enum TargetKind { Ram(Arc), RomDevice(Arc, Arc), + RamDevice(Arc, Arc), Mmio(Arc), Iommu(Arc), /// Containers and aliases, which never appear in a FlatView. @@ -68,7 +73,9 @@ pub(crate) enum TargetKind { impl RegionTarget { pub(crate) fn ram_block(&self) -> Option<&Arc> { match &self.kind { - TargetKind::Ram(b) | TargetKind::RomDevice(b, _) => Some(b), + TargetKind::Ram(b) | TargetKind::RomDevice(b, _) | TargetKind::RamDevice(b, _) => { + Some(b) + } _ => None, } } diff --git a/crates/mem/src/system.rs b/crates/mem/src/system.rs index fe8f7c9d..db781563 100644 --- a/crates/mem/src/system.rs +++ b/crates/mem/src/system.rs @@ -12,8 +12,9 @@ use std::fmt; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; -use crate::access::MmioOps; +use crate::access::{AccessConstraints, AccessSize, Endian, MmioOps}; use crate::address_space::AddressSpace; +use crate::attrs::{AccessCtx, MemResult, MemTxResult}; use crate::dirty::{DirtyClient, DirtySnapshot}; use crate::error::MemError; use crate::flatview::{FlatView, RenderFlags, SPACE_END, dirty_log_mask, render}; @@ -21,6 +22,7 @@ use crate::iommu::IommuOps; use crate::listener::{ListenerId, MemoryListener}; use crate::ram::RamBlock; use crate::region::{Arena, Node, RegionId, RegionInfo, RegionTarget, RegionType, TargetKind}; +use ruvm_sys::{HostMemory, hostatomic}; /// `GLOBAL_DIRTY_MIGRATION`: global dirty tracking for live migration. pub const GLOBAL_DIRTY_MIGRATION: u32 = 1 << 0; @@ -304,6 +306,21 @@ impl MemorySystem { ) } + /// Host memory of a device, `memory_region_init_ram_device_ptr()`. The region takes `mem`, + /// which is usually a mapped BAR, and unmaps it when the region goes. CPU accesses go + /// through [`RamDeviceOps`] so the device sees each one at its width. + pub fn new_ram_device_ptr(&self, name: &str, mem: HostMemory) -> Result { + let size = mem.len() as u64; + let block = Self::track(RamBlock::from_memory(name, mem, self.config.page_bits)); + let ops: Arc = Arc::new(RamDeviceOps(Arc::clone(&block))); + self.create( + name, + u128::from(size), + RegionType::RamDevice, + TargetKind::RamDevice(block, ops), + ) + } + /// A device, `memory_region_init_io()`. pub fn new_io( &self, @@ -953,3 +970,56 @@ fn topology_pass( } } } + +/// `ram_device_mem_ops`: each access is one host load or store of its width, in host byte +/// order, so a device register sees the size the guest used. Unaligned accesses are allowed +/// and copied a byte at a time, as `ldl_he_p()` would. +pub(crate) struct RamDeviceOps(Arc); + +impl RamDeviceOps { + fn bytes(&self, offset: u64, size: AccessSize) -> MemResult<&[std::sync::atomic::AtomicU8]> { + let start = usize::try_from(offset).map_err(|_| MemTxResult::ERROR)?; + let end = start.checked_add(size.bytes() as usize).ok_or(MemTxResult::ERROR)?; + self.0.atomic_bytes().get(start..end).ok_or(MemTxResult::ERROR) + } +} + +impl MmioOps for RamDeviceOps { + fn read(&self, _cx: &AccessCtx, offset: u64, size: AccessSize) -> MemResult { + let mem = self.bytes(offset, size)?; + if let Some(v) = hostatomic::load(mem) { + return Ok(v); + } + let mut b = [0u8; 8]; + let n = mem.len(); + self.0.read(offset, &mut b[..n]).map_err(|_| MemTxResult::ERROR)?; + Ok(if cfg!(target_endian = "big") { + u64::from_be_bytes(b) >> (64 - 8 * n) + } else { + u64::from_le_bytes(b) + }) + } + + fn write(&self, _cx: &AccessCtx, offset: u64, size: AccessSize, value: u64) -> MemResult<()> { + let mem = self.bytes(offset, size)?; + if hostatomic::store(mem, value) { + return Ok(()); + } + let n = mem.len(); + let b = value.to_ne_bytes(); + let b = if cfg!(target_endian = "big") { &b[8 - n..] } else { &b[..n] }; + self.0.write(offset, b).map_err(|_| MemTxResult::ERROR) + } + + fn valid(&self) -> AccessConstraints { + AccessConstraints::any_size(1, 8).allow_unaligned() + } + + fn impl_constraints(&self) -> AccessConstraints { + AccessConstraints::any_size(1, 8).allow_unaligned() + } + + fn endianness(&self) -> Endian { + Endian::HOST + } +} diff --git a/crates/mem/tests/api.rs b/crates/mem/tests/api.rs index 2e652c45..53e897fc 100644 --- a/crates/mem/tests/api.rs +++ b/crates/mem/tests/api.rs @@ -247,6 +247,33 @@ fn rom_device_switches_between_ram_and_callbacks() { assert!(matches!(sys.set_romd(root, false), Err(MemError::WrongKind(_)))); } +#[test] +fn ram_device_is_ram_for_listeners_and_sized_for_the_cpu() { + let (sys, root, space) = setup(); + let bar = sys.new_ram_device_ptr("bar", ruvm_sys::HostMemory::new(0x1000).unwrap()).unwrap(); + sys.add_subregion(root, 0x4000, bar).unwrap(); + assert_eq!(sys.region(bar).unwrap().ty, ruvm_mem::RegionType::RamDevice); + assert_eq!(space.write_u32(0x4010, U, 0x1122_3344), MemTxResult::OK); + let mut b = [0; 4]; + sys.ram_block(bar).unwrap().read(0x10, &mut b).unwrap(); + assert_eq!(b, 0x1122_3344u32.to_ne_bytes()); + assert_eq!(space.load(0x4010, 8, Endian::HOST, U), (0x1122_3344, MemTxResult::OK)); + // Unaligned accesses are allowed and copied a byte at a time. + assert_eq!(space.store(0x4021, 4, 0xa1b2_c3d4, Endian::Little, U), MemTxResult::OK); + assert_eq!(space.load(0x4021, 4, Endian::Little, U), (0xa1b2_c3d4, MemTxResult::OK)); + assert_eq!(space.load(0x4022, 2, Endian::Little, U), (0xb2c3, MemTxResult::OK)); + // The memory attribute lets it through, like RAM. + let mem = MemTxAttrs::new().with_memory(true); + assert_eq!(space.read_u32(0x4010, mem), (0x1122_3344, MemTxResult::OK)); + // A mapped BAR is never migrated, so migration does not log it. + sys.global_dirty_log_start(GLOBAL_DIRTY_MIGRATION).unwrap(); + let fv = space.flatview(); + let r = fv.ranges().iter().find(|r| r.name() == "bar").unwrap(); + assert!(!r.dirty_log_mask().contains(DirtyClient::Migration)); + assert!(r.ram_block().is_some()); + sys.global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION); +} + struct Iommu { target: Arc, } diff --git a/crates/sys/Cargo.toml b/crates/sys/Cargo.toml index 875c8243..c02e5137 100644 --- a/crates/sys/Cargo.toml +++ b/crates/sys/Cargo.toml @@ -11,7 +11,7 @@ authors.workspace = true publish = false [package.metadata.ruvm] -unsafe-budget = 33 +unsafe-budget = 37 [dependencies] diff --git a/crates/sys/src/hostmem.rs b/crates/sys/src/hostmem.rs index 3192e606..00069274 100644 --- a/crates/sys/src/hostmem.rs +++ b/crates/sys/src/hostmem.rs @@ -141,6 +141,99 @@ impl HostMemory { Ok(HostMemory { ptr, len, fd }) } + /// Maps the `areas` of a device region, each an offset and a size in the region, the way + /// `vfio_region_mmap()` does: the region of `size` bytes at `fd_offset` of `fd` gets an + /// address aligned to its size rounded up to a power of two, capped at 1 GiB, so the kernel + /// can use huge mappings, and each area is mapped shared at its place in it. The areas must + /// be sorted, apart and inside the region. Each one comes back as memory of its own, which + /// unmaps it when it goes. If one fails, none stay mapped. + #[cfg(any(target_os = "linux", target_os = "android"))] + pub fn map_device( + fd: std::os::fd::BorrowedFd<'_>, + fd_offset: u64, + size: u64, + areas: &[(u64, u64)], + read: bool, + write: bool, + ) -> std::io::Result> { + use std::os::fd::AsRawFd; + + let invalid = || std::io::Error::from(std::io::ErrorKind::InvalidInput); + let size = usize::try_from(size).map_err(|_| invalid())?; + let mut end = 0; + let mut spans = Vec::with_capacity(areas.len()); + for &(off, len) in areas { + let off = usize::try_from(off).map_err(|_| invalid())?; + let len = usize::try_from(len).map_err(|_| invalid())?; + if off < end || len == 0 || off.checked_add(len).is_none_or(|e| e > size) { + return Err(invalid()); + } + let file_off = fd_offset.checked_add(off as u64).ok_or_else(invalid)?; + let file_off = libc::off_t::try_from(file_off).map_err(|_| invalid())?; + spans.push((off, len, file_off)); + end = off + len; + } + let align = size.checked_next_power_of_two().unwrap_or(usize::MAX).min(1 << 30); + let total = size.checked_add(align).ok_or_else(invalid)?; + // SAFETY: a new inaccessible mapping at an address the kernel picks aliases nothing in + // this process. + let base = unsafe { + libc::mmap( + std::ptr::null_mut(), + total, + libc::PROT_NONE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_NORESERVE, + -1, + 0, + ) + }; + if base == libc::MAP_FAILED { + return Err(std::io::Error::last_os_error()); + } + let base = base as usize; + let start = base.next_multiple_of(align); + unmap_reserved(base, start - base); + unmap_reserved(start + size, align - (start - base)); + + let mut prot = 0; + if read { + prot |= libc::PROT_READ; + } + if write { + prot |= libc::PROT_WRITE; + } + let mut out = Vec::with_capacity(spans.len()); + let mut done = 0; + for (off, len, file_off) in spans { + unmap_reserved(start + done, off - done); + // SAFETY: the target is the part of the reservation above for this area, which + // nothing else in the process maps, and the device memory is only ever reached + // through the `AtomicU8` view. + let p = unsafe { + libc::mmap( + (start + off) as *mut libc::c_void, + len, + prot, + libc::MAP_SHARED | libc::MAP_FIXED, + fd.as_raw_fd(), + file_off, + ) + }; + if p == libc::MAP_FAILED { + let err = std::io::Error::last_os_error(); + // The areas mapped so far go with `out`. + unmap_reserved(start + off, size - off); + return Err(err); + } + let ptr = + std::ptr::NonNull::new(p.cast::()).expect("mmap never returns null"); + out.push(HostMemory { ptr, len, fd: None }); + done = off + len; + } + unmap_reserved(start + done, size - done); + Ok(out) + } + /// The descriptor of shared memory, which another process can map. #[cfg(unix)] pub fn fd(&self) -> Option> { @@ -392,6 +485,16 @@ impl HostMemory { } } +/// Gives back `len` bytes at `addr` of a reservation [`HostMemory::map_device`] made and no +/// [`HostMemory`] owns. +#[cfg(any(target_os = "linux", target_os = "android"))] +fn unmap_reserved(addr: usize, len: usize) { + if len != 0 { + // SAFETY: the range is part of an inaccessible reservation nothing points into. + unsafe { libc::munmap(addr as *mut libc::c_void, len) }; + } +} + #[cfg(unix)] impl Drop for HostMemory { fn drop(&mut self) { @@ -408,6 +511,34 @@ mod tests { use super::*; use std::sync::atomic::Ordering; + #[cfg(any(target_os = "linux", target_os = "android"))] + #[test] + fn device_areas_are_aligned_and_shared() { + use std::os::fd::AsFd; + + let file = HostMemory::shared("dev", 0x8000).unwrap(); + file.as_slice()[0x5008].store(0x5a, Ordering::Relaxed); + let fd = file.fd().unwrap(); + let areas = [(0, 0x2000), (0x5000, 0x1000)]; + let maps = HostMemory::map_device(fd.as_fd(), 0, 0x8000, &areas, true, true).unwrap(); + assert_eq!(maps.len(), 2); + assert_eq!(maps[0].host_addr() % 0x8000, 0); + assert_eq!(maps[1].host_addr(), maps[0].host_addr() + 0x5000); + assert_eq!(maps[1].len(), 0x1000); + assert_eq!(maps[1].as_slice()[8].load(Ordering::Relaxed), 0x5a); + maps[0].as_slice()[1].store(3, Ordering::Relaxed); + assert_eq!(file.as_slice()[1].load(Ordering::Relaxed), 3); + // Out of order, overlapping and empty areas are refused. + for bad in [ + [(0x5000, 0x1000), (0, 0x1000)], + [(0, 0x2000), (0x1000, 0x1000)], + [(0, 0), (0x1000, 0x1000)], + ] { + assert!(HostMemory::map_device(fd.as_fd(), 0, 0x8000, &bad, true, true).is_err()); + } + assert!(HostMemory::map_device(fd.as_fd(), 0, 0x1000, &[(0, 0x2000)], true, true).is_err()); + } + #[test] fn zeroed_and_writable() { let m = HostMemory::new(1 << 20).unwrap(); diff --git a/crates/sys/src/vfio.rs b/crates/sys/src/vfio.rs index 0ec82adf..3304efb7 100644 --- a/crates/sys/src/vfio.rs +++ b/crates/sys/src/vfio.rs @@ -15,7 +15,7 @@ use std::ffi::CString; use std::fs::{File, OpenOptions}; use std::io; -use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd}; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, OwnedFd}; use std::os::unix::fs::{FileExt, OpenOptionsExt}; /// `VFIO_API_VERSION`. @@ -41,6 +41,17 @@ pub const REGION_INFO_FLAG_READ: u32 = 1; pub const REGION_INFO_FLAG_WRITE: u32 = 1 << 1; /// `VFIO_REGION_INFO_FLAG_MMAP`. pub const REGION_INFO_FLAG_MMAP: u32 = 1 << 2; +/// `VFIO_REGION_INFO_FLAG_CAPS`: the region info has a capability chain. +pub const REGION_INFO_FLAG_CAPS: u32 = 1 << 3; +/// `VFIO_REGION_INFO_CAP_SPARSE_MMAP`: only the listed areas of the region can be mapped. +pub const REGION_INFO_CAP_SPARSE_MMAP: u16 = 1; +/// `VFIO_REGION_INFO_CAP_MSIX_MAPPABLE`: the MSI-X table of the BAR can be mapped too. +pub const REGION_INFO_CAP_MSIX_MAPPABLE: u16 = 3; + +/// `VFIO_DEVICE_FEATURE_GET`. +pub const DEVICE_FEATURE_GET: u32 = 1 << 16; +/// `VFIO_DEVICE_FEATURE_DMA_BUF`. +pub const DEVICE_FEATURE_DMA_BUF: u32 = 11; /// `VFIO_PCI_BAR0_REGION_INDEX`. pub const PCI_BAR0_REGION_INDEX: u32 = 0; @@ -96,6 +107,7 @@ const VFIO_DEVICE_SET_IRQS: u64 = io(10); const VFIO_DEVICE_RESET: u64 = io(11); const VFIO_IOMMU_MAP_DMA: u64 = io(13); const VFIO_IOMMU_UNMAP_DMA: u64 = io(14); +const VFIO_DEVICE_FEATURE: u64 = io(17); #[repr(C)] #[derive(Default)] @@ -132,6 +144,54 @@ pub struct RegionInfo { pub offset: u64, } +/// The capability chain of a region's info, the bytes `VFIO_DEVICE_GET_REGION_INFO` returned +/// when its `argsz` was large enough for them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RegionCaps { + buf: Vec, + first: u32, +} + +impl RegionCaps { + /// A chain from the whole `vfio_region_info` buffer and its `cap_offset`. + pub fn new(buf: Vec, first: u32) -> Self { + RegionCaps { buf, first } + } + + /// `vfio_get_cap()`: the capability `id`, from its header on. The chain ends at a `next` + /// of 0, which is where the info starts. + pub fn find(&self, id: u16) -> Option<&[u8]> { + let mut off = self.first as usize; + // A chain that loops back on itself would never reach 0. + for _ in 0..self.buf.len() / 8 { + if off == 0 { + return None; + } + let hdr = self.buf.get(off..off.checked_add(8)?)?; + if u16::from_ne_bytes([hdr[0], hdr[1]]) == id { + return Some(&self.buf[off..]); + } + off = u32::from_ne_bytes([hdr[4], hdr[5], hdr[6], hdr[7]]) as usize; + } + None + } + + /// Whether the chain has capability `id`. + pub fn has(&self, id: u16) -> bool { + self.find(id).is_some() + } + + /// The areas of `struct vfio_region_info_cap_sparse_mmap` as offset and size pairs, in the + /// order the kernel gave them. + pub fn sparse_mmap(&self) -> Option> { + let cap = self.find(REGION_INFO_CAP_SPARSE_MMAP)?; + let nr = u32::from_ne_bytes(cap.get(8..12)?.try_into().ok()?) as usize; + let u64_at = + |o: usize| cap.get(o..o + 8).map(|b| u64::from_ne_bytes(b.try_into().unwrap())); + (0..nr).map(|i| Some((u64_at(16 + 16 * i)?, u64_at(24 + 16 * i)?))).collect() + } +} + /// `struct vfio_irq_info`. #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -177,8 +237,10 @@ struct IrqSet { } /// The argument structures of the ioctls this module issues. A `c_char` is the first byte of a -/// string and a `u32` the first word of a `vfio_irq_set` followed by its eventfds. +/// string, a `u32` the first word of a `vfio_irq_set` followed by its eventfds, and a `u64` the +/// first word of a buffer as long as the `argsz` it starts with. trait Arg {} +impl Arg for u64 {} impl Arg for IrqSet {} impl Arg for u32 {} impl Arg for GroupStatus {} @@ -208,6 +270,13 @@ fn ioctl_ptr(file: &File, req: u64, arg: *mut T) -> io::Result { if ret < 0 { Err(io::Error::last_os_error()) } else { Ok(ret) } } +/// Two `u32` fields that follow each other in a structure, as the `u64` word they fill. +fn word(first: u32, second: u32) -> u64 { + let [a, b, c, d] = first.to_ne_bytes(); + let [e, f, g, h] = second.to_ne_bytes(); + u64::from_ne_bytes([a, b, c, d, e, f, g, h]) +} + fn argsz() -> u32 { size_of::() as u32 } @@ -328,6 +397,13 @@ pub struct VfioDevice { file: File, } +impl AsFd for VfioDevice { + /// The device descriptor, which the regions that allow it are mapped from. + fn as_fd(&self) -> BorrowedFd<'_> { + self.file.as_fd() + } +} + impl VfioDevice { /// `VFIO_DEVICE_GET_INFO`. pub fn info(&self) -> io::Result { @@ -343,6 +419,59 @@ impl VfioDevice { Ok(info) } + /// `vfio_device_get_region_info()`: `VFIO_DEVICE_GET_REGION_INFO` for region `index` with + /// its capability chain, asking again with the `argsz` the kernel wants when the chain did + /// not fit. + pub fn region_info_caps(&self, index: u32) -> io::Result<(RegionInfo, RegionCaps)> { + let mut argsz = argsz::(); + loop { + let mut buf = vec![0u64; (argsz as usize).div_ceil(8)]; + buf[0] = word(argsz, 0); + buf[1] = word(index, 0); + ioctl_ptr(&self.file, VFIO_DEVICE_GET_REGION_INFO, buf.as_mut_ptr())?; + let bytes: Vec = buf.iter().flat_map(|w| w.to_ne_bytes()).collect(); + let u32_at = |o: usize| u32::from_ne_bytes(bytes[o..o + 4].try_into().unwrap()); + let u64_at = |o: usize| u64::from_ne_bytes(bytes[o..o + 8].try_into().unwrap()); + let want = u32_at(0); + if want > argsz { + argsz = want; + continue; + } + let info = RegionInfo { + argsz: want, + flags: u32_at(4), + index: u32_at(8), + cap_offset: u32_at(12), + size: u64_at(16), + offset: u64_at(24), + }; + let first = if info.flags & REGION_INFO_FLAG_CAPS != 0 { info.cap_offset } else { 0 }; + return Ok((info, RegionCaps::new(bytes, first))); + } + } + + /// `VFIO_DEVICE_FEATURE` with `VFIO_DEVICE_FEATURE_DMA_BUF`: a dma-buf of the `ranges` of + /// region `index`, each an offset and a length, as `vfio_region_create_dma_buf()` asks for + /// one. + pub fn dma_buf( + &self, + index: u32, + open_flags: u32, + ranges: &[(u64, u64)], + ) -> io::Result { + let bytes = 8 + 16 + 16 * ranges.len(); + let argsz = + u32::try_from(bytes).map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + let mut buf = Vec::with_capacity(bytes / 8); + buf.push(word(argsz, DEVICE_FEATURE_GET | DEVICE_FEATURE_DMA_BUF)); + buf.push(word(index, open_flags)); + buf.push(word(0, ranges.len() as u32)); + buf.extend(ranges.iter().flat_map(|&(offset, length)| [offset, length])); + let fd = ioctl_ptr(&self.file, VFIO_DEVICE_FEATURE, buf.as_mut_ptr())?; + // SAFETY: the kernel just returned this descriptor and nothing else owns it. + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + } + /// `VFIO_DEVICE_GET_IRQ_INFO` for interrupt `index`. pub fn irq_info(&self, index: u32) -> io::Result { let mut info = IrqInfo { argsz: argsz::(), index, ..IrqInfo::default() }; @@ -417,6 +546,42 @@ mod tests { assert_eq!(size_of::(), 24); assert_eq!(size_of::(), 32); assert_eq!(size_of::(), 16); + assert_eq!(VFIO_DEVICE_FEATURE, 0x3b75); + } + + /// A region info of 32 bytes followed by a chain at `first`. + fn chain(caps: &[(usize, u16, u32, &[u8])], len: usize) -> Vec { + let mut buf = vec![0u8; len]; + for &(off, id, next, body) in caps { + buf[off..off + 2].copy_from_slice(&id.to_ne_bytes()); + buf[off + 2..off + 4].copy_from_slice(&1u16.to_ne_bytes()); + buf[off + 4..off + 8].copy_from_slice(&next.to_ne_bytes()); + buf[off + 8..off + 8 + body.len()].copy_from_slice(body); + } + buf + } + + #[test] + fn region_caps() { + let mut sparse = Vec::new(); + sparse.extend(2u32.to_ne_bytes()); + sparse.extend(0u32.to_ne_bytes()); + for v in [0x3000u64, 0x1000, 0, 0x2000] { + sparse.extend(v.to_ne_bytes()); + } + let buf = chain(&[(32, 3, 40, &[]), (40, 1, 0, &sparse)], 96); + let caps = RegionCaps::new(buf.clone(), 32); + assert!(caps.has(REGION_INFO_CAP_MSIX_MAPPABLE)); + assert_eq!(caps.sparse_mmap(), Some(vec![(0x3000, 0x1000), (0, 0x2000)])); + assert!(!caps.has(2)); + // No chain at all, and a chain that points back at itself. + assert_eq!(RegionCaps::new(buf, 0).sparse_mmap(), None); + let looped = chain(&[(32, 3, 32, &[])], 40); + assert!(!RegionCaps::new(looped, 32).has(1)); + // A sparse cap cut short by the buffer. + let short = chain(&[(32, 1, 0, &sparse[..24])], 64); + assert_eq!(RegionCaps::new(short, 32).sparse_mmap(), None); + assert_eq!(word(1, 2).to_ne_bytes()[..4], 1u32.to_ne_bytes()); } #[test] diff --git a/crates/system/src/vfio.rs b/crates/system/src/vfio.rs index 4912153e..365b8382 100644 --- a/crates/system/src/vfio.rs +++ b/crates/system/src/vfio.rs @@ -25,7 +25,6 @@ const UNSUPPORTED: &[&str] = &[ "display", "xres", "yres", - "x-intx-mmap-timeout-ms", "x-vga", "x-req", "x-igd-opregion", @@ -107,10 +106,8 @@ pub(crate) fn plan_device( "x-balloon-allowed" => conf.balloon_allowed = boolean(k, v)?, "skip-vsc-check" => conf.skip_vsc_check = boolean(k, v)?, "multifunction" => conf.multifunction = boolean(k, v)?, - // The BARs are always trapped. - "x-no-mmap" => { - boolean(k, v)?; - } + "x-no-mmap" => conf.no_mmap = boolean(k, v)?, + "x-intx-mmap-timeout-ms" => conf.intx_mmap_timeout_ms = uint32(k, v)?, "ats" => { let ats = OnOffAuto::parse(v).ok_or_else(|| { Located::new(loc, format!("Parameter '{k}' does not accept value '{v}'")) @@ -153,6 +150,7 @@ pub(crate) fn realize_x86( board: &mut X86Board, plug: &VfioPlug, firmware: &FirmwareSearch, + clock: &std::sync::Arc, ) -> Result<(), Located> { use std::sync::Arc; @@ -179,6 +177,7 @@ pub(crate) fn realize_x86( &dma_as, m.memory_as(), Some(&viommu), + clock, &plug.conf, ) .map_err(|e| Located(plug.loc.clone(), e))?; @@ -220,6 +219,7 @@ pub(crate) fn realize_x86( _board: &mut X86Board, plug: &VfioPlug, _firmware: &FirmwareSearch, + _clock: &std::sync::Arc, ) -> Result<(), Located> { Err(Located::new(&plug.loc, format!("'{TYPE_VFIO_PCI}' is not a valid device model name"))) } diff --git a/crates/system/src/x86.rs b/crates/system/src/x86.rs index 98f1a3e1..c9c910d4 100644 --- a/crates/system/src/x86.rs +++ b/crates/system/src/x86.rs @@ -2347,7 +2347,7 @@ fn build( // The vfio-pci devices that come before virtio plug `at`, or all that are left. let mut plug_vfio = |board: &mut X86Board, at: Option| { while let Some((_, plug)) = vfio.next_if(|(vi, _)| at.is_none_or(|i| *vi == i)) { - crate::vfio::realize_x86(board, plug, &firmware).map_err(one)?; + crate::vfio::realize_x86(board, plug, &firmware, &clock).map_err(one)?; } Ok::<(), Vec>(()) }; @@ -3249,6 +3249,10 @@ mod tests { assert_eq!(plug.conf.host.unwrap().to_string(), "0000:00:04.0"); assert_eq!((plug.conf.device_id, plug.conf.vendor_id), (0x10d4, ruvm_hw_vfio::PCI_ANY_ID)); assert_eq!(plug.conf.ats, Some(false)); + assert_eq!((plug.conf.no_mmap, plug.conf.intx_mmap_timeout_ms), (false, 1100)); + let args = ["vfio-pci,host=0000:00:04.0,x-no-mmap=on,x-intx-mmap-timeout-ms=0"]; + let p = plan(q, &[], &devices(&args), true).unwrap(); + assert_eq!((p.vfio[0].1.conf.no_mmap, p.vfio[0].1.conf.intx_mmap_timeout_ms), (true, 0)); let no_pci = "No 'PCI' bus found for device 'vfio-pci'"; assert_eq!(err(Some(BoardKind::Microvm), &["vfio-pci"]), no_pci); @@ -3266,6 +3270,10 @@ mod tests { "Parameter 'x-pci-vendor-id' expects integer" ); assert_eq!(err(q, &["vfio-pci,rombar=x"]), "Parameter 'rombar' expects integer"); + assert_eq!( + err(q, &["vfio-pci,x-intx-mmap-timeout-ms=x"]), + "Parameter 'x-intx-mmap-timeout-ms' expects integer" + ); assert_eq!(err(q, &["vfio-pci,ats=x"]), "Parameter 'ats' does not accept value 'x'"); assert_eq!(err(q, &["vfio-pci,foo=1"]), "Property 'vfio-pci.foo' not found"); assert_eq!( From d64344dae0804a420ac583d9d0c533de0aea1ba6 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:25:54 +0700 Subject: [PATCH 2/2] hw-vfio: test the INTx mmap timer --- crates/hw-vfio/src/pci.rs | 103 +++++++++++++++++++++++++++++--------- 1 file changed, 80 insertions(+), 23 deletions(-) diff --git a/crates/hw-vfio/src/pci.rs b/crates/hw-vfio/src/pci.rs index 8c27279e..6a6e9e25 100644 --- a/crates/hw-vfio/src/pci.rs +++ b/crates/hw-vfio/src/pci.rs @@ -859,6 +859,30 @@ impl Mmaps { t.modify((clock.get_ms() + i64::from(self.timeout_ms)) * 1_000_000); } } + + /// The mmap side of `vfio_intx_interrupt()`: the mmaps go off before `raise` puts the pin + /// up, so that the guest's first BAR access after it traps and is taken as the EOI, and + /// they stay locked until the pin is pending, so the timer cannot turn them on in between. + fn intx_fired(&self, raise: impl FnOnce()) { + let mut state = lock(&self.state); + self.set_enabled(&mut state, false); + raise(); + drop(state); + if self.timeout_ms != 0 { + self.arm(); + } + } + + /// `vfio_intx_mmap_enable()`: the timer fired. The mmaps go back on unless INTx is still + /// `pending`, in which case the timer waits another timeout. + fn timer_fired(&self, pending: impl FnOnce() -> bool) { + let mut state = lock(&self.state); + if pending() { + self.arm(); + } else { + self.set_enabled(&mut state, true); + } + } } /// What the BAR, ROM and config callbacks share: the device, where its config space is and the @@ -1015,19 +1039,13 @@ impl Shared { self.mmaps.set_enabled(&mut lock(&self.mmaps.state), true); } - /// `vfio_intx_mmap_enable()`: the mmap timer fired. The mmaps go back on unless INTx is - /// still pending, in which case the timer waits another timeout. + /// `vfio_intx_mmap_enable()`, with the pending flag of INTx if that is what the device + /// uses. fn intx_mmap_enable(&self) { - let mut state = lock(&self.mmaps.state); - let pending = match &*lock(&self.interrupt) { + self.mmaps.timer_fired(|| match &*lock(&self.interrupt) { Interrupt::Intx(i) => *lock(&i.pending), _ => false, - }; - if pending { - self.mmaps.arm(); - } else { - self.mmaps.set_enabled(&mut state, true); - } + }); } /// The part of `vfio_intx_enable()` after the pin is known: an eventfd for the kernel to @@ -1037,23 +1055,14 @@ impl Shared { let notifier = { let (pending, irq) = (Arc::clone(&pending), irq.clone()); let mmaps = Arc::clone(&self.mmaps); - // vfio_intx_interrupt(). The mmaps go off before the pin goes up, so that the - // guest's first BAR access after it traps and is taken as the EOI, and they stay - // locked until the pin is pending, so the timer cannot turn them on in between. - // The pending lock is held over the raise so that an EOI cannot lower the pin in - // between. + // vfio_intx_interrupt(). The pending lock is held over the raise so that an EOI + // cannot lower the pin in between. Notifiers::new("intx-interrupt", 1, move |_| { - let mut state = lock(&mmaps.state); - mmaps.set_enabled(&mut state, false); - { + mmaps.intx_fired(|| { let mut p = lock(&pending); *p = true; irq.raise(); - } - drop(state); - if mmaps.timeout_ms != 0 { - mmaps.arm(); - } + }); })? }; // vfio_device_irq_set_signaling(). @@ -2566,6 +2575,54 @@ mod tests { assert!(region_mmaps(0, &info, &none, false).unwrap().is_empty()); } + #[test] + fn intx_turns_the_mmaps_off_until_it_is_quiet() { + use ruvm_base::ClockType; + let memory = Arc::new(MemorySystem::new()); + let r = memory.new_container("BAR 0 mmaps[0]", 0x1000).unwrap(); + let on = || memory.region(r).unwrap().enabled; + let clock = Clock::manual(ClockType::Virtual); + let mmaps = Arc::new(Mmaps::new(&memory, 1100)); + lock(&mmaps.state).regions.push(r); + let pending = Arc::new(Mutex::new(false)); + let timer = { + let (m, p) = (Arc::downgrade(&mmaps), Arc::clone(&pending)); + clock.new_timer(move || { + if let Some(m) = m.upgrade() { + m.timer_fired(|| *lock(&p)); + } + }) + }; + mmaps.timer.set(timer).unwrap(); + + mmaps.intx_fired(|| *lock(&pending) = true); + assert!(!on()); + // Still pending when the timer fires, so it waits another timeout. + clock.advance_to(1_100_000_000); + assert!(!on()); + *lock(&pending) = false; + clock.advance_to(2_199_000_000); + assert!(!on()); + clock.advance_to(2_200_000_000); + assert!(on()); + // Each interrupt pushes the timeout out again. + mmaps.intx_fired(|| *lock(&pending) = true); + *lock(&pending) = false; + clock.advance_to(3_000_000_000); + mmaps.intx_fired(|| ()); + clock.advance_to(3_300_000_000); + assert!(!on()); + clock.advance_to(4_100_000_000); + assert!(on()); + + // With no timeout they stay off. + let off = Mmaps::new(&memory, 0); + lock(&off.state).regions.push(r); + off.intx_fired(|| ()); + assert!(!on()); + assert!(mmaps.timer.get().unwrap().expire_time().is_none()); + } + #[test] fn msix_table_pages_are_cut_out_of_the_mmap() { let fix = |size, offset, entries| {