From 19f4b6252e44a57a6a062cec034403ba17a7f19a Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Wed, 14 Oct 2026 23:22:05 +0700 Subject: [PATCH 1/3] vfio-pci: hot reset when the device has no FLR, and the system reset handler --- crates/hw-vfio/src/pci.rs | 234 +++++++++++++++++++++++++++++++------- crates/sys/src/vfio.rs | 87 ++++++++++++++ crates/system/src/vfio.rs | 9 +- crates/system/src/x86.rs | 4 +- 4 files changed, 292 insertions(+), 42 deletions(-) diff --git a/crates/hw-vfio/src/pci.rs b/crates/hw-vfio/src/pci.rs index 6a6e9e2..22b0b40 100644 --- a/crates/hw-vfio/src/pci.rs +++ b/crates/hw-vfio/src/pci.rs @@ -19,8 +19,10 @@ //! a BAR the kernel lets be mapped are mapped into the process and laid over the trapped BAR as //! RAM device regions, as `vfio_region_mmap()` does, leaving the MSI-X table trapped. While INTx //! is pending they are turned off, so that the guest's next BAR access traps and serves as the -//! EOI, and a timer turns them on again once INTx has gone quiet. Hot reset and the device -//! quirks are not here yet. +//! EOI, and a timer turns them on again once INTx has gone quiet. A device without FLR gets a +//! bus or slot reset when every device it would take along is in a group opened here, the way +//! `vfio_legacy_pci_hot_reset()` does it, and on a system reset [`reset_handler`] resets the +//! devices that share a bus together. The device quirks are not here yet. use std::fmt; use std::io; @@ -65,10 +67,10 @@ use ruvm_mem::{ }; use ruvm_sys::HostMemory; use ruvm_sys::vfio::{ - API_VERSION, DEVICE_FLAGS_PCI, DEVICE_FLAGS_RESET, DeviceInfo, GROUP_FLAGS_VIABLE, - IRQ_INFO_NORESIZE, IRQ_SET_ACTION_TRIGGER, IRQ_SET_ACTION_UNMASK, PCI_CONFIG_REGION_INDEX, - PCI_ERR_IRQ_INDEX, PCI_INTX_IRQ_INDEX, PCI_MSI_IRQ_INDEX, PCI_MSIX_IRQ_INDEX, - PCI_ROM_REGION_INDEX, REGION_INFO_CAP_MSIX_MAPPABLE, REGION_INFO_FLAG_MMAP, + API_VERSION, DEVICE_FLAGS_PCI, DEVICE_FLAGS_RESET, DependentDevice, DeviceInfo, + GROUP_FLAGS_VIABLE, IRQ_INFO_NORESIZE, IRQ_SET_ACTION_TRIGGER, IRQ_SET_ACTION_UNMASK, + PCI_CONFIG_REGION_INDEX, PCI_ERR_IRQ_INDEX, PCI_INTX_IRQ_INDEX, PCI_MSI_IRQ_INDEX, + PCI_MSIX_IRQ_INDEX, PCI_ROM_REGION_INDEX, REGION_INFO_CAP_MSIX_MAPPABLE, REGION_INFO_FLAG_MMAP, REGION_INFO_FLAG_READ, REGION_INFO_FLAG_WRITE, RegionCaps, RegionInfo, TYPE1_IOMMU, TYPE1V2_IOMMU, UNMAP_ALL, VfioContainer, VfioDevice, VfioGroup, }; @@ -100,6 +102,8 @@ const PCI_MSIX_ENTRY_SIZE: u64 = 16; const NUM_BARS: usize = PCI_ROM_SLOT; /// What `read()` gives back for a short config read, `EFAULT`. const EFAULT: i32 = 14; +/// What `VFIO_DEVICE_GET_PCI_HOT_RESET_INFO` fails with when the devices do not fit. +const ENOSPC: i32 = 28; /// How deep a capability chain is followed. A chain that loops would have QEMU recurse until /// it crashes; this stops early instead. const MAX_CAPS: u32 = 48; @@ -427,6 +431,47 @@ fn groups() -> MutexGuard<'static, Vec> { GROUPS.lock().unwrap_or_else(PoisonError::into_inner) } +/// `vfio_device_list`: the realized devices. +static DEVICES: Mutex>> = Mutex::new(Vec::new()); + +/// The realized devices that are still there, each with its config callbacks. +fn devices() -> Vec<(Arc, Arc)> { + let mut list = lock(&DEVICES); + list.retain(|w| w.strong_count() > 0); + list.iter() + .filter_map(|w| { + let ops = w.upgrade()?; + let dev = ops.dev.get()?.upgrade()?; + Some((ops, dev)) + }) + .collect() +} + +/// The realized device that `host` names, a PCI address as in sysfs. +fn device_in_use(host: &str) -> Option<(Arc, Arc)> { + devices().into_iter().find(|(ops, _)| ops.shared.name == host) +} + +fn group_owned(id: u32) -> bool { + groups().iter().any(|g| g.id == id) +} + +/// `vfio_device_reset_handler()`, which the board runs after it has reset its devices. Each +/// device that only a bus or slot reset resets properly gets one, and so do the other devices +/// in use here that the reset takes along. A device that was reset together with an earlier +/// one is not reset again. +pub fn reset_handler() { + let list = devices(); + for (ops, _) in &list { + ops.compute_needs_reset(); + } + for (ops, dev) in &list { + if ops.needs_reset.load(Ordering::Relaxed) { + ops.hot_reset(dev, false); + } + } +} + /// A device's place in its group. Dropping it is `vfio_device_put()` plus `vfio_group_put()`: /// the group goes once it has no devices left. #[derive(Debug)] @@ -1244,6 +1289,8 @@ struct VfioPciOps { reset_works: bool, has_flr: bool, has_pm_reset: bool, + /// `VFIODevice::needs_reset`: the system reset still has to give the device a bus reset. + needs_reset: AtomicBool, } impl VfioPciOps { @@ -1430,6 +1477,137 @@ impl VfioPciOps { error_report(&format!("vfio: Unable to power on device, stuck in D{state}")); } } + + /// `vfio_pci_pre_reset()`: stops interrupts and DMA, puts INTx Disable in a known state and + /// brings the device to D0. + fn pre_reset(&self, dev: &PciDevice) { + self.disable_interrupts(dev); + let val = self.config_read(dev, PCI_COMMAND as u32, 2); + if val != u32::MAX { + let cmd = val as u16 + & !(PCI_COMMAND_IO + | PCI_COMMAND_MEMORY + | PCI_COMMAND_MASTER + | PCI_COMMAND_INTX_DISABLE); + self.config_write(dev, PCI_COMMAND as u32, u32::from(cmd), 2); + } + self.power_on(dev); + } + + /// `vfio_pci_post_reset()`: INTx back on and the BARs of the device cleared. + fn post_reset(&self, dev: &PciDevice) { + if let Err(e) = self.intx_enable(dev) { + report_error(&e.prepend(format_args!("vfio {}: ", self.shared.name))); + } + for nr in 0..NUM_BARS { + if let Err(e) = self.shared.config_write(PCI_BASE_ADDRESS_0 + 4 * nr, &[0; 4]) { + error_report(&format!( + "vfio_pci_post_reset({}) reset bar {nr} failed: {e}", + self.shared.name + )); + } + } + } + + /// `vfio_pci_compute_needs_reset()`: a device with no reset of its own, or with only a PM + /// reset, gets a bus reset on a system reset. + fn compute_needs_reset(&self) { + if !self.reset_works || (!self.has_flr && self.has_pm_reset) { + self.needs_reset.store(true, Ordering::Relaxed); + } + } + + /// `vfio_pci_get_pci_hot_reset_info()`: the devices a hot reset of this one resets. + fn hot_reset_info(&self) -> Option> { + let device = &self.shared.device; + let (count, r) = device.pci_hot_reset_info(0); + match r { + Err(e) if e.raw_os_error() != Some(ENOSPC) => { + if !self.has_pm_reset { + error_report(&format!( + "vfio: Cannot reset device {}, no available reset mechanism.", + self.shared.name + )); + } + return None; + } + _ => {} + } + match device.pci_hot_reset_info(count).1 { + Ok(devices) => Some(devices), + Err(e) => { + error_report(&format!("vfio: hot reset info failed: {}", strerror(&e))); + None + } + } + } + + /// `vfio_legacy_pci_hot_reset()`: a bus or slot reset of the device, which works when the + /// group of every device it takes along is open here. With `single`, the device's own reset + /// does it as long as no other device in use here is taken along. Otherwise the system + /// reset does it for all of them at once, and only when there is such a device, since the + /// device's own reset already did the rest. Says whether the reset was done. + fn hot_reset(&self, dev: &PciDevice, single: bool) -> bool { + if !single { + self.pre_reset(dev); + } + self.needs_reset.store(false, Ordering::Relaxed); + let done = self.hot_reset_info().is_some_and(|d| self.hot_reset_devices(single, &d)); + if !single { + self.post_reset(dev); + } + done + } + + /// The part of `vfio_legacy_pci_hot_reset()` once the affected devices are known. + fn hot_reset_devices(&self, single: bool, devices: &[DependentDevice]) -> bool { + let others: Vec<_> = devices.iter().filter(|d| d.host_name() != self.shared.name).collect(); + let mut multi = false; + let done = 'reset: { + // Check that every group is owned here and ready the other devices in use. + for d in &others { + if !group_owned(d.group_id) { + if !self.has_pm_reset { + error_report(&format!( + "vfio: Cannot reset device {}, depends on group {} which is not \ + owned.", + self.shared.name, d.group_id + )); + } + break 'reset false; + } + if let Some((ops, dev)) = device_in_use(&d.host_name()) { + if single { + return false; + } + ops.pre_reset(&dev); + ops.needs_reset.store(false, Ordering::Relaxed); + multi = true; + } + } + if !single && !multi { + return false; + } + // The descriptor of each group the reset reaches, in the order they were opened. + let groups = groups(); + let fds: Vec<_> = groups + .iter() + .filter(|g| devices.iter().any(|d| d.group_id == g.id)) + .map(|g| g.group.as_fd()) + .collect(); + self.shared.device.pci_hot_reset(&fds).is_ok() + }; + // INTx back on for the other devices in use, up to the first group not owned here. + for d in &others { + if !group_owned(d.group_id) { + break; + } + if let Some((ops, dev)) = device_in_use(&d.host_name()) { + ops.post_reset(&dev); + } + } + done + } } impl PciDeviceOps for VfioPciOps { @@ -1500,41 +1678,15 @@ impl PciDeviceOps for VfioPciOps { } } - /// `vfio_pci_reset()`. + /// `vfio_pci_reset()`: FLR, or whatever the kernel has when there is neither FLR nor a PM + /// reset, then a bus or slot reset, and a PM reset only when nothing else worked. fn reset(&self, dev: &PciDevice) { - // vfio_pci_pre_reset(): stop interrupts and DMA and put INTx Disable in a known state. - self.disable_interrupts(dev); - let val = self.config_read(dev, PCI_COMMAND as u32, 2); - if val != u32::MAX { - let cmd = val as u16 - & !(PCI_COMMAND_IO - | PCI_COMMAND_MEMORY - | PCI_COMMAND_MASTER - | PCI_COMMAND_INTX_DISABLE); - self.config_write(dev, PCI_COMMAND as u32, u32::from(cmd), 2); - } - self.power_on(dev); - - let device = &self.shared.device; - let flr = self.reset_works && (self.has_flr || !self.has_pm_reset); - // The bus reset QEMU tries next is not done, so a device that only has a PM reset - // gets that. - if !(flr && device.reset().is_ok()) && self.reset_works && self.has_pm_reset { - let _ = device.reset(); - } - - // vfio_pci_post_reset(). - if let Err(e) = self.intx_enable(dev) { - report_error(&e.prepend(format_args!("vfio {}: ", self.shared.name))); - } - for nr in 0..NUM_BARS { - if let Err(e) = self.shared.config_write(PCI_BASE_ADDRESS_0 + 4 * nr, &[0; 4]) { - error_report(&format!( - "vfio_pci_post_reset({}) reset bar {nr} failed: {e}", - self.shared.name - )); - } - } + self.pre_reset(dev); + let device_reset = |works: bool| works && self.shared.device.reset().is_ok(); + let _ = device_reset(self.reset_works && (self.has_flr || !self.has_pm_reset)) + || self.hot_reset(dev, true) + || device_reset(self.reset_works && self.has_pm_reset); + self.post_reset(dev); } } @@ -1651,6 +1803,7 @@ pub fn realize( bus.unregister_device(&dev); return Err(prefix(e)); } + lock(&DEVICES).push(Arc::downgrade(&ops)); // QEMU resets every device once before the guest runs, and a hotplugged one right // after realize. The boards here are built in their reset state, so a passed through // device has to be reset now or it keeps whatever the last user left in it. @@ -1839,6 +1992,7 @@ impl<'a> Setup<'a> { reset_works: self.reset_works, has_flr: self.has_flr, has_pm_reset: self.has_pm_reset, + needs_reset: AtomicBool::new(false), }) } diff --git a/crates/sys/src/vfio.rs b/crates/sys/src/vfio.rs index 3304efb..43b635e 100644 --- a/crates/sys/src/vfio.rs +++ b/crates/sys/src/vfio.rs @@ -89,6 +89,44 @@ pub const IRQ_SET_ACTION_TRIGGER: u32 = 1 << 5; /// `VFIO_IRQ_INFO_NORESIZE`: the MSI-X vector count cannot change while enabled. pub const IRQ_INFO_NORESIZE: u32 = 1 << 3; +/// A device a PCI hot reset resets too, `struct vfio_pci_dependent_device` as a device opened +/// through its group sees it. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct DependentDevice { + /// The IOMMU group the device is in. + pub group_id: u32, + pub segment: u16, + pub bus: u8, + pub devfn: u8, +} + +impl DependentDevice { + /// The device's name in sysfs, `0000:01:00.0`, which is what `vfio_pci_host_match()` + /// compares. + pub fn host_name(&self) -> String { + format!( + "{:04x}:{:02x}:{:02x}.{:x}", + self.segment, + self.bus, + self.devfn >> 3, + self.devfn & 7 + ) + } +} + +/// The devices of a `vfio_pci_hot_reset_info` that `count` says were filled in: two words each, +/// the group and then the segment, bus and devfn, after the three words of the header. +fn dependent_devices(words: &[u32], count: u32) -> Vec { + words[3..] + .chunks_exact(2) + .take(count as usize) + .map(|d| { + let [s0, s1, bus, devfn] = d[1].to_ne_bytes(); + DependentDevice { group_id: d[0], segment: u16::from_ne_bytes([s0, s1]), bus, devfn } + }) + .collect() +} + /// `_IO(';', 100 + nr)`. Every VFIO ioctl is `_IO`, with the size in the `argsz` field instead. const fn io(nr: u64) -> u64 { ((b';' as u64) << 8) | (100 + nr) @@ -105,6 +143,8 @@ const VFIO_DEVICE_GET_REGION_INFO: u64 = io(8); const VFIO_DEVICE_GET_IRQ_INFO: u64 = io(9); const VFIO_DEVICE_SET_IRQS: u64 = io(10); const VFIO_DEVICE_RESET: u64 = io(11); +const VFIO_DEVICE_GET_PCI_HOT_RESET_INFO: u64 = io(12); +const VFIO_DEVICE_PCI_HOT_RESET: u64 = io(13); const VFIO_IOMMU_MAP_DMA: u64 = io(13); const VFIO_IOMMU_UNMAP_DMA: u64 = io(14); const VFIO_DEVICE_FEATURE: u64 = io(17); @@ -361,6 +401,13 @@ pub struct VfioGroup { file: File, } +impl AsFd for VfioGroup { + /// The group descriptor, which a hot reset takes as proof that the group is owned. + fn as_fd(&self) -> BorrowedFd<'_> { + self.file.as_fd() + } +} + impl VfioGroup { /// Opens group `id`. pub fn open(id: u32) -> io::Result { @@ -515,6 +562,28 @@ impl VfioDevice { ioctl_val(&self.file, VFIO_DEVICE_RESET, 0).map(drop) } + /// `VFIO_DEVICE_GET_PCI_HOT_RESET_INFO` with room for `room` devices: how many a hot reset + /// of this device would reset, and the devices. The kernel says how many there are even + /// when they do not fit, which it fails with `ENOSPC`. + pub fn pci_hot_reset_info(&self, room: u32) -> (u32, io::Result>) { + let mut buf = vec![0u32; 3 + 2 * room as usize]; + buf[0] = 12 + 8 * room; + let r = ioctl_ptr(&self.file, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, buf.as_mut_ptr()); + (buf[2], r.map(|_| dependent_devices(&buf, buf[2].min(room)))) + } + + /// `VFIO_DEVICE_PCI_HOT_RESET`: a bus or slot reset of this device and every device + /// [`VfioDevice::pci_hot_reset_info`] lists, with the descriptors of all of their groups as + /// proof that they are owned. + pub fn pci_hot_reset(&self, group_fds: &[BorrowedFd<'_>]) -> io::Result<()> { + let argsz = u32::try_from(12 + 4 * group_fds.len()) + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + let mut buf = Vec::with_capacity(3 + group_fds.len()); + buf.extend([argsz, 0, group_fds.len() as u32]); + buf.extend(group_fds.iter().map(|fd| fd.as_raw_fd() as u32)); + ioctl_ptr(&self.file, VFIO_DEVICE_PCI_HOT_RESET, buf.as_mut_ptr()).map(drop) + } + /// Reads at `offset` of the descriptor, which is a region's offset plus the offset inside /// it. Returns how many bytes were read. pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result { @@ -538,6 +607,8 @@ mod tests { assert_eq!(VFIO_GROUP_GET_DEVICE_FD, 0x3b6a); assert_eq!(VFIO_DEVICE_SET_IRQS, 0x3b6e); assert_eq!(VFIO_DEVICE_RESET, 0x3b6f); + assert_eq!(VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, 0x3b70); + assert_eq!(VFIO_DEVICE_PCI_HOT_RESET, 0x3b71); assert_eq!(size_of::(), 20); assert_eq!(VFIO_IOMMU_MAP_DMA, 0x3b71); assert_eq!(VFIO_IOMMU_UNMAP_DMA, 0x3b72); @@ -584,6 +655,22 @@ mod tests { assert_eq!(word(1, 2).to_ne_bytes()[..4], 1u32.to_ne_bytes()); } + #[test] + fn hot_reset_info() { + let dev = |segment: u16, bus: u8, devfn: u8| { + let [a, b] = segment.to_ne_bytes(); + u32::from_ne_bytes([a, b, bus, devfn]) + }; + let words = [28, 0, 2, 7, dev(0, 1, 0), 9, dev(1, 0x3a, 0x1b)]; + let devices = dependent_devices(&words, 2); + assert_eq!(devices[0], DependentDevice { group_id: 7, segment: 0, bus: 1, devfn: 0 }); + assert_eq!(devices[0].host_name(), "0000:01:00.0"); + assert_eq!(devices[1].host_name(), "0001:3a:03.3"); + // The kernel never fills in more than there was room for. + assert_eq!(dependent_devices(&words, 1).len(), 1); + assert!(dependent_devices(&words[..3], 0).is_empty()); + } + #[test] fn container() { // Hosts without the vfio module, or without access to it, have nothing to check. diff --git a/crates/system/src/vfio.rs b/crates/system/src/vfio.rs index 9664ced..24154c8 100644 --- a/crates/system/src/vfio.rs +++ b/crates/system/src/vfio.rs @@ -144,13 +144,14 @@ pub(crate) fn plan_device( Ok(plug) } -/// Realizes a planned `vfio-pci` on q35. +/// Realizes a planned `vfio-pci` on q35. The `first` one also hooks the system reset. #[cfg(target_os = "linux")] pub(crate) fn realize_x86( board: &mut X86Board, plug: &VfioPlug, firmware: &FirmwareSearch, clock: &std::sync::Arc, + first: bool, ) -> Result<(), Located> { use std::sync::Arc; @@ -185,6 +186,11 @@ pub(crate) fn realize_x86( &plug.conf, ) .map_err(|e| Located(plug.loc.clone(), e))?; + // vfio_address_space_get() registers vfio_device_reset_handler() along with the first + // device, so it runs after the devices themselves are reset. + if first { + m.add_reset_hook(Box::new(ruvm_hw_vfio::pci::reset_handler)); + } // pci_add_option_rom() if let Some(name) = plug.conf.romfile.as_deref().filter(|f| !f.is_empty()) { let at = |e: String| Located::new(&plug.loc, e); @@ -224,6 +230,7 @@ pub(crate) fn realize_x86( plug: &VfioPlug, _firmware: &FirmwareSearch, _clock: &std::sync::Arc, + _first: bool, ) -> Result<(), Located> { Err(Located::new(&plug.loc, format!("'{TYPE_VFIO_PCI}' is not a valid device model name"))) } diff --git a/crates/system/src/x86.rs b/crates/system/src/x86.rs index f9f9fb0..b5c302e 100644 --- a/crates/system/src/x86.rs +++ b/crates/system/src/x86.rs @@ -2344,10 +2344,12 @@ fn build( Ok::<(), Vec>(()) }; let mut vfio = p.vfio.iter().peekable(); + let mut vfio_first = true; // The vfio-pci devices that come before virtio plug `at`, or all that are left. let mut plug_vfio = |board: &mut X86Board, at: Option| { while let Some((_, plug)) = vfio.next_if(|(vi, _)| at.is_none_or(|i| *vi == i)) { - crate::vfio::realize_x86(board, plug, &firmware, &clock).map_err(one)?; + let first = std::mem::take(&mut vfio_first); + crate::vfio::realize_x86(board, plug, &firmware, &clock, first).map_err(one)?; } Ok::<(), Vec>(()) }; From 9ed3489f482f2738e07634501ce89bc7f66f67e7 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:15:12 +0700 Subject: [PATCH 2/3] vfio-pci: run the reset handler before the devices reset, as QEMU registers it before the system bus --- crates/hw-vfio/src/pci.rs | 2 +- crates/machine-x86/src/q35.rs | 13 +++++++++++++ crates/system/src/vfio.rs | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/hw-vfio/src/pci.rs b/crates/hw-vfio/src/pci.rs index 22b0b40..e4eee96 100644 --- a/crates/hw-vfio/src/pci.rs +++ b/crates/hw-vfio/src/pci.rs @@ -456,7 +456,7 @@ fn group_owned(id: u32) -> bool { groups().iter().any(|g| g.id == id) } -/// `vfio_device_reset_handler()`, which the board runs after it has reset its devices. Each +/// `vfio_device_reset_handler()`, which the board runs before it resets its devices. Each /// device that only a bus or slot reset resets properly gets one, and so do the other devices /// in use here that the reset takes along. A device that was reset together with an earlier /// one is not reset again. diff --git a/crates/machine-x86/src/q35.rs b/crates/machine-x86/src/q35.rs index 742c03f..1b397b7 100644 --- a/crates/machine-x86/src/q35.rs +++ b/crates/machine-x86/src/q35.rs @@ -846,6 +846,8 @@ pub struct Q35 { pci_dma: Arc, /// What devices outside the board reset with it. reset_hooks: Mutex>, + /// The `qemu_register_reset()` handlers of devices, which run before any device is reset. + early_reset_hooks: Mutex>, done: bool, } @@ -1470,6 +1472,7 @@ impl Q35 { acpi_src, pci_dma, reset_hooks: Mutex::new(Vec::new()), + early_reset_hooks: Mutex::new(Vec::new()), done: false, }) } @@ -1673,6 +1676,9 @@ impl Q35 { /// rebuilt on their next read and reloads the firmware and PVH segments into RAM. pub fn system_reset(&mut self) -> Result<(), String> { // qemu_devices_reset() + for hook in self.early_reset_hooks.lock().unwrap_or_else(PoisonError::into_inner).iter() { + hook(); + } if let Some(p) = &self.pic { p.master.reset(); p.slave.reset(); @@ -2181,6 +2187,13 @@ impl Q35 { self.reset_hooks.lock().unwrap_or_else(PoisonError::into_inner).push(hook); } + /// Has `hook` run on every system reset before any device is reset, as a handler a device + /// registers with `qemu_register_reset()` while it is realized does: QEMU only registers + /// the reset of the system bus once every device on the command line is realized. + pub fn add_early_reset_hook(&self, hook: ResetHook) { + self.early_reset_hooks.lock().unwrap_or_else(PoisonError::into_inner).push(hook); + } + /// The HPET, with `hpet=on`. pub fn hpet(&self) -> Option<&Arc> { self.hpet.as_ref() diff --git a/crates/system/src/vfio.rs b/crates/system/src/vfio.rs index 24154c8..fb3bd95 100644 --- a/crates/system/src/vfio.rs +++ b/crates/system/src/vfio.rs @@ -187,9 +187,9 @@ pub(crate) fn realize_x86( ) .map_err(|e| Located(plug.loc.clone(), e))?; // vfio_address_space_get() registers vfio_device_reset_handler() along with the first - // device, so it runs after the devices themselves are reset. + // device, before the system bus, so it runs before the devices themselves are reset. if first { - m.add_reset_hook(Box::new(ruvm_hw_vfio::pci::reset_handler)); + m.add_early_reset_hook(Box::new(ruvm_hw_vfio::pci::reset_handler)); } // pci_add_option_rom() if let Some(name) = plug.conf.romfile.as_deref().filter(|f| !f.is_empty()) { From c64ade12cbe71fc44e230d28c69a69af7e0489f1 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:46:34 +0700 Subject: [PATCH 3/3] vfio-pci: drop the reset at realize, the first system reset does it as in QEMU --- crates/hw-vfio/src/pci.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/hw-vfio/src/pci.rs b/crates/hw-vfio/src/pci.rs index e4eee96..d6d7351 100644 --- a/crates/hw-vfio/src/pci.rs +++ b/crates/hw-vfio/src/pci.rs @@ -1804,10 +1804,8 @@ pub fn realize( return Err(prefix(e)); } lock(&DEVICES).push(Arc::downgrade(&ops)); - // QEMU resets every device once before the guest runs, and a hotplugged one right - // after realize. The boards here are built in their reset state, so a passed through - // device has to be reset now or it keeps whatever the last user left in it. - dev.reset(); + // No reset here. Like QEMU, the device is reset by the first system reset, which + // `machine_done()` runs once every device is plugged. Ok(dev) } Err(e) => {