From d5cd39acf6f7a1e2c5c44fee01eec53434c4a323 Mon Sep 17 00:00:00 2001 From: Riatre Foo Date: Mon, 22 Jun 2026 16:32:40 +0800 Subject: [PATCH 1/8] feat(snapshot): save nested vCPU state Persist KVM nested state in x86 vCPU snapshots and restore it before MSRs are applied. Store the buffer only when KVM reports non-empty nested state. On x86_64, this, along with exposing VMX/SVM bits and EPT features, should be enough to make L2 KVM work, with the usual caveats of nested virtualization. Use a private snapshot major version for this fork so the changed layout does not collide with future upstream snapshot versions. Allow KVM_GET_NESTED_STATE in the x86 vCPU seccomp policy for snapshot creation. --- .../seccomp/x86_64-unknown-linux-musl.json | 12 +++++++++ src/vmm/src/arch/x86_64/vcpu.rs | 27 +++++++++++++++++++ src/vmm/src/persist.rs | 4 +-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/resources/seccomp/x86_64-unknown-linux-musl.json b/resources/seccomp/x86_64-unknown-linux-musl.json index 4ccbfbd8e..ffe835ad3 100644 --- a/resources/seccomp/x86_64-unknown-linux-musl.json +++ b/resources/seccomp/x86_64-unknown-linux-musl.json @@ -1140,6 +1140,18 @@ } ] }, + { + "syscall": "ioctl", + "args": [ + { + "index": 1, + "type": "dword", + "op": "eq", + "val": 3229658814, + "comment": "KVM_GET_NESTED_STATE" + } + ] + }, { "syscall": "ioctl", "args": [ diff --git a/src/vmm/src/arch/x86_64/vcpu.rs b/src/vmm/src/arch/x86_64/vcpu.rs index 64bff7fb0..087408b59 100644 --- a/src/vmm/src/arch/x86_64/vcpu.rs +++ b/src/vmm/src/arch/x86_64/vcpu.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use kvm_bindings::{ CpuId, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES, Msrs, Xsave, kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_regs, kvm_sregs, kvm_vcpu_events, kvm_xcrs, kvm_xsave, kvm_xsave2, + nested::KvmNestedStateBuffer, }; use kvm_ioctls::{VcpuExit, VcpuFd}; use log::{error, warn}; @@ -68,6 +69,8 @@ pub enum KvmVcpuError { VcpuGetMsr(u32), /// Failed to get KVM vcpu msrs: {0} VcpuGetMsrs(kvm_ioctls::Error), + /// Failed to get KVM vcpu nested state: {0} + VcpuGetNestedState(kvm_ioctls::Error), /// Failed to get KVM vcpu regs: {0} VcpuGetRegs(kvm_ioctls::Error), /// Failed to get KVM vcpu sregs: {0} @@ -96,6 +99,8 @@ pub enum KvmVcpuError { VcpuSetMsrs(kvm_ioctls::Error), /// Failed to set all KVM MSRs for this vCPU. Only a partial write was done. VcpuSetMsrsIncomplete, + /// Failed to set KVM vcpu nested state: {0} + VcpuSetNestedState(kvm_ioctls::Error), /// Failed to set KVM vcpu regs: {0} VcpuSetRegs(kvm_ioctls::Error), /// Failed to set KVM vcpu sregs: {0} @@ -593,6 +598,12 @@ impl KvmVcpu { None }); let cpuid = self.get_cpuid()?; + let mut nested_state = KvmNestedStateBuffer::empty(); + let nested_state = self + .fd + .nested_state(&mut nested_state) + .map_err(KvmVcpuError::VcpuGetNestedState)? + .map(|_| nested_state); let saved_msrs = self.get_msr_chunks(self.msrs_to_save.iter().copied())?; let vcpu_events = self .fd @@ -611,6 +622,7 @@ impl KvmVcpu { xcrs, xsave, tsc_khz, + nested_state, }) } @@ -703,6 +715,11 @@ impl KvmVcpu { self.fd .set_lapic(&state.lapic) .map_err(KvmVcpuError::VcpuSetLapic)?; + if let Some(nested_state) = state.nested_state.as_ref() { + self.fd + .set_nested_state(nested_state) + .map_err(KvmVcpuError::VcpuSetNestedState)?; + } for msrs in &state.saved_msrs { let nmsrs = self.fd.set_msrs(msrs).map_err(KvmVcpuError::VcpuSetMsrs)?; if nmsrs < msrs.as_fam_struct_ref().nmsrs as usize { @@ -781,6 +798,8 @@ pub struct VcpuState { pub xsave: Xsave, /// Tsc khz. pub tsc_khz: Option, + /// Nested guest state. + pub nested_state: Option, } impl Debug for VcpuState { @@ -802,6 +821,13 @@ impl Debug for VcpuState { .field("xcrs", &self.xcrs) .field("xsave", &self.xsave) .field("tsc_khz", &self.tsc_khz) + .field( + "nested_state", + &self + .nested_state + .as_ref() + .map(|state| (state.format, state.flags, state.size)), + ) .finish() } } @@ -840,6 +866,7 @@ mod tests { xcrs: Default::default(), xsave: Xsave::new(0).unwrap(), tsc_khz: Some(0), + nested_state: None, } } } diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs index 8dc8fd85f..d404812e6 100644 --- a/src/vmm/src/persist.rs +++ b/src/vmm/src/persist.rs @@ -160,8 +160,8 @@ pub enum CreateSnapshotError { SnapshotBackingFile(&'static str, io::Error), } -/// Snapshot version -pub const SNAPSHOT_VERSION: Version = Version::new(9, 0, 0); +/// Snapshot version. +pub const SNAPSHOT_VERSION: Version = Version::new(1000, 0, 0); /// Creates a Microvm snapshot. pub fn create_snapshot( From af3ee416cfe267aee43a4124e496eb24403f0c77 Mon Sep 17 00:00:00 2001 From: Riatre Foo Date: Thu, 16 Jul 2026 21:27:27 +0800 Subject: [PATCH 2/8] fix(snapshot): save/restore MSR_VM_HSAVE_PA for nested SVM On AMD, KVM_SET_NESTED_STATE restores the VMCB but not L1's host save area address (MSR_VM_HSAVE_PA). Firecracker's MSR whitelist dropped it, so a restored L1 vCPU had svm->nested.hsave_msr == 0 and KVM injected #GP into L1 on its next VMRUN, hanging both L1 and L2. Intel is unaffected (VMCS GPAs live in the nested-state blob itself) and ignores the MSR write. Add MSR_VM_HSAVE_PA to the serializable MSR ranges so it round-trips through snapshots. --- src/vmm/src/arch/x86_64/msr.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vmm/src/arch/x86_64/msr.rs b/src/vmm/src/arch/x86_64/msr.rs index 39bf0d748..87fc798c5 100644 --- a/src/vmm/src/arch/x86_64/msr.rs +++ b/src/vmm/src/arch/x86_64/msr.rs @@ -234,6 +234,7 @@ static SERIALIZABLE_MSR_RANGES: &[MsrRange] = &[ MSR_RANGE!(MSR_TSC_AUX), MSR_RANGE!(MSR_MISC_FEATURES_ENABLES), MSR_RANGE!(MSR_K7_HWCR), + MSR_RANGE!(MSR_VM_HSAVE_PA), MSR_RANGE!(MSR_KVM_POLL_CONTROL), MSR_RANGE!(MSR_KVM_ASYNC_PF_INT), MSR_RANGE!(MSR_IA32_TSX_CTRL), From c7e1cceb6bf643f239ddac804a535ceb8ac93cb1 Mon Sep 17 00:00:00 2001 From: Riatre Foo Date: Wed, 16 Sep 2026 00:34:44 +0800 Subject: [PATCH 3/8] fix(snapshot): keep VMXON-only nested vCPU state kvm-ioctls' VcpuFd::nested_state() reports None whenever KVM_GET_NESTED_STATE returns only the fixed-size header, but that header alone carries the "in VMX operation, no current VMCS" state in hdr.vmx.vmxon_pa. Saving None for it skips KVM_SET_NESTED_STATE on restore, leaving the new vCPU out of VMX operation on the host while the guest kernel still believes VMX is on (that state lives in restored guest memory), so it never re-executes VMXON. The guest's first VMPTRLD then faults with #UD and hits the BUG() in kvm_spurious_fault(). Decide whether the buffer holds state by inspecting the header GPAs, mirroring QEMU's nested_state_needed(). --- src/vmm/src/arch/x86_64/vcpu.rs | 36 +++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/vmm/src/arch/x86_64/vcpu.rs b/src/vmm/src/arch/x86_64/vcpu.rs index 087408b59..9c83d4f8a 100644 --- a/src/vmm/src/arch/x86_64/vcpu.rs +++ b/src/vmm/src/arch/x86_64/vcpu.rs @@ -10,9 +10,9 @@ use std::fmt::Debug; use std::sync::Arc; use kvm_bindings::{ - CpuId, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES, Msrs, Xsave, kvm_debugregs, kvm_lapic_state, - kvm_mp_state, kvm_regs, kvm_sregs, kvm_vcpu_events, kvm_xcrs, kvm_xsave, kvm_xsave2, - nested::KvmNestedStateBuffer, + CpuId, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES, KVM_STATE_NESTED_FORMAT_VMX, Msrs, Xsave, + kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_nested_state, kvm_regs, kvm_sregs, + kvm_vcpu_events, kvm_xcrs, kvm_xsave, kvm_xsave2, nested::KvmNestedStateBuffer, }; use kvm_ioctls::{VcpuExit, VcpuFd}; use log::{error, warn}; @@ -171,6 +171,30 @@ pub struct Peripherals { pub mmio_bus: Option>, } +/// Returns whether a buffer filled by `KVM_GET_NESTED_STATE` holds state that +/// must be restored with `KVM_SET_NESTED_STATE`. +/// +/// kvm-ioctls' `VcpuFd::nested_state()` reports `None` whenever KVM returns +/// only the fixed-size header, but the header alone carries meaningful state: +/// a guest in VMX operation without a currently loaded VMCS reports just +/// `hdr.vmx.vmxon_pa` there, without inflating `size`. Dropping that state +/// leaves the restored vCPU out of VMX operation on the host while the guest +/// kernel still believes VMX is on (that state lives in restored guest +/// memory), so the guest never re-executes VMXON and its first VMPTRLD faults. +/// Mirror QEMU's `nested_state_needed()` and inspect the header GPAs instead. +fn has_nested_state(state: &KvmNestedStateBuffer) -> bool { + // Any variable-size data following the header is state (a current + // VMCS/VMCB always inflates `size`). + if state.size as usize > std::mem::size_of::() { + return true; + } + // The VMXON-only case described above is header-only. SVM has no VMXON + // equivalent, so there the header alone carries no state. + u32::from(state.format) == KVM_STATE_NESTED_FORMAT_VMX + // SAFETY: the format check selects the valid union member. + && unsafe { state.hdr.vmx.vmxon_pa != u64::MAX } +} + impl KvmVcpu { /// Constructs a new kvm vcpu with arch specific functionality. /// @@ -599,11 +623,11 @@ impl KvmVcpu { }); let cpuid = self.get_cpuid()?; let mut nested_state = KvmNestedStateBuffer::empty(); - let nested_state = self + let _ = self .fd .nested_state(&mut nested_state) - .map_err(KvmVcpuError::VcpuGetNestedState)? - .map(|_| nested_state); + .map_err(KvmVcpuError::VcpuGetNestedState)?; + let nested_state = has_nested_state(&nested_state).then_some(nested_state); let saved_msrs = self.get_msr_chunks(self.msrs_to_save.iter().copied())?; let vcpu_events = self .fd From 7009054d00016bb67adb4242dd4d2c55707ecf5d Mon Sep 17 00:00:00 2001 From: Yicheng Lin Date: Thu, 22 Oct 2026 00:03:21 +0800 Subject: [PATCH 4/8] fix(snapshot): make the v1000 runtime compatible with legacy v9 snapshots Decode the historical x86_64 v9 layout into current VM state and accept the unchanged ARM v9 layout while retaining current-format writes. Share validated loading with recovery and inspection tools, and query KVM nested-state support before capture. Cover conversion and rejection with regression tests and an authentic v9 fixture. Signed-off-by: Yicheng Lin --- .gitignore | 3 + src/snapshot-editor/src/info.rs | 13 +- src/snapshot-editor/src/utils.rs | 6 +- src/vmm/src/arch/x86_64/vcpu.rs | 38 ++- src/vmm/src/persist.rs | 2 +- src/vmm/src/snapshot/legacy.rs | 69 +++++ src/vmm/src/snapshot/mod.rs | 350 +++++++++++++++++++++--- src/vmm/testdata/README.md | 32 +++ src/vmm/testdata/snapshot-v9-x86_64.bin | Bin 0 -> 25465 bytes 9 files changed, 459 insertions(+), 54 deletions(-) create mode 100644 src/vmm/src/snapshot/legacy.rs create mode 100644 src/vmm/testdata/README.md create mode 100644 src/vmm/testdata/snapshot-v9-x86_64.bin diff --git a/.gitignore b/.gitignore index 65051a0c2..6dc6c1f3d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ tags /resources/linux /resources/x86_64 /resources/aarch64 + +# Keep the immutable legacy snapshot regression fixture. +!/src/vmm/testdata/snapshot-v9-x86_64.bin diff --git a/src/snapshot-editor/src/info.rs b/src/snapshot-editor/src/info.rs index 97d06a3e5..1f50e3420 100644 --- a/src/snapshot-editor/src/info.rs +++ b/src/snapshot-editor/src/info.rs @@ -4,8 +4,7 @@ use std::path::PathBuf; use clap::Subcommand; -use vmm::persist::MicrovmState; -use vmm::snapshot::Snapshot; +use vmm::snapshot::LoadedMicrovmSnapshot; use crate::utils::*; @@ -50,19 +49,19 @@ pub fn info_vmstate_command(command: InfoVmStateSubCommand) -> Result<(), InfoVm fn info( vmstate_path: &PathBuf, - f: impl Fn(&Snapshot) -> Result<(), InfoVmStateError>, + f: impl Fn(&LoadedMicrovmSnapshot) -> Result<(), InfoVmStateError>, ) -> Result<(), InfoVmStateError> { let snapshot = open_vmstate(vmstate_path)?; f(&snapshot)?; Ok(()) } -fn info_version(snapshot: &Snapshot) -> Result<(), InfoVmStateError> { - println!("v{}", snapshot.version()); +fn info_version(snapshot: &LoadedMicrovmSnapshot) -> Result<(), InfoVmStateError> { + println!("v{}", snapshot.format_version); Ok(()) } -fn info_vcpu_states(snapshot: &Snapshot) -> Result<(), InfoVmStateError> { +fn info_vcpu_states(snapshot: &LoadedMicrovmSnapshot) -> Result<(), InfoVmStateError> { for (i, state) in snapshot.data.vcpu_states.iter().enumerate() { println!("vcpu {i}:"); println!("{state:#?}"); @@ -70,7 +69,7 @@ fn info_vcpu_states(snapshot: &Snapshot) -> Result<(), InfoVmState Ok(()) } -fn info_vmstate(snapshot: &Snapshot) -> Result<(), InfoVmStateError> { +fn info_vmstate(snapshot: &LoadedMicrovmSnapshot) -> Result<(), InfoVmStateError> { println!("{:#?}", snapshot.data); Ok(()) } diff --git a/src/snapshot-editor/src/utils.rs b/src/snapshot-editor/src/utils.rs index 1d3d053d2..adcbc4016 100644 --- a/src/snapshot-editor/src/utils.rs +++ b/src/snapshot-editor/src/utils.rs @@ -5,7 +5,7 @@ use std::fs::{File, OpenOptions}; use std::path::PathBuf; use vmm::persist::MicrovmState; -use vmm::snapshot::Snapshot; +use vmm::snapshot::{LoadedMicrovmSnapshot, Snapshot, load_microvm_snapshot}; // Some errors are only used in aarch64 code #[allow(unused)] @@ -22,9 +22,9 @@ pub enum UtilsError { } #[allow(unused)] -pub fn open_vmstate(snapshot_path: &PathBuf) -> Result, UtilsError> { +pub fn open_vmstate(snapshot_path: &PathBuf) -> Result { let mut snapshot_reader = File::open(snapshot_path).map_err(UtilsError::VmStateFileOpen)?; - Snapshot::load(&mut snapshot_reader).map_err(UtilsError::VmStateLoad) + load_microvm_snapshot(&mut snapshot_reader).map_err(UtilsError::VmStateLoad) } // This method is used only in aarch64 code so far diff --git a/src/vmm/src/arch/x86_64/vcpu.rs b/src/vmm/src/arch/x86_64/vcpu.rs index 9c83d4f8a..05200e00b 100644 --- a/src/vmm/src/arch/x86_64/vcpu.rs +++ b/src/vmm/src/arch/x86_64/vcpu.rs @@ -69,6 +69,8 @@ pub enum KvmVcpuError { VcpuGetMsr(u32), /// Failed to get KVM vcpu msrs: {0} VcpuGetMsrs(kvm_ioctls::Error), + /// Failed to check KVM nested-state capability: {0} + VcpuCheckNestedState(kvm_ioctls::Error), /// Failed to get KVM vcpu nested state: {0} VcpuGetNestedState(kvm_ioctls::Error), /// Failed to get KVM vcpu regs: {0} @@ -160,6 +162,8 @@ pub struct KvmVcpu { /// /// `None` if `KVM_CAP_XSAVE2` not supported. xsave2_size: Option, + /// Whether KVM supports saving nested state on this VM. + nested_state_supported: bool, } /// Vcpu peripherals @@ -208,12 +212,24 @@ impl KvmVcpu { .create_vcpu(index.into()) .map_err(KvmVcpuError::VcpuFd)?; + let nested_state_supported = match vm.fd().check_extension_int(kvm_ioctls::Cap::NestedState) + { + ..=-1 => { + return Err(KvmVcpuError::VcpuCheckNestedState( + vmm_sys_util::errno::Error::last(), + )); + } + 0 => false, + _ => true, + }; + Ok(KvmVcpu { index, fd: kvm_vcpu, peripherals: Default::default(), msrs_to_save: vm.msrs_to_save().to_vec(), xsave2_size: vm.xsave2_size(), + nested_state_supported, }) } @@ -622,12 +638,15 @@ impl KvmVcpu { None }); let cpuid = self.get_cpuid()?; - let mut nested_state = KvmNestedStateBuffer::empty(); - let _ = self - .fd - .nested_state(&mut nested_state) - .map_err(KvmVcpuError::VcpuGetNestedState)?; - let nested_state = has_nested_state(&nested_state).then_some(nested_state); + let nested_state = if self.nested_state_supported { + let mut state = KvmNestedStateBuffer::empty(); + self.fd + .nested_state(&mut state) + .map_err(KvmVcpuError::VcpuGetNestedState)?; + has_nested_state(&state).then_some(state) + } else { + None + }; let saved_msrs = self.get_msr_chunks(self.msrs_to_save.iter().copied())?; let vcpu_events = self .fd @@ -1201,6 +1220,13 @@ mod tests { } } + #[test] + fn test_save_state_without_nested_capability() { + let (_kvm, _vm, mut vcpu) = setup_vcpu(0x1000); + vcpu.nested_state_supported = false; + assert!(vcpu.save_state().unwrap().nested_state.is_none()); + } + #[test] fn test_set_tsc() { let (kvm, _, vcpu) = setup_vcpu(0x1000); diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs index d404812e6..d94f1a2d3 100644 --- a/src/vmm/src/persist.rs +++ b/src/vmm/src/persist.rs @@ -463,7 +463,7 @@ fn snapshot_state_from_file( snapshot_path: &Path, ) -> Result { let mut snapshot_reader = File::open(snapshot_path)?; - let snapshot = Snapshot::load(&mut snapshot_reader)?; + let snapshot = crate::snapshot::load_microvm_snapshot(&mut snapshot_reader)?; Ok(snapshot.data) } diff --git a/src/vmm/src/snapshot/legacy.rs b/src/vmm/src/snapshot/legacy.rs new file mode 100644 index 000000000..6644e7558 --- /dev/null +++ b/src/vmm/src/snapshot/legacy.rs @@ -0,0 +1,69 @@ +// Copyright 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Frozen x86_64 v9.0.0 wire layout from aenv-deps@90288c397d6d3fa576f9d367d997bdeebf835f10. +//! Do not add current-layout fields here. Ordinary v9 snapshots contain no nested guest state. + +use crate::arch::x86_64::vcpu::VcpuState; +use crate::device_manager::DevicesState; +use crate::persist::{MicrovmState, VmInfo}; +use crate::vstate::kvm::KvmState; +use crate::vstate::vm::VmState; +use kvm_bindings::{ + CpuId, Msrs, Xsave, kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_regs, kvm_sregs, + kvm_vcpu_events, kvm_xcrs, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub(super) struct VcpuStateV9 { + pub cpuid: CpuId, + pub saved_msrs: Vec, + pub debug_regs: kvm_debugregs, + pub lapic: kvm_lapic_state, + pub mp_state: kvm_mp_state, + pub regs: kvm_regs, + pub sregs: kvm_sregs, + pub vcpu_events: kvm_vcpu_events, + pub xcrs: kvm_xcrs, + pub xsave: Xsave, + pub tsc_khz: Option, +} + +#[derive(Serialize, Deserialize)] +pub(super) struct MicrovmStateV9 { + pub vm_info: VmInfo, + pub kvm_state: KvmState, + pub vm_state: VmState, + pub vcpu_states: Vec, + pub device_states: DevicesState, +} + +impl From for MicrovmState { + fn from(old: MicrovmStateV9) -> Self { + Self { + vm_info: old.vm_info, + kvm_state: old.kvm_state, + vm_state: old.vm_state, + vcpu_states: old + .vcpu_states + .into_iter() + .map(|old| VcpuState { + cpuid: old.cpuid, + saved_msrs: old.saved_msrs, + debug_regs: old.debug_regs, + lapic: old.lapic, + mp_state: old.mp_state, + regs: old.regs, + sregs: old.sregs, + vcpu_events: old.vcpu_events, + xcrs: old.xcrs, + xsave: old.xsave, + tsc_khz: old.tsc_khz, + nested_state: None, + }) + .collect(), + device_states: old.device_states, + } + } +} diff --git a/src/vmm/src/snapshot/mod.rs b/src/vmm/src/snapshot/mod.rs index 556646e2c..d99bdc1ec 100644 --- a/src/vmm/src/snapshot/mod.rs +++ b/src/vmm/src/snapshot/mod.rs @@ -24,6 +24,8 @@ //! The snapshot format uses a version value in the form of `MAJOR.MINOR.PATCH`. The version is //! provided by the library clients (it is not tied to this crate). pub mod crc; +#[cfg(target_arch = "x86_64")] +mod legacy; mod persist; use std::fmt::Debug; use std::io::{Read, Write}; @@ -82,46 +84,85 @@ struct SnapshotHdr { /// Assumes the raw bytes stream read from the given [`Read`] instance is a snapshot file, /// and returns the version of it. pub fn get_format_version(reader: &mut R) -> Result { - // Check size limit before reading the full file to prevent DOS attacks - let mut buf = Vec::new(); - let bytes_read = reader - .take((SNAPSHOT_DESERIALIZATION_BYTES_LIMIT + 1) as u64) - .read_to_end(&mut buf)?; - - if bytes_read > SNAPSHOT_DESERIALIZATION_BYTES_LIMIT { - return Err(SnapshotError::SizeLimitExceeded( - SNAPSHOT_DESERIALIZATION_BYTES_LIMIT, - )); - } - - // The last 8 bytes are the CRC, so we need to separate them for deserialization - if buf.len() < 8 { - return Err(SnapshotError::Io(std::io::Error::new( - std::io::ErrorKind::UnexpectedEof, - "File too short to contain CRC", - ))); - } - - let (data_buf, _crc_buf) = buf.split_at(buf.len() - 8); + Ok(load_microvm_snapshot(reader)?.format_version) +} - // Since bitcode requires exact type matching, we need to try deserializing - // as the specific snapshot type we know about. In practice, all snapshots - // in Firecracker use MicrovmState as the data type. - use crate::persist::MicrovmState; +/// Decoded VM state and its original file format. This type cannot be serialized: +/// use `Snapshot::new(data)` to write the current layout with the current version. +#[derive(Debug)] +pub struct LoadedMicrovmSnapshot { + /// Original file format, before in-memory conversion. + pub format_version: Version, + /// State converted to the current runtime layout. + pub data: crate::persist::MicrovmState, +} - match bitcode::deserialize::>(data_buf) { - Ok(snapshot) => Ok(snapshot.header.version), - Err(e) => { - // If deserialization fails, it could be due to: - // 1. The snapshot was created with bincode (older versions) - // 2. The MicrovmState structure has changed and is incompatible - // 3. The snapshot file is corrupted - // Since supporting bincode is out of scope, we return a descriptive error. - Err(SnapshotError::Bitcode(e)) +/// Load the explicitly supported VM wire layouts with size, CRC, architecture and version checks. +pub fn load_microvm_snapshot( + reader: &mut R, +) -> Result { + #[cfg(target_arch = "aarch64")] + { + let snapshot = Snapshot::::load(reader)?; + Ok(LoadedMicrovmSnapshot { + format_version: snapshot.header.version, + data: snapshot.data, + }) + } + #[cfg(target_arch = "x86_64")] + { + let mut buf = Vec::new(); + reader + .take((SNAPSHOT_DESERIALIZATION_BYTES_LIMIT + 1) as u64) + .read_to_end(&mut buf)?; + if buf.len() > SNAPSHOT_DESERIALIZATION_BYTES_LIMIT { + return Err(SnapshotError::SizeLimitExceeded( + SNAPSHOT_DESERIALIZATION_BYTES_LIMIT, + )); + } + if buf.len() < 8 { + return Err(SnapshotError::Io(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "File too short to contain CRC", + ))); + } + if crc64(0, &buf) != 0 { + return Err(SnapshotError::Crc64); + } + let data_buf = &buf[..buf.len() - 8]; + match bitcode::deserialize::>(data_buf) { + Ok(snapshot) if snapshot.header.version != Version::new(9, 0, 0) => { + validate_microvm_header(&snapshot.header, &SNAPSHOT_VERSION)?; + Ok(LoadedMicrovmSnapshot { + format_version: snapshot.header.version, + data: snapshot.data, + }) + } + _ => { + // Empty vCPU vectors can match both schemas. Version 9 always + // selects the frozen wire type, even if current decoding succeeds. + let old: Snapshot = bitcode::deserialize(data_buf)?; + validate_microvm_header(&old.header, &Version::new(9, 0, 0))?; + Ok(LoadedMicrovmSnapshot { + format_version: old.header.version, + data: old.data.into(), + }) + } } } } +#[cfg(target_arch = "x86_64")] +fn validate_microvm_header(header: &SnapshotHdr, expected: &Version) -> Result<(), SnapshotError> { + if header.magic != SNAPSHOT_MAGIC_ID { + return Err(SnapshotError::InvalidMagic(header.magic)); + } + if &header.version != expected { + return Err(SnapshotError::InvalidFormatVersion(header.version.clone())); + } + Ok(()) +} + /// Firecracker snapshot type /// /// A type used to store and load Firecracker snapshots of a particular version @@ -166,9 +207,13 @@ impl Snapshot { return Err(SnapshotError::InvalidMagic(snapshot.header.magic)); } - if snapshot.header.version.major != SNAPSHOT_VERSION.major - || snapshot.header.version.minor > SNAPSHOT_VERSION.minor - { + let supported_version = snapshot.header.version.major == SNAPSHOT_VERSION.major + && snapshot.header.version.minor <= SNAPSHOT_VERSION.minor; + // ARM's wire layout is unchanged from v9; only x86 added nested vCPU state. + #[cfg(target_arch = "aarch64")] + let supported_version = + supported_version || snapshot.header.version == Version::new(9, 0, 0); + if !supported_version { return Err(SnapshotError::InvalidFormatVersion( snapshot.header.version.clone(), )); @@ -233,6 +278,236 @@ mod tests { use super::*; use crate::persist::MicrovmState; + #[cfg(target_arch = "x86_64")] + fn old_snapshot() -> Snapshot { + let old = crate::arch::x86_64::vcpu::VcpuState::default(); + Snapshot { + header: SnapshotHdr { + magic: SNAPSHOT_MAGIC_ID, + version: Version::new(9, 0, 0), + }, + data: legacy::MicrovmStateV9 { + vm_info: Default::default(), + kvm_state: Default::default(), + vm_state: Default::default(), + device_states: Default::default(), + vcpu_states: vec![legacy::VcpuStateV9 { + cpuid: old.cpuid, + saved_msrs: old.saved_msrs, + debug_regs: old.debug_regs, + lapic: old.lapic, + mp_state: old.mp_state, + regs: old.regs, + sregs: old.sregs, + vcpu_events: old.vcpu_events, + xcrs: old.xcrs, + xsave: old.xsave, + tsc_khz: old.tsc_khz, + }], + }, + } + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn test_microvm_v9_conversion_and_current_output() { + let mut bytes = Vec::new(); + old_snapshot().save(&mut bytes).unwrap(); + let loaded = load_microvm_snapshot(&mut bytes.as_slice()).unwrap(); + assert_eq!(loaded.format_version, Version::new(9, 0, 0)); + assert_eq!( + get_format_version(&mut bytes.as_slice()).unwrap(), + loaded.format_version + ); + assert_eq!(loaded.data.vcpu_states.len(), 1); + assert!(loaded.data.vcpu_states[0].nested_state.is_none()); + assert_eq!(loaded.data.vcpu_states[0].tsc_khz, Some(0)); + let mut upgraded = Vec::new(); + Snapshot::new(loaded.data).save(&mut upgraded).unwrap(); + let loaded = load_microvm_snapshot(&mut upgraded.as_slice()).unwrap(); + assert_eq!(loaded.format_version, SNAPSHOT_VERSION); + assert!(loaded.data.vcpu_states[0].nested_state.is_none()); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn test_microvm_v9_rejects_invalid_files() { + let mut old = old_snapshot(); + for version in [ + Version::new(9, 1, 0), + Version::new(9, 0, 1), + Version::new(10, 0, 0), + SNAPSHOT_VERSION, + ] { + old.header.version = version; + let mut bytes = Vec::new(); + old.save(&mut bytes).unwrap(); + assert!(matches!( + load_microvm_snapshot(&mut bytes.as_slice()), + Err(SnapshotError::InvalidFormatVersion(_)) + )); + } + old.header.version = Version::new(9, 0, 0); + old.header.magic = 0x0710_1984_AAAA_0000; + let mut bytes = Vec::new(); + old.save(&mut bytes).unwrap(); + assert!(matches!( + load_microvm_snapshot(&mut bytes.as_slice()), + Err(SnapshotError::InvalidMagic(_)) + )); + old.header.magic = SNAPSHOT_MAGIC_ID; + bytes.clear(); + old.save(&mut bytes).unwrap(); + let mut corrupt = bytes.clone(); + *corrupt.last_mut().unwrap() ^= 1; + assert!(matches!( + load_microvm_snapshot(&mut corrupt.as_slice()), + Err(SnapshotError::Crc64) + )); + for length in [0, 7, bytes.len() / 2, bytes.len() - 1] { + load_microvm_snapshot(&mut &bytes[..length]).unwrap_err(); + } + let oversized = vec![0; SNAPSHOT_DESERIALIZATION_BYTES_LIMIT + 1]; + assert!(matches!( + load_microvm_snapshot(&mut oversized.as_slice()), + Err(SnapshotError::SizeLimitExceeded(_)) + )); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn test_microvm_v9_checked_in_fixture() { + // Produced by the pinned old executable, never by the current serializer. + let bytes = include_bytes!("../../testdata/snapshot-v9-x86_64.bin"); + assert_eq!(bytes.len(), 25_465); + let digest = aws_lc_rs::digest::digest(&aws_lc_rs::digest::SHA256, bytes); + let digest = digest + .as_ref() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + assert_eq!( + digest, + "27c2e0a2b6ca8d979050def88f773082d69a653b29487c76896b445ec08c1bad" + ); + let loaded = load_microvm_snapshot(&mut bytes.as_slice()).unwrap(); + assert_eq!(loaded.format_version, Version::new(9, 0, 0)); + assert_eq!( + get_format_version(&mut bytes.as_slice()).unwrap(), + loaded.format_version + ); + assert_eq!(loaded.data.vm_info.mem_size_mib, 256); + assert_eq!(loaded.data.vcpu_states.len(), 2); + for state in &loaded.data.vcpu_states { + assert!(!state.cpuid.as_slice().is_empty()); + assert!(state.tsc_khz.is_some()); + assert!(state.nested_state.is_none()); + } + let expected = serde_json::to_value(&loaded.data).unwrap(); + let mut upgraded = Vec::new(); + Snapshot::new(loaded.data).save(&mut upgraded).unwrap(); + let restored = load_microvm_snapshot(&mut upgraded.as_slice()).unwrap(); + assert_eq!(restored.format_version, Version::new(1000, 0, 0)); + assert_eq!(serde_json::to_value(&restored.data).unwrap(), expected); + } + + #[cfg(target_arch = "x86_64")] + #[test] + #[ignore = "exports negative fixture files for external hardware tests"] + fn test_export_microvm_v9_negative_fixtures() { + let path = std::env::var("FC_V9_FIXTURE") + .expect("set FC_V9_FIXTURE to an authentic v9 state file"); + let bytes = std::fs::read(path).unwrap(); + let loaded = load_microvm_snapshot(&mut bytes.as_slice()).unwrap(); + assert_eq!(loaded.format_version, Version::new(9, 0, 0)); + assert!(!loaded.data.vcpu_states.is_empty()); + assert!( + loaded + .data + .vcpu_states + .iter() + .all(|state| state.nested_state.is_none()) + ); + // Emit valid-CRC negative fixtures using the historical state's exact layout. + // The actual VMM and both inspection tools must reject these files, too. + let directory = std::env::var("FC_V9_NEGATIVE_DIR").expect("set FC_V9_NEGATIVE_DIR"); + let directory = std::path::Path::new(&directory); + std::fs::create_dir_all(directory).unwrap(); + let mut old: Snapshot = + bitcode::deserialize(&bytes[..bytes.len() - 8]).unwrap(); + old.header.version = Version::new(9, 1, 0); + old.save(&mut std::fs::File::create(directory.join("unknown-version.bin")).unwrap()) + .unwrap(); + old.header.version = Version::new(9, 0, 0); + old.header.magic = 0x0710_1984_AAAA_0000; + old.save(&mut std::fs::File::create(directory.join("wrong-architecture.bin")).unwrap()) + .unwrap(); + let mut corrupt = bytes.clone(); + *corrupt.last_mut().unwrap() ^= 1; + std::fs::write(directory.join("bad-crc.bin"), corrupt).unwrap(); + std::fs::write(directory.join("truncated.bin"), &bytes[..bytes.len() / 2]).unwrap(); + for entry in std::fs::read_dir(directory).unwrap() { + let negative = std::fs::read(entry.unwrap().path()).unwrap(); + load_microvm_snapshot(&mut negative.as_slice()).unwrap_err(); + } + } + + #[cfg(target_arch = "aarch64")] + #[test] + fn test_arm_v9_restore_and_current_output() { + let mut old = Snapshot::new(MicrovmState::default()); + old.header.version = Version::new(9, 0, 0); + let mut bytes = Vec::new(); + old.save(&mut bytes).unwrap(); + + let loaded = load_microvm_snapshot(&mut bytes.as_slice()).unwrap(); + assert_eq!(loaded.format_version, Version::new(9, 0, 0)); + assert_eq!( + get_format_version(&mut bytes.as_slice()).unwrap(), + loaded.format_version + ); + let expected = serde_json::to_value(&loaded.data).unwrap(); + let mut upgraded = Vec::new(); + Snapshot::new(loaded.data).save(&mut upgraded).unwrap(); + let restored = load_microvm_snapshot(&mut upgraded.as_slice()).unwrap(); + assert_eq!(restored.format_version, Version::new(1000, 0, 0)); + assert_eq!(serde_json::to_value(&restored.data).unwrap(), expected); + + // Accepting v9 must not bypass the existing integrity or architecture checks. + *bytes.last_mut().unwrap() ^= 1; + assert!(matches!( + load_microvm_snapshot(&mut bytes.as_slice()), + Err(SnapshotError::Crc64) + )); + old.header.magic = 0x0710_1984_8664_0000; + bytes.clear(); + old.save(&mut bytes).unwrap(); + assert!(matches!( + load_microvm_snapshot(&mut bytes.as_slice()), + Err(SnapshotError::InvalidMagic(_)) + )); + } + + #[cfg(target_arch = "aarch64")] + #[test] + fn test_arm_rejects_unknown_legacy_versions() { + for version in [ + Version::new(8, 0, 0), + Version::new(9, 0, 1), + Version::new(9, 1, 0), + Version::new(10, 0, 0), + ] { + let mut snapshot = Snapshot::new(MicrovmState::default()); + snapshot.header.version = version; + let mut bytes = Vec::new(); + snapshot.save(&mut bytes).unwrap(); + assert!(matches!( + load_microvm_snapshot(&mut bytes.as_slice()), + Err(SnapshotError::InvalidFormatVersion(_)) + )); + } + } + #[test] fn test_snapshot_restore() { let state = MicrovmState::default(); @@ -364,3 +639,4 @@ mod tests { Snapshot::<()>::load_without_crc_check(&data).unwrap(); } } + diff --git a/src/vmm/testdata/README.md b/src/vmm/testdata/README.md new file mode 100644 index 000000000..de145dff8 --- /dev/null +++ b/src/vmm/testdata/README.md @@ -0,0 +1,32 @@ +# Historical snapshot fixture + +snapshot-v9-x86_64.bin is an unmodified, 25,465-byte ordinary x86_64 state file +created by the official old Firecracker executable, not a current serializer. +Only VMM state is included; guest memory, disks and private AgentENV state are +not included. The isolated BusyBox guest had no credentials; MMDS is absent. + +- Generator source: aenv-deps@90288c397d6d3fa576f9d367d997bdeebf835f10. +- Official archive: https://github.com/kvcache-ai/firecracker/releases/download/aenv-deps/firecracker-1.15.1-patch-v1-x86_64.tgz +- Archive SHA256: ac69af73ba5300332a9914397c8e81bdc6764711b5c190e6b8d748578bbae96b. +- Executable SHA256: 280fda256adc3093d9e14d8291b77eb3b8bab33f734a7fb96f0ab048a8aa6f51. +- Guest kernel: 6.1.175, SHA256 cd7c5a06a2026a2d8a0cddf60b41a5d76a6ac025e54439eb3ee5c9e57a60c67b. +- Fixture SHA256: 27c2e0a2b6ca8d979050def88f773082d69a653b29487c76896b445ec08c1bad. +- Intel same-host control: two vCPUs, 256 MiB RAM, one ext4 root disk and one TAP. + Boot the old guest, save a full snapshot, restore it with that same old + executable and verify its memory/disk witness and gateway ping before using + the state file. The separate implementation acceptance evidence records this + old-runtime validation and its subsequent new-runtime restore. + +The embedded file keeps the original CPU and virtio state, generated MAC/IP and +isolated backing paths. The regression test reads bytes without opening those +paths, KVM, guest memory or disks. It checks field/content invariants and full +converted-state equality after saving and reloading v1000.0.0. + +Run the always-enabled regression with: + +```sh +cargo test --locked --target x86_64-unknown-linux-gnu -p vmm --lib snapshot::tests::test_microvm_v9_checked_in_fixture -- --exact +``` + +The external-fixture test remains an optional generator for separate negative +hardware fixtures; it is not required for the embedded historical regression. diff --git a/src/vmm/testdata/snapshot-v9-x86_64.bin b/src/vmm/testdata/snapshot-v9-x86_64.bin new file mode 100644 index 0000000000000000000000000000000000000000..93a9b10b7d72e56ef2ea1dbcbcd07440558e6795 GIT binary patch literal 25465 zcmeI5YitzP6@c%|j@N60@sjW`kKpB16O#3=ZC*AptO3U+*x12f-WWgDWx?Kcch@%1 z1}`)MOYIL*wUL@g5eib&DwT?={;5<6^RNBUibk#4s0~;}Re!W9uF@Y>p`M;|=ic3! zvk%)14@2hKJNJC&o_p_^Gk13GJa(5c)^qvoCGPyZ>T-X%UtsfV3fyc=)%BbI zP*mnY(Z|LVkh!`PWX7Dxeqja;ITa_i$5;RtI;0WE!4Oga7g7bH(@-a*S_dJ6id(Ae zVF3eQiWu}aUIgNO5Eb=(rI4wAh6TvG{tG6T8JD^FPEZIRuYg*85A6&e6}40a%@J5Y z&3o9&3Y3jCI*Mib-6Hf1*U0IBUMHZ}```dLsZKy{cq0t70o0?tk#sXy^=F*-_Wj{U z^T5ucXWvWeWh1$BKl}WZeN||KZDBg@RgNl!cS^iXjX#|C)9)Bc(^A)5#>U|~^f;CN zof2*7%MbxR7)&G3yv}#bV9U>c1o&V`y*G?a;wqPRtgq5RL)%g z@$1u8$W~;Cfa-Fw4MLyHHx#VPU@BD0Aa4kUVUiPs2}^Ld*-~z6)P3Z1G6u(>vdaJH z`h$;4&338L2jf7WWE18>lk70bxkODAV*e`@rfr8w%JvKu&<<_zwaTzlve}`JXt&d_ z)AS6^V0)-WyY4!pU%i7d=?|!RLY93CRmLnYqc&X$kxeGrSF!+vd;#bmuFv1fZvOK` zM;&u(?B}A0XK05u_{x>gPR7M-N9(!Fg*oJc9A)OBLNnx$=ZG@t56>l->Y*K~@r5zx z>x2uE<`T|^eDVl|*xp~Eol$TYUo`OvB`DFZ7+#}U0)TeOID>H@gKuEM&<_2__5%3I zG8oVf#|+vrXe*J!PL2f}unF@(!@442d+0j0hub=lVO>F;oUEdpD`6}!asLc4U{8Cq@h_Dk$1EmpzAqkuc;^k2xF@5|TZw(MWeN7K=nZ0Z&!AU+OW}Om_Z;#`ESfvkd!q!pzxZ z*j>}?0w>es1N(TI9d8rY*_6yNE3YnR?347iOy*pi;ru|c=Lg2z0LH?U!L{cHbhH_q zWe+&nn$-0n!Tv>K7xA9NMaITtLvFU3ebdCKSCv#b^4p^P zcOz>N*(JdVX0#{!eJ#}deJme_c_JpKlFyuZ`C5&E$-}#YwBJa|;nD<8cxeICr*eG^ zWPc?5C&Hf*zC&1PZr_7VDOcYzJQeIp7_18Z)sHXoGc$+ai-I-5y9GN1e=hvnDsq`! z6a7m!&PzcUd>{?^#T1sjASKnmmV%hJ21xyfguj)t=P_T%lrfWv=B3wYYknkMBa|ss_o=3Qt zFkIfVr1^wjAp9cX1%ww8UPO2?;S$122rnhPjPP>8FA-irxRkJm@Jhm~2(Ko*hVWX# zUm;vZcpc%F39l#o3gK4?dkL2l_7R5X5iF^Ka3$d?!W#&0B)p06X2M$tR}-!wyp`~4 zglh?JBfOn(fN&k*9fWrh-bJ{c@au#d2saYmP52GMdk8lXZYI2!a0}s9!fz7ZN4Smf ze!>R`w-Y`{_z>a4gpUwDO86Mz>2nPxG68Qse&kx37hW!N358hbZG^6taIX`_`=LdCDo*%qJ&JPSuogWx%pC4Eb$IL!Iu*Sna zKd{EbK0mO=!#+Q-#=|~8u*SnaKd{EbK0mO=!#+Q-#=|~8Fye9g{$JihDUmQZ$8w^0 zQs>3wR3k@~M_Tl_``2}i;onTt0{;BjOlxJgm$y=~Sxi+<>&a6nsmI$tQpQ}T2IfLr z{?bL@0j#lkNSI{|#^2NS5o$p=pD@|Q_`~I$WbEBKUeBsz?0wPy>tY9dfLMZUsy`Zezx+gQ?O3{`mp8~cX+*RviKKOnJU)NylTo)Zjo?mouuUx-5$@9dUM$Q2ilx2N$ zjV%Jn{#Lk7gSzBXMdsA&CYiwz*hoePveRIpc!yN~A@84TLReihMj({D3rx$_! z2M!!~dcE5{+YzuMU`N1?z-+$`Ah*zX-Lis*Bk-j%+jel(7_j+neGpvC+hVs+1?GWaS5|^fsxC4{`FRt`q`e_*8yhx7J9z?bK}>>m{O@pwnaMvk2?Yv zJgYtCKHAz0nxDGVZ7xoe3*f7ARn3|2Dkv;kx@`GND@u?2Rmw1n^hNt`Ypad)^>~M>z5T)dNc7xRPwJX`5FM{7~hKb`E=SDiQ1KALt8udqd&QJ|DJM9y}Z0!0!`!M!Dj!FK|Ah$D~+1KFvKQuVWfS+p6n4AB$Loj~Iui>IFHdJl6fe(hk&B*!wO*MYM z{I9O@S1@+{GiLNF1vp{h&j-N*`V3rmu(6+j& Date: Thu, 22 Oct 2026 00:03:22 +0800 Subject: [PATCH 5/8] build: prepare patch-v2 for x86_64 and aarch64 Align package, API and changelog versions at 1.15.1-patch-v2. Signed-off-by: Yicheng Lin --- CHANGELOG.md | 7 +++++++ Cargo.lock | 12 ++++++------ src/cpu-template-helper/Cargo.toml | 2 +- src/firecracker/Cargo.toml | 2 +- src/firecracker/swagger/firecracker.yaml | 2 +- src/jailer/Cargo.toml | 2 +- src/rebase-snap/Cargo.toml | 2 +- src/seccompiler/Cargo.toml | 2 +- src/snapshot-editor/Cargo.toml | 2 +- 9 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0084d5498..8d9583d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.15.1-patch-v2] + +### Added + +- x86_64 nested KVM state preservation across snapshots. +- Legacy v9 snapshot loading on x86_64 and aarch64; new snapshots use v1000. + ## [1.15.1] ### Added diff --git a/Cargo.lock b/Cargo.lock index 3750df127..2b4b420d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -383,7 +383,7 @@ dependencies = [ [[package]] name = "cpu-template-helper" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "clap", "displaydoc", @@ -550,7 +550,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "firecracker" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "cargo_toml", "displaydoc", @@ -704,7 +704,7 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jailer" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "libc", "log-instrument", @@ -1073,7 +1073,7 @@ dependencies = [ [[package]] name = "rebase-snap" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "displaydoc", "libc", @@ -1157,7 +1157,7 @@ dependencies = [ [[package]] name = "seccompiler" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "bitcode", "clap", @@ -1254,7 +1254,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "snapshot-editor" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "clap", "clap-num", diff --git a/src/cpu-template-helper/Cargo.toml b/src/cpu-template-helper/Cargo.toml index a856bf216..689a5388e 100644 --- a/src/cpu-template-helper/Cargo.toml +++ b/src/cpu-template-helper/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cpu-template-helper" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" diff --git a/src/firecracker/Cargo.toml b/src/firecracker/Cargo.toml index 1e3a7b483..20f999738 100644 --- a/src/firecracker/Cargo.toml +++ b/src/firecracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "firecracker" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" build = "build.rs" diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index d74a6fa84..39eb32d67 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -5,7 +5,7 @@ info: The API is accessible through HTTP calls on specific URLs carrying JSON modeled data. The transport medium is a Unix Domain Socket. - version: 1.15.1-patch-v1 + version: 1.15.1-patch-v2 termsOfService: "" contact: email: "firecracker-maintainers@amazon.com" diff --git a/src/jailer/Cargo.toml b/src/jailer/Cargo.toml index 012c98af5..98c3513b4 100644 --- a/src/jailer/Cargo.toml +++ b/src/jailer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jailer" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" description = "Process for starting Firecracker in production scenarios; applies a cgroup/namespace isolation barrier and then drops privileges." diff --git a/src/rebase-snap/Cargo.toml b/src/rebase-snap/Cargo.toml index d6abd826b..c292623cb 100644 --- a/src/rebase-snap/Cargo.toml +++ b/src/rebase-snap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rebase-snap" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" diff --git a/src/seccompiler/Cargo.toml b/src/seccompiler/Cargo.toml index a0c66ab13..83dcd27d0 100644 --- a/src/seccompiler/Cargo.toml +++ b/src/seccompiler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "seccompiler" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" description = "Program that compiles multi-threaded seccomp-bpf filters expressed as JSON into raw BPF programs, serializing them and outputting them to a file." diff --git a/src/snapshot-editor/Cargo.toml b/src/snapshot-editor/Cargo.toml index 8f08a9ac9..46949f60d 100644 --- a/src/snapshot-editor/Cargo.toml +++ b/src/snapshot-editor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "snapshot-editor" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" From 1054c5ee441f245adb2a0aa25d8a41b7bccf59da Mon Sep 17 00:00:00 2001 From: Jack Thomson Date: Tue, 16 Jun 2026 09:18:56 +0000 Subject: [PATCH 6/8] fix(x86_64): preserve IA32_XSS across snapshot restore IA32_XSS controls which supervisor XSTATE components (CET_U, CET_S, PT, HDC, etc.) are valid for XSAVES/XRSTORS. Without it the guest kernel's context-switch path triggers #GP when it encounters supervisor components in a process's XSAVE buffer that don't match the (now zeroed) XSS register: Bad FPU state detected at restore_fpregs_from_fpstate, \ reinitializing FPU registers. KVM unconditionally includes IA32_XSS in its msrs_to_save_base[] list (it's not CPUID-gated), but Firecracker's serializable MSR allowlist previously excluded it. Add it so snapshots round-trip any guest relying on supervisor XSTATE features. Signed-off-by: Jack Thomson (cherry picked from commit 490577590dc9ddd59ba8b9f13868cd56d0c3f9c2) Backport the MSR allowlist change and document it in the release changelog. Validated against the AgentENV lifecycle failure on AMD EPYC 9V45: unmodified runtime fails; the XSS candidate passes all 21 active Rust integration tests with the same Linux 6.18 kernel. Signed-off-by: Yicheng Lin --- CHANGELOG.md | 4 ++++ src/vmm/src/arch/x86_64/msr.rs | 3 +++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d9583d8d..173d4deb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to - x86_64 nested KVM state preservation across snapshots. - Legacy v9 snapshot loading on x86_64 and aarch64; new snapshots use v1000. +### Fixed + +- Preserve IA32_XSS across x86_64 snapshot restore for guests using supervisor XSTATE. + ## [1.15.1] ### Added diff --git a/src/vmm/src/arch/x86_64/msr.rs b/src/vmm/src/arch/x86_64/msr.rs index 87fc798c5..4c51c39a2 100644 --- a/src/vmm/src/arch/x86_64/msr.rs +++ b/src/vmm/src/arch/x86_64/msr.rs @@ -238,6 +238,9 @@ static SERIALIZABLE_MSR_RANGES: &[MsrRange] = &[ MSR_RANGE!(MSR_KVM_POLL_CONTROL), MSR_RANGE!(MSR_KVM_ASYNC_PF_INT), MSR_RANGE!(MSR_IA32_TSX_CTRL), + // IA32_XSS controls which supervisor XSTATE components are enabled. Without it, the guest + // kernel's XRSTORS of process FPU state containing CET/PT supervisor components triggers #GP. + MSR_RANGE!(MSR_IA32_XSS), ]; /// Specifies whether a particular MSR should be included in vcpu serialization. From 70889647fcf31a12860a553673f98dbdad040ebc Mon Sep 17 00:00:00 2001 From: Yicheng Lin Date: Fri, 23 Oct 2026 17:30:53 +0800 Subject: [PATCH 7/8] fix(x86_64): preserve AMD GIF in header-only nested state Keep SVM nested-state headers so snapshot restore preserves both GIF values. Cover SVM headers and the existing VMX cases with a focused regression test. --- src/vmm/src/arch/x86_64/vcpu.rs | 47 ++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src/vmm/src/arch/x86_64/vcpu.rs b/src/vmm/src/arch/x86_64/vcpu.rs index 05200e00b..0666a77c0 100644 --- a/src/vmm/src/arch/x86_64/vcpu.rs +++ b/src/vmm/src/arch/x86_64/vcpu.rs @@ -10,9 +10,10 @@ use std::fmt::Debug; use std::sync::Arc; use kvm_bindings::{ - CpuId, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES, KVM_STATE_NESTED_FORMAT_VMX, Msrs, Xsave, - kvm_debugregs, kvm_lapic_state, kvm_mp_state, kvm_nested_state, kvm_regs, kvm_sregs, - kvm_vcpu_events, kvm_xcrs, kvm_xsave, kvm_xsave2, nested::KvmNestedStateBuffer, + CpuId, KVM_MAX_CPUID_ENTRIES, KVM_MAX_MSR_ENTRIES, KVM_STATE_NESTED_FORMAT_SVM, + KVM_STATE_NESTED_FORMAT_VMX, Msrs, Xsave, kvm_debugregs, kvm_lapic_state, kvm_mp_state, + kvm_nested_state, kvm_regs, kvm_sregs, kvm_vcpu_events, kvm_xcrs, kvm_xsave, kvm_xsave2, + nested::KvmNestedStateBuffer, }; use kvm_ioctls::{VcpuExit, VcpuFd}; use log::{error, warn}; @@ -185,18 +186,23 @@ pub struct Peripherals { /// leaves the restored vCPU out of VMX operation on the host while the guest /// kernel still believes VMX is on (that state lives in restored guest /// memory), so the guest never re-executes VMXON and its first VMPTRLD faults. -/// Mirror QEMU's `nested_state_needed()` and inspect the header GPAs instead. +/// SVM also stores GIF in the header, even outside nested guest mode. fn has_nested_state(state: &KvmNestedStateBuffer) -> bool { // Any variable-size data following the header is state (a current // VMCS/VMCB always inflates `size`). if state.size as usize > std::mem::size_of::() { return true; } - // The VMXON-only case described above is header-only. SVM has no VMXON - // equivalent, so there the header alone carries no state. - u32::from(state.format) == KVM_STATE_NESTED_FORMAT_VMX - // SAFETY: the format check selects the valid union member. - && unsafe { state.hdr.vmx.vmxon_pa != u64::MAX } + // Preserve both GIF values for SVM. The absence of GIF_SET means that + // interrupts are disabled, not that there is no state to restore. + match u32::from(state.format) { + KVM_STATE_NESTED_FORMAT_SVM => true, + KVM_STATE_NESTED_FORMAT_VMX => { + // SAFETY: the format check selects the valid union member. + unsafe { state.hdr.vmx.vmxon_pa != u64::MAX } + } + _ => false, + } } impl KvmVcpu { @@ -1220,6 +1226,29 @@ mod tests { } } + #[test] + fn test_has_nested_state_header() { + use kvm_bindings::{KVM_STATE_NESTED_FORMAT_SVM, KVM_STATE_NESTED_GIF_SET}; + + let mut state = KvmNestedStateBuffer::empty(); + state.size = std::mem::size_of::() as u32; + state.format = KVM_STATE_NESTED_FORMAT_SVM as u16; + for flags in [0, KVM_STATE_NESTED_GIF_SET as u16] { + state.flags = flags; + assert!(has_nested_state(&state), "SVM GIF must survive snapshots"); + } + + state.format = KVM_STATE_NESTED_FORMAT_VMX as u16; + state.flags = 0; + state.hdr.vmx.vmxon_pa = u64::MAX; + assert!(!has_nested_state(&state)); + state.hdr.vmx.vmxon_pa = 0x1000; + assert!(has_nested_state(&state)); + state.hdr.vmx.vmxon_pa = u64::MAX; + state.size += 1; + assert!(has_nested_state(&state)); + } + #[test] fn test_save_state_without_nested_capability() { let (_kvm, _vm, mut vcpu) = setup_vcpu(0x1000); From 9b08a504b3bdabad1b8a8872f2c8f0f67f5afd30 Mon Sep 17 00:00:00 2001 From: Yicheng Lin Date: Wed, 28 Oct 2026 17:28:03 +0800 Subject: [PATCH 8/8] fix(x86_64): preserve MSRs across repeated snapshot restores Rebuild the ongoing MSR save set from captured state and guest CPUID, retaining baseline indices and deduplicating additions without reordering. Select VM_CR and HSAVE_PA when the guest exposes SVM, since KVM does not fully enumerate the SVM state registers. Include VIRT_SPEC_CTRL in the serialization whitelist to retain the guest SSBD setting. Add focused whitelist, CPUID-selection and repeated-restore regressions. --- CHANGELOG.md | 3 + src/vmm/src/arch/x86_64/msr.rs | 1 + src/vmm/src/arch/x86_64/vcpu.rs | 91 +++++++++++++++++-- src/vmm/src/cpu_config/x86_64/cpuid/common.rs | 29 +++++- 4 files changed, 116 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 173d4deb3..2af87b751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to ### Fixed +- Retain the x86_64 MSR save set across repeated snapshot restores. +- Save AMD SVM VM_CR and HSAVE_PA when the guest exposes SVM. +- Preserve the guest VIRT_SPEC_CTRL setting across snapshot restore. - Preserve IA32_XSS across x86_64 snapshot restore for guests using supervisor XSTATE. ## [1.15.1] diff --git a/src/vmm/src/arch/x86_64/msr.rs b/src/vmm/src/arch/x86_64/msr.rs index 4c51c39a2..6f9774ee2 100644 --- a/src/vmm/src/arch/x86_64/msr.rs +++ b/src/vmm/src/arch/x86_64/msr.rs @@ -235,6 +235,7 @@ static SERIALIZABLE_MSR_RANGES: &[MsrRange] = &[ MSR_RANGE!(MSR_MISC_FEATURES_ENABLES), MSR_RANGE!(MSR_K7_HWCR), MSR_RANGE!(MSR_VM_HSAVE_PA), + MSR_RANGE!(MSR_AMD64_VIRT_SPEC_CTRL), MSR_RANGE!(MSR_KVM_POLL_CONTROL), MSR_RANGE!(MSR_KVM_ASYNC_PF_INT), MSR_RANGE!(MSR_IA32_TSX_CTRL), diff --git a/src/vmm/src/arch/x86_64/vcpu.rs b/src/vmm/src/arch/x86_64/vcpu.rs index 0666a77c0..59de867b6 100644 --- a/src/vmm/src/arch/x86_64/vcpu.rs +++ b/src/vmm/src/arch/x86_64/vcpu.rs @@ -156,8 +156,7 @@ pub struct KvmVcpu { pub fd: VcpuFd, /// Vcpu peripherals, such as buses pub peripherals: Peripherals, - /// The list of MSRs to include in a VM snapshot, in the same order as KVM returned them - /// from KVM_GET_MSR_INDEX_LIST + /// MSRs to include in snapshots, extended by guest features and captured state. msrs_to_save: Vec, /// Size in bytes requiring to hold the dynamically-sized `kvm_xsave` struct. /// @@ -239,6 +238,14 @@ impl KvmVcpu { }) } + fn add_msrs_to_save(&mut self, indices: impl IntoIterator) { + for index in indices { + if !self.msrs_to_save.contains(&index) { + self.msrs_to_save.push(index); + } + } + } + /// Configures a x86_64 specific vcpu for booting Linux and should be called once per vcpu. /// /// # Arguments @@ -276,7 +283,7 @@ impl KvmVcpu { // Clone MSR entries that are modified by CPU template from `VcpuConfig`. let mut msrs = vcpu_config.cpu_config.msrs.clone(); - self.msrs_to_save.extend(msrs.keys()); + self.add_msrs_to_save(msrs.keys().copied()); // Apply MSR modification to comply the linux boot protocol. create_boot_msr_entries().into_iter().for_each(|entry| { @@ -292,7 +299,7 @@ impl KvmVcpu { // Since CPUID tells us what features are enabled for the Guest, we can infer // the extra MSRs that we need to save based on a dependency map. let extra_msrs = cpuid::common::msrs_to_save_by_cpuid(&kvm_cpuid); - self.msrs_to_save.extend(extra_msrs); + self.add_msrs_to_save(extra_msrs); // TODO: Some MSRs depend on values of other MSRs. This dependency will need to // be implemented. @@ -710,7 +717,7 @@ impl KvmVcpu { } /// Use provided state to populate KVM internal state. - pub fn restore_state(&self, state: &VcpuState) -> Result<(), KvmVcpuError> { + pub fn restore_state(&mut self, state: &VcpuState) -> Result<(), KvmVcpuError> { // Ordering requirements: // // KVM_GET_VCPU_EVENTS/KVM_SET_VCPU_EVENTS is unsafe if other vCPUs are @@ -779,6 +786,16 @@ impl KvmVcpu { .set_vcpu_events(&state.vcpu_events) .map_err(KvmVcpuError::VcpuSetVcpuEvents)?; + // Keep captured and CPUID-dependent registers in subsequent snapshots. + // The baseline list also covers required registers absent from older snapshots. + self.add_msrs_to_save( + state + .saved_msrs + .iter() + .flat_map(|msrs| msrs.as_slice().iter().map(|entry| entry.index)), + ); + self.add_msrs_to_save(cpuid::common::msrs_to_save_by_cpuid(&state.cpuid)); + self.kvmclock_ctrl(); Ok(()) } @@ -1052,7 +1069,7 @@ mod tests { #[test] fn test_vcpu_cpuid_restore() { - let (kvm, _, vcpu) = setup_vcpu(0x10000); + let (kvm, _, mut vcpu) = setup_vcpu(0x10000); vcpu.fd.set_cpuid2(&kvm.supported_cpuid).unwrap(); // Mutate the CPUID. @@ -1070,7 +1087,7 @@ mod tests { drop(vcpu); // Restore the state into a new vcpu. - let (_, _vm, vcpu) = setup_vcpu(0x10000); + let (_, _vm, mut vcpu) = setup_vcpu(0x10000); let result2 = vcpu.restore_state(&state); assert!(result2.is_ok(), "{}", result2.unwrap_err()); @@ -1087,6 +1104,66 @@ mod tests { assert!(leaf3.result.eax == 0x1234_5678); } + #[test] + fn test_msr_values_survive_repeated_restore() { + use crate::arch::x86_64::generated::msr_index::{ + MSR_AMD64_VIRT_SPEC_CTRL, MSR_IA32_MC0_CTL, MSR_VM_CR, MSR_VM_HSAVE_PA, + }; + + let (kvm, vm, mut vcpu) = setup_vcpu(0x10000); + let config = create_vcpu_config(&kvm, &vcpu, &CustomCpuTemplate::default()).unwrap(); + vcpu.configure( + vm.guest_memory(), + EntryPoint { + entry_addr: GuestAddress(0), + protocol: BootProtocol::LinuxBoot, + }, + &config, + ) + .unwrap(); + let mut expected = BTreeMap::from([(MSR_IA32_MC0_CTL, u64::MAX)]); + if kvm + .fd + .get_msr_index_list() + .unwrap() + .as_slice() + .contains(&MSR_AMD64_VIRT_SPEC_CTRL) + { + expected.insert(MSR_AMD64_VIRT_SPEC_CTRL, 1 << 2); + } + if cpuid::common::msrs_to_save_by_cpuid(&vcpu.get_cpuid().unwrap()).contains(&MSR_VM_CR) { + expected.insert(MSR_VM_CR, 1); + expected.insert(MSR_VM_HSAVE_PA, 0x1000); + } + let entries: Vec<_> = expected + .iter() + .map(|(&index, &data)| kvm_bindings::kvm_msr_entry { + index, + data, + ..Default::default() + }) + .collect(); + crate::arch::x86_64::msr::set_msrs(&vcpu.fd, &entries).unwrap(); + let mut state = vcpu.save_state().unwrap(); + for _ in 0..2 { + let (_, _vm, mut restored) = setup_vcpu(0x10000); + restored.restore_state(&state).unwrap(); + state = restored.save_state().unwrap(); + let saved: BTreeMap<_, _> = state + .saved_msrs + .iter() + .flat_map(|msrs| { + msrs.as_slice() + .iter() + .map(|entry| (entry.index, entry.data)) + }) + .collect(); + for (&index, &value) in &expected { + assert_eq!(saved.get(&index), Some(&value), "MSR {index:#x}"); + } + } + } + #[test] fn test_empty_cpuid_entries_removed() { // Test that `get_cpuid()` removes zeroed empty entries from the `KVM_GET_CPUID2` result. diff --git a/src/vmm/src/cpu_config/x86_64/cpuid/common.rs b/src/vmm/src/cpu_config/x86_64/cpuid/common.rs index 388953022..f11aca5a6 100644 --- a/src/vmm/src/cpu_config/x86_64/cpuid/common.rs +++ b/src/vmm/src/cpu_config/x86_64/cpuid/common.rs @@ -6,7 +6,7 @@ use crate::arch::x86_64::generated::msr_index::{ MSR_IA32_BNDCFGS, MSR_IA32_CR_PAT, MSR_MTRRdefType, MSR_MTRRfix4K_C0000, MSR_MTRRfix4K_C8000, MSR_MTRRfix4K_D0000, MSR_MTRRfix4K_D8000, MSR_MTRRfix4K_E0000, MSR_MTRRfix4K_E8000, MSR_MTRRfix4K_F0000, MSR_MTRRfix4K_F8000, MSR_MTRRfix16K_80000, MSR_MTRRfix16K_A0000, - MSR_MTRRfix64K_00000, + MSR_MTRRfix64K_00000, MSR_VM_CR, MSR_VM_HSAVE_PA, }; /// Error type for [`get_cpuid`]. @@ -67,6 +67,9 @@ pub(crate) fn msrs_to_save_by_cpuid(cpuid: &kvm_bindings::CpuId) -> Vec { /// Memory Type Range Registers const MTRR_BITINDEX: u32 = 12; + /// Secure Virtual Machine + const SVM_BITINDEX: u32 = 2; + /// Memory Check Exception const MCE_BITINDEX: u32 = 7; @@ -99,6 +102,15 @@ pub(crate) fn msrs_to_save_by_cpuid(cpuid: &kvm_bindings::CpuId) -> Vec { }; } + // SVM state is not fully enumerated by KVM_GET_MSR_INDEX_LIST. + cpuid_msr_dep!( + 0x8000_0001, + 0, + ecx, + SVM_BITINDEX, + [MSR_VM_CR, MSR_VM_HSAVE_PA] + ); + // TODO: Add more dependencies. cpuid_msr_dep!(0x7, 0, ebx, MPX_BITINDEX, [MSR_IA32_BNDCFGS]); @@ -142,6 +154,21 @@ pub(crate) fn msrs_to_save_by_cpuid(cpuid: &kvm_bindings::CpuId) -> Vec { mod tests { use super::*; + #[test] + fn test_svm_msr_dependencies() { + let mut cpuid = kvm_bindings::CpuId::new(1).unwrap(); + assert!(msrs_to_save_by_cpuid(&cpuid).is_empty()); + cpuid.as_mut_slice()[0].function = 1; + cpuid.as_mut_slice()[0].edx = 1 << 12; + let msrs = msrs_to_save_by_cpuid(&cpuid); + assert!(!msrs.contains(&MSR_VM_CR)); + assert!(!msrs.contains(&MSR_VM_HSAVE_PA)); + cpuid.as_mut_slice()[0].function = 0x8000_0001; + assert!(msrs_to_save_by_cpuid(&cpuid).is_empty()); + cpuid.as_mut_slice()[0].ecx = 1 << 2; + assert_eq!(msrs_to_save_by_cpuid(&cpuid), [MSR_VM_CR, MSR_VM_HSAVE_PA]); + } + #[test] fn get_cpuid_unsupported_leaf() { let max_leaf =