diff --git a/.github/actions/check-quality/action.yml b/.github/actions/check-quality/action.yml index 9f735e8eb..e583f935e 100644 --- a/.github/actions/check-quality/action.yml +++ b/.github/actions/check-quality/action.yml @@ -38,6 +38,7 @@ runs: guest/common/nvx-init-agent \ guest/common/nvx-container-enter-azurelinux \ guest/common/nvx-container-launch-azurelinux \ + guest/common/nvx-image-slot \ guest/common/nvx-sandbox-smoke guest/common/nvx-snapshot \ guest/common/nvx-virtio-restore-probe \ scripts/setup/setup-linux-mshv.sh \ @@ -54,6 +55,7 @@ runs: guest/common/nvx-init-agent \ guest/common/nvx-container-enter-azurelinux \ guest/common/nvx-container-launch-azurelinux \ + guest/common/nvx-image-slot \ guest/common/nvx-sandbox-smoke guest/common/nvx-snapshot \ guest/common/nvx-virtio-restore-probe \ scripts/setup/setup-linux-mshv.sh \ diff --git a/.github/actions/run-benchmark/action.yml b/.github/actions/run-benchmark/action.yml index 19f1fae28..a9b5c6da9 100644 --- a/.github/actions/run-benchmark/action.yml +++ b/.github/actions/run-benchmark/action.yml @@ -140,6 +140,31 @@ runs: --output-dir "${run_dir}" done + - name: Run Linux image-slot boot cost + if: runner.os != 'Windows' + shell: bash + env: + NVX_BENCHMARK_CPUS: ${{ inputs.cpus }} + NVX_BENCHMARK_HOST_CPU_RESERVE: ${{ inputs.host-cpu-reserve }} + NVX_BENCHMARK_MARKER_TIMEOUT: ${{ inputs.marker-timeout }} + run: | + set -euo pipefail + cpu_args=(--host-cpu-reserve "${NVX_BENCHMARK_HOST_CPU_RESERVE}") + if [[ -n "${NVX_BENCHMARK_CPUS}" ]]; then + cpu_args+=(--cpus "${NVX_BENCHMARK_CPUS}") + fi + python3 scripts/nvx.py benchmark \ + --suite image-slot-boot \ + --backend "${{ inputs.backend }}" \ + --platform "${{ inputs.platform }}" \ + --processors 1 \ + "${cpu_args[@]}" \ + --warmups "${{ inputs.warmups }}" \ + --runs "${{ inputs.runs }}" \ + --timeout "${NVX_BENCHMARK_MARKER_TIMEOUT}" \ + --skip-build \ + --output-dir "data/runs/${{ inputs.platform }}/image-slot-boot" + - name: Run Linux device operation rates if: runner.os != 'Windows' shell: bash @@ -324,6 +349,32 @@ runs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } + - name: Run Windows image-slot boot cost + if: runner.os == 'Windows' + shell: powershell + env: + NVX_BENCHMARK_CPUS: ${{ inputs.cpus }} + NVX_BENCHMARK_HOST_CPU_RESERVE: ${{ inputs.host-cpu-reserve }} + NVX_BENCHMARK_MARKER_TIMEOUT: ${{ inputs.marker-timeout }} + run: | + $ErrorActionPreference = "Stop" + $CpuArgs = @("--host-cpu-reserve", $env:NVX_BENCHMARK_HOST_CPU_RESERVE) + if ($env:NVX_BENCHMARK_CPUS) { + $CpuArgs += @("--cpus", $env:NVX_BENCHMARK_CPUS) + } + python scripts\nvx.py benchmark ` + --suite image-slot-boot ` + --backend "${{ inputs.backend }}" ` + --platform "${{ inputs.platform }}" ` + --processors 1 ` + --warmups "${{ inputs.warmups }}" ` + --runs "${{ inputs.runs }}" ` + --timeout $env:NVX_BENCHMARK_MARKER_TIMEOUT ` + --skip-build ` + --output-dir "data/runs/${{ inputs.platform }}/image-slot-boot" ` + @CpuArgs + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - name: Run Windows device operation rates if: runner.os == 'Windows' shell: powershell diff --git a/doc/benchmarks.md b/doc/benchmarks.md index d3302977a..e2c1386c8 100644 --- a/doc/benchmarks.md +++ b/doc/benchmarks.md @@ -33,7 +33,9 @@ ABI version and processor count are also separate history dimensions. Legacy CSV rows are interpreted as ABI v1 with one vCPU; they are never used as an ABI-v2 one-vCPU baseline. New runs always emit ABI value 2 and remain stored under `microvm-v2` paths so -they cannot collide with legacy unsuffixed ABI-1 history. +they cannot collide with legacy unsuffixed ABI-1 history. The image-slot boot +suite records its configurations with and without image slots together under +the dedicated `image-slot-boot` path. The OpenVMM benchmark coordinator is implemented in `scripts/nvx_tools/benchmark.py` and exposed through the supported NVX CLI. Guest payloads live in `scripts/nvx_tools/benchmark_scripts`; the coordinator fills the `.sh.in` templates before use. @@ -261,6 +263,15 @@ python3 scripts/nvx.py performance collect --platform linux-kvm-baremetal --comm | Network snapshot | `benchmark --suite network-snapshot` | Compares a network-ready cold boot with snapshot restore and verifies gateway connectivity. | | Snapshot lifecycle profile | `benchmark --suite snapshot-profile` | Retains raw capture and restore phase records and summarizes 128/256/512/1024 MiB warm/cold restores. | | Virtio device restore profile | `benchmark --suite device-restore-profile` | Verifies active and driver-unbound deferred restore for console, network, and virtio-fs; emits standalone diagnostic JSON and raw logs. | +| Image-slot boot cost | `benchmark --suite image-slot-boot` | Measures launches without image slots and with image slots at `B = 1` and `B = 4` using matched launches, and reports both slot deltas against the launch without slots. | + +Image-slot boot-cost results must use the same host, backend, kernel, +initramfs, processor count, memory size, warmups, runs, and readiness marker. +The suite records three series: `no-slots`, `slots-b1` (`B = 1`), and +`slots-b4` (`B = 4`). Report both slot deltas against `no-slots` rather than +comparing only the two slot series. Run the matrix on KVM, MSHV, and WHP; +performance runners retain the raw commands and results under their normal +`data/runs` output directory. ## Kernel command lines diff --git a/doc/design/current-limits.md b/doc/design/current-limits.md index eb3762f71..03d5c7603 100644 --- a/doc/design/current-limits.md +++ b/doc/design/current-limits.md @@ -18,6 +18,8 @@ The current ABI family intentionally does not provide: without reflink support; - direct-claimed paired-scratch restore on filesystems without exact-file hard-link support; +- more than four image slots, writable image slots, binding an inactive slot, + rebinding or ejecting media, or activating a slot after restore readiness; - sandbox-block, network, or control-console construction through the management RPC, or its restore of snapshots that contain them; - host networking other than the in-process portable endpoint, unrestricted @@ -47,7 +49,10 @@ transport foundation for that protocol, not completion of it. The shell bootstrap does not implement a fleet-safe platform build point, warm-runtime handoff, arbitrary OCI policy, or production agent resource guarantees. -Changing a guest-visible address, IRQ, command-line token, feature mask, queue +Image slots extend ABI 2 in place: a machine that does not declare them keeps +the original device topology, and development snapshots from before this +extension are not guaranteed to remain compatible. Otherwise, changing a +guest-visible address, IRQ, command-line token, feature mask, queue shape, time policy, or device behavior requires a new microVM ABI version. A backend-specific difference is valid only when it is explicitly part of that versioned contract, such as the virtio-net IRQ. diff --git a/doc/design/integrated-implementation-status.md b/doc/design/integrated-implementation-status.md index e64da5c1d..661c0e81f 100644 --- a/doc/design/integrated-implementation-status.md +++ b/doc/design/integrated-implementation-status.md @@ -6,7 +6,7 @@ The implementation and remaining proposals separate as follows: | Area | Current implementation | | --- | --- | -| Base machine | ACPI-free Linux direct boot with Intel MP tables, allowlisted chipset/PMIO, fixed-role sandbox blocks, nine reserved virtio-mmio slots, and 1/2/4/8-vCPU SMP on KVM, MSHV, and WHP, selected explicitly from the command line or the management RPC. Persisted ABI and boot layout remain value 2. | +| Base machine | ACPI-free Linux direct boot with Intel MP tables, allowlisted chipset/PMIO, fixed-role sandbox blocks, nine reserved virtio-mmio slots, optional four-slot image capacity that replaces the second virtio-fs slot, and 1/2/4/8-vCPU SMP on KVM, MSHV, and WHP, selected explicitly from the command line or the management RPC. Persisted ABI and boot layout remain value 2, with or without image slots. | | Snapshot and restore | Guest-requested version-5 publication, same-backend restore, exact saved inventory, private COW RAM, optional fresh RAM expansion, fresh/paired scratch, three sandbox tiers, input gating, single-use resume claims, a single-use restore readiness event, and management exclusion at the capture boundary. The management RPC drives the same capture and restore for blockless machines without a NIC or control console. Older supported manifests remain readable subject to their recorded capabilities. | | Resource activation | Opt-in CPU-prefix and 128-MiB-aligned RAM targets within immutable capacity. Explicit MSHV targets materialize only that VP prefix and cannot be saved again; KVM, WHP, and untargeted MSHV retain full VP capacity. | | Console | Boot virtio-console with private RX/TX state and reconnect policies; bounded portb/host-relay drain at process exit, with guest exit status propagated to management-RPC waiters. The separate control console has authenticated Linux and Windows endpoints, bounded framing with receive credits, per-instance epochs, broker saved state, and managed workload RPC. | diff --git a/doc/design/machine-and-device-abi.md b/doc/design/machine-and-device-abi.md index 51a2583da..cb693b3b9 100644 --- a/doc/design/machine-and-device-abi.md +++ b/doc/design/machine-and-device-abi.md @@ -103,10 +103,11 @@ restore, the same write acknowledges completion of guest repair. ## Fixed virtio-mmio transport -All nine fixed address slots are reserved, including the dedicated control -console at `0xd0007000..0xd0007fff` on IRQ 3 (shared status at `0x3001c`) and -the second virtio-fs slot at `0xd0008000..0xd0008fff` on IRQ 13 (shared status -at `0x30020`). +Without image slots, ABI 2 reserves nine fixed address slots, including the +dedicated control console at `0xd0007000..0xd0007fff` on IRQ 3 (shared status +at `0x3001c`) and the second virtio-fs slot at `0xd0008000..0xd0008fff` on +IRQ 13 (shared status at `0x30020`). A slot-declaring machine also reserves +`0xd0009000..0xd000bfff`, as described below. Every microVM cold-booted from the command line or the management RPC instantiates the first virtio-fs slot, so it is discoverable before capture even without a host attachment; a restore keeps the slot only when the snapshot @@ -117,8 +118,8 @@ The control slot is instantiated for a control console, either a live authenticated local endpoint on Linux or Windows or an explicitly disconnected console. Every device uses virtio-mmio, is discovered only through profile-owned command-line tokens, and has packed-ring support masked. -The MP table routes only the 16 ISA interrupts, and no other device uses -IRQ 13 on KVM, MSHV, or WHP. +The MP table routes only the 16 ISA interrupts, and no other device of a +machine without image slots uses IRQ 13 on KVM, MSHV, or WHP. | Device | Stable identity | MMIO range | IRQ | Availability | | --- | --- | ---: | ---: | --- | @@ -132,6 +133,31 @@ IRQ 13 on KVM, MSHV, or WHP. | Control virtio-console | `console:microvm-control0` | `0xd0007000..0xd0007fff` | 3 | Optional authenticated local endpoint | | Second virtio-fs | `fs:microvm1` | `0xd0008000..0xd0008fff` | 13 | Optional; requires a HostFs attachment | +Image slots are an optional part of ABI 2. A slot-declaring machine has four +permanent image-slot transports. Image slot 0 takes the second virtio-fs slot's +MMIO window and shared-status word, and image slot 1 takes its IRQ 13, so a +slot-declaring machine attaches at most one virtio-fs share. Omitting image +slots keeps the original topology. A slot-declaring launch +always has capacity `S = 4`; `B` is the active cold-boot prefix, and a restore +may activate a larger prefix `N` only while restore input remains gated. +Inactive slots are virtio-mmio placeholders with device ID 0. Active slots are +empty read-only virtio-blk devices with 512-byte logical and 4096-byte physical +blocks until the host binds media once. + +| Device | Stable identity | MMIO range | IRQ | Shared status | +| --- | --- | ---: | ---: | ---: | +| Image slot 0 | `blk:image0` | `0xd0008000..0xd0008fff` | 1 | `0x30020` | +| Image slot 1 | `blk:image1` | `0xd0009000..0xd0009fff` | 13 | `0x30024` | +| Image slot 2 | `blk:image2` | `0xd000a000..0xd000afff` | 14 | `0x30028` | +| Image slot 3 | `blk:image3` | `0xd000b000..0xd000bfff` | 15 | `0x3002c` | + +The discovery token is `microvm_image_slots=4`. Binding sets capacity and +raises a virtio configuration-change interrupt; the guest block driver updates +capacity through its normal config-change path. The host-control protocol +permits query and bind-once operations only. The same identity is idempotent, +a different second bind fails, and there is no unbind, eject, rebind, device +addition, or activation after readiness. + Explicit placement metadata bypasses the standard sequential MMIO allocator. The worker validates the complete device count, kind, bus, address, IRQ, and feature policy before resolving devices. @@ -165,7 +191,10 @@ per fixed slot resides in the reserved shared-status page: | `custom` block | `0x30014` | | `scratch` block | `0x30018` | | Control virtio-console | `0x3001c` | -| Second virtio-fs | `0x30020` | +| Second virtio-fs, or image slot 0 | `0x30020` | +| Image slot 1 | `0x30024` | +| Image slot 2 | `0x30028` | +| Image slot 3 | `0x3002c` | OpenVMM publishes config-change and used-buffer bits with a sequentially consistent compare-exchange loop. It pulses the device IRQ only when the old diff --git a/doc/design/remaining-production-work.md b/doc/design/remaining-production-work.md index 6182ff262..06628c183 100644 --- a/doc/design/remaining-production-work.md +++ b/doc/design/remaining-production-work.md @@ -28,10 +28,11 @@ the required stop signal. Such images may need `CAP_MKNOD` together with a cgroup device filter. They are not covered by the current drop-all-capabilities bootstrap, and its private mount policy must not be described as that profile. -Azure Files, secret/content volumes, and other optional mounts remain undesigned. -Guest-side CIFS, a scratch-backed temporary directory, or an explicit small -content attachment are possible directions, not supported extra device slots -in the fixed ABI. Managed executions deliver bounded environments through a +Secret/content volumes and other optional mounts remain undesigned. +Guest-side network filesystems, a scratch-backed temporary directory, or an +explicit small content attachment are separate directions. ABI 2 can declare +four fixed read-only image slots, but they are bind-once block media rather +than general volume or device hotplug. Managed executions deliver bounded environments through a sealed anonymous file inside the guest; snapshot-time launch configuration and secret-volume delivery remain distinct proposed features. Scratch cannot currently grow online; capacity exhaustion and whole-file copy-up remain diff --git a/doc/design/sandbox-filesystem-and-agent-architecture.md b/doc/design/sandbox-filesystem-and-agent-architecture.md index 8b7b5fb8a..c6dde0ecf 100644 --- a/doc/design/sandbox-filesystem-and-agent-architecture.md +++ b/doc/design/sandbox-filesystem-and-agent-architecture.md @@ -32,9 +32,10 @@ supplied native library and an image-backed edge guest that NVX does not build. ## Implemented filesystem bootstrap The public `nvx sandbox` command accepts one to three role-bearing EROFS lower -images, a preformatted ext4 scratch image, up to two -[live host shares](#live-host-shares), an absolute entrypoint, and -individual argument tokens. It supplies non-secret kernel-command-line +images (none for a managed launch that declares image slots), a preformatted +ext4 scratch image, up to two [live host shares](#live-host-shares) (one when +the launch declares image slots), an absolute entrypoint, and individual +argument tokens. It supplies non-secret kernel-command-line configuration for one-shot runs. Managed execution carries bounded arguments and a per-execution environment over the authenticated control channel; one-shot environment variables, secrets, and sandbox snapshot orchestration @@ -84,10 +85,14 @@ virtio-fs tag microvm (optional share) ---> /run/nvx/rootfs/TARGET virtio-fs tag microvm1 (optional share) ---> /run/nvx/rootfs/TARGET ``` -At least one lower role is required by the bootstrap; distro plus runtime is -the intended curated-image shape, not a requirement that all three lower -slots be populated. The initramfs remains the supervisor's root and is not -another container lower layer. +At least one lower role is required by the bootstrap unless the launch +declares image slots; distro plus runtime is the intended curated-image shape, +not a requirement that all three lower slots be populated. A slot-declaring +launch without lower roles bind-mounts `/run/nvx/scratch/upper` as +`/run/nvx/rootfs` instead of assembling an overlay, and skips the image's +user-database checks: the workload runs with the configured UID and GID and +home `/`. The initramfs remains the supervisor's root and is not another +container lower layer. The container launch helper releases the barrier into private mount, PID, and UTS namespaces. The container entry helper makes mounts @@ -122,9 +127,11 @@ enforces the share's mode and its denied, allowed, and writable paths, and accesses host files as the identity that `--mount-owner` selects. Guest mount flags are therefore not a security boundary, and the access policy adds no guest configuration. Before OpenVMM starts, NVX rejects a third share and -policy paths outside their share. It also rejects guest targets or host -directories that equal or contain one another, because one share could -otherwise hide the other or reach its files under a different policy. +policy paths outside their share. A launch that declares image slots accepts +only one share, because image slot 0 takes the second share's MMIO window and +status word, and image slot 1 takes its IRQ. NVX also rejects guest targets +or host directories that equal or contain one another, because one share +could otherwise hide the other or reach its files under a different policy. [Machine and device ABI](machine-and-device-abi.md#filesystem) defines the device contract. diff --git a/doc/design/snapshot-and-restore.md b/doc/design/snapshot-and-restore.md index 2dff7b11c..b46d264d9 100644 --- a/doc/design/snapshot-and-restore.md +++ b/doc/design/snapshot-and-restore.md @@ -496,6 +496,27 @@ Performance comparisons should separate VP binding and worker construction from guest resume-to-readiness. Matching host-side phase costs do not require the total restore latencies or tail behavior to match. +## Restore-time image-slot activation + +Snapshots of slot-declaring machines record image-slot capacity `S = 4`, the +cold-boot active prefix `B`, and the empty state of every active slot. Capture +is rejected before the guest is quiesced if any slot is bound. Snapshots +without image slots record zero capacity and active count, and their restore +packet leaves header byte 7 zero. + +An explicit `--restore-image-slots N` target is process-local and must satisfy +`B <= N <= 4`. During gated repair the guest binds the virtio-mmio driver to +newly active placeholders `B..N-1`, verifies that all slots in `0..N-1` are +empty read-only block devices, and only then acknowledges the restore gate. +The target does not modify the snapshot, and no operation activates a slot +after readiness. + +The [restore packet](time-abi.md#restore-packet) carries `N` in header byte 7 +and sets its `IMAGE_SLOT_TARGET` flag; the target is always in `1..4`, and no +portb status bit advertises it. Every explicit target holds the restore gate, +so the guest always verifies the active prefix and acknowledges before +readiness, even when no new slot becomes active. + ## Restore-time memory activation Restore-time memory activation separates the immutable RAM geometry recorded diff --git a/doc/design/time-abi.md b/doc/design/time-abi.md index d30bb0180..f497f3597 100644 --- a/doc/design/time-abi.md +++ b/doc/design/time-abi.md @@ -1189,7 +1189,7 @@ offset size field 4 1 flags 5 1 online_vp_count 0 = no processor target, else 1, 2, 4, or 8 6 1 memory_range_count 0 unless MEMORY_TARGET - 7 1 reserved = 0 + 7 1 image_slot_target 0 unless IMAGE_SLOT_TARGET, else 1 to 4 8 4 generation u32, little-endian, g >= 1 12 4 rate_deviation i32, little-endian, ppm scaled by 2^16 16 8 downtime_ns u64, little-endian, D @@ -1206,13 +1206,15 @@ Flags: | 1 | `MEMORY_TARGET` | An explicit RAM target was requested; `memory_range_count` is valid and may be 0 | | 2 | `ACK_REQUIRED` | Host input is gated; the guest must acknowledge through `0x605` after repair | | 3 | `TEST_HOOKS` | A test hook is active: it can alter the downtime source, `D`, UTC, the rate deviation, the handling of time samples, or the CPU profile | -| 4–7 | | Zero; the guest rejects a packet with any of them set | +| 4 | `IMAGE_SLOT_TARGET` | An explicit image-slot target was requested; `image_slot_target` is set exactly when this flag is, and `ACK_REQUIRED` is always set with it | +| 5–7 | | Zero; the guest rejects a packet with any of them set | `utc_ns` is latched when the guest first writes selector `0xa5` after the restore; every other field is sealed before the first restored VP runs (step 16 of the [restore algorithm](#restore-algorithm)). A packet without ranges is 96 bytes, 24 four-byte reads. Status bits 2 to 4 remain and agree with the -packet; the packet is authoritative. `ACK_REQUIRED` replaces the guest's +packet; the packet is authoritative, and the image-slot target has no status +bit. `ACK_REQUIRED` replaces the guest's inference of gating from the tier and targets, so an untiered guest never writes `0x605` after an ungated restore. @@ -1558,7 +1560,8 @@ acknowledgement. discontinuity and the new `g`, with `restore_status=pending` and `restore_generation=g` in the same state-file update, so that `nvx-time status` waits for this restore. -9. Run the existing processor and memory activation. +9. Run the existing processor activation, the image-slot activation that + `IMAGE_SLOT_TARGET` requests, and the existing memory activation. 10. Run the existing entropy and identity repair for the restore path ([restore entropy](#restore-entropy)). The RTC-based wall-clock refresh is removed; `instance-checkpoint` restores also get step 8. @@ -1583,9 +1586,12 @@ No restore check runs before the acknowledgement. The post-acknowledgement checks are `C1`, `C2`, `C3`, `C5`, `C6`, `C7`, and `C10`. Steps 6, 7, and 8 run in one helper process. For an untiered restore with no -processor or memory to activate, that helper runs every step through 12 and -signals the daemon, so no other process starts on the readiness path. Each -extra process costs 2 to 4 ms after a restore (fork and exec under demand +processor, image slot, or memory to activate, that helper runs every step +through 12 and signals the daemon, so no other process starts on the +readiness path. Any of these targets, an image-slot target alone included, +returns the packet metadata to `nvx-snapshot`, which finishes the restore; +image-slot activation also starts short processes for each slot. Each extra +process costs 2 to 4 ms after a restore (fork and exec under demand faulting, measured on KVM). Step 13 starts 150 ms after the acknowledgement (or after step 10, when none @@ -1627,7 +1633,7 @@ policies of [Time and entropy](snapshot-and-restore.md#time-and-entropy): | `platform` or `workload-start` tier (clone policy) | Reseeded before the acknowledgement: `nvx-reseed` credits the 64 bytes to the kernel entropy pool (`RNDADDENTROPY`) and forces a CRNG reseed (`RNDRESEEDCRNG`) | Machine ID and hostname refreshed; the runtime hook runs if present, and `workload-start` requires it | | Untiered, with processors or memory to activate | Reseeded before the acknowledgement, as for the clone policy | Refreshed as for the clone policy | | `instance-checkpoint` tier (resume policy) | Not reseeded, by design: only the generation ID is refreshed | Preserved | -| Untiered, with nothing to activate (`online_vp_count` and `memory_range_count` both 0) | Not reseeded: `nvx-time` writes the 64 bytes to `/run/nvx/restore-entropy` for the caller of `nvx-snapshot` | Preserved | +| Untiered, with no processors or memory to activate (`online_vp_count` and `memory_range_count` both 0), including a restore that activates only image slots | Not reseeded: `nvx-time` writes the 64 bytes to `/run/nvx/restore-entropy` for the caller of `nvx-snapshot` | Preserved | Each path's handling is unchanged from before the time ABI. v1 changes only the delivery: the agent used to read packets v1 to v3 a byte at a time, and @@ -2014,7 +2020,7 @@ Expected effects: | Counting LAPIC instead of TSC-deadline on KVM | Different timer-programming exits; covered by the gate | | v1 profiles without `ITS_NO` | Linux's ITS mitigation at boot, about 6 ms of a one-vCPU cold boot where the host's KVM advertises `ITS_NO` (the bare-metal KVM host: +6.3 [5.5, 7.3] ms against the same build booted with `indirect_target_selection=off`). The gated Azure KVM, MSHV, and WHP guests were already mitigated and are unchanged; restore is unaffected | | The profile's CPU view at 2 or more vCPUs on KVM | None at the integration head. Before the L3 fix, multi-vCPU cold boots were slower: +9.2 [5.0, 13.7] ms at 2 vCPUs on the bare-metal KVM host against the pre-profile head, and about +25 ms on nested Azure KVM. OpenVMM's `CPUID.4` reported a private L3 cache per vCPU, which made Linux's cache-info initialization wait a 10 ms tick for CPU 1 on many boots. With the L3 shared by the socket, the bare-metal host's cold boots match the pre-profile head: −0.3 [−3.2, +2.7], −0.0 [−2.1, +2.0], and −1.0 [−3.6, +1.7] ms at 2, 4, and 8 vCPUs. The gate's cold boots are one-vCPU; a boot-only comparison against the `dev` base on nested Azure KVM found no significant difference: +1.7 [−2.5, +6.9] ms at 2 vCPUs, +3.7 [−5.4, +13.6] at 4, and +3.3 [−7.0, +11.5] at 8, which excludes the earlier +25 ms. At 8 vCPUs, the time ABI's VMM boots 9.5 [4.6, 16.4] ms faster and its guest 8.2 [0.4, 15.9] ms slower, which cancel; the guest's part may be the KVM paravirtual features that the identity forgoes (see [Hypervisor identity](#hypervisor-identity)). The msr driver's per-CPU hotplug callback (`msr_init`) waits one or two ticks on multi-vCPU boots, as it did before the profiles; it competes for CPU with the asynchronous initramfs unpack and is off the critical path | -| Restore work before the acknowledgement | Only the packet read, the clock set, CPU and memory activation, and entropy and identity repair, in one helper process. The restore checks, the RCU release, and the deferred `C7` start at `SCHED_IDLE` 150 ms after the acknowledgement, after the readiness path | +| Restore work before the acknowledgement | Only the packet read, the clock set, CPU and memory activation, and entropy and identity repair, in one helper process, plus image-slot activation when the packet requests it. The restore checks, the RCU release, and the deferred `C7` start at `SCHED_IDLE` 150 ms after the acknowledgement, after the readiness path | | Monotonic time that includes the downtime | Right after resume, the guest kernel runs the timer work that came due during the downtime (kworkers, softirqs, and RCU) and first-touches cold pages doing so: at one vCPU about 1.4 ms on MSHV, 2.2 ms on KVM, and 17 ms on WHP (26 ms at two vCPUs). On WHP each restored 4 KiB page that the guest first touches costs about 50 µs of wall time on bare metal and about 70 to 95 µs on the Azure runners: a nested page fault, about 11 µs of it in the hypervisor on bare metal and the rest resolved by the root, which the VMM never sees. WHP's lazily registered 2 MiB chunks add about 90 µs each, all before readiness, and on bare metal a WHP restore's latency after the VPs are released is roughly the pages it touches times 53 µs. It falls inside the gated restore metric whatever the helper's priority. On MSHV the cost is per 2 MiB chunk: the root driver resolves the first touch of each chunk with 512 copy-on-write copies of the private snapshot mapping, about 1.1 ms. A one-vCPU restore touches about 24 chunks, 25 ms of its restore on both the time ABI and the legacy path, and the time ABI's catch-up adds about one chunk | | No time ABI console lines in production | Each console byte costs one or two port exits, about 10 to 12 µs on bare-metal KVM and an estimated 20 to 45 µs nested on Azure. Without the boot marker (about 110 bytes) and the restore marker (about 117 bytes), boots and restores save 1 to 5 ms, enough to fail the gate on nested Azure KVM; violation events still print | @@ -2029,9 +2035,10 @@ kernel follows with its RTC write, and the restore checks start 150 ms after the acknowledgement. Readiness doesn't wait for this work. The first process that a restored guest starts pays a separate cost. A plain -(untiered) restore with nothing to activate starts no process before -readiness, while the legacy path's status read forked and executed one there, -which first-touched the guest kernel's fork, exec, and exit paths. Under the +(untiered) restore with no processor, image slot, or memory to activate +starts no process before readiness, while the legacy path's status read +forked and executed one there, which first-touched the guest kernel's fork, +exec, and exit paths. Under the time ABI the first process after readiness touches them instead, about 80 to 110 more pages than on the legacy path, which matters where first touches of restored RAM are expensive: on the Azure WHP runners, the first process after diff --git a/doc/run.md b/doc/run.md index d06f0448f..2ddb11b9a 100644 --- a/doc/run.md +++ b/doc/run.md @@ -382,6 +382,17 @@ For restore-time memory expansion, capture a fresh snapshot with remains exactly 512 MiB; selected expansion ranges receive fresh per-launch backing and are onlined before restore readiness. +For a snapshot with image slots, select the process-local active prefix with: + +```console +python3 scripts/nvx.py run \ + --restore-snapshot build/snapshot \ + --restore-image-slots 4 +``` + +The target must be between the snapshot's cold-boot prefix and four. Activation +finishes before restore readiness; slots cannot be activated later. + ## virtio-fs host mapping The microVM has two mapping slots with the fixed tags `microvm` and @@ -536,6 +547,32 @@ policy, creates `/nonexistent` for UID/GID 65534, and invokes `mkfs.erofs`. Systemd entrypoints are explicitly unsupported and do not relax the non-root, drop-all-capabilities sandbox policy. +Managed sandboxes can opt into image slots at provision time: + +```console +python3 scripts/nvx.py sandbox provision \ + --state-dir build/sandbox \ + --layer distro,build/distro.erofs,EROFS_UUID \ + --scratch build/scratch.ext4 \ + --image-slot-boot-count 4 +python3 scripts/nvx.py sandbox start --state-dir build/sandbox +python3 scripts/nvx.py sandbox bind-image-slot \ + --state-dir build/sandbox \ + --image-slot 0 \ + --image-path build/image.raw \ + --image-identity sha256:IMAGE_DIGEST +python3 scripts/nvx.py sandbox query-image-slots --state-dir build/sandbox +``` + +Capacity is always four; `--image-slot-boot-count` selects `B`. Workloads that +need every slot immediately use `B = 4`, while other slot-declaring launches +use `B = 1`. A slot-declaring managed launch may omit a `distro` layer and boot +with scratch plus empty image slots. Image slot 0 uses the second virtio-fs +slot's transport, so a slot-declaring launch accepts at most one `--mount`. +A slot-declaring configuration uses format 6, which earlier NVX releases reject +rather than start without its image slots. Launches without image slots retain +the existing layer requirement, behavior, and configuration formats. + This is the cold-filesystem bootstrap described in [the sandbox design](design/sandbox-filesystem-and-agent-architecture.md#implemented-filesystem-bootstrap), not the final production agent. It accepts no environment variables or secrets and does not diff --git a/doc/setup.md b/doc/setup.md index c080e6e21..1dba0a99d 100644 --- a/doc/setup.md +++ b/doc/setup.md @@ -190,6 +190,7 @@ shellcheck --shell=sh \ guest/common/nvx-init-agent \ guest/common/nvx-container-enter-azurelinux \ guest/common/nvx-container-launch-azurelinux \ + guest/common/nvx-image-slot \ guest/common/nvx-sandbox-smoke \ guest/common/nvx-snapshot guest/common/nvx-virtio-restore-probe \ scripts/setup/setup-linux-mshv.sh scripts/setup/setup-linux-runner.sh @@ -205,6 +206,7 @@ shfmt -d -ln posix -i 4 -ci \ guest/common/nvx-init-agent \ guest/common/nvx-container-enter-azurelinux \ guest/common/nvx-container-launch-azurelinux \ + guest/common/nvx-image-slot \ guest/common/nvx-sandbox-smoke \ guest/common/nvx-snapshot guest/common/nvx-virtio-restore-probe \ scripts/setup/setup-linux-mshv.sh scripts/setup/setup-linux-runner.sh diff --git a/doc/usage.md b/doc/usage.md index 39a1832e5..4ec8bc029 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -487,6 +487,7 @@ python3 scripts/nvx.py run [--cmdline TEXT] [--restore-snapshot PATH] [--restore-processors {1,2,4,8}] + [--restore-image-slots {1,2,3,4}] [--restore-memory-mib MIB] [--restore-ready-path PATH] [--dry-run] @@ -520,6 +521,7 @@ python3 scripts/nvx.py run | `--cmdline TEXT` | empty | Append kernel parameters; `nvx_*` and `tsc=` tokens are reserved. | | `--restore-snapshot PATH` | none | Restore the immutable machine contract and saved state from a snapshot directory. | | `--restore-processors {1,2,4,8}` | none | Bring this contiguous processor prefix online before restore readiness. Requires an opt-in microVM snapshot and cannot exceed `--processors` capacity. | +| `--restore-image-slots {1,2,3,4}` | none | Activate this contiguous image-slot prefix during gated restore repair. | | `--restore-memory-mib MIB` | none | Select the 128 MiB-aligned RAM target for an expansion-capable snapshot restore. | | `--restore-ready-path PATH` | none | Publish one restore-readiness event to an existing Unix socket or Windows named pipe. | | `--dry-run` | off | Print the generated OpenVMM command without running it. | @@ -548,9 +550,13 @@ error itself names `--cpu-profile host`. ```text python3 scripts/nvx.py sandbox - [{run,provision,start,exec,stop,deprovision}] + [{run,provision,start,exec,query-image-slots,bind-image-slot,stop,deprovision}] [--layer ROLE,PATH,EROFS_UUID]... [--scratch PATH] + [--image-slot-boot-count {1,2,3,4}] + [--image-slot {0,1,2,3}] + [--image-path PATH] + [--image-identity IDENTITY] [--state-dir PATH] [--entrypoint PATH] [--arg VALUE]... @@ -591,9 +597,13 @@ launches. | Option | Default | Description | | --- | --- | --- | -| `{run,provision,start,exec,stop,deprovision}` | `run` | Select a one-shot run or a managed lifecycle operation. | -| `--layer ROLE,PATH,EROFS_UUID` | required for `run` and `provision` | Attach a `distro`, `runtime`, or `custom` EROFS layer. Repeat once per distinct role. | +| `{run,provision,start,exec,query-image-slots,bind-image-slot,stop,deprovision}` | `run` | Select a one-shot run or a managed lifecycle or image-slot operation. | +| `--layer ROLE,PATH,EROFS_UUID` | required for `run`, and for `provision` without `--image-slot-boot-count` | Attach a `distro`, `runtime`, or `custom` EROFS layer. Repeat once per distinct role. | | `--scratch PATH` | required for `run` and `provision` | Attach a preformatted ext4 scratch image as the writable overlay. | +| `--image-slot-boot-count {1,2,3,4}` | none | On `provision`, declare the four image slots and activate this cold-boot prefix. A slot-declaring launch may omit `--layer` and accepts at most one `--mount`. | +| `--image-slot {0,1,2,3}` | none | Select the slot for `bind-image-slot`. | +| `--image-path PATH` | none | Select read-only regular-file media for `bind-image-slot`. | +| `--image-identity IDENTITY` | none | Record the immutable media identity, 1 to 1024 UTF-8 bytes, for idempotent bind recovery. | | `--state-dir PATH` | required for managed operations | Select persistent sandbox state. One-shot `run` rejects this option. | | `--entrypoint PATH` | `/bin/sh` | Select an absolute workload entrypoint without whitespace. | | `--arg VALUE` | none | Append one whitespace-free entrypoint argument. Repeat to pass multiple arguments. | diff --git a/docker/Dockerfile b/docker/Dockerfile index 2dea1185a..94cc525ba 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -110,6 +110,7 @@ install -m 0755 /repo/guest/common/nvx-hostmount \ /repo/guest/common/nvx-identity-probe \ /repo/guest/common/nvx-snapshot \ /repo/guest/common/nvx-virtio-restore-probe \ + /repo/guest/common/nvx-image-slot \ /repo/guest/common/nvx-container-enter-azurelinux /rootfs/sbin/ install -m 0755 /repo/guest/common/nvx-container-launch-azurelinux \ /rootfs/sbin/nvx-container-launch diff --git a/guest/common/nvx-image-slot b/guest/common/nvx-image-slot new file mode 100644 index 000000000..9658c6d84 --- /dev/null +++ b/guest/common/nvx-image-slot @@ -0,0 +1,95 @@ +#!/bin/sh +set -eu + +usage() { + echo "usage: nvx-image-slot --slot 0|1|2|3 [--timeout SECONDS] [--allow-empty]" >&2 + exit 2 +} + +slot= +timeout=30 +allow_empty=false +while [ "$#" -gt 0 ]; do + case "$1" in + --slot) + [ "$#" -ge 2 ] || usage + slot=$2 + shift 2 + ;; + --timeout) + [ "$#" -ge 2 ] || usage + timeout=$2 + shift 2 + ;; + --allow-empty) + allow_empty=true + shift + ;; + *) usage ;; + esac +done + +case "$slot" in 0 | 1 | 2 | 3) ;; *) usage ;; esac +case "$timeout" in '' | *[!0-9]*) usage ;; esac +[ "$timeout" -gt 0 ] || usage + +mmio_address=$((0xd0008000 + slot * 0x1000)) +address_decimal=$(printf '%d' "$mmio_address") +platform= +while IFS=' ' read -r range separator candidate _; do + [ "$separator" = : ] || continue + case "$candidate" in virtio-mmio.*) ;; *) continue ;; esac + start=${range%%-*} + start_decimal=$(printf '%d' "0x$start") || continue + if [ "$start_decimal" -eq "$address_decimal" ]; then + platform=$candidate + break + fi +done &2 + exit 1 +} + +# The virtio-mmio driver binds only an active slot: an inactive slot is a +# placeholder with device ID 0. The window may instead hold the second live +# share, which is not a block device. Neither gains a disk by waiting. +device=/sys/devices/virtio-mmio-cmdline/$platform +[ -e "$device/driver" ] || { + echo "nvx-image-slot: image$slot is inactive" >&2 + exit 1 +} +for virtio in "$device"/virtio*; do + [ -e "$virtio/device" ] || continue + [ "$(cat "$virtio/device")" = 0x0002 ] || { + echo "nvx-image-slot: image$slot transport is not a block device" >&2 + exit 1 + } +done + +deadline=$((timeout * 20)) +attempt=0 +while [ "$attempt" -lt "$deadline" ]; do + for block in "$device"/virtio*/block/*; do + [ -e "$block" ] || continue + name=${block##*/} + [ "$(cat "/sys/class/block/$name/ro")" = 1 ] || { + echo "nvx-image-slot: image$slot is not read-only" >&2 + exit 1 + } + sectors=$(cat "/sys/class/block/$name/size") + if [ "$sectors" -gt 0 ] || [ "$allow_empty" = true ]; then + if [ "$sectors" -gt 0 ]; then + blockdev --rereadpt "/dev/$name" 2>/dev/null || true + fi + ln -sfn "$name" "/dev/nvx-image$slot" + printf '/dev/nvx-image%s\n' "$slot" + exit 0 + fi + done + attempt=$((attempt + 1)) + sleep 0.05 +done + +echo "nvx-image-slot: image$slot capacity remained zero for $timeout seconds" >&2 +exit 1 diff --git a/guest/common/nvx-init-agent b/guest/common/nvx-init-agent index 8322bbd96..078de25e0 100644 --- a/guest/common/nvx-init-agent +++ b/guest/common/nvx-init-agent @@ -353,7 +353,10 @@ for token in $cmdline; do esac done set +f -[ "$layer_count" -gt 0 ] || fatal "at least one EROFS layer is required" +[ "$layer_count" -gt 0 ] || case " $cmdline " in + *" microvm_image_slots=4 "*) ;; + *) fatal "at least one EROFS layer is required" ;; +esac lowerdirs= if [ "$distro" = true ]; then @@ -382,9 +385,14 @@ mount -t ext4 -o rw,nosuid,nodev "$scratch_device" "$scratch" || fatal "failed to mount scratch from $scratch_device" mkdir -p "$scratch/upper" "$scratch/work" -mount -t overlay overlay \ - -o "lowerdir=$lowerdirs,upperdir=$scratch/upper,workdir=$scratch/work,metacopy=on,xino=on" \ - "$rootfs" || fatal "failed to assemble the container overlay" +if [ -n "$lowerdirs" ]; then + mount -t overlay overlay \ + -o "lowerdir=$lowerdirs,upperdir=$scratch/upper,workdir=$scratch/work,metacopy=on,xino=on" \ + "$rootfs" || fatal "failed to assemble the container overlay" +else + mount --bind "$scratch/upper" "$rootfs" || + fatal "failed to expose scratch-only workload root" +fi entrypoint=$(cmdline_value nvx_entrypoint || true) [ -n "$entrypoint" ] || entrypoint=/bin/sh @@ -395,31 +403,36 @@ workload_uid=$(cmdline_value nvx_workload_uid || true) workload_gid=$(cmdline_value nvx_workload_gid || true) case "$workload_uid" in '' | 0 | *[!0-9]*) fatal "workload UID is unavailable" ;; esac case "$workload_gid" in '' | 0 | *[!0-9]*) fatal "workload GID is unavailable" ;; esac -[ -f "$rootfs/etc/passwd" ] && [ ! -L "$rootfs/etc/passwd" ] || - fatal "workload user database is unavailable" -[ -f "$rootfs/etc/group" ] && [ ! -L "$rootfs/etc/group" ] || - fatal "workload group database is unavailable" -workload_record=$( - awk -F: -v uid="$workload_uid" ' - $3 == uid { count++; record = $1 ":" $3 ":" $4 ":" $6 } - END { if (count == 1) print record; else exit 1 } - ' "$rootfs/etc/passwd" -) || fatal "configured workload UID is unavailable" -workload_name=${workload_record%%:*} -workload_remainder=${workload_record#*:} -resolved_uid=${workload_remainder%%:*} -workload_remainder=${workload_remainder#*:} -resolved_gid=${workload_remainder%%:*} -workload_home=${workload_remainder#*:} -[ -n "$workload_name" ] && [ "$workload_home" != "$workload_remainder" ] || - fatal "configured workload user record is invalid" -[ "$resolved_uid" = "$workload_uid" ] && [ "$resolved_gid" = "$workload_gid" ] || - fatal "configured workload identity does not match the user database" -awk -F: -v gid="$workload_gid" '$3 == gid { found = 1 } END { exit !found }' \ - "$rootfs/etc/group" || fatal "configured workload GID is unavailable" -case "$workload_home" in /*) ;; *) fatal "configured workload home is invalid" ;; esac -[ -d "$rootfs$workload_home" ] && [ ! -L "$rootfs$workload_home" ] || - fatal "configured workload home is unavailable" +if [ "$layer_count" -gt 0 ]; then + [ -f "$rootfs/etc/passwd" ] && [ ! -L "$rootfs/etc/passwd" ] || + fatal "workload user database is unavailable" + [ -f "$rootfs/etc/group" ] && [ ! -L "$rootfs/etc/group" ] || + fatal "workload group database is unavailable" + workload_record=$( + awk -F: -v uid="$workload_uid" ' + $3 == uid { count++; record = $1 ":" $3 ":" $4 ":" $6 } + END { if (count == 1) print record; else exit 1 } + ' "$rootfs/etc/passwd" + ) || fatal "configured workload UID is unavailable" + workload_name=${workload_record%%:*} + workload_remainder=${workload_record#*:} + resolved_uid=${workload_remainder%%:*} + workload_remainder=${workload_remainder#*:} + resolved_gid=${workload_remainder%%:*} + workload_home=${workload_remainder#*:} + [ -n "$workload_name" ] && [ "$workload_home" != "$workload_remainder" ] || + fatal "configured workload user record is invalid" + [ "$resolved_uid" = "$workload_uid" ] && [ "$resolved_gid" = "$workload_gid" ] || + fatal "configured workload identity does not match the user database" + awk -F: -v gid="$workload_gid" '$3 == gid { found = 1 } END { exit !found }' \ + "$rootfs/etc/group" || fatal "configured workload GID is unavailable" + case "$workload_home" in /*) ;; *) fatal "configured workload home is invalid" ;; esac + [ -d "$rootfs$workload_home" ] && [ ! -L "$rootfs$workload_home" ] || + fatal "configured workload home is unavailable" +else + workload_name=nvx + workload_home=/ +fi workload_lifecycle=$(cmdline_value nvx_lifecycle || true) [ -n "$workload_lifecycle" ] || workload_lifecycle=one-shot @@ -434,6 +447,20 @@ case "$workload_lifecycle" in esac mount_live_shares +# Link the active empty image slots of a slot-declaring launch; launches +# without image slots skip the probe. +case " $cmdline " in + *" microvm_image_slots=4 "*) image_slots=true ;; + *) image_slots=false ;; +esac +if [ "$image_slots" = true ] && [ -x /sbin/nvx-image-slot ]; then + slot=0 + while [ "$slot" -lt 4 ]; do + /sbin/nvx-image-slot --slot "$slot" --timeout 1 --allow-empty \ + >/dev/null 2>&1 || true + slot=$((slot + 1)) + done +fi workload_machine_id=$(tr -d - "$runtime/workload-machine-id" diff --git a/guest/common/nvx-snapshot b/guest/common/nvx-snapshot index 21c641c89..a214d4c1c 100644 --- a/guest/common/nvx-snapshot +++ b/guest/common/nvx-snapshot @@ -154,6 +154,47 @@ activate_restore_processors() { echo "NVX-CPU-ONLINE-OK: count=$target" } +image_slot_platform() { + slot=$1 + address=$((0xd0008000 + slot * 0x1000)) + while IFS=' ' read -r range separator platform _; do + [ "$separator" = : ] || continue + case "$platform" in virtio-mmio.*) ;; *) continue ;; esac + start=${range%%-*} + start=$(printf '%d' "0x$start") || continue + if [ "$start" -eq "$address" ]; then + printf '%s\n' "$platform" + return 0 + fi + done /dev/null 2>&1; then + bind=/sys/bus/platform/drivers/virtio-mmio/bind + [ -w "$bind" ] || fail_closed "image$slot transport cannot be activated" + printf '%s\n' "$platform" >"$bind" || + fail_closed "failed to activate image$slot" + fi + device=$( + /sbin/nvx-image-slot --slot "$slot" --timeout 5 --allow-empty + ) || fail_closed "image$slot did not become active" + name=$(readlink "$device") || + fail_closed "image$slot stable device link is invalid" + [ "$(cat "/sys/class/block/$name/size")" = 0 ] || + fail_closed "image$slot unexpectedly has media after restore" + slot=$((slot + 1)) + done + echo "NVX-IMAGE-SLOTS-ACTIVE-OK: count=$target" +} + uptime_microseconds() { IFS=' .' read -r uptime_seconds uptime_fraction _ online_vp_count = online; packet->range_count = bytes[6]; + packet->image_slot_target = bytes[7]; packet->generation = read_le32(bytes + 8); packet->rate_deviation = (int32_t)read_le32(bytes + 12); packet->downtime_ns = read_le64(bytes + 16); @@ -1166,8 +1170,26 @@ static int parse_packet_header(const uint8_t *bytes, packet->range_count); return -1; } - if (bytes[7] != 0) { - snprintf(detail, size, "restore packet reserved byte is %u", bytes[7]); + // Header byte 7 is the image-slot target exactly when flag bit 4 is set. + if (((packet->flags & PACKET_IMAGE_SLOT_TARGET) != 0) != + (packet->image_slot_target != 0)) { + snprintf(detail, size, + "restore packet image-slot flag and target %u disagree", + packet->image_slot_target); + return -1; + } + if (packet->image_slot_target > PACKET_MAX_IMAGE_SLOT_TARGET) { + snprintf(detail, size, "restore packet image-slot target is %u", + packet->image_slot_target); + return -1; + } + // The guest verifies the active slots before it acknowledges, so a + // target always holds the restore gate. + if (packet->image_slot_target != 0 && + (packet->flags & PACKET_ACK_REQUIRED) == 0) { + snprintf(detail, size, + "restore packet image-slot target %u is not acknowledged", + packet->image_slot_target); return -1; } if (packet->generation == 0) { @@ -4078,10 +4100,10 @@ static int repair_failed(const char *code, const char *detail) // and set the wall clock from its bracketed UTC, then write the entropy to // ENTROPY_FD, the caller's file at ENTROPY_PATH. START_NS is when the capture // request returned. The caller holds the portb lock. With FINISH (untiered) -// and a DAEMON, a restore with no processors or memory to activate needs no -// shell work before the acknowledgement, so steps 12 and 13 start here too -// and no second helper process starts; the metadata then begins with 2 -// instead of 1. +// and a DAEMON, a restore with no processors, image slots, or memory to +// activate needs no shell work before the acknowledgement, so steps 12 and 13 +// start here too and no second helper process starts; the metadata then +// begins with 2 instead of 1. static int repair_restore(const uint8_t *previous_id, const char *entropy_path, int entropy_fd, int64_t start_ns, bool finish, pid_t daemon) @@ -4156,7 +4178,7 @@ static int repair_restore(const uint8_t *previous_id, const char *entropy_path, // A plain untiered restore needs no shell work: this helper also // acknowledges and hands step 13 to the daemon. finishing = finish && daemon > 0 && packet.online_vp_count == 0 && - packet.range_count == 0 && + packet.range_count == 0 && packet.image_slot_target == 0 && (packet.flags & PACKET_MEMORY_TARGET) == 0; // The entropy goes to the caller in every case, an untiered restore's // too; the file already exists, empty, so only its bytes are written. @@ -4181,8 +4203,9 @@ static int repair_restore(const uint8_t *previous_id, const char *entropy_path, fail_fatal(STATUS_CONFORMANCE, "G_CONFORMANCE_C10", "conformance", PHASE_RESTORE, "the time daemon is not running"); } - printf("%d %u %u %u %s", finished >= 0 ? 2 : 1, packet.flags, - packet.online_vp_count, packet.range_count, id); + printf("%d %u %u %u %u %s", finished >= 0 ? 2 : 1, packet.flags, + packet.online_vp_count, packet.range_count, + packet.image_slot_target, id); for (unsigned index = 0; index < packet.range_count; index++) printf(" %" PRIu64 " %" PRIu64, packet.ranges[index][0], packet.ranges[index][1]); @@ -4516,11 +4539,11 @@ static int test_packet(int argc, char **argv) return 0; } format_hex(packet.entropy, GENERATION_ID_SIZE, id); - printf("ok flags=%u online=%u ranges=%u generation=%" PRIu32 + printf("ok flags=%u online=%u ranges=%u image_slots=%u generation=%" PRIu32 " rate_deviation=%" PRId32 " frequency=%" PRId64 " downtime_ns=%" PRIu64 " utc_ns=%" PRIu64 " generation_id=%s", packet.flags, packet.online_vp_count, packet.range_count, - packet.generation, packet.rate_deviation, + packet.image_slot_target, packet.generation, packet.rate_deviation, restore_frequency(packet.rate_deviation), packet.downtime_ns, packet.utc_ns, id); for (unsigned index = 0; index < packet.range_count; index++) diff --git a/guest/common/nvx-virtio-restore-probe b/guest/common/nvx-virtio-restore-probe index d9491449e..b996aebac 100644 --- a/guest/common/nvx-virtio-restore-probe +++ b/guest/common/nvx-virtio-restore-probe @@ -2,7 +2,7 @@ set -eu usage() { - echo "usage: nvx-virtio-restore-probe --device console|net|virtiofs --mode active|deferred" >&2 + echo "usage: nvx-virtio-restore-probe --device console|net|virtiofs|image0|image1|image2|image3 --mode active|deferred" >&2 exit 2 } @@ -146,6 +146,14 @@ perform_io() { console) perform_console_io ;; net) configure_network ;; virtiofs) perform_virtiofs_io ;; + image*) + slot=${device#image} + image_device=$(/sbin/nvx-image-slot --slot "$slot" --timeout 30) || + fatal image-capacity-timeout + dd if="$image_device" bs=512 count=1 2>/dev/null | + grep -q '^NVX-IMAGE-SLOT-TEST$' || + fatal image-content-invalid + ;; esac } @@ -180,21 +188,32 @@ case "$device" in mmio_address=0xd0001000 expected_device_id=26 ;; + image0 | image1 | image2 | image3) + slot=${device#image} + mmio_address=$((0xd0008000 + slot * 0x1000)) + expected_device_id=2 + ;; *) usage ;; esac case "$mode" in active | deferred) ;; *) usage ;; esac device_path= device_name= -resolve_virtio_device || fatal device-not-found -driver_path=$(readlink -f "$device_path/driver") || fatal driver-not-bound -[ -d "$driver_path" ] || fatal driver-not-found +if ! resolve_virtio_device; then + case "$device:$mode" in image*:deferred) ;; *) fatal device-not-found ;; esac +fi +if [ -n "$device_path" ]; then + driver_path=$(readlink -f "$device_path/driver") || fatal driver-not-bound + [ -d "$driver_path" ] || fatal driver-not-found +else + driver_path= +fi emit_marker "NVX-VIRTIO-RESTORE-PROBE phase=located device=$device mode=$mode mmio=$mmio_address id=$expected_device_id name=$device_name" -perform_io +case "$device" in image*) ;; *) perform_io ;; esac emit_marker "NVX-VIRTIO-RESTORE-PROBE phase=pre-io-success device=$device mode=$mode" -if [ "$mode" = deferred ]; then +if [ "$mode" = deferred ] && [ -n "$device_path" ]; then if [ "$device" = virtiofs ]; then mount_dir=$(cmdline_value virtfs_dir || true) [ -n "$mount_dir" ] || mount_dir=/mnt/host @@ -208,7 +227,7 @@ emit_marker "NVX-VIRTIO-RESTORE-PROBE phase=capture-ready device=$device mode=$m emit_marker "NVX-VIRTIO-RESTORE-PROBE phase=restore-ready device=$device mode=$mode" emit_marker "NVX-VIRTIO-RESTORE-PROBE phase=trigger device=$device mode=$mode" -if [ "$mode" = deferred ]; then +if [ "$mode" = deferred ] && [ -n "$device_path" ]; then [ -c /dev/mem ] || fatal devmem-missing [ -w /dev/mem ] || fatal devmem-not-writable notify_address=$((mmio_address + 0x50)) @@ -220,6 +239,13 @@ if [ "$mode" = deferred ]; then emit_marker "NVX-VIRTIO-RESTORE-PROBE phase=kick-staged device=$device mode=$mode" bind_device fi +case "$device" in + image*) + slot=${device#image} + /sbin/nvx-image-slot --slot "$slot" --timeout 5 --allow-empty >/dev/null || + fatal image-restore-activation-failed + ;; +esac perform_io emit_marker "NVX-VIRTIO-RESTORE-PROBE phase=io-success device=$device mode=$mode" /sbin/nvx-exit 0 diff --git a/kernel/patches/0003-microvm-virtio-mmio-shared-status.patch b/kernel/patches/0003-microvm-virtio-mmio-shared-status.patch index d579cb087..62a9b242c 100644 --- a/kernel/patches/0003-microvm-virtio-mmio-shared-status.patch +++ b/kernel/patches/0003-microvm-virtio-mmio-shared-status.patch @@ -48,7 +48,7 @@ path. /* The alignment to use between consumer and producer parts of vring. -@@ -56,7 +60,92 @@ struct virtio_mmio_device { +@@ -56,7 +60,104 @@ struct virtio_mmio_device { void __iomem *base; unsigned long version; +#ifdef CONFIG_VIRTIO_MMIO_SHARED_STATUS @@ -69,7 +69,10 @@ path. +#define MICROVM_VIRTIO_CUSTOM_BLK_MMIO_BASE 0xd0005000 +#define MICROVM_VIRTIO_SCRATCH_BLK_MMIO_BASE 0xd0006000 +#define MICROVM_VIRTIO_CONTROL_CONSOLE_MMIO_BASE 0xd0007000 -+#define MICROVM_VIRTIO_FS1_MMIO_BASE 0xd0008000 ++#define MICROVM_VIRTIO_FS1_MMIO_BASE 0xd0008000 /* or image slot 0 */ ++#define MICROVM_VIRTIO_IMAGE1_BLK_MMIO_BASE 0xd0009000 ++#define MICROVM_VIRTIO_IMAGE2_BLK_MMIO_BASE 0xd000a000 ++#define MICROVM_VIRTIO_IMAGE3_BLK_MMIO_BASE 0xd000b000 + +static int vm_setup_shared_status(struct virtio_mmio_device *vm_dev) +{ @@ -113,6 +116,15 @@ path. + case MICROVM_VIRTIO_FS1_MMIO_BASE: + offset = 0x20; + break; ++ case MICROVM_VIRTIO_IMAGE1_BLK_MMIO_BASE: ++ offset = 0x24; ++ break; ++ case MICROVM_VIRTIO_IMAGE2_BLK_MMIO_BASE: ++ offset = 0x28; ++ break; ++ case MICROVM_VIRTIO_IMAGE3_BLK_MMIO_BASE: ++ offset = 0x2c; ++ break; + default: + dev_err(&pdev->dev, + "shared-status ABI has no slot for MMIO base %#llx\n", @@ -141,7 +153,7 @@ path. /* Configuration interface */ static u64 vm_get_features(struct virtio_device *vdev) -@@ -291,8 +376,16 @@ static irqreturn_t vm_interrupt(int irq, void *opaque) +@@ -291,8 +388,16 @@ static irqreturn_t vm_interrupt(int irq, void *opaque) irqreturn_t ret = IRQ_NONE; /* Read and acknowledge interrupts */ @@ -158,7 +170,7 @@ path. if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)) { virtio_config_changed(&vm_dev->vdev); -@@ -594,6 +687,10 @@ static int virtio_mmio_probe(struct platform_device *pdev) +@@ -594,6 +699,10 @@ static int virtio_mmio_probe(struct platform_device *pdev) vm_dev->vdev.config = &virtio_mmio_config_ops; vm_dev->pdev = pdev; @@ -169,7 +181,7 @@ path. vm_dev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(vm_dev->base)) { rc = PTR_ERR(vm_dev->base); -@@ -817,4 +914,11 @@ static struct platform_driver virtio_mmio_driver = { +@@ -817,4 +926,11 @@ static struct platform_driver virtio_mmio_driver = { static int __init virtio_mmio_init(void) { +#ifdef CONFIG_VIRTIO_MMIO_SHARED_STATUS diff --git a/openvmm b/openvmm index f4566bab0..67b641d84 160000 --- a/openvmm +++ b/openvmm @@ -1 +1 @@ -Subproject commit f4566bab0a30324de0a2ac292a65d0081ba319e0 +Subproject commit 67b641d849619513e86128d718767b8b82bf8136 diff --git a/scripts/nvx.py b/scripts/nvx.py index dea9cfc0d..56997f9b6 100644 --- a/scripts/nvx.py +++ b/scripts/nvx.py @@ -454,6 +454,8 @@ def command_run(args: argparse.Namespace) -> None: raise ScriptError("--restore-ready-path requires --restore-snapshot") if args.restore_processors is not None and args.restore_snapshot is None: raise ScriptError("--restore-processors requires --restore-snapshot") + if args.restore_image_slots is not None and args.restore_snapshot is None: + raise ScriptError("--restore-image-slots requires --restore-snapshot") if args.restore_memory_mib is not None and args.restore_snapshot is None: raise ScriptError("--restore-memory-mib requires --restore-snapshot") if args.memory_capacity_mib is not None and args.restore_snapshot is not None: @@ -489,6 +491,8 @@ def command_run(args: argparse.Namespace) -> None: command.extend(["--restore-snapshot", str(args.restore_snapshot)]) if args.restore_processors is not None: command.extend(["--restore-processors", str(args.restore_processors)]) + if args.restore_image_slots is not None: + command.extend(["--restore-image-slots", str(args.restore_image_slots)]) if args.restore_memory_mib is not None: command.extend(["--restore-memory", f"{args.restore_memory_mib}M"]) if args.restore_ready_path is not None: @@ -652,11 +656,29 @@ def command_sandbox(args: argparse.Namespace) -> None: raise ScriptError("--mount-owner requires --mount") if args.mount and operation not in ("run", "provision"): raise ScriptError("--mount is only valid for sandbox run or provision") + image_operation_options = ( + args.image_slot, + args.image_path, + args.image_identity, + ) + if operation != "bind-image-slot" and any( + value is not None for value in image_operation_options + ): + raise ScriptError( + "--image-slot, --image-path, and --image-identity require " + "sandbox bind-image-slot" + ) + if args.image_slot_boot_count is not None and operation != "provision": + raise ScriptError("--image-slot-boot-count is only valid for sandbox provision") if operation in ("run", "provision"): if (args.net is None) != (args.network_profile is None): raise ScriptError("--net and --network-profile must be specified together") - if not args.layer or args.scratch is None: - raise ScriptError(f"sandbox {operation} requires --layer and --scratch") + if args.scratch is None: + raise ScriptError(f"sandbox {operation} requires --scratch") + if not args.layer and args.image_slot_boot_count is None: + raise ScriptError( + f"sandbox {operation} requires --layer unless image slots are declared" + ) network_egress_allow, network_egress_deny = _resolve_network_egress_rules(args) launch = SandboxLaunch( layers=tuple(args.layer), @@ -668,6 +690,7 @@ def command_sandbox(args: argparse.Namespace) -> None: memory_max=args.memory_max, pids_max=args.pids_max, mounts=_sandbox_mounts(args), + image_slot_boot_count=args.image_slot_boot_count, ).validated() _validate_sandbox_systemd_policy(launch) else: @@ -722,6 +745,32 @@ def command_sandbox(args: argparse.Namespace) -> None: if args.outcome_report is not None: sandbox_lifecycle.write_exec_outcome(args.outcome_report, result) raise SystemExit(result.returncode) + if operation == "query-image-slots": + if args.state_dir is None: + raise ScriptError("sandbox query-image-slots requires --state-dir") + slots = sandbox_lifecycle.query_image_slots(args.state_dir, args.timeout) + print(json.dumps(slots, indent=2, sort_keys=True)) + return + if operation == "bind-image-slot": + if args.state_dir is None: + raise ScriptError("sandbox bind-image-slot requires --state-dir") + if ( + args.image_slot is None + or args.image_path is None + or args.image_identity is None + ): + raise ScriptError( + "sandbox bind-image-slot requires --image-slot, --image-path, " + "and --image-identity" + ) + sandbox_lifecycle.bind_image_slot( + args.state_dir, + args.image_slot, + args.image_path, + args.image_identity, + args.timeout, + ) + return if operation == "stop": if args.state_dir is None: raise ScriptError("sandbox stop requires --state-dir") @@ -1139,6 +1188,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: run.add_argument("--cmdline", default="") run.add_argument("--restore-snapshot", type=Path) run.add_argument("--restore-processors", type=int, choices=(1, 2, 4, 8)) + run.add_argument("--restore-image-slots", type=int, choices=(1, 2, 3, 4)) run.add_argument("--restore-memory-mib", type=int) run.add_argument("--restore-ready-path", type=Path) run.add_argument("--dry-run", action="store_true") @@ -1164,7 +1214,16 @@ def sandbox_identity(value: str) -> tuple[int, int]: sandbox.add_argument( "sandbox_operation", nargs="?", - choices=("run", "provision", "start", "exec", "stop", "deprovision"), + choices=( + "run", + "provision", + "start", + "exec", + "query-image-slots", + "bind-image-slot", + "stop", + "deprovision", + ), default="run", ) sandbox.add_argument( @@ -1175,6 +1234,15 @@ def sandbox_identity(value: str) -> tuple[int, int]: metavar="ROLE,PATH,EROFS_UUID", ) sandbox.add_argument("--scratch", type=Path) + sandbox.add_argument( + "--image-slot-boot-count", + type=int, + choices=(1, 2, 3, 4), + help="declare four image slots and activate this prefix at cold boot", + ) + sandbox.add_argument("--image-slot", type=int, choices=(0, 1, 2, 3)) + sandbox.add_argument("--image-path", type=Path) + sandbox.add_argument("--image-identity") sandbox.add_argument("--state-dir", type=Path) sandbox.add_argument("--entrypoint", default="/bin/sh") sandbox.add_argument("--arg", action="append", default=[], dest="sandbox_arg") diff --git a/scripts/nvx_tools/benchmark.py b/scripts/nvx_tools/benchmark.py index a39be7f3e..b0fb9815d 100644 --- a/scripts/nvx_tools/benchmark.py +++ b/scripts/nvx_tools/benchmark.py @@ -18,6 +18,7 @@ import os import queue import re +import secrets import select import selectors import shutil @@ -42,6 +43,7 @@ OpenVMMBuildConstants, ) from .common import bytes_to_mib, sha256_file +from .control_session import capability_pipe from .time_abi import TimeAbiFailure, TimeAbiMonitor, status_script BOOT_MARKER = b"ALPINE-MICROVM-BOOT-OK" @@ -74,6 +76,7 @@ { "cold-start", "device-io", + "image-slot-boot", "virtfs", "shell-snapshot", "shell-snapshot-restore", @@ -3098,6 +3101,170 @@ def benchmark_cold_start_workload( print(f" {label:<25}: {format_sample_summary(result['samples_ms'])}") +def _image_slot_boot_sample( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + memory_mib: int, + processors: int, + boot_count: int | None, + timeout: float, + *, + command_prefix: Sequence[str], + windows_cpus: set[int] | None, +) -> float: + with tempfile.TemporaryDirectory(prefix="nvx-image-slot-boot-") as temporary: + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + console_address = cast(tuple[str, int], listener.getsockname()) + listener.close() + command = workload_boot_command( + executable, + backend, + kernel, + initrd, + memory_mib, + "quiet loglevel=0", + processors=processors, + command_prefix=command_prefix, + ) + command.extend( + ( + "--virtio-console", + f"listen=tcp:{console_address[0]}:{console_address[1]}", + ) + ) + capability: bytes | None = None + if boot_count is not None: + endpoint = ( + "//./pipe/openvmm-image-slot-benchmark-" + f"{Path(temporary).name}-{secrets.token_hex(16)}" + if os.name == "nt" + else os.fspath(Path(temporary) / "host-control.sock") + ) + capability = secrets.token_bytes(32) + command.extend( + ( + "--microvm-image-slots", + "--microvm-image-slot-boot-count", + str(boot_count), + "--microvm-host-control", + f"listen={endpoint}", + "--microvm-control-auth-stdin", + ) + ) + capability_input: int | None = None + if capability is not None: + # OpenVMM reads its capability as soon as it starts. + capability_input = capability_pipe(capability) + stdin = subprocess.DEVNULL if capability_input is None else capability_input + started = time.perf_counter_ns() + try: + process = subprocess.Popen( + command, + stdin=stdin, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + finally: + if capability_input is not None: + os.close(capability_input) + try: + if windows_cpus is not None: + set_windows_affinity(process.pid, windows_cpus) + deadline = time.monotonic() + timeout + while True: + try: + console = socket.create_connection(console_address, timeout=0.25) + break + except OSError as error: + if time.monotonic() >= deadline: + raise TimeoutError( + "image-slot benchmark console did not become available" + ) from error + with console: + output = bytearray() + while BOOT_MARKER not in output: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("image-slot benchmark boot marker timed out") + console.settimeout(min(remaining, 0.25)) + try: + chunk = console.recv(4096) + except TimeoutError: + continue + if not chunk: + raise RuntimeError( + "image-slot benchmark console closed before readiness" + ) + output.extend(chunk) + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 + console.sendall(b"/sbin/nvx-exit 0\n") + # Closing the console before the guest consumes the exit + # command discards it, so keep it open until OpenVMM exits. + if process.wait(timeout=timeout) != 0: + raise RuntimeError("image-slot benchmark VM exited unsuccessfully") + return elapsed_ms + finally: + if process.poll() is None: + terminate(process) + + +def benchmark_image_slot_boot_workload( + args: argparse.Namespace, + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + *, + command_prefix: Sequence[str] = (), + windows_cpus: set[int] | None = None, +) -> None: + results: dict[str, list[float]] = {} + for label, boot_count in (("no-slots", None), ("slots-b1", 1), ("slots-b4", 4)): + samples: list[float] = [] + for index in range(args.warmups + args.runs): + elapsed = _image_slot_boot_sample( + executable, + kernel, + initrd, + backend, + args.memory_mib, + args.processors, + boot_count, + args.timeout, + command_prefix=command_prefix, + windows_cpus=windows_cpus, + ) + if index >= args.warmups: + samples.append(elapsed) + print( + "NVX_IMAGE_SLOT_BOOT_RESULT=" + + json.dumps( + { + "schema_version": 1, + "backend": backend, + "configuration": label, + "warmup": index < args.warmups, + "elapsed_ms": elapsed, + }, + separators=(",", ":"), + sort_keys=True, + ) + ) + results[label] = samples + baseline = statistics.median(results["no-slots"]) + print("image-slot cold boot:") + for label in ("no-slots", "slots-b1", "slots-b4"): + median = statistics.median(results[label]) + delta = median - baseline + print( + f" {label:<8}: {format_sample_summary(results[label])}; " + f"delta-vs-no-slots={delta:+.3f} ms" + ) + + def _format_rate_summary(samples: Sequence[float]) -> str: if not samples: raise ValueError("cannot summarize an empty throughput sample set") @@ -4614,6 +4781,18 @@ def run_workload_benchmarks( windows_cpus=windows_cpus, ), ), + "image-slot-boot": ( + "image-slot-boot.log", + lambda: benchmark_image_slot_boot_workload( + args, + executable, + kernel, + initrd, + backend, + command_prefix=command_prefix, + windows_cpus=windows_cpus, + ), + ), "snapshot-restore-memory": ( "snapshot-restore-memory.log", lambda: benchmark_snapshot_restore_memory_workload( @@ -5047,6 +5226,7 @@ def snapshot_restore_command( *, processors: int = 1, restore_processors: int | None = None, + restore_image_slots: int | None = None, restore_memory_mib: int | None = None, network_profile: str | None = None, ) -> list[str]: @@ -5064,6 +5244,8 @@ def snapshot_restore_command( ] if restore_processors is not None: command.extend(("--restore-processors", str(restore_processors))) + if restore_image_slots is not None: + command.extend(("--restore-image-slots", str(restore_image_slots))) if restore_memory_mib is not None: command.extend(("--restore-memory", f"{restore_memory_mib}M")) if network_profile is not None: diff --git a/scripts/nvx_tools/build.py b/scripts/nvx_tools/build.py index dd917075b..54246d493 100644 --- a/scripts/nvx_tools/build.py +++ b/scripts/nvx_tools/build.py @@ -817,6 +817,11 @@ def _install_guest_files( common / "nvx-virtio-restore-probe", root / "sbin" / "nvx-virtio-restore-probe", ), + ( + "nvx-image-slot", + common / "nvx-image-slot", + root / "sbin" / "nvx-image-slot", + ), ] if descriptor.sandbox_control: alpine = BuildConstants.REPO_ROOT / AlpineBuildConstants.GUEST_DIRECTORY diff --git a/scripts/nvx_tools/control_session.py b/scripts/nvx_tools/control_session.py index 624d6188d..97a5d0130 100644 --- a/scripts/nvx_tools/control_session.py +++ b/scripts/nvx_tools/control_session.py @@ -10,7 +10,7 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Any, cast +from typing import Any, Protocol, cast from .common import ScriptError, remaining_timeout @@ -294,8 +294,24 @@ def close(self) -> None: os.close(self._fd) +class LocalStream(Protocol): + def read_exact(self, length: int, deadline: float) -> bytes: ... + + def write_all(self, data: bytes) -> None: ... + + def close(self) -> None: ... + + +def connect_local_stream(endpoint: Path, timeout: float) -> LocalStream: + return ( + _NamedPipeStream.connect(endpoint, timeout) + if os.name == "nt" + else _SocketStream.connect(endpoint, timeout) + ) + + class ControlSession: - def __init__(self, stream: _SocketStream | _NamedPipeStream) -> None: + def __init__(self, stream: LocalStream) -> None: self._stream = stream self._instance_id = bytes(16) self._epoch = 0 @@ -311,11 +327,7 @@ def connect( ) -> ControlSession: if len(capability) != 32 or capability == bytes(32): raise ValueError("control capability must be 32 nonzero bytes") - stream = ( - _NamedPipeStream.connect(endpoint, timeout) - if os.name == "nt" - else _SocketStream.connect(endpoint, timeout) - ) + stream = connect_local_stream(endpoint, timeout) session = cls(stream) try: session._write_outer( diff --git a/scripts/nvx_tools/host_control.py b/scripts/nvx_tools/host_control.py new file mode 100644 index 000000000..40897493d --- /dev/null +++ b/scripts/nvx_tools/host_control.py @@ -0,0 +1,98 @@ +"""Client for OpenVMM's authenticated local host-control protocol.""" + +from __future__ import annotations + +import json +import struct +import time +from pathlib import Path +from typing import Any, cast + +from .common import ScriptError +from .control_session import connect_local_stream + +FRAME_HEADER = struct.Struct(" None: + if len(capability) != 32 or capability == bytes(32): + raise ValueError("control capability must be 32 nonzero bytes") + self._stream = connect_local_stream(endpoint, timeout) + try: + self._stream.write_all(capability) + except BaseException: + self._stream.close() + raise + self._timeout = timeout + self._next_request_id = 1 + + def __enter__(self) -> HostControl: + return self + + def __exit__(self, *_: object) -> None: + self._stream.close() + + @classmethod + def connect(cls, endpoint: Path, capability: bytes, timeout: float) -> HostControl: + return cls(endpoint, capability, timeout) + + def request(self, operation: str, **arguments: object) -> dict[str, Any]: + request_id = self._next_request_id + self._next_request_id += 1 + request = { + "version": 1, + "request_id": request_id, + "operation": operation, + **arguments, + } + payload = json.dumps(request, separators=(",", ":")).encode("utf-8") + if len(payload) > MAX_FRAME_SIZE: + raise ValueError("host-control request exceeds the protocol limit") + self._stream.write_all(FRAME_HEADER.pack(len(payload)) + payload) + deadline = time.monotonic() + self._timeout + (length,) = FRAME_HEADER.unpack( + self._stream.read_exact(FRAME_HEADER.size, deadline) + ) + if length > MAX_FRAME_SIZE: + raise ScriptError("host-control response exceeds the protocol limit") + raw = self._stream.read_exact(length, deadline) + try: + value: object = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ScriptError("host-control returned invalid JSON") from error + if not isinstance(value, dict): + raise ScriptError("host-control returned a non-object response") + response = cast(dict[str, Any], value) + if response.get("version") != 1 or response.get("request_id") != request_id: + raise ScriptError("host-control returned a mismatched response") + if response.get("status") != "ok": + code = response.get("code", "unknown") + message = response.get("message", "host-control operation failed") + raise ScriptError(f"host-control {operation} failed ({code}): {message}") + result = response.get("result") + if not isinstance(result, dict): + raise ScriptError("host-control returned an invalid success response") + return cast(dict[str, Any], result) + + def query_image_slots(self) -> list[dict[str, Any]]: + response = self.request("query_image_slots") + slots = response.get("slots") + if not isinstance(slots, list): + raise ScriptError("host-control returned an invalid image-slot table") + typed_slots = cast(list[object], slots) + result: list[dict[str, Any]] = [] + for slot in typed_slots: + if not isinstance(slot, dict): + raise ScriptError("host-control returned an invalid image-slot table") + result.append(cast(dict[str, Any], slot)) + return result + + def bind_image_slot(self, slot: int, path: Path, identity: str) -> None: + self.request( + "bind_image_slot", + slot=slot, + path=str(path), + identity=identity, + ) diff --git a/scripts/nvx_tools/microvm_tests.py b/scripts/nvx_tools/microvm_tests.py index c858cf97e..9bccf41c0 100644 --- a/scripts/nvx_tools/microvm_tests.py +++ b/scripts/nvx_tools/microvm_tests.py @@ -17,8 +17,8 @@ import threading import time import uuid -from collections.abc import Sequence -from contextlib import ExitStack +from collections.abc import Generator, Sequence +from contextlib import ExitStack, contextmanager from dataclasses import dataclass from pathlib import Path from stat import S_ISUID @@ -62,6 +62,7 @@ from .control_session import ControlSession, ManagedExecRefused, capability_pipe from .egress_policy import CompiledEgressPolicy, compile_policy_file from .guests import GUEST_NAMES, GuestDescriptor, guest_descriptor +from .host_control import HostControl from .managed_exec_tests import run_managed_exec_configuration from .openvmm_process import OpenvmmProcess, TcpConsole from .sandbox_lifecycle_tests import run_sandbox_lifecycle @@ -101,6 +102,7 @@ "guest-boot", "guest-identity", "host-loopback-policy", + "image-slots", "lifecycle", "l3-l4-egress-policy", "l3-l4-egress-port-ranges", @@ -239,6 +241,8 @@ ) MANAGED_REFUSED_CWD_MARKER = "/tmp/nvx-refused-cwd-ran" BOOT_MARKER = b"NVX-GUEST-BOOT-OK:" +IMAGE_SLOT_READ_MARKER = b"NVX-IMAGE-SLOT-READ-OK" +IMAGE_SLOT_RESTORE_MARKER = b"NVX-IMAGE-SLOT-RESTORE-OK" GUEST_BOOT_COMPLETION_MARKER = b"NVX-GUEST-BOOT-CHECK-OK" GUEST_IDENTITY_COMPLETION_MARKER = b"NVX-GUEST-IDENTITY-OK" RESTORE_PROCESSORS_FAILURE_MARKER = b"NVX-RESTORE-PROCESSORS-FAIL" @@ -1131,6 +1135,260 @@ def run_managed_lifecycle( ) +def _host_control_endpoint(root: Path, name: str) -> Path: + return Path( + f"//./pipe/openvmm-{name}-{uuid.uuid4().hex}" + if os.name == "nt" + else root / f"{name}.sock" + ) + + +def _image_slot_command( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + memory_mib: int, + endpoint: Path, + console_address: tuple[str, int], + *, + snapshot_path: Path | None = None, + restore_image_slots: int | None = None, +) -> list[str]: + if snapshot_path is None: + command = workload_boot_command( + executable, + backend, + kernel, + initrd, + memory_mib, + "quiet loglevel=0", + ) + command.extend( + ( + "--microvm-image-slots", + "--microvm-image-slot-boot-count", + "1", + ) + ) + else: + command = snapshot_restore_command( + executable, + backend, + snapshot_path, + restore_image_slots=restore_image_slots, + ) + command.extend( + ( + "--virtio-console", + f"listen=tcp:{console_address[0]}:{console_address[1]}", + "--microvm-host-control", + f"listen={endpoint}", + "--microvm-control-auth-stdin", + ) + ) + return command + + +def _write_image_slot_media(path: Path, label: str) -> None: + payload = f"NVX-IMAGE-SLOT-TEST {label}\n".encode() + path.write_bytes(payload + bytes(4096 - len(payload))) + + +@contextmanager +def _image_slot_process( + command: Sequence[str], + capability: bytes, + log_path: Path, +) -> Generator[subprocess.Popen[bytes]]: + environment = os.environ.copy() + environment["OPENVMM_LOG"] = "off" + with log_path.open("wb") as log: + # OpenVMM reads its capability as soon as it starts. + capability_input = capability_pipe(capability) + try: + process = subprocess.Popen( + command, + stdin=capability_input, + stdout=log, + stderr=subprocess.STDOUT, + env=environment, + ) + finally: + os.close(capability_input) + record_adversarial_openvmm_pid(process.pid, environment) + try: + yield process + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def run_image_slots( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + *, + memory_mib: int, + timeout: float, + output_dir: Path, +) -> None: + with tempfile.TemporaryDirectory(prefix="nvx-image-slots-") as temporary: + root = Path(temporary) + image_a = root / "image-a.raw" + _write_image_slot_media(image_a, "A") + # After the restore activates every slot, slots 1 to 3 each get their + # own media, so each slot's IRQ and shared-status word must deliver its + # configuration change before the guest can read it. + restored_media = {1: "B", 2: "C", 3: "D"} + for label in restored_media.values(): + _write_image_slot_media(root / f"image-{label.lower()}.raw", label) + # Valid media, so that only the bind-once rule can refuse slot 0's + # different-identity bind. + image_b = root / "image-b.raw" + + endpoint = _host_control_endpoint(root, "image-slots") + console_address = _available_tcp_address() + capability = secrets.token_bytes(32) + command = _image_slot_command( + executable, + kernel, + initrd, + backend, + memory_mib, + endpoint, + console_address, + ) + with _image_slot_process( + command, capability, output_dir / "image-slots-bind.log" + ) as process: + # The guest's shell is on this virtio console, so it answers the + # cold boot's status query there. + console = TcpConsole.connect( + console_address, + timeout, + monitor=TimeAbiMonitor(command), + time_abi_status=True, + ) + console.wait_for(BOOT_MARKER, timeout) + with HostControl.connect(endpoint, capability, timeout) as control: + slots = control.query_image_slots() + if [slot["state"] for slot in slots] != [ + "empty", + "inactive", + "inactive", + "inactive", + ]: + raise RuntimeError(f"unexpected cold-boot image slots: {slots!r}") + control.bind_image_slot(0, image_a, "image-a") + control.bind_image_slot(0, image_a, "image-a") + try: + control.bind_image_slot(0, image_b, "image-b") + except ScriptError as error: + if "(already_bound)" not in str(error): + raise + else: + raise RuntimeError("a second different image-slot bind succeeded") + console.send_line( + "d=$(/sbin/nvx-image-slot --slot 0 --timeout 10); " + "head -c 22 \"$d\" | grep -q '^NVX-IMAGE-SLOT-TEST A' && " + "echo NVX-IMAGE-SLOT-READ-OK; /sbin/nvx-exit 0" + ) + console.wait_for_line(IMAGE_SLOT_READ_MARKER, timeout) + result = process.wait(timeout=timeout) + (output_dir / "image-slots-bind-guest.log").write_bytes(console.finish()) + if result != 0: + raise RuntimeError("image-slot read VM exited unsuccessfully") + + snapshot = root / "snapshot" + capture_endpoint = _host_control_endpoint(root, "image-slots-capture") + capture_console_address = _available_tcp_address() + capture_capability = secrets.token_bytes(32) + capture_command = _image_slot_command( + executable, + kernel, + initrd, + backend, + memory_mib, + capture_endpoint, + capture_console_address, + ) + capture_command.extend(("--snapshot-destination", str(snapshot))) + with _image_slot_process( + capture_command, + capture_capability, + output_dir / "image-slots-capture.log", + ) as process: + console = TcpConsole.connect( + capture_console_address, + timeout, + monitor=TimeAbiMonitor(capture_command), + time_abi_status=True, + ) + console.wait_for(BOOT_MARKER, timeout) + console.send_line("/sbin/nvx-snapshot") + result = process.wait(timeout=timeout) + (output_dir / "image-slots-capture-guest.log").write_bytes(console.finish()) + if result != 0: + raise RuntimeError("empty image-slot snapshot capture failed") + + restore_endpoint = _host_control_endpoint(root, "image-slots-restore") + restore_console_address = _available_tcp_address() + restore_capability = secrets.token_bytes(32) + restore_command = _image_slot_command( + executable, + kernel, + initrd, + backend, + memory_mib, + restore_endpoint, + restore_console_address, + snapshot_path=snapshot, + restore_image_slots=4, + ) + with _image_slot_process( + restore_command, + restore_capability, + output_dir / "image-slots-restore.log", + ) as process: + console = TcpConsole.connect( + restore_console_address, + timeout, + monitor=TimeAbiMonitor(restore_command), + ) + console.wait_for(b"NVX-POST-RESTORE-OK:", timeout) + with HostControl.connect( + restore_endpoint, restore_capability, timeout + ) as control: + slots = control.query_image_slots() + if [slot["state"] for slot in slots] != ["empty"] * 4: + raise RuntimeError(f"unexpected restored image slots: {slots!r}") + for slot, label in restored_media.items(): + control.bind_image_slot( + slot, + root / f"image-{label.lower()}.raw", + f"image-{label.lower()}", + ) + console.send_line( + "ok=1; for p in " + + " ".join(f"{slot}:{label}" for slot, label in restored_media.items()) + + "; do d=$(/sbin/nvx-image-slot --slot ${p%%:*} --timeout 10) && " + 'head -c 22 "$d" | grep -q "^NVX-IMAGE-SLOT-TEST ${p#*:}" || ok=0; ' + 'done; [ "$ok" = 1 ] && echo NVX-IMAGE-SLOT-RESTORE-OK; /sbin/nvx-exit 0' + ) + console.wait_for_line(IMAGE_SLOT_RESTORE_MARKER, timeout) + result = process.wait(timeout=timeout) + (output_dir / "image-slots-restore-guest.log").write_bytes(console.finish()) + if result != 0: + raise RuntimeError("restored image-slot VM exited unsuccessfully") + + def run_structured_outcome( executable: Path, kernel: Path, @@ -6333,6 +6591,17 @@ def run(args: argparse.Namespace) -> int: timeout=args.timeout, output_dir=output_dir, ) + if "image-slots" in scenarios: + print(f"Running microVM image-slot correctness on OpenVMM/{args.backend}") + run_image_slots( + executable, + kernel, + initrd, + args.backend, + memory_mib=args.memory_mib, + timeout=args.timeout, + output_dir=output_dir, + ) if "structured-outcome" in scenarios: print(f"Running structured microVM outcomes on OpenVMM/{args.backend}") run_structured_outcome( diff --git a/scripts/nvx_tools/sandbox.py b/scripts/nvx_tools/sandbox.py index c986b6bff..6eaf1ceb9 100644 --- a/scripts/nvx_tools/sandbox.py +++ b/scripts/nvx_tools/sandbox.py @@ -377,10 +377,15 @@ class SandboxLaunch: memory_max: int | None = None pids_max: int | None = None mounts: tuple[SandboxMount, ...] = () + image_slot_boot_count: int | None = None def __post_init__(self) -> None: - if not 1 <= len(self.layers) <= len(LAYER_ROLES): - raise ScriptError("a sandbox requires one to three read-only layers") + minimum_layers = 0 if self.image_slot_boot_count is not None else 1 + if not minimum_layers <= len(self.layers) <= len(LAYER_ROLES): + raise ScriptError( + "a sandbox requires one to three read-only layers unless it " + "declares image slots" + ) roles = [layer.role for layer in self.layers] duplicates = sorted({role for role in roles if roles.count(role) > 1}) if duplicates: @@ -412,6 +417,14 @@ def __post_init__(self) -> None: if value is not None and value <= 0: raise ScriptError(f"--{name} must be positive") validate_mounts(self.mounts) + if self.image_slot_boot_count is not None: + if not 1 <= self.image_slot_boot_count <= 4: + raise ScriptError("--image-slot-boot-count must be between 1 and 4") + # Image slot 0 uses the transport of the second live share. + if len(self.mounts) > 1: + raise ScriptError( + "a sandbox with image slots permits at most one --mount" + ) def validated(self) -> SandboxLaunch: for layer in self.layers: @@ -446,6 +459,14 @@ def openvmm_arguments(self) -> list[str]: ) ) arguments.extend(mounts_openvmm_arguments(self.mounts)) + if self.image_slot_boot_count is not None: + arguments.extend( + ( + "--microvm-image-slots", + "--microvm-image-slot-boot-count", + str(self.image_slot_boot_count), + ) + ) return arguments def kernel_command_line(self, user_command_line: str = "") -> str: diff --git a/scripts/nvx_tools/sandbox_lifecycle.py b/scripts/nvx_tools/sandbox_lifecycle.py index f2594f08a..9adfb775d 100644 --- a/scripts/nvx_tools/sandbox_lifecycle.py +++ b/scripts/nvx_tools/sandbox_lifecycle.py @@ -29,6 +29,7 @@ ManagedExecResult, capability_pipe, ) +from .host_control import HostControl from .sandbox import SandboxLaunch, SandboxLayer, SandboxMount CONFIG_NAME = "config.json" @@ -36,6 +37,7 @@ CAPABILITY_NAME = "control.capability" LOG_NAME = "openvmm.log" CONTROL_SOCKET_NAME = "control.sock" +HOST_CONTROL_SOCKET_NAME = "host-control.sock" OUTCOME_NAME = "outcome.json" STATE_FORMAT = 1 CONFIG_FORMAT = 1 @@ -52,13 +54,26 @@ # configuration with either lists every share under `mounts` in a format that # they reject. POLICY_CONFIG_FORMAT = 5 +# Earlier readers would start a sandbox without its image slots, so a +# configuration that declares them lists every share under `mounts`, with its +# policy, in a format that they reject. +IMAGE_SLOT_CONFIG_FORMAT = 6 CONFIG_FORMATS = ( CONFIG_FORMAT, MOUNT_CONFIG_FORMAT, OWNER_CONFIG_FORMAT, MULTI_MOUNT_CONFIG_FORMAT, POLICY_CONFIG_FORMAT, + IMAGE_SLOT_CONFIG_FORMAT, ) +# These formats list every share under `mounts`. +MOUNTS_CONFIG_FORMATS = ( + MULTI_MOUNT_CONFIG_FORMAT, + POLICY_CONFIG_FORMAT, + IMAGE_SLOT_CONFIG_FORMAT, +) +# These formats also carry each share's allowed and writable paths. +MOUNT_POLICY_CONFIG_FORMATS = (POLICY_CONFIG_FORMAT, IMAGE_SLOT_CONFIG_FORMAT) OUTCOME_SCHEMA_VERSION = 1 @@ -199,7 +214,7 @@ def _serialize_launch( cmdline: str, ) -> dict[str, Any]: config: dict[str, Any] = { - "format": _config_format(launch.mounts), + "format": _config_format(launch.mounts, launch.image_slot_boot_count), "layers": [ { "role": layer.role, @@ -227,18 +242,24 @@ def _serialize_launch( "host_loopback_forward": list(host_loopback_forward), "cmdline": cmdline, } - config_format = _config_format(launch.mounts) - if config_format in (MULTI_MOUNT_CONFIG_FORMAT, POLICY_CONFIG_FORMAT): + config_format = config["format"] + if config_format in MOUNTS_CONFIG_FORMATS: config["mounts"] = [ - _serialize_mount(mount, policy=config_format == POLICY_CONFIG_FORMAT) + _serialize_mount(mount, policy=config_format in MOUNT_POLICY_CONFIG_FORMATS) for mount in launch.mounts ] else: config["mount"] = _serialize_mount(launch.mounts[0]) if launch.mounts else None + if config_format == IMAGE_SLOT_CONFIG_FORMAT: + config["image_slot_boot_count"] = launch.image_slot_boot_count return config -def _config_format(mounts: tuple[SandboxMount, ...]) -> int: +def _config_format( + mounts: tuple[SandboxMount, ...], image_slot_boot_count: int | None = None +) -> int: + if image_slot_boot_count is not None: + return IMAGE_SLOT_CONFIG_FORMAT if any(mount.allowed_paths or mount.writable_paths for mount in mounts): return POLICY_CONFIG_FORMAT if len(mounts) > 1: @@ -287,11 +308,11 @@ def _deserialize_mount(value: object, *, policy: bool = False) -> SandboxMount: def _deserialize_mounts(config: dict[str, Any]) -> tuple[SandboxMount, ...]: - if config.get("format") in (MULTI_MOUNT_CONFIG_FORMAT, POLICY_CONFIG_FORMAT): + if config.get("format") in MOUNTS_CONFIG_FORMATS: mounts = config["mounts"] if not isinstance(mounts, list): raise TypeError("sandbox mounts configuration must be a list") - policy = config.get("format") == POLICY_CONFIG_FORMAT + policy = config.get("format") in MOUNT_POLICY_CONFIG_FORMATS return tuple( _deserialize_mount(mount, policy=policy) for mount in cast(list[object], mounts) @@ -321,11 +342,21 @@ def _deserialize_launch(config: dict[str, Any]) -> SandboxLaunch: ), pids_max=None if config["pids_max"] is None else int(config["pids_max"]), mounts=_deserialize_mounts(config), + # The format check below rejects a count in any other format. + image_slot_boot_count=( + None + if config.get("image_slot_boot_count") is None + else int(config["image_slot_boot_count"]) + ), ) except (KeyError, TypeError, ValueError) as error: raise ScriptError("sandbox configuration is malformed") from error - if config.get("format") != _config_format(launch.mounts): - raise ScriptError("sandbox configuration format does not match its mounts") + if config.get("format") != _config_format( + launch.mounts, launch.image_slot_boot_count + ): + raise ScriptError( + "sandbox configuration format does not match its mounts or image slots" + ) return launch.validated() @@ -516,6 +547,13 @@ def _startup_exit_message(outcome_path: Path, status: int) -> str: return f"OpenVMM exited during startup with status {status}" +def _host_endpoint(runtime: dict[str, Any]) -> Path: + endpoint = runtime.get("host_control_endpoint") + if endpoint is None: + raise ScriptError("sandbox was not provisioned with image slots") + return Path(str(endpoint)) + + def provision( state_path: Path, launch: SandboxLaunch, @@ -585,6 +623,14 @@ def start(state_path: Path, timeout: float) -> None: if os.name == "nt" else os.fspath(state_dir / CONTROL_SOCKET_NAME) ) + # Only a slot-declaring launch has a host-control endpoint. + host_endpoint_value: str | None = None + if launch.image_slot_boot_count is not None: + host_endpoint_value = ( + f"//./pipe/openvmm-microvm-host-{uuid.uuid4().hex}" + if os.name == "nt" + else os.fspath(state_dir / HOST_CONTROL_SOCKET_NAME) + ) command = [ os.fspath(executable), *launch.openvmm_arguments(), @@ -609,6 +655,8 @@ def start(state_path: Path, timeout: float) -> None: "--microvm-report", os.fspath(outcome_path), ] + if host_endpoint_value is not None: + command.extend(("--microvm-host-control", f"listen={host_endpoint_value}")) net = config.get("net") network_profile = config.get("network_profile") if net is not None: @@ -669,15 +717,15 @@ def start(state_path: Path, timeout: float) -> None: if process.poll() is not None else "cannot identify the OpenVMM process" ) - _write_json( - state_dir / RUNTIME_NAME, - { - "format": STATE_FORMAT, - "pid": process.pid, - "start_time": start_time, - "control_endpoint": endpoint_value, - }, - ) + runtime: dict[str, Any] = { + "format": STATE_FORMAT, + "pid": process.pid, + "start_time": start_time, + "control_endpoint": endpoint_value, + } + if host_endpoint_value is not None: + runtime["host_control_endpoint"] = host_endpoint_value + _write_json(state_dir / RUNTIME_NAME, runtime) with ControlSession.connect( Path(endpoint_value), capability, timeout ) as session: @@ -696,6 +744,7 @@ def start(state_path: Path, timeout: float) -> None: (state_dir / RUNTIME_NAME).unlink(missing_ok=True) capability_path.unlink(missing_ok=True) (state_dir / CONTROL_SOCKET_NAME).unlink(missing_ok=True) + (state_dir / HOST_CONTROL_SOCKET_NAME).unlink(missing_ok=True) if status is not None and isinstance(error, Exception): raise ScriptError(_startup_exit_message(outcome_path, status)) from error raise @@ -728,6 +777,32 @@ def exec_workload( ) +def query_image_slots(state_path: Path, timeout: float) -> list[dict[str, Any]]: + state_dir = _prepare_state_directory(state_path, create=False) + runtime, capability = _load_running(state_dir) + with HostControl.connect(_host_endpoint(runtime), capability, timeout) as control: + return control.query_image_slots() + + +def bind_image_slot( + state_path: Path, + slot: int, + image: Path, + identity: str, + timeout: float, +) -> None: + state_dir = _prepare_state_directory(state_path, create=False) + runtime, capability = _load_running(state_dir) + # OpenVMM records at most 1024 identity bytes per slot. + if not identity or len(identity.encode("utf-8")) > 1024: + raise ScriptError("image slot identity must be 1..1024 UTF-8 bytes") + # OpenVMM opens the media itself, as a read-only regular file that is not a + # symbolic link, and only when the slot is empty. A retry with the bound + # identity therefore succeeds even after the media's path is gone. + with HostControl.connect(_host_endpoint(runtime), capability, timeout) as control: + control.bind_image_slot(slot, image.absolute(), identity) + + def stop(state_path: Path, timeout: float) -> dict[str, Any]: state_dir = _prepare_state_directory(state_path, create=False) runtime, capability = _load_running(state_dir) @@ -745,6 +820,7 @@ def stop(state_path: Path, timeout: float) -> dict[str, Any]: (state_dir / RUNTIME_NAME).unlink(missing_ok=True) (state_dir / CAPABILITY_NAME).unlink(missing_ok=True) (state_dir / CONTROL_SOCKET_NAME).unlink(missing_ok=True) + (state_dir / HOST_CONTROL_SOCKET_NAME).unlink(missing_ok=True) return outcome @@ -759,6 +835,7 @@ def deprovision(state_path: Path) -> None: RUNTIME_NAME, CAPABILITY_NAME, CONTROL_SOCKET_NAME, + HOST_CONTROL_SOCKET_NAME, OUTCOME_NAME, LOG_NAME, CONFIG_NAME, diff --git a/scripts/test_control_session.py b/scripts/test_control_session.py index 87db7f20e..ebf36e6c1 100644 --- a/scripts/test_control_session.py +++ b/scripts/test_control_session.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # pyright: reportPrivateUsage=false +import json import os import socket import struct @@ -9,10 +10,13 @@ import time import unittest import uuid +from collections.abc import Callable from pathlib import Path +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).parent)) -from nvx_tools import control_session # noqa: E402 +from nvx_tools import control_session, host_control # noqa: E402 +from nvx_tools.common import ScriptError # noqa: E402 def _read_like_openvmm(descriptor: int) -> bytes | None: @@ -723,5 +727,148 @@ def test_write_reports_a_closed_endpoint(self): stream.close() +def _host_control_frame(value: object) -> bytes: + payload = value if isinstance(value, bytes) else json.dumps(value).encode() + return host_control.FRAME_HEADER.pack(len(payload)) + payload + + +class HostControlTests(unittest.TestCase): + CAPABILITY = bytes(range(1, 33)) + + def _exchange( + self, + response: bytes, + operation: Callable[[host_control.HostControl], object], + ) -> tuple[object, dict[str, object]]: + """Runs `operation` against a peer that answers one request with + `response`, and returns its result and the request it received.""" + client, server = socket.socketpair() + received: list[dict[str, object]] = [] + + def serve() -> None: + with server: + if _read_exact(server, 32) != self.CAPABILITY: + raise RuntimeError("host-control client sent a wrong capability") + (length,) = host_control.FRAME_HEADER.unpack(_read_exact(server, 4)) + received.append(json.loads(_read_exact(server, length))) + server.sendall(response) + + thread = threading.Thread(target=serve) + thread.start() + try: + with ( + patch.object( + host_control, + "connect_local_stream", + return_value=control_session._SocketStream(client), + ), + host_control.HostControl( + Path("endpoint"), self.CAPABILITY, 5 + ) as control, + ): + result = operation(control) + finally: + thread.join(5) + return result, received[0] + + def test_query_sends_a_versioned_request_and_returns_the_slots(self): + slots = [{"slot": 0, "state": "empty"}] + result, request = self._exchange( + _host_control_frame( + { + "version": 1, + "request_id": 1, + "status": "ok", + "result": {"slots": slots}, + } + ), + lambda control: control.query_image_slots(), + ) + self.assertEqual(result, slots) + self.assertEqual( + request, {"version": 1, "request_id": 1, "operation": "query_image_slots"} + ) + + def test_bind_sends_the_path_and_identity_and_reports_error_codes(self): + bound = { + "version": 1, + "request_id": 1, + "status": "ok", + "result": {"slot": 2, "capacity_sectors": 8}, + } + _, request = self._exchange( + _host_control_frame(bound), + lambda control: control.bind_image_slot(2, Path("image.raw"), "id"), + ) + self.assertEqual( + request, + { + "version": 1, + "request_id": 1, + "operation": "bind_image_slot", + "slot": 2, + "path": str(Path("image.raw")), + "identity": "id", + }, + ) + response = { + "version": 1, + "request_id": 1, + "status": "error", + "code": "already_bound", + "message": "image slot 2 is already bound to a different identity", + } + with self.assertRaisesRegex(ScriptError, r"\(already_bound\): image slot 2"): + self._exchange( + _host_control_frame(response), + lambda control: control.bind_image_slot(2, Path("image.raw"), "id"), + ) + + def test_invalid_responses_are_rejected(self): + ok = {"version": 1, "request_id": 1, "status": "ok"} + for response, message in ( + ( + host_control.FRAME_HEADER.pack(host_control.MAX_FRAME_SIZE + 1), + "exceeds the protocol limit", + ), + (_host_control_frame(b"{not json"), "invalid JSON"), + (_host_control_frame([1]), "non-object response"), + (_host_control_frame({**ok, "request_id": 2, "result": {}}), "mismatched"), + (_host_control_frame({**ok, "version": 2, "result": {}}), "mismatched"), + (_host_control_frame({**ok, "result": []}), "invalid success response"), + (_host_control_frame({**ok, "result": {"slots": {}}}), "image-slot table"), + (_host_control_frame({**ok, "result": {"slots": [1]}}), "image-slot table"), + ): + with self.subTest(message=message, response=response[:40]): + with self.assertRaisesRegex(ScriptError, message): + self._exchange( + response, lambda control: control.query_image_slots() + ) + + def test_oversized_requests_and_invalid_capabilities_fail_before_sending(self): + with self.assertRaisesRegex(ValueError, "32 nonzero bytes"): + host_control.HostControl(Path("endpoint"), bytes(32), 5) + client, server = socket.socketpair() + with ( + server, + patch.object( + host_control, + "connect_local_stream", + return_value=control_session._SocketStream(client), + ), + host_control.HostControl(Path("endpoint"), self.CAPABILITY, 5) as control, + ): + with self.assertRaisesRegex(ValueError, "exceeds the protocol limit"): + control.bind_image_slot( + 0, Path("x" * host_control.MAX_FRAME_SIZE), "identity" + ) + server.settimeout(5) + # Only the capability reached the peer. + self.assertEqual(_read_exact(server, 32), self.CAPABILITY) + server.setblocking(False) + with self.assertRaises(BlockingIOError): + server.recv(1) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_guest_time.py b/scripts/test_guest_time.py index 7de7daf3b..1f0632437 100644 --- a/scripts/test_guest_time.py +++ b/scripts/test_guest_time.py @@ -24,6 +24,7 @@ MEMORY_TARGET = 0x02 ACK_REQUIRED = 0x04 TEST_HOOKS = 0x08 +IMAGE_SLOT_TARGET = 0x10 UTC_NS = 1_790_841_600_123_456_789 ENTROPY = bytes(range(64)) PREVIOUS_ID = "ff" * 16 @@ -42,12 +43,12 @@ def encode_packet( utc_ns: int = UTC_NS, magic: bytes = b"OVR", version: int = 4, - reserved: int = 0, + image_slots: int = 0, entropy: bytes = ENTROPY, ) -> bytes: """Encode a restore packet v4 (spec: "Restore packet").""" count = len(ranges) if range_count is None else range_count - header = magic + bytes((version, flags, online, count, reserved)) + header = magic + bytes((version, flags, online, count, image_slots)) header += struct.pack(" None: with self.assertRaises(OSError): os.fstat(prepared[0]) + def test_image_slots_spawns_openvmm_with_a_prepared_capability_pipe(self): + # The image-slot scenario authenticates its host-control endpoint with + # the same capability, so it also hands OpenVMM a prepared pipe (#440). + prepared: list[int] = [] + spawned: list[object] = [] + + def prepare(capability: bytes) -> int: + descriptor = control_session.capability_pipe(capability) + prepared.append(descriptor) + return descriptor + + def launch(_command: list[str], *, stdin: object, **_options: object) -> None: + # The pipe stays open until OpenVMM has started with it. + self.assertIsInstance(stdin, int) + os.fstat(cast(int, stdin)) + spawned.append(stdin) + raise RuntimeError("stopped after the spawn") + + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object(microvm_tests, "capability_pipe", side_effect=prepare), + patch.object(microvm_tests.subprocess, "Popen", side_effect=launch), + self.assertRaisesRegex(RuntimeError, "^stopped after the spawn$"), + ): + microvm_tests.run_image_slots( + Path("openvmm"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + "kvm", + memory_mib=256, + timeout=5, + output_dir=Path(temporary), + ) + + self.assertEqual(len(prepared), 1) + self.assertEqual(spawned, prepared) + # The scenario closes its copy of the pipe once OpenVMM has it. + with self.assertRaises(OSError): + os.fstat(prepared[0]) + def test_host_loopback_listener_pair_retries_protocol_port_conflict(self): first_udp = MagicMock() first_udp.getsockname.return_value = ("127.0.0.1", 50000) @@ -3104,7 +3144,7 @@ def test_plain_untiered_restore_prints_nothing(self): functions = functions.replace("/sbin/nvx-time", "nvx_time") generation_id = "0123456789abcdef0123456789abcdef" - def restore(flags: int) -> tuple[str, str, str]: + def restore(flags: int, image_slots: int = 0) -> tuple[str, str, str]: with tempfile.TemporaryDirectory() as temporary: result = subprocess.run( [shell, "-s", "--", temporary], @@ -3123,7 +3163,8 @@ def restore(flags: int) -> tuple[str, str, str]: 'nvx_time() { printf "%s\\n" "$*" >>"$calls"; }\n' 'activate_restore_memory() { echo "memory $*"; }\n' f"{functions}\n" - f"post_restore {flags} 0 0 {generation_id}\n" + 'activate_restore_image_slots() { echo "image-slots $*"; }\n' + f"post_restore {flags} 0 0 {image_slots} {generation_id}\n" 'echo "pending=$post_restore_pending"\n' ), text=True, @@ -3157,6 +3198,75 @@ def restore(flags: int) -> tuple[str, str, str]: stdout, "memory 0\nNVX-POST-RESTORE-OK: tier=legacy\npending=false\n" ) self.assertEqual(calls, "restore-finish --new-cpus none\n") + # So does an image-slot target, which activates the slots first. + console, stdout, calls = restore(20, image_slots=4) + self.assertEqual( + console, + "NVX-POST-RESTORE-STAGE: packet\nNVX-POST-RESTORE-STAGE: acknowledge\n", + ) + self.assertEqual( + stdout, + "image-slots 4\nNVX-POST-RESTORE-OK: tier=legacy\npending=false\n", + ) + self.assertEqual(calls, "restore-finish --ack --new-cpus none\n") + + def test_image_slot_helper_fails_fast_without_an_active_block_device(self): + if sys.platform != "linux": + self.skipTest("the guest helper's device links need Linux") + shell = _posix_shell() + if shell is None: + self.skipTest("POSIX shell is unavailable") + helper = ( + Path(__file__).parents[1] / "guest" / "common" / "nvx-image-slot" + ).read_text(encoding="utf-8") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + iomem = root / "iomem" + iomem.write_bytes(b"d0008000-d0008fff : virtio-mmio.8\n") + devices = root / "devices" + disks = root / "block" + links = root / "dev" + links.mkdir() + script = ( + helper.replace("/proc/iomem", iomem.as_posix()) + .replace("/sys/devices/virtio-mmio-cmdline", devices.as_posix()) + .replace("/sys/class/block", disks.as_posix()) + .replace("/dev/nvx-image", (links / "nvx-image").as_posix()) + ) + platform = devices / "virtio-mmio.8" + + def probe() -> tuple[int, str, str]: + result = subprocess.run( + [shell, "-s", "--", "--slot", "0", "--timeout", "30"] + + ["--allow-empty"], + input=script, + text=True, + capture_output=True, + timeout=10, + check=False, + ) + return result.returncode, result.stdout, result.stderr + + # An inactive slot is a placeholder that no driver binds. + platform.mkdir(parents=True) + self.assertEqual(probe(), (1, "", "nvx-image-slot: image0 is inactive\n")) + # Without image slots, the window may hold the second live share. + (platform / "driver").mkdir() + virtio = platform / "virtio8" + virtio.mkdir() + (virtio / "device").write_bytes(b"0x001a\n") + self.assertEqual( + probe(), + (1, "", "nvx-image-slot: image0 transport is not a block device\n"), + ) + # An active slot is an empty read-only disk until the host binds it. + (virtio / "device").write_bytes(b"0x0002\n") + (virtio / "block" / "vdb").mkdir(parents=True) + (disks / "vdb").mkdir(parents=True) + (disks / "vdb" / "ro").write_bytes(b"1\n") + (disks / "vdb" / "size").write_bytes(b"0\n") + self.assertEqual(probe(), (0, f"{(links / 'nvx-image0').as_posix()}\n", "")) + self.assertEqual(os.readlink(links / "nvx-image0"), "vdb") def test_capture_metadata_2_leaves_no_restore_work_to_the_shell(self): shell = _posix_shell() @@ -3192,12 +3302,12 @@ def dispatch_metadata(metadata: str) -> tuple[int, str]: self.assertEqual(dispatch_metadata("0"), (0, "suppressed=true\n")) # nvx-time finished the restore and handed it to the daemon. self.assertEqual( - dispatch_metadata(f"2 4 0 0 {generation_id}"), + dispatch_metadata(f"2 4 0 0 0 {generation_id}"), (0, "suppressed=false\n"), ) self.assertEqual( - dispatch_metadata(f"1 4 2 0 {generation_id}"), - (0, f"post_restore 4 2 0 {generation_id}\nsuppressed=false\n"), + dispatch_metadata(f"1 20 2 0 4 {generation_id}"), + (0, f"post_restore 20 2 0 4 {generation_id}\nsuppressed=false\n"), ) self.assertEqual( dispatch_metadata("3"), @@ -3468,9 +3578,10 @@ def test_every_tcp_console_is_monitored_and_cold_boot_shells_are_queried(self): if isinstance(node, ast.Call) and ast.unparse(node.func) == "TcpConsole.connect" ] - # The managed lifecycle, the console-snapshot capture and restore, and - # the snapshot-tier capture, restore, and gate-timeout restore. - self.assertEqual(len(connects), 6) + # The managed lifecycle, the console-snapshot capture and restore, the + # snapshot-tier capture, restore, and gate-timeout restore, and the + # image-slot bind, capture, and restore. + self.assertEqual(len(connects), 9) for keywords in connects: self.assertRegex( keywords.get("monitor", ""), r"^TimeAbiMonitor\(\w*command\)$" @@ -3479,8 +3590,10 @@ def test_every_tcp_console_is_monitored_and_cold_boot_shells_are_queried(self): # nvx-time status. Restores never ask, and in the managed lifecycle # init starts the managed agent instead of a shell. self.assertEqual( - [k["monitor"] for k in connects if k.get("time_abi_status") == "True"], - ["TimeAbiMonitor(capture_command)"] * 2, + sorted( + k["monitor"] for k in connects if k.get("time_abi_status") == "True" + ), + ["TimeAbiMonitor(capture_command)"] * 3 + ["TimeAbiMonitor(command)"], ) def test_snapshot_core_script_handles_no_clocksource(self): diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index 190a23c52..2b79ac3c1 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -31,6 +31,7 @@ import uuid import zipfile from collections.abc import Callable +from contextlib import redirect_stdout from pathlib import Path, PurePosixPath from typing import Any, cast from unittest.mock import MagicMock, call, patch @@ -575,6 +576,21 @@ def test_benchmark_exposes_non_python_performance_suite(self): self.assertEqual(args.output_dir, Path("results")) self.assertIs(args.handler, nvx.command_benchmark) + def test_benchmark_exposes_image_slot_boot_suite(self): + args = nvx.parse_args( + [ + "benchmark", + "--suite", + "image-slot-boot", + "--output-dir", + "results", + ] + ) + + self.assertEqual(args.suite, "image-slot-boot") + self.assertEqual(args.output_dir, Path("results")) + self.assertIs(args.handler, nvx.command_benchmark) + def test_device_io_uses_canonical_sampling_defaults(self): args = nvx.parse_args(["benchmark", "--suite", "device-io"]) @@ -10838,7 +10854,9 @@ def _write_managed_runtime(state: Path, pid: int, start_time: int | None) -> Non (state / sandbox_lifecycle.CAPABILITY_NAME).write_bytes(b"x" * 32) -def _provision_managed_sandbox(root: Path) -> Path: +def _provision_managed_sandbox( + root: Path, image_slot_boot_count: int | None = None +) -> Path: layer_path = root / "distro.erofs" scratch_path = root / "scratch.ext4" layer_path.write_bytes(b"layer") @@ -10855,6 +10873,7 @@ def _provision_managed_sandbox(root: Path) -> Path: ), ), scratch=scratch_path, + image_slot_boot_count=image_slot_boot_count, ), hypervisor="whp", memory_mib=256, @@ -11000,6 +11019,77 @@ def wait(timeout: float) -> int: class SandboxTests(unittest.TestCase): + def test_image_slots_allow_scratch_only_managed_launch(self): + launch = sandbox.SandboxLaunch( + layers=(), + scratch=Path("scratch.ext4"), + image_slot_boot_count=4, + ) + + self.assertEqual( + launch.openvmm_arguments(), + [ + "--machine", + "microvm", + "--microvm-sandbox-block", + "scratch:file:scratch.ext4", + "--microvm-workload-identity", + "65534:65534", + "--microvm-image-slots", + "--microvm-image-slot-boot-count", + "4", + ], + ) + self.assertNotIn("nvx_layer=", launch.kernel_command_line()) + + def test_scratch_only_launch_requires_image_slots(self): + with self.assertRaisesRegex(common.ScriptError, "unless it declares"): + sandbox.SandboxLaunch(layers=(), scratch=Path("scratch.ext4")) + with self.assertRaisesRegex(common.ScriptError, "between 1 and 4"): + sandbox.SandboxLaunch( + layers=(), + scratch=Path("scratch.ext4"), + image_slot_boot_count=5, + ) + + def test_image_slots_permit_at_most_one_mount(self): + work = sandbox.SandboxMount.parse("/workspace,work,rw", ()) + tools = sandbox.SandboxMount.parse("/tools,tools,ro", ()) + launch = sandbox.SandboxLaunch( + layers=(), + scratch=Path("scratch.ext4"), + mounts=(work,), + image_slot_boot_count=1, + ) + + arguments = launch.openvmm_arguments() + self.assertEqual(arguments.count("--mount"), 1) + self.assertEqual( + arguments[-3:], + ["--microvm-image-slots", "--microvm-image-slot-boot-count", "1"], + ) + # Image slot 0 uses the transport of the second live share. + with self.assertRaisesRegex(common.ScriptError, "at most one --mount"): + sandbox.SandboxLaunch( + layers=(), + scratch=Path("scratch.ext4"), + mounts=(work, tools), + image_slot_boot_count=1, + ) + + def test_run_parser_accepts_restore_image_slot_target(self): + args = nvx.parse_args( + [ + "run", + "--restore-snapshot", + "snapshot", + "--restore-image-slots", + "4", + "--dry-run", + ] + ) + self.assertEqual(args.restore_image_slots, 4) + def test_launch_contract_orders_roles_and_builds_agent_command_line(self): custom = sandbox.SandboxLayer.parse( "custom,custom.erofs,22222222-2222-2222-2222-222222222222" @@ -12258,6 +12348,77 @@ def identity(path: Path) -> tuple[int, int]: with self.assertRaisesRegex(common.ScriptError, "does not match"): sandbox_lifecycle._deserialize_launch(config) + def test_managed_configuration_format_binds_image_slots(self): + def entry(target: str, path: str) -> dict[str, object]: + return { + "guest_target": target, + "host_path": path, + "access": "rw", + "denied_paths": [], + "allowed_paths": [], + "writable_paths": [], + "owner": "vmm", + } + + config: dict[str, object] = { + "format": sandbox_lifecycle.IMAGE_SLOT_CONFIG_FORMAT, + "layers": [], + "scratch": "scratch.ext4", + "hostname": "nvx-sandbox", + "workload_uid": 65534, + "workload_gid": 65534, + "memory_max": None, + "pids_max": None, + "mounts": [entry("/workspace", "work")], + "image_slot_boot_count": 2, + } + + def require(path: Path, _description: str) -> Path: + return path + + def unchanged(mount: sandbox.SandboxMount) -> sandbox.SandboxMount: + return mount + + with ( + patch.object(sandbox, "require_file", side_effect=require), + patch.object(sandbox.SandboxMount, "validated", unchanged), + ): + launch = sandbox_lifecycle._deserialize_launch(config) + self.assertEqual(launch.image_slot_boot_count, 2) + self.assertEqual( + [mount.guest_target for mount in launch.mounts], ["/workspace"] + ) + self.assertEqual( + sandbox_lifecycle._config_format( + launch.mounts, launch.image_slot_boot_count + ), + sandbox_lifecycle.IMAGE_SLOT_CONFIG_FORMAT, + ) + # Image slot 0 uses the transport of the second live share. + config["mounts"] = [entry("/workspace", "work"), entry("/tools", "tools")] + with self.assertRaisesRegex(common.ScriptError, "at most one --mount"): + sandbox_lifecycle._deserialize_launch(config) + config["image_slot_boot_count"] = 5 + config["mounts"] = [] + with self.assertRaisesRegex(common.ScriptError, "between 1 and 4"): + sandbox_lifecycle._deserialize_launch(config) + # Only format 6 holds image slots, and format 6 always does. + config["layers"] = [ + { + "role": "distro", + "path": "distro.erofs", + "uuid": "11111111-1111-1111-1111-111111111111", + } + ] + del config["image_slot_boot_count"] + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + config["image_slot_boot_count"] = 2 + config["format"] = sandbox_lifecycle.POLICY_CONFIG_FORMAT + config["mounts"] = [entry("/workspace", "work")] + with self.assertRaisesRegex(common.ScriptError, "does not match"): + sandbox_lifecycle._deserialize_launch(config) + def test_managed_lifecycle_persists_and_replays_mount(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary).resolve() @@ -12957,6 +13118,7 @@ def acknowledge_stop(_timeout: float) -> None: _exit_unreaped(child) session.stop.side_effect = acknowledge_stop + (state / sandbox_lifecycle.HOST_CONTROL_SOCKET_NAME).touch() with patch.object( sandbox_lifecycle.ControlSession, "connect", @@ -12967,6 +13129,9 @@ def acknowledge_stop(_timeout: float) -> None: session.stop.assert_called_once_with(10) self.assertFalse((state / sandbox_lifecycle.RUNTIME_NAME).exists()) self.assertFalse((state / sandbox_lifecycle.CAPABILITY_NAME).exists()) + self.assertFalse( + (state / sandbox_lifecycle.HOST_CONTROL_SOCKET_NAME).exists() + ) finally: _finish_child(child) @@ -12981,6 +13146,8 @@ def test_managed_lifecycle_treats_a_reused_process_id_as_stopped(self): # OpenVMM is gone, and this test process now has its process ID. _write_managed_runtime(state, os.getpid(), start_time + 1) (state / sandbox_lifecycle.CONFIG_NAME).write_text("{}", encoding="utf-8") + # A killed slot-declaring OpenVMM leaves its host-control socket. + (state / sandbox_lifecycle.HOST_CONTROL_SOCKET_NAME).touch() with patch.object(sandbox_lifecycle.ControlSession, "connect") as connect: with self.assertRaisesRegex( @@ -13279,6 +13446,100 @@ def launch(_command: list[str], *, stdin: object, **_options: object) -> Any: self.assertEqual(connect.call_args.args[1], capability) session.ping.assert_called_once_with(10) + def test_managed_start_records_host_control_only_for_image_slots(self): + def require(path: Path, _description: str) -> Path: + return path + + process = MagicMock() + process.pid = 123 + session = MagicMock() + context = MagicMock() + context.__enter__.return_value = session + context.__exit__.return_value = False + for image_slot_boot_count in (None, 1): + with ( + self.subTest(image_slot_boot_count=image_slot_boot_count), + tempfile.TemporaryDirectory() as temporary, + ): + state = _provision_managed_sandbox( + Path(temporary).resolve(), image_slot_boot_count + ) + with ( + patch.object( + sandbox_lifecycle, "require_file", side_effect=require + ), + patch.object( + sandbox_lifecycle.subprocess, "Popen", return_value=process + ) as popen, + patch.object( + sandbox_lifecycle, "_process_start_time", return_value=456 + ), + patch.object( + sandbox_lifecycle.ControlSession, + "connect", + return_value=context, + ), + ): + sandbox_lifecycle.start(state, 10) + + command = popen.call_args.args[0] + runtime = json.loads( + (state / sandbox_lifecycle.RUNTIME_NAME).read_text(encoding="utf-8") + ) + if image_slot_boot_count is not None: + self.assertEqual( + command[command.index("--microvm-host-control") + 1], + f"listen={runtime['host_control_endpoint']}", + ) + continue + self.assertNotIn("--microvm-host-control", command) + self.assertNotIn("host_control_endpoint", runtime) + # A sandbox without image slots fails at once rather than + # retrying a host-control endpoint that never exists. + with ( + patch.object( + sandbox_lifecycle, "_process_running", return_value=True + ), + patch.object(sandbox_lifecycle.HostControl, "connect") as connect, + self.assertRaisesRegex( + common.ScriptError, "not provisioned with image slots" + ), + ): + sandbox_lifecycle.query_image_slots(state, 10) + connect.assert_not_called() + + def test_bind_image_slot_leaves_media_checks_to_openvmm(self): + # OpenVMM checks the media only when the slot is empty, so an + # idempotent retry succeeds even after the bound path is gone. + control = MagicMock() + context = MagicMock() + context.__enter__.return_value = control + context.__exit__.return_value = False + missing = Path("missing") / "image.raw" + runtime = {"host_control_endpoint": "host-control.sock"} + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object( + sandbox_lifecycle, + "_load_running", + return_value=(runtime, b"x" * 32), + ), + patch.object( + sandbox_lifecycle.HostControl, "connect", return_value=context + ), + ): + sandbox_lifecycle.bind_image_slot( + Path(temporary), 1, missing, "image-a", 10 + ) + with self.assertRaisesRegex(common.ScriptError, "1..1024 UTF-8 bytes"): + sandbox_lifecycle.bind_image_slot( + Path(temporary), 1, missing, "x" * 1025, 10 + ) + + control.bind_image_slot.assert_called_once_with( + 1, missing.absolute(), "image-a" + ) + def test_launch_contract_rejects_disk_option_delimiters(self): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) @@ -13300,6 +13561,130 @@ def test_launch_contract_rejects_disk_option_delimiters(self): class BenchmarkTests(unittest.TestCase): + def test_image_slot_boot_benchmark_compares_both_prefixes_to_no_slots(self): + args = argparse.Namespace( + warmups=0, + runs=1, + memory_mib=128, + processors=1, + timeout=60.0, + ) + output = io.StringIO() + with ( + patch.object( + benchmark, + "_image_slot_boot_sample", + side_effect=(10.0, 11.0, 12.0), + ) as sample, + redirect_stdout(output), + ): + benchmark.benchmark_image_slot_boot_workload( + args, + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "whp", + ) + + self.assertEqual( + [call.args[6] for call in sample.call_args_list], + [None, 1, 4], + ) + self.assertIn("delta-vs-no-slots=+1.000 ms", output.getvalue()) + self.assertIn("delta-vs-no-slots=+2.000 ms", output.getvalue()) + + def test_image_slot_boot_sample_keeps_console_open_until_exit(self): + events: list[str] = [] + + class FakeConsole: + def __enter__(self): + return self + + def __exit__(self, *_): + events.append("close") + + def settimeout(self, _): + pass + + def recv(self, _): + return benchmark.BOOT_MARKER + + def sendall(self, data: bytes) -> None: + events.append(f"send:{data.decode()}") + + class FakeProcess: + stdin = None + + def wait(self, timeout: float | None = None) -> int: + events.append("wait") + return 0 + + def poll(self): + return 0 + + with ( + patch.object(benchmark.subprocess, "Popen", return_value=FakeProcess()), + patch.object( + benchmark.socket, + "create_connection", + return_value=FakeConsole(), + ), + ): + benchmark._image_slot_boot_sample( + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "kvm", + 128, + 1, + None, + 5.0, + command_prefix=(), + windows_cpus=None, + ) + + self.assertEqual(events, ["send:/sbin/nvx-exit 0\n", "wait", "close"]) + + def test_image_slot_boot_sample_hands_openvmm_a_prepared_capability_pipe(self): + # OpenVMM reads its capability as soon as it starts (#440). + prepared: list[int] = [] + spawned: list[object] = [] + + def prepare(value: bytes) -> int: + descriptor = control_session.capability_pipe(value) + prepared.append(descriptor) + return descriptor + + def launch(_command: list[str], *, stdin: object, **_options: object) -> None: + # The pipe stays open until OpenVMM has started with it. + self.assertIsInstance(stdin, int) + os.fstat(cast(int, stdin)) + spawned.append(stdin) + raise RuntimeError("stopped after the spawn") + + with ( + patch.object(benchmark, "capability_pipe", side_effect=prepare), + patch.object(benchmark.subprocess, "Popen", side_effect=launch), + self.assertRaisesRegex(RuntimeError, "^stopped after the spawn$"), + ): + benchmark._image_slot_boot_sample( + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "kvm", + 128, + 1, + 1, + 5.0, + command_prefix=(), + windows_cpus=None, + ) + + self.assertEqual(len(prepared), 1) + self.assertEqual(spawned, prepared) + with self.assertRaises(OSError): + os.fstat(prepared[0]) + def test_kvm_worker_result_decoding(self): completed = subprocess.CompletedProcess( ["worker"],