diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 663c63204..724e73c20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -609,6 +609,54 @@ jobs: if-no-files-found: error retention-days: 1 + # The test-only programs that the microVM guest library scenarios, such as + # gpt-device, share into an Alpine guest rather than grow its initramfs. + # Beside the guest artifacts, so that their build delays only the microVM + # guest and sandbox jobs, which run them. + guest-test-programs: + name: Build guest test programs + needs: [openvmm-changes] + if: >- + ${{ + always() && + !cancelled() && + ( + github.event_name != 'pull_request' || + github.event.pull_request.draft == false + ) && + ( + needs.openvmm-changes.result != 'success' || + needs.openvmm-changes.outputs.run-guest-artifacts == 'true' + ) + }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Build guest test programs + shell: bash + run: | + set -euo pipefail + toolchain=$(sed -n 's/^channel = "\([^"]*\)".*/\1/p' rust-toolchain.toml) + test -n "${toolchain}" + rustup toolchain install "${toolchain}" --profile minimal --no-self-update \ + --target x86_64-unknown-linux-musl + python3 scripts/nvx.py build-guest-tests --output build/guest-tests + + # A one-file artifact keeps its file at its root, so the jobs that run + # the programs download it into build/guest-tests. + - name: Upload guest test programs + uses: actions/upload-artifact@v7 + with: + name: guest-test-programs + path: build/guest-tests/gpt-device + if-no-files-found: error + retention-days: 1 + build-openvmm-linux-gnu: name: OpenVMM binary / Linux GNU needs: [openvmm-changes] @@ -1444,11 +1492,12 @@ jobs: # The steps that follow each backend's default suite, from the public # managed sandbox acceptance to the runtime adapter scenarios, depend on # none of its results, so a second job runs them beside it, on the same - # runner pool. They alone read the Ubuntu and Azure Linux artifacts, so - # only this job waits for distro-artifacts. + # runner pool. They alone read the Ubuntu and Azure Linux artifacts and the + # guest test programs, so only this job waits for distro-artifacts and + # guest-test-programs. nvx-microvm-guests-kvm: name: NVX microVM tests / Linux / KVM / Guests and sandboxes - needs: [artifacts, distro-artifacts, build-openvmm-linux-gnu, nvx-cli-linux, openvmm-changes] + needs: [artifacts, distro-artifacts, guest-test-programs, build-openvmm-linux-gnu, nvx-cli-linux, openvmm-changes] if: >- ${{ always() && @@ -1463,6 +1512,7 @@ jobs: ) && needs.artifacts.result == 'success' && needs.distro-artifacts.result == 'success' && + needs.guest-test-programs.result == 'success' && needs.build-openvmm-linux-gnu.result == 'success' && needs.nvx-cli-linux.result == 'success' && ( @@ -1483,7 +1533,7 @@ jobs: nvx-microvm-guests-mshv: name: NVX microVM tests / Linux / MSHV / Guests and sandboxes - needs: [artifacts, distro-artifacts, build-openvmm-linux-gnu, nvx-cli-linux, openvmm-changes] + needs: [artifacts, distro-artifacts, guest-test-programs, build-openvmm-linux-gnu, nvx-cli-linux, openvmm-changes] if: >- ${{ always() && @@ -1498,6 +1548,7 @@ jobs: ) && needs.artifacts.result == 'success' && needs.distro-artifacts.result == 'success' && + needs.guest-test-programs.result == 'success' && needs.build-openvmm-linux-gnu.result == 'success' && needs.nvx-cli-linux.result == 'success' && ( @@ -1518,7 +1569,7 @@ jobs: nvx-microvm-guests-whp: name: NVX microVM tests / Windows / WHP / Guests and sandboxes - needs: [artifacts, distro-artifacts, build-openvmm-windows-msvc, nvx-cli-windows, openvmm-changes] + needs: [artifacts, distro-artifacts, guest-test-programs, build-openvmm-windows-msvc, nvx-cli-windows, openvmm-changes] if: >- ${{ always() && @@ -1533,6 +1584,7 @@ jobs: ) && needs.artifacts.result == 'success' && needs.distro-artifacts.result == 'success' && + needs.guest-test-programs.result == 'success' && needs.build-openvmm-windows-msvc.result == 'success' && needs.nvx-cli-windows.result == 'success' && ( @@ -1780,6 +1832,7 @@ jobs: name: Publish development release needs: - debug-kernel + - guest-test-programs - quality - aci-edge-sandboxes - nvx-rs @@ -1811,6 +1864,7 @@ jobs: github.ref == 'refs/heads/dev' && !cancelled() && needs.debug-kernel.result == 'success' && + needs.guest-test-programs.result == 'success' && needs.quality.result == 'success' && needs.aci-edge-sandboxes.result == 'success' && needs.nvx-rs.result == 'success' && @@ -1930,6 +1984,7 @@ jobs: - distro-artifacts - debug-kernel - guest-artifact-reuse + - guest-test-programs - build-openvmm-linux-gnu - build-openvmm-linux-musl - build-openvmm-windows-msvc @@ -1984,6 +2039,7 @@ jobs: DISTRO_ARTIFACTS_RESULT: ${{ needs.distro-artifacts.result }} DEBUG_KERNEL_RESULT: ${{ needs.debug-kernel.result }} GUEST_ARTIFACT_REUSE_RESULT: ${{ needs.guest-artifact-reuse.result }} + GUEST_TEST_PROGRAMS_RESULT: ${{ needs.guest-test-programs.result }} BUILD_LINUX_GNU_RESULT: ${{ needs.build-openvmm-linux-gnu.result }} BUILD_LINUX_MUSL_RESULT: ${{ needs.build-openvmm-linux-musl.result }} BUILD_WINDOWS_MSVC_RESULT: ${{ needs.build-openvmm-windows-msvc.result }} @@ -2019,6 +2075,7 @@ jobs: name: Persist performance baseline needs: - debug-kernel + - guest-test-programs - nvx-cli-linux - nvx-cli-windows - openvmm-changes @@ -2047,6 +2104,7 @@ jobs: github.ref == 'refs/heads/dev' && !cancelled() && needs.debug-kernel.result == 'success' && + needs.guest-test-programs.result == 'success' && needs.nvx-cli-linux.result == 'success' && needs.nvx-cli-windows.result == 'success' && ( diff --git a/.github/workflows/run-nvx-microvm-tests.yml b/.github/workflows/run-nvx-microvm-tests.yml index e79a811ee..c4b5963ef 100644 --- a/.github/workflows/run-nvx-microvm-tests.yml +++ b/.github/workflows/run-nvx-microvm-tests.yml @@ -35,10 +35,10 @@ on: description: >- Run the steps that follow the default suite instead of it, in a second job of the backend beside the default suite: the public - managed sandbox acceptance, the Ubuntu and Azure Linux guest tests, - the Ubuntu sandbox steps, the aci_edge_sandboxes lifecycle, and the - runtime adapter scenarios. Not for the debug-kernel jobs, which skip - these steps + managed sandbox acceptance, the guest library scenarios, the Ubuntu + and Azure Linux guest tests, the Ubuntu sandbox steps, the + aci_edge_sandboxes lifecycle, and the runtime adapter scenarios. Not + for the debug-kernel jobs, which skip these steps required: false type: boolean default: false @@ -89,7 +89,8 @@ jobs: # qualifies its runner with the kernel and the Alpine initramfs, the # default suite boots only those, and the debug-kernel jobs run only # Alpine scenarios on the debug kernel, so only the guest and sandbox - # steps take the Ubuntu and Azure Linux artifacts. + # steps take the Ubuntu and Azure Linux artifacts and the guest test + # programs. - name: Download guest kernel uses: actions/download-artifact@v8 with: @@ -137,6 +138,15 @@ jobs: name: guest-azurelinux path: build + # The artifact holds the programs at its root, and test-microvm looks + # for them in build/guest-tests. + - name: Download guest test programs + if: ${{ inputs.guests-and-sandboxes && !inputs.debug-kernel }} + uses: actions/download-artifact@v8 + with: + name: guest-test-programs + path: build/guest-tests + - name: Download OpenVMM executable uses: actions/download-artifact@v8 with: @@ -215,6 +225,26 @@ jobs: --scenario sandbox-lifecycle --output-dir "build/test-results/public-exec-${{ inputs.backend }}" + # The guest library scenarios run the test-only guest programs that the + # guest-test-programs job builds. + - name: Run guest library scenarios on Linux + if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel + shell: bash + run: >- + python3 scripts/nvx.py test-microvm + --backend "${{ inputs.backend }}" + --scenario gpt-device + --output-dir "build/test-results/guest-library-${{ inputs.backend }}" + + - name: Run guest library scenarios on Windows + if: runner.os == 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel + shell: powershell + run: >- + python scripts\nvx.py test-microvm + --backend "${{ inputs.backend }}" + --scenario gpt-device + --output-dir "build/test-results/guest-library-${{ inputs.backend }}" + - name: Run Ubuntu guest smoke tests on Linux if: runner.os != 'Windows' && inputs.guests-and-sandboxes && !inputs.debug-kernel shell: bash @@ -1275,6 +1305,7 @@ jobs: name: nvx-microvm-tests-${{ inputs.backend }}${{ inputs.debug-kernel && '-debug' || '' }}${{ inputs.guests-and-sandboxes && '-guests' || '' }}${{ inputs.shard && format('-shard-{0}', inputs.shard) || '' }} path: | build/test-results/public-exec-${{ inputs.backend }} + build/test-results/guest-library-${{ inputs.backend }} build/test-results/microvm-${{ inputs.backend }} build/test-results/microvm-${{ inputs.backend }}-debug build/test-results/ubuntu-${{ inputs.backend }} diff --git a/doc/ci.md b/doc/ci.md index 8c8e61f4c..5ef7ef715 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -191,8 +191,9 @@ production kernels leave out. It selects the same-host restore scenarios `smp`, `smp-snapshot`, `restore-processors`, `restore-downtime`, and `snapshot-tiers`. The jobs skip every other test step, because those boot the production kernel, which the `nvx-microvm-guests-{kvm,mshv,whp}` jobs already -cover: the public managed execution configuration, the Ubuntu and Azure Linux -guest tests, the Ubuntu sandbox layer and live-share smoke tests, the +cover: the public managed execution configuration, the guest library +scenarios, the Ubuntu and Azure Linux guest tests, the Ubuntu sandbox layer +and live-share smoke tests, the caller-owned live-share test, and the `aci_edge_sandboxes` lifecycle test. Any RCU stall, soft lockup, or hung task makes the guest's time ABI watcher power off with status 194, which fails the run. Every guest those scenarios restore @@ -294,7 +295,16 @@ all. Ubuntu keys also include the Canonical archive pin, supplemental package lock, common guest sources, shared download and guest descriptor modules, and converter implementation. Each component is handed to the jobs that read it as a workflow artifact of its own, under its build filenames (see -[Guest artifact handoff](#guest-artifact-handoff)). Each +[Guest artifact handoff](#guest-artifact-handoff)). Beside them, the +GitHub-hosted `guest-test-programs` job builds the test-only guest programs +with `nvx.py build-guest-tests`, and each backend's +`nvx-microvm-guests-{kvm,mshv,whp}` job runs the guest library scenario +`gpt-device` with them: as root in an Alpine guest, the `gpt-device` program +validates a crafted GPT disk through the `nvx-rs` guest library, and rewrites +a scratch disk so that the kernel's partitions after a reread are missing, +extra, moved, or late, or come from an MBR that the kernel reads in place of +the table, all of which the validation must refuse or wait for. Its logs are +part of the backend's failure logs. Each backend also boots the Ubuntu initramfs and runs `/sbin/nvx-sandbox-smoke` from the Ubuntu EROFS layer as UID/GID 65534 over a fresh ext4 scratch copy, with a 32 MiB `--memory-max` and an 8-process @@ -490,7 +500,8 @@ GitHub-hosted runners, and `openvmm-changes`, which classifies a push or pull request and selects every suite for a full run (see [Change classification](#change-classification)). When it finishes, the jobs that it selects start: `artifacts`, `distro-artifacts`, and `debug-kernel` -restore or build the guest artifacts on GitHub-hosted runners (see +restore or build the guest artifacts, and `guest-test-programs` builds the +guest test programs, on GitHub-hosted runners (see [Guest artifact handoff](#guest-artifact-handoff)), the three OpenVMM binary producers restore or build OpenVMM on a KVM, an MSHV, and a WHP runner, `build-openvmm-vmm-tests-linux` builds the Linux VMM tests on a KVM runner, and @@ -500,7 +511,8 @@ Alpine initramfs, and the Linux VMM-test build, and `openvmm-vmm-tests-windows` for `artifacts`. Each microVM test and platform job waits for `artifacts`, its OpenVMM producer, and the NVX CLI job of its operating system. The microVM guest and sandbox jobs and the platform jobs also wait for `distro-artifacts`, -and the debug-kernel jobs for `debug-kernel`. `Performance regression gate`, +the microVM guest and sandbox jobs for `guest-test-programs`, and the +debug-kernel jobs for `debug-kernel`. `Performance regression gate`, `Publish development release`, `Persist performance baseline`, and `Required status check` wait for the jobs whose results they check. Required handoffs between these jobs use same-run workflow artifacts; caches only skip @@ -706,10 +718,13 @@ each with the action, whose `components` input selects what it builds: `artifacts` the kernel and the Alpine initramfs, `distro-artifacts` the Ubuntu initramfs, its EROFS distro layer, and the sandbox scratch template, together with the Azure -Linux initramfs, and `debug-kernel` the CI debug kernel. Each job uploads each -of its components as a one-day workflow artifact of its own. Each job that -boots or reads guest artifacts downloads, by name and into `build/`, only the -components that its steps read, and waits only for the jobs that upload them: +Linux initramfs, and `debug-kernel` the CI debug kernel. Beside them, +`guest-test-programs` builds the test-only guest programs with +`nvx.py build-guest-tests`. Each job uploads each of its components as a +one-day workflow artifact of its own. Each job that boots or reads guest +artifacts downloads, by name and into the directory of `build/` that holds +its files, only the components that its steps read, and waits only for the +jobs that upload them: | Artifact | Files in `build/` | Producer | Jobs that download it | | --- | --- | --- | --- | @@ -720,14 +735,16 @@ components that its steps read, and waits only for the jobs that upload them: | `guest-ubuntu-scratch` | `ubuntu-smoke-scratch.ext4`, the sandbox scratch template | `distro-artifacts` | microVM guest and sandbox jobs | | `guest-azurelinux` | `initramfs-azurelinux.cpio.gz` and its `.packages.json` | `distro-artifacts` | microVM guest and sandbox jobs, platform jobs | | `guest-debug-kernel` | `vmlinux-debug`, `vmlinux-debug.config`, `vmlinux-debug.provenance.json` | `debug-kernel` | debug-kernel jobs | +| `guest-test-programs` | `guest-tests/gpt-device` | `guest-test-programs` | microVM guest and sandbox jobs | The OpenVMM VMM tests boot only the kernel and the Alpine initramfs, and their Linux build boots no guest, so it downloads no component. The debug-kernel jobs run the Alpine scenarios on the debug kernel and qualify their runner, through the doctor's preflight (H3), with the kernel and the Alpine initramfs. The microVM default suites boot only those two, and the -microVM guest and sandbox jobs boot every guest and run sandboxes from the -Ubuntu distro layer over copies of the scratch template. The platform +microVM guest and sandbox jobs boot every guest, run sandboxes from the +Ubuntu distro layer over copies of the scratch template, and run the guest +test programs in the Alpine guest. The platform jobs boot the kernel and the Alpine initramfs to qualify their runner and to benchmark, and package every guest image but the scratch template into their release. A missing artifact fails its download step, so no job runs without a @@ -944,8 +961,9 @@ other path selects each suite that takes it as an input: of the guest binaries, which `GuestRustBuildConstants.SOURCE_PATTERN` matches, so a change to its host side or tests runs the NVX microVM tests and the platform jobs but not the VMM tests. -- `run-guest-artifacts` enables `artifacts`, `distro-artifacts`, and - `debug-kernel`, which build the guest artifacts and the CI debug kernel, and +- `run-guest-artifacts` enables `artifacts`, `distro-artifacts`, + `debug-kernel`, and `guest-test-programs`, which build the guest artifacts, + the CI debug kernel, and the guest test programs, and `run-openvmm-binaries` the three OpenVMM binary producers. Neither has a use of its own, so the classifier sets each exactly when a job that uses its artifacts runs: the guest artifacts for the OpenVMM VMM tests, the NVX @@ -954,17 +972,18 @@ other path selects each suite that takes it as an input: of each job that uses it. Every input of the VMM tests is also an input of the microVM tests, which read the Ubuntu and Azure Linux images of `distro-artifacts`, so the three guest artifact jobs share the output. The - debug kernel is built with the guest artifacts, as before, because the - development release and performance persistence require its build too. + debug kernel and the guest test programs are built with the guest + artifacts, because the development release and performance persistence + require their builds too. Every CLI module is an input of the microVM tests, and of the platform jobs, unless none of that suite's jobs runs it, so a new module selects both until the classifier excludes it. `microvm_tests.py` and the modules that only it, -`nvx.py sandbox`, or `nvx.py test-aci-edge-sandboxes` use, with the -`microvm_test_scripts/` that it runs in guests and the partition of its -[shards](#microvm-test-shards), `microvm_shards.json`, are inputs of the -microVM tests alone, and `performance.py` is an input of the platform jobs -alone. The +`nvx.py sandbox`, `nvx.py test-aci-edge-sandboxes`, or +`nvx.py build-guest-tests` use, with the `microvm_test_scripts/` that it runs +in guests and the partition of its [shards](#microvm-test-shards), +`microvm_shards.json`, are inputs of the microVM tests alone, and +`performance.py` is an input of the platform jobs alone. The modules that both kinds of job run, such as `benchmark.py`, which `microvm_tests.py` imports, `doctor.py`, which every runner validation runs, and `release.py`, whose `verify` command the OpenVMM builds run, select both. diff --git a/doc/project-structure.md b/doc/project-structure.md index 492ce6f9a..37a0a6a8c 100644 --- a/doc/project-structure.md +++ b/doc/project-structure.md @@ -103,11 +103,12 @@ nvx/ | | |-- protocol/ Control-console codec, NVXS session, and the guest targets that live shares may not take, shared by the host and the guest | | |-- contract.rs Control-contract versions and release compatibility | | |-- net/ CIDR parsing and subtraction, and hostname and guest-network forms +| | |-- gpt/ Strict reading of GPT disks whose first partition configures an adapter and whose other partitions hold a root's read-only layers, shared by the host and the guest | | |-- host/ Artifacts, launch, network and egress-policy lowering, sessions, VM instances, sandbox stores, the sandbox launch profile, and the `nvx-rs` CLI (feature `cli`) | | |-- testing/ Fake OpenVMM (feature `testing`) -| | |-- guest/ Programs that run in Linux guests, the guest's NVXS connection, the libraries through which any guest init or agent starts a program, collects its children's statuses, settles its executions' cgroups, drives each execution, bind-mounts host mappings below the roots that it names, and assembles and tears down a layered root, and the typed calls through which it runs the time ABI's steps and the reseed (feature `guest`) +| | |-- guest/ Programs that run in Linux guests, the guest's NVXS connection, the libraries through which any guest init or agent starts a program, collects its children's statuses, settles its executions' cgroups, drives each execution, bind-mounts host mappings below the roots that it names, assembles and tears down a layered root, and validates a GPT disk's block device against the kernel's partitions, and the typed calls through which it runs the time ABI's steps and the reseed (feature `guest`) | | `-- bin/ nvx-rs CLI, nvx-fake-openvmm test double, and the guest binaries -| |-- tests/ Fake-OpenVMM instance lifecycle, sandbox store, CLI, NVXS session, guest-binary, and hypervisor managed-lifecycle tests, the store's record fixtures, and the golden fixtures that Python recorded +| |-- tests/ Fake-OpenVMM instance lifecycle, sandbox store, CLI, NVXS session, guest-binary, hypervisor managed-lifecycle, and guest GPT device tests, the store's record fixtures, and the golden fixtures that Python recorded | |-- build.rs Stages the bundled artifacts (feature `bundled`) | `-- artifacts.json Release package pinned for the bundled artifacts |-- openvmm/ OpenVMM Git submodule @@ -134,6 +135,8 @@ nvx/ | | |-- development_release.py Resumable development release publication | | |-- doctor.py Time ABI host qualification | | |-- elf.py Static-executable check of ELF binaries +| | |-- gpt_image.py GPT disk images that the gpt-device microVM test attaches +| | |-- guest_test_programs.py Test-only guest programs that microVM tests run in a guest | | |-- guests.py Typed guest descriptors | | |-- host_telemetry.py Benchmark host provenance and Windows capture storage telemetry | | |-- host_time_probe.rs Host TSC probe that doctor builds and runs diff --git a/doc/usage.md b/doc/usage.md index 282d066a8..6e8157934 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -68,6 +68,7 @@ stale build never ships or runs unnoticed. | `check-guest-binaries` | Lint and test the Rust guest executables for musl. | | `build-guest-binaries` | Build and digest the pinned Rust executables for Linux guests. | | `test-guest-binaries` | Run the musl integration tests against prebuilt Rust guest executables. | +| `build-guest-tests` | Build the test-only Rust programs that microVM scenarios run in a guest. | | `build-kernel` | Build the pinned and patched Linux kernel natively. | | `build-initramfs` | Build the selected Alpine, Ubuntu, or Azure Linux initramfs. | | `build-distro-layer` | Build a deterministic Ubuntu EROFS distro layer. | @@ -370,6 +371,21 @@ The Linux-only test command validates the nine binaries and build manifest, sets `NVX_GUEST_BINARIES`, and runs the musl `guest_tools` integration test against that exact directory. +### `build-guest-tests` + +```text +python3 scripts/nvx.py build-guest-tests [--output DIRECTORY] +``` + +The Linux-only build command builds the test-only programs that +[`test-microvm`](#test-microvm) scenarios share into a guest and run there, as +root, rather than ship in a guest image: `gpt-device`, the `nvx-rs` integration +test `tests/gpt_device.rs`. It builds each with the `guest` feature for +`x86_64-unknown-linux-musl`, with the exact `rust-toolchain.toml` pin and +Cargo's locked mode, in the test profile, whose debug assertions it keeps, but +without debug information, runs the static-ELF check, and copies the programs to +`DIRECTORY`, by default `build/guest-tests`, where the scenarios look for them. + See [Setup](setup.md) for host prerequisites. ### `materialize-kernel-provenance-inputs` @@ -589,12 +605,19 @@ python3 scripts/nvx.py test-microvm Runs NVX-owned Linux, SMP, virtio, sandbox, and snapshot correctness scenarios against the public OpenVMM CLI. Repeat `--scenario` to select a subset; without -it, every scenario supported by the selected guest runs, except three that run +it, every scenario supported by the selected guest runs, except four that run only when named. `smp-lapic` repeats `smp` and also asserts that every CPU uses the one-shot counting LAPIC. `managed-exec-config` and `sandbox-lifecycle` drive the public `sandbox` lifecycle commands (see [Run](run.md#managed-lifecycle)) and also require the guest-artifact build's -Ubuntu layer and scratch template, which packages omit. Alpine remains the +Ubuntu layer and scratch template, which packages omit. `gpt-device` validates +crafted GPT disks in the guest through the `nvx-rs` guest library: it attaches +a read-only disk that holds a GPT and a writable scratch disk, shares the +test-only `gpt-device` program, which [`build-guest-tests`](#build-guest-tests) +builds into `build/guest-tests`, with disk images, and runs the program as root, +which validates the read-only disk and overwrites the scratch disk with tables +whose kernel partitions are missing, extra, moved, or late, or with an MBR that +the kernel reads in place of the table. Alpine remains the default. Ubuntu and Azure Linux cannot act as sandbox control, so they reject the Alpine-control-only `sandbox-blocks` and `scratch-snapshot` scenarios and the sandbox-control-dependent `snapshot-tiers`, `managed-exec-config`, and diff --git a/nvx-rs/README.md b/nvx-rs/README.md index a484f3bdb..c6c0138de 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -29,9 +29,9 @@ nvx-rs/ │ │ no dependencies, shared by the host and the guest │ ├── contract.rs control-contract versions and release-manifest compatibility │ ├── net/ CIDR parsing and subtraction, and hostname and guest-network forms -│ ├── gpt/ strict reading of GPT disks whose p1 configures an adapter and whose other -│ │ partitions hold a root's read-only layers; no dependencies, shared by the -│ │ host and the guest +│ ├── gpt/ strict reading of GPT disks whose first partition configures an adapter and +│ │ whose other partitions hold a root's read-only layers; no dependencies, +│ │ shared by the host and the guest │ ├── uuid.rs UUIDs in their canonical text and in GPT's layout; no dependencies, shared │ │ by the host and the guest │ ├── hypervisor.rs @@ -45,13 +45,14 @@ nvx-rs/ │ ├── guest/ feature `guest`: the programs that run in Linux guests, the guest's NVXS │ │ connection, the libraries through which a guest init or agent starts a │ │ program, collects its children's statuses, settles its executions' -│ │ cgroups, bind-mounts host mappings below the roots that it names, and -│ │ assembles and tears down a layered root, and the typed calls through which -│ │ it runs the time ABI's steps and the reseed +│ │ cgroups, bind-mounts host mappings below the roots that it names, +│ │ assembles and tears down a layered root, and validates a GPT disk's block +│ │ device, and the typed calls through which it runs the time ABI's steps and +│ │ the reseed │ └── bin/ nvx-rs, nvx-fake-openvmm, and the guest binaries -├── tests/ fake-OpenVMM, CLI, NVXS session, process-supervision, store, and guest-binary -│ tests, the store's record fixtures, and the golden fixtures that Python -│ recorded for the ports of its code, in fixtures/ +├── tests/ fake-OpenVMM, CLI, NVXS session, process-supervision, store, guest-binary, and +│ guest GPT device tests, the store's record fixtures, and the golden fixtures +│ that Python recorded for the ports of its code, in fixtures/ ├── build.rs bundled-release staging └── artifacts.json the NVX release that the `bundled` feature stages ``` @@ -65,7 +66,7 @@ nvx-rs/ | `nvx::protocol::share` | The guest targets that a sandbox's live shares may not take, which the host's `sandbox::mount` and the guest's `root` both refuse: `RESERVED_MOUNT_TARGETS`, the trees where the container runtime mounts or binds, `RESERVED_EXACT_MOUNT_TARGETS`, the directories that it creates, `RESERVED_TARGET_CHARACTERS`, and `reservation`, which tells why a target is reserved | | `nvx::contract` | The OpenVMM microVM ABI, broker protocol, and control-contract revision that the crate speaks, and `manifest_is_compatible` for a release's `SOURCE-MANIFEST.json` | | `nvx::net` | `Cidr`: IPv4 and IPv6 networks and their subtraction; `is_valid_hostname` and `is_valid_guest_network`, the forms of a VM's hostname and guest address | -| `nvx::gpt` | `Disk::read`: the protective MBR, primary header, and primary partition entry array of a GPT disk, read through a `ReadAt` source that the caller gives and checked strictly, as UEFI specifies them and as Linux requires them, without ever reading the backup: 128-byte entries, at most 1 MiB of them, used entries first and without holes, p1 and 1 to 254 layers, extents in the usable blocks without overlap, and unique GUIDs, with checked arithmetic in the disk's logical blocks. `Expected` adds the disk GUID and the partitions' unique GUIDs and sizes that a caller knows, and `Disk::read_p1` returns p1's opaque bytes, up to a bound and without trailing NUL bytes and whitespace. GUIDs are `nvx::uuid::Uuid`s. It has no dependencies, so the guest and the host's checks of the images that it stages share it | +| `nvx::gpt` | `Disk::read`: the protective MBR, primary header, and primary partition entry array of a GPT disk, read through a `ReadAt` source that the caller gives and checked strictly, as UEFI specifies them and as Linux requires them, without ever reading the backup: 128-byte entries, at most 1 MiB of them, used entries first and without holes, a first partition and 1 to 254 layers, extents in the usable blocks without overlap, and unique GUIDs, with checked arithmetic in the disk's logical blocks. `Expected` adds the disk GUID and the partitions' unique GUIDs and sizes that a caller knows, and `Disk::read_first_partition` returns the first partition's opaque bytes, up to a bound and without trailing NUL bytes and whitespace. GUIDs are `nvx::uuid::Uuid`s. It has no dependencies, so the guest and the host's checks of the images that it stages share it | | `nvx::uuid` | `Uuid`: a UUID, such as the GUID of a GPT disk or partition or the UUID of a filesystem, in the order of its canonical text, the one text that it displays and parses: 32 lowercase hexadecimal digits in groups of 8, 4, 4, 4, and 12, joined by hyphens. `from_bytes_le` and `to_bytes_le` convert the layout in which GPT and other EFI GUIDs store their first three groups little-endian. It has no dependencies, so the host and the guest share it | | `nvx::Hypervisor` | KVM, MSHV, or WHP, with its serde form behind the `serde` feature | | `nvx::host::artifacts` | `Artifacts` (also `nvx::Artifacts`): the OpenVMM executable, guest kernel, and Alpine initramfs, from a release, a repository checkout, the environment, a directory next to the executable, or the bundle. `ArtifactDir` and `Guest`: every artifact of the guests that `nvx.py` runs, the OpenVMM executable, the kernel, the Alpine, Ubuntu, and Azure Linux initramfs images and their package manifests, and the Ubuntu distro layer and its manifest, in a checkout's `build/` and `openvmm/target/release/` or a release's `guest/` and `bin/`, checked in the order and with the messages of `nvx.py`. `ArtifactPaths` lists them all and serializes as JSON | @@ -93,6 +94,7 @@ nvx-rs/ | `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | | `nvx::guest::reseed` | `Seed::read` and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. `nvx-reseed` runs them | | `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their expected sizes and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already: a read-only EROFS image, whose expected `nvx::uuid::Uuid` it checks in its superblock, under an overlay with `metacopy=on,xino=on`, mounted `nosuid,nodev`, whose upper and work directories are on an ext4 scratch mounted `nosuid,nodev` too. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, whose children it binds through `mappings::Mapping`. `Root::teardown`: the shares in reverse mount order, then the overlay, the image, and the scratch, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | +| `nvx::guest::gpt` | `BlockDevice`: a whole-disk block device that the caller opened, whose GPT `validate` reads with `nvx::gpt` at the device's logical block size and size, and checks against the caller's `Expected` values; it then checks that the kernel's partitions in sysfs are exactly the table's, with their numbers, starts, and sizes, and only when they differ asks the kernel to reread the table and waits, until a deadline, for them to match. `ValidatedDisk` returns the partitions' devices: the first partition's, and the layers' to mount | Lowering functions take the names by which messages refer to their inputs, such as `EgressRules::name` and `PathList::name`, so callers' messages name their own @@ -602,6 +604,8 @@ mount of the host's export and `/` as the roots, and any other guest init or agent can use them too. Any guest init or agent can also run the steps of `nvx-time` and `nvx-reseed` in its own process, through the `nvx::guest::time` and `nvx::guest::reseed` calls, on which both programs run. +Nor is the `gpt` library a program: it validates a GPT disk's block device +against the partitions that the kernel created from its table. | Binary | Program | | --- | --- | @@ -699,3 +703,16 @@ that it leaves: ```bash python3 ../scripts/nvx.py test-microvm --backend kvm --scenario managed-lifecycle ``` + +`tests/gpt_device.rs` holds ignored tests of `nvx::guest::gpt` that run as +root in a microVM, on the block devices that it attaches: a crafted GPT disk, +missing, extra, moved, and late partitions, and an MBR that the kernel reads in +place of a table. `nvx.py build-guest-tests` builds the test as a static musl +program in `build/guest-tests`, rather than adding it to a guest image, and +`nvx.py test-microvm --scenario gpt-device` shares it into the guest with the +disk images, and runs it there: + +```bash +python3 ../scripts/nvx.py build-guest-tests +python3 ../scripts/nvx.py test-microvm --backend kvm --scenario gpt-device +``` diff --git a/nvx-rs/src/gpt/error.rs b/nvx-rs/src/gpt/error.rs index edb149b01..3c12973d1 100644 --- a/nvx-rs/src/gpt/error.rs +++ b/nvx-rs/src/gpt/error.rs @@ -149,9 +149,9 @@ pub enum GptError { /// The partition's size, in bytes. found: u64, }, - /// p1 is larger than the caller's bound. - P1TooLarge { - /// The size of p1, in bytes. + /// The first partition is larger than the caller's bound. + FirstPartitionTooLarge { + /// The size of the first partition, in bytes. size: u64, /// The caller's bound, in bytes. bound: usize, @@ -250,7 +250,7 @@ impl fmt::Display for GptError { Self::PartitionCount(count) => write!( formatter, "the disk has {count} partitions, not {MIN_PARTITIONS} through {MAX_PARTITIONS}: \ - p1 and at least one layer" + the first partition and at least one layer" ), Self::Extent { number, @@ -295,9 +295,9 @@ impl fmt::Display for GptError { formatter, "partition {number} has {found} bytes, not the expected {expected}" ), - Self::P1TooLarge { size, bound } => write!( + Self::FirstPartitionTooLarge { size, bound } => write!( formatter, - "p1 has {size} bytes, more than the bound of {bound}" + "the first partition has {size} bytes, more than the bound of {bound}" ), } } diff --git a/nvx-rs/src/gpt/mod.rs b/nvx-rs/src/gpt/mod.rs index 00618b32d..834be2e92 100644 --- a/nvx-rs/src/gpt/mod.rs +++ b/nvx-rs/src/gpt/mod.rs @@ -1,4 +1,4 @@ -//! GUID Partition Table (GPT) disks that hold a layered root: a first partition, p1, whose bytes +//! GUID Partition Table (GPT) disks that hold a layered root: a first partition whose bytes //! configure an adapter, and the read-only filesystem layers of the root in the partitions after //! it. //! @@ -17,11 +17,11 @@ //! - **Entry array:** entries of [`ENTRY_SIZE`] bytes, the only size that Linux reads, at most //! [`MAX_ENTRY_ARRAY_BYTES`] of them in all, and a matching CRC-32. //! - **Partitions:** the used entries, those whose type GUID is not nil, come first, with no unused -//! entry among them, and number from [`MIN_PARTITIONS`], p1 and at least one layer, through -//! [`MAX_PARTITIONS`], the most that Linux creates. Each lies in the usable blocks, shares no -//! block with another, and has a unique GUID that is not nil and repeats no other GUID of the -//! disk. Linux numbers each partition one more than its entry's index, as [`Partition::number`] -//! does. +//! entry among them, and number from [`MIN_PARTITIONS`], the first partition and at least one +//! layer, through [`MAX_PARTITIONS`], the most that Linux creates. Each lies in the usable +//! blocks, shares no block with another, and has a unique GUID that is not nil and repeats no +//! other GUID of the disk. Linux numbers each partition one more than its entry's index, as +//! [`Partition::number`] does. //! //! The backup header and entry array are never read: a disk whose primary table is invalid fails //! even when its backup is intact. Linux, too, falls back to the backup only when its command @@ -31,8 +31,8 @@ //! the entry array, which it allocates only once the header has bounded its size, and computes //! positions with checked arithmetic, in the disk's logical blocks. [`Expected`] adds the //! identities and sizes that a caller knows, such as the disk GUID that a boot parameter names. -//! [`Disk::read_p1`] then reads p1 as opaque bytes, up to a bound that the caller sets, which it -//! neither interprets nor probes as a filesystem. +//! [`Disk::read_first_partition`] then reads the first partition as opaque bytes, up to a bound +//! that the caller sets, which it neither interprets nor probes as a filesystem. //! //! GUIDs are [`Uuid`]s, which GPT stores in the layout that [`Uuid::from_bytes_le`] reads. //! @@ -48,10 +48,10 @@ //! # fn main() -> Result<(), Box> { //! let image = File::open("disk.img")?; //! let disk = Disk::read(&image, 512, image.metadata()?.len(), &Expected::new())?; -//! let configuration = disk.read_p1(&image, 64 * 1024)?; -//! println!("p1 holds {} bytes of configuration", configuration.len()); +//! let configuration = disk.read_first_partition(&image, 64 * 1024)?; +//! println!("the first partition holds {} bytes of configuration", configuration.len()); //! for layer in disk.lowers() { -//! println!("p{}: {} bytes at {}", layer.number(), layer.size(), layer.offset()); +//! println!("partition {}: {} bytes at {}", layer.number(), layer.size(), layer.offset()); //! } //! # Ok(()) //! # } @@ -61,7 +61,7 @@ mod crc; mod error; mod source; #[cfg(test)] -mod tests; +pub(crate) mod tests; use crc::{Crc32, crc32}; pub use error::GptError; @@ -84,7 +84,7 @@ pub const ENTRY_SIZE: u32 = 128; /// allocation and the work of its CRC: 8,192 entries. UEFI's default array holds 128. pub const MAX_ENTRY_ARRAY_BYTES: u32 = 1024 * 1024; -/// The fewest partitions of a disk: p1 and one layer. +/// The fewest partitions of a disk: the first partition and one layer. pub const MIN_PARTITIONS: usize = 2; /// The most partitions of a disk: Linux creates partitions 1 through 255 only. @@ -182,7 +182,7 @@ mod entry { pub struct Expected { /// The disk's GUID, if the caller expects one. disk_guid: Option, - /// The partitions, p1 first, if the caller expects them. + /// The partitions, in the order of their numbers, if the caller expects them. partitions: Option>, } @@ -198,8 +198,8 @@ impl Expected { self } - /// Expects exactly as many partitions as `partitions` holds, p1 first, and each to have the - /// unique GUID and size that its expectation sets. + /// Expects exactly as many partitions as `partitions` holds, in the order of their numbers, + /// and each to have the unique GUID and size that its expectation sets. pub fn partitions(mut self, partitions: Vec) -> Self { self.partitions = Some(partitions); self @@ -277,7 +277,7 @@ impl ExpectedPartition { } /// A GPT disk whose protective MBR, primary header, and primary partition entry array -/// [`Disk::read`] checked, and its partitions: p1 and the layers after it. +/// [`Disk::read`] checked, and its partitions: the first partition and the layers after it. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Disk { /// The size of a logical block, in bytes. @@ -331,44 +331,43 @@ impl Disk { self.guid } - /// Returns every partition, p1 first, in the order of their numbers, which run from 1 - /// without a gap. + /// Returns every partition, in the order of their numbers, which run from 1 without a gap. pub fn partitions(&self) -> &[Partition] { &self.partitions } - /// Returns p1, the partition whose bytes configure the adapter. - pub fn p1(&self) -> &Partition { + /// Returns the first partition, whose bytes configure the adapter. + pub fn first_partition(&self) -> &Partition { &self.partitions[0] } - /// Returns the partitions after p1, which hold the read-only layers of the root: at least - /// one. + /// Returns the partitions after the first, which hold the read-only layers of the root: at + /// least one. pub fn lowers(&self) -> &[Partition] { &self.partitions[1..] } - /// Reads p1 from `source`, the disk that [`Disk::read`] read: every byte of the partition, - /// less the trailing NUL bytes and ASCII whitespace (space, tab, line feed, form feed, and - /// carriage return), as opaque bytes that need not be UTF-8. + /// Reads the first partition from `source`, the disk that [`Disk::read`] read: every byte of + /// the partition, less the trailing NUL bytes and ASCII whitespace (space, tab, line feed, + /// form feed, and carriage return), as opaque bytes that need not be UTF-8. /// - /// A p1 larger than `bound` bytes fails before any of it is read, so that no more than - /// `bound` bytes are read or allocated. - pub fn read_p1( + /// A first partition larger than `bound` bytes fails before any of it is read, so that no + /// more than `bound` bytes are read or allocated. + pub fn read_first_partition( &self, source: &S, bound: usize, ) -> Result, GptError> { - let p1 = self.p1(); - let length = usize::try_from(p1.size) + let partition = self.first_partition(); + let length = usize::try_from(partition.size) .ok() .filter(|length| *length <= bound) - .ok_or(GptError::P1TooLarge { - size: p1.size, + .ok_or(GptError::FirstPartitionTooLarge { + size: partition.size, bound, })?; let mut bytes = vec![0; length]; - source.read_exact_at(p1.offset, &mut bytes)?; + source.read_exact_at(partition.offset, &mut bytes)?; let end = bytes .iter() .rposition(|byte| *byte != 0 && !byte.is_ascii_whitespace()) @@ -401,7 +400,7 @@ pub struct Partition { impl Partition { /// Returns the partition's number, as Linux numbers it: one more than its entry's index, so - /// p1 is 1, and the first layer 2. + /// the first partition is 1, and the first layer 2. pub fn number(&self) -> u8 { self.number } diff --git a/nvx-rs/src/gpt/tests.rs b/nvx-rs/src/gpt/tests.rs index 8de0d7bab..2d18a021c 100644 --- a/nvx-rs/src/gpt/tests.rs +++ b/nvx-rs/src/gpt/tests.rs @@ -101,14 +101,14 @@ struct Spec { entry_array_crc: Option, /// The partition entries by index; `None` leaves an entry unused. entries: Vec>, - /// Bytes written at the start of a block, such as p1's contents. + /// Bytes written at the start of a block, such as the first partition's contents. contents: Vec<(u64, Vec)>, } impl Spec { /// Returns a valid disk of `blocks` blocks of `block_size` bytes, whose `entry_count` - /// entries start at LBA 2, as do UEFI's defaults, and with three partitions: p1, one block, - /// and two layers of 8 and 16 blocks after it. + /// entries start at LBA 2, as do UEFI's defaults, and with three partitions: the first, one + /// block, and two layers of 8 and 16 blocks after it. fn new(block_size: u32, blocks: u64, entry_count: u32) -> Self { let array_blocks = (u64::from(entry_count) * 128).div_ceil(u64::from(block_size)); let first_usable_lba = 2 + array_blocks; @@ -250,6 +250,13 @@ impl Spec { } } +/// Returns [`Spec::standard`]'s disk, for the tests of other modules: the first partition in LBA +/// 34, and layers in LBAs 35 through 42 and 43 through 58, of 512 bytes each. +#[cfg(all(feature = "guest", target_os = "linux"))] +pub(crate) fn standard_disk() -> Disk { + Spec::standard().read().unwrap() +} + /// A source that records the byte ranges that the reader reads. struct Recording<'a> { /// The disk. @@ -307,12 +314,12 @@ fn a_disk_reads_with_its_partitions_in_number_order() { (3, 43, 58, 43 * 512, 16 * 512), ] ); - assert_eq!(disk.p1().number(), 1); - assert_eq!(disk.p1().unique_guid(), guid(1)); - assert_eq!(disk.p1().type_guid(), LINUX_DATA); + assert_eq!(disk.first_partition().number(), 1); + assert_eq!(disk.first_partition().unique_guid(), guid(1)); + assert_eq!(disk.first_partition().type_guid(), LINUX_DATA); // GPT stores the first three groups of a GUID little-endian. assert_eq!( - disk.p1().type_guid().to_string(), + disk.first_partition().type_guid().to_string(), "0fc63daf-8483-4772-8e79-3d69d8477de4" ); let lowers: Vec<_> = disk.lowers().iter().map(Partition::number).collect(); @@ -516,12 +523,12 @@ fn the_entries_are_128_bytes_and_bounded_in_number() { // The largest array, 1 MiB in 256 4096-byte blocks, and the smallest, one entry with the // others of its block outside the array, which its CRC does not cover. let spec = Spec::new(4096, 600, 8192); - assert_eq!(spec.read().unwrap().p1().first_lba(), 258); + assert_eq!(spec.read().unwrap().first_partition().first_lba(), 258); let mut spec = Spec::new(512, 2048, 3); spec.contents .push((2, [spec.array(), vec![0xFF; 128]].concat())); let disk = spec.read().unwrap(); - assert_eq!(disk.p1().first_lba(), 3); + assert_eq!(disk.first_partition().first_lba(), 3); } #[test] @@ -544,7 +551,7 @@ fn the_entry_array_lies_after_the_header_and_before_the_usable_blocks() { spec.first_usable_lba = 35; spec.entry(1).first_lba = 59; spec.entry(1).last_lba = 59; - assert_eq!(spec.read().unwrap().p1().offset(), 59 * 512); + assert_eq!(spec.read().unwrap().first_partition().offset(), 59 * 512); } #[test] @@ -639,9 +646,12 @@ fn the_reader_never_reads_past_the_usable_blocks() { reads: RefCell::new(Vec::new()), }; let disk = Disk::read(&source, 512, image.len() as u64, &Expected::new()).unwrap(); - assert_eq!(disk.read_p1(&source, 512).unwrap(), b"configuration"); + assert_eq!( + disk.read_first_partition(&source, 512).unwrap(), + b"configuration" + ); let reads = source.reads.into_inner(); - // The MBR, the header, the array, and p1, each read once. + // The MBR, the header, the array, and the first partition, each read once. assert_eq!( reads, [(0, 512), (512, 512), (1024, 128 * 128), (34 * 512, 512)] @@ -694,7 +704,7 @@ fn partitions_share_no_block() { spec.entry(3).first_lba = 34; spec.entry(3).last_lba = 34; let disk = spec.read().unwrap(); - assert_eq!(disk.p1().first_lba(), 100); + assert_eq!(disk.first_partition().first_lba(), 100); assert_eq!(disk.lowers()[1].first_lba(), 34); } @@ -728,7 +738,7 @@ fn used_entries_come_first_without_holes() { } #[test] -fn a_disk_has_p1_and_one_to_254_layers() { +fn a_disk_has_a_first_partition_and_one_to_254_layers() { let mut spec = Spec::standard(); spec.entries.clear(); assert_fails!(spec.read(), GptError::PartitionCount(0)); @@ -774,7 +784,7 @@ fn partition_guids_are_unique_and_not_nil() { let mut spec = Spec::standard(); spec.entry(1).type_guid = guid(0x77); let disk = spec.read().unwrap(); - assert_eq!(disk.p1().type_guid(), guid(0x77)); + assert_eq!(disk.first_partition().type_guid(), guid(0x77)); assert_eq!(disk.lowers()[0].type_guid(), LINUX_DATA); } @@ -847,9 +857,9 @@ fn expectations_follow_the_format_checks() { ); } -/// Returns the standard disk whose p1, of `blocks` blocks of `block_size` bytes, starts with -/// `contents`. -fn disk_with_p1(block_size: u32, blocks: u64, contents: &[u8]) -> (Disk, Vec) { +/// Returns the standard disk whose first partition, of `blocks` blocks of `block_size` bytes, +/// starts with `contents`. +fn disk_with_first_partition(block_size: u32, blocks: u64, contents: &[u8]) -> (Disk, Vec) { let mut spec = if block_size == 512 { Spec::standard() } else { @@ -874,60 +884,84 @@ fn disk_with_p1(block_size: u32, blocks: u64, contents: &[u8]) -> (Disk, Vec } #[test] -fn p1_is_its_opaque_bytes_without_trailing_padding() { +fn the_first_partition_is_its_opaque_bytes_without_trailing_padding() { // One sector of configuration, padded with NUL bytes. - let (disk, image) = disk_with_p1(512, 1, b"{\"layers\":2}\n"); - assert_eq!(disk.p1().size(), 512); + let (disk, image) = disk_with_first_partition(512, 1, b"{\"layers\":2}\n"); + assert_eq!(disk.first_partition().size(), 512); assert_eq!( - disk.read_p1(image.as_slice(), 512).unwrap(), + disk.read_first_partition(image.as_slice(), 512).unwrap(), b"{\"layers\":2}" ); // Bytes that are not UTF-8, with NUL bytes and whitespace inside, are kept. let bytes = [0xFF, 0x00, 0xC3, 0x28, b' ', 0x80, b'\t', 0x01]; - let (disk, image) = disk_with_p1(512, 1, &bytes); - assert_eq!(disk.read_p1(image.as_slice(), 512).unwrap(), bytes); + let (disk, image) = disk_with_first_partition(512, 1, &bytes); + assert_eq!( + disk.read_first_partition(image.as_slice(), 512).unwrap(), + bytes + ); // Trailing NUL bytes and whitespace go, in any order; leading ones stay, as does a // vertical tab, which is not ASCII whitespace. - let (disk, image) = disk_with_p1(512, 1, b" \tx \t\r\n\x0c\0 \0\n"); - assert_eq!(disk.read_p1(image.as_slice(), 512).unwrap(), b" \tx"); - let (disk, image) = disk_with_p1(512, 1, b"x\x0b\0"); - assert_eq!(disk.read_p1(image.as_slice(), 512).unwrap(), b"x\x0b"); - let (disk, image) = disk_with_p1(512, 1, b"\0\n \0"); - assert_eq!(disk.read_p1(image.as_slice(), 512).unwrap(), b""); - - // A full sector, and p1 across several sectors. + let (disk, image) = disk_with_first_partition(512, 1, b" \tx \t\r\n\x0c\0 \0\n"); + assert_eq!( + disk.read_first_partition(image.as_slice(), 512).unwrap(), + b" \tx" + ); + let (disk, image) = disk_with_first_partition(512, 1, b"x\x0b\0"); + assert_eq!( + disk.read_first_partition(image.as_slice(), 512).unwrap(), + b"x\x0b" + ); + let (disk, image) = disk_with_first_partition(512, 1, b"\0\n \0"); + assert_eq!( + disk.read_first_partition(image.as_slice(), 512).unwrap(), + b"" + ); + + // A full sector, and a first partition across several sectors. let full = vec![b'a'; 512]; - let (disk, image) = disk_with_p1(512, 1, &full); - assert_eq!(disk.read_p1(image.as_slice(), 512).unwrap(), full); + let (disk, image) = disk_with_first_partition(512, 1, &full); + assert_eq!( + disk.read_first_partition(image.as_slice(), 512).unwrap(), + full + ); let long: Vec = (0..1500).map(|index| (index % 251) as u8 + 1).collect(); - let (disk, image) = disk_with_p1(512, 3, &long); - assert_eq!(disk.read_p1(image.as_slice(), 1536).unwrap(), long); + let (disk, image) = disk_with_first_partition(512, 3, &long); + assert_eq!( + disk.read_first_partition(image.as_slice(), 1536).unwrap(), + long + ); // One sector of a disk of 4096-byte blocks. - let (disk, image) = disk_with_p1(4096, 1, b"config\n"); - assert_eq!(disk.p1().size(), 4096); - assert_eq!(disk.read_p1(image.as_slice(), 4096).unwrap(), b"config"); + let (disk, image) = disk_with_first_partition(4096, 1, b"config\n"); + assert_eq!(disk.first_partition().size(), 4096); + assert_eq!( + disk.read_first_partition(image.as_slice(), 4096).unwrap(), + b"config" + ); } #[test] -fn p1_over_the_bound_fails_before_it_is_read() { - let (disk, image) = disk_with_p1(512, 2, b"x"); +fn a_first_partition_over_the_bound_fails_before_it_is_read() { + let (disk, image) = disk_with_first_partition(512, 2, b"x"); for bound in [0, 1, 1023] { assert_fails!( - disk.read_p1(image.as_slice(), bound), - GptError::P1TooLarge { size: 1024, bound: found } if found == bound + disk.read_first_partition(image.as_slice(), bound), + GptError::FirstPartitionTooLarge { size: 1024, bound: found } if found == bound ); } let source = Recording { bytes: &image, reads: RefCell::new(Vec::new()), }; - assert!(disk.read_p1(&source, 1023).is_err()); + assert!(disk.read_first_partition(&source, 1023).is_err()); assert!(source.reads.borrow().is_empty()); - assert_eq!(disk.read_p1(&source, 1024).unwrap(), b"x"); - assert_eq!(disk.read_p1(&source, usize::MAX).unwrap(), b"x"); + assert_eq!(disk.read_first_partition(&source, 1024).unwrap(), b"x"); + assert_eq!( + disk.read_first_partition(&source, usize::MAX).unwrap(), + b"x" + ); } #[test] @@ -945,7 +979,7 @@ fn a_short_source_fails_its_read() { } let disk = Disk::read(image.as_slice(), 512, size, &Expected::new()).unwrap(); assert_fails!( - disk.read_p1(&image[..34 * 512 + 100], 512), + disk.read_first_partition(&image[..34 * 512 + 100], 512), GptError::Io(error) if error.kind() == io::ErrorKind::UnexpectedEof ); } @@ -981,7 +1015,10 @@ fn files_read_at_offsets() { let mut file = tempfile::tempfile().unwrap(); file.write_all(&spec.image()).unwrap(); let disk = Disk::read(&file, 512, 2048 * 512, &Expected::new()).unwrap(); - assert_eq!(disk.read_p1(&file, 512).unwrap(), b"from a file"); + assert_eq!( + disk.read_first_partition(&file, 512).unwrap(), + b"from a file" + ); } #[test] @@ -993,7 +1030,7 @@ fn errors_describe_the_check_and_keep_their_cause() { ); assert_eq!( GptError::PartitionCount(256).to_string(), - "the disk has 256 partitions, not 2 through 255: p1 and at least one layer" + "the disk has 256 partitions, not 2 through 255: the first partition and at least one layer" ); assert_eq!( GptError::EntryCount(0).to_string(), diff --git a/nvx-rs/src/guest/gpt/error.rs b/nvx-rs/src/guest/gpt/error.rs new file mode 100644 index 000000000..82194a2d3 --- /dev/null +++ b/nvx-rs/src/guest/gpt/error.rs @@ -0,0 +1,178 @@ +//! Why a disk's block device failed its validation. + +use std::fmt; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::gpt::GptError; + +/// An operation on a block device. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DeviceOperation { + /// Reading the device's status: its file type and device number. + Status, + /// Reading the device's logical block size. + LogicalBlockSize, + /// Reading the device's size. + Size, + /// Asking the kernel to reread the device's partition table. + RereadPartitions, +} + +/// Describes the operation as what failed, such as "reread the device's partition table". +impl fmt::Display for DeviceOperation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Status => "read the device's status", + Self::LogicalBlockSize => "read the device's logical block size", + Self::Size => "read the device's size", + Self::RereadPartitions => "reread the device's partition table", + }) + } +} + +/// How the partitions that the kernel created for a disk differ from the partitions of its table: +/// the first difference, in the order of the partition numbers, or partitions that changed while +/// they were read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Mismatch { + /// The kernel has no partition of this number, which the table has. + Missing { + /// The partition's number. + number: u8, + }, + /// The kernel has a partition of this number, which the table lacks. + Unexpected { + /// The partition's number. + number: u8, + }, + /// The kernel's partition of this number spans other bytes of the disk than the table's. + Extent { + /// The partition's number. + number: u8, + /// The first byte of the kernel's partition. + start: u64, + /// The size of the kernel's partition, in bytes. + size: u64, + /// The first byte of the table's partition. + expected_start: u64, + /// The size of the table's partition, in bytes. + expected_size: u64, + }, + /// The kernel's partitions of the disk changed while they were read, as they do when the + /// kernel rereads the partition table, so they could not be compared. + Changed, +} + +impl fmt::Display for Mismatch { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Missing { number } => write!( + formatter, + "the kernel has no partition {number}, which the table has" + ), + Self::Unexpected { number } => write!( + formatter, + "the kernel has a partition {number}, which the table lacks" + ), + Self::Extent { + number, + start, + size, + expected_start, + expected_size, + } => write!( + formatter, + "the kernel's partition {number} has {size} bytes from byte {start}, but the \ + table's has {expected_size} bytes from byte {expected_start}" + ), + Self::Changed => formatter.write_str("they changed while they were read"), + } + } +} + +/// Why a disk's block device failed its validation: the device, its sysfs description, its +/// partition table, or the partitions that the kernel created from the table. +#[derive(Debug)] +#[non_exhaustive] +pub enum DeviceError { + /// The file is not a block device. + NotBlockDevice, + /// The block device is a partition rather than a whole disk. + NotWholeDisk, + /// An operation on the device failed. + Device { + /// The operation. + operation: DeviceOperation, + /// Why it failed. + source: io::Error, + }, + /// A file of the device's sysfs description, or of one of its partitions', could not be read, + /// or held text in another form than the kernel writes, which fails with + /// [`io::ErrorKind::InvalidData`]. + Sysfs { + /// The file. + path: PathBuf, + /// Why it could not be read. + source: io::Error, + }, + /// The disk's partition table is invalid, or does not meet the caller's expected values. + Table(GptError), + /// The kernel's partitions still did not match the table's when the timeout passed. + Partitions(Mismatch), +} + +impl DeviceError { + /// Returns the error for `operation` on the device, which failed with `source`. + pub(super) fn device(operation: DeviceOperation, source: io::Error) -> Self { + Self::Device { operation, source } + } + + /// Returns the error for the sysfs file at `path`, which could not be read. + pub(super) fn sysfs(path: &Path, source: io::Error) -> Self { + Self::Sysfs { + path: path.to_owned(), + source, + } + } +} + +impl fmt::Display for DeviceError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotBlockDevice => formatter.write_str("the file is not a block device"), + Self::NotWholeDisk => { + formatter.write_str("the block device is a partition, not a whole disk") + } + Self::Device { operation, source } => { + write!(formatter, "cannot {operation}: {source}") + } + Self::Sysfs { path, source } => { + write!(formatter, "cannot read {}: {source}", path.display()) + } + Self::Table(error) => write!(formatter, "invalid partition table: {error}"), + Self::Partitions(mismatch) => write!( + formatter, + "the kernel's partitions do not match the table's: {mismatch}" + ), + } + } +} + +impl std::error::Error for DeviceError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Device { source, .. } | Self::Sysfs { source, .. } => Some(source), + Self::Table(error) => Some(error), + _ => None, + } + } +} + +impl From for DeviceError { + fn from(error: GptError) -> Self { + Self::Table(error) + } +} diff --git a/nvx-rs/src/guest/gpt/mod.rs b/nvx-rs/src/guest/gpt/mod.rs new file mode 100644 index 000000000..1a44d2283 --- /dev/null +++ b/nvx-rs/src/guest/gpt/mod.rs @@ -0,0 +1,648 @@ +//! GPT disks that the guest's kernel attached as block devices: a disk's partition table, which +//! [`crate::gpt`] reads and checks, validated against the partitions that the kernel created +//! from it. +//! +//! A guest that boots from such a disk, such as one whose first partition configures an adapter +//! and whose later partitions hold the read-only layers of its root, first resolves the disk's +//! block device by its role, checks the device's access mode and size, flushes its cached +//! buffers, and opens it. [`BlockDevice::validate`] then: +//! +//! 1. reads the disk's protective MBR, primary GPT header, and primary partition entry array +//! through the device, at its logical block size and size, and checks them and the caller's +//! [`Expected`] values, as [`Disk::read`] describes; +//! 2. compares the kernel's partitions, as sysfs describes them, with the table's: the kernel +//! must have one partition for each of the table's, with its number, its first byte, and its +//! size, and no other. The kernel created its partitions from the disk's table when it found +//! the disk, so they usually match, and the validation ends there; +//! 3. only when they differ, asks the kernel to reread the disk's partition table, which drops +//! the disk's partitions and creates them again from what the device holds, and compares them +//! again, until they match or a deadline passes. +//! +//! It returns the block devices of the partitions: the first partition's, whose bytes the caller +//! reads as opaque configuration with [`Disk::read_first_partition`], and the layers', which the +//! caller mounts. When the kernel's partitions still differ from the table that the validation +//! read after the reread, for example because the table changed after the read, or because the +//! kernel read an MBR in its place, the validation fails, so the caller never uses a partition +//! that the table does not describe. +//! +//! The kernel creates a disk's partitions before its reread returns, but another program, such as +//! a device manager that probes the disk, may make it drop and create them again at any time. The +//! comparison therefore repeats, after pauses that grow from 1 ms to 4 ms, until the deadline, +//! which also bounds a caller that waits for partitions that the kernel creates late. Each +//! comparison takes partitions that the kernel held together: it reads each partition's +//! description through one descriptor of the partition's sysfs directory, then lists the disk's +//! directories again, and checks each one again through its first descriptor, so that it never +//! combines partitions, or the attributes of one partition, from before and after a reread. +//! +//! The reread needs `CAP_SYS_ADMIN`, and fails while any partition of the disk is open, for +//! example while a layer is mounted. The validation of a disk whose partitions already match its +//! table needs neither. +//! +//! ```no_run +//! use std::fs::File; +//! use std::time::Duration; +//! +//! use nvx::gpt::Expected; +//! use nvx::guest::gpt::BlockDevice; +//! +//! # fn main() -> Result<(), Box> { +//! let file = File::open("/dev/vda")?; +//! let expected = Expected::new().disk_guid("8d2f3a4e-1c5b-4e7a-9f60-2b3c4d5e6f70".parse()?); +//! let validated = BlockDevice::new(&file)?.validate(&expected, Duration::from_secs(5))?; +//! let configuration = validated.disk().read_first_partition(&file, 64 * 1024)?; +//! println!("the first partition holds {} bytes of configuration", configuration.len()); +//! for layer in validated.lowers() { +//! println!("layer {:?} is device {}", layer.name(), layer.device_number()); +//! } +//! # Ok(()) +//! # } +//! ``` + +mod error; +#[cfg(test)] +mod tests; + +use std::ffi::{CStr, OsStr, OsString}; +use std::fmt; +use std::fs::{self, File}; +use std::io::{self, Read}; +use std::os::fd::AsFd; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::{FileTypeExt, MetadataExt}; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::{Duration, Instant}; + +pub use error::{DeviceError, DeviceOperation, Mismatch}; + +use super::parse::parse_decimal; +use super::sys::{self, Errno, FileDescriptor, Mode, OpenFlags}; +use crate::gpt::{Disk, Expected, Partition}; + +/// The first pause between two comparisons of the kernel's partitions with a table. +const FIRST_PAUSE: Duration = Duration::from_millis(1); + +/// The longest pause between two comparisons of the kernel's partitions with a table, and so about +/// the longest that a wait takes to see partitions that match once the kernel has created them. A +/// comparison of a few partitions takes well under a millisecond, so the wait stays cheap. +const LONGEST_PAUSE: Duration = Duration::from_millis(4); + +/// A device number: the major and minor numbers that name a device to the kernel. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DeviceNumber { + /// The major number, which names the device's driver. + major: u32, + /// The minor number, which names the device among the driver's. + minor: u32, +} + +impl DeviceNumber { + /// Returns the major number, which names the device's driver. + pub fn major(self) -> u32 { + self.major + } + + /// Returns the minor number, which names the device among the driver's. + pub fn minor(self) -> u32 { + self.minor + } +} + +/// Formats the number as sysfs does, `MAJOR:MINOR`. +impl fmt::Display for DeviceNumber { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}:{}", self.major, self.minor) + } +} + +/// A whole-disk block device that the caller opened, and its geometry, to validate the GPT that +/// it holds. +#[derive(Debug)] +pub struct BlockDevice<'a> { + /// The device, as the caller opened it. + file: &'a File, + /// The device's number. + number: DeviceNumber, + /// The device's logical block size, in bytes. + logical_block_size: u32, + /// The device's size, in bytes. + size: u64, + /// The device's sysfs directory, which holds a directory for each of its partitions. + directory: PathBuf, +} + +impl<'a> BlockDevice<'a> { + /// Takes the block device that `file` opens, which must be a whole disk rather than one of + /// its partitions, and reads its number, logical block size, and size. + /// + /// The device may be open for reading only: the validation never writes it. + pub fn new(file: &'a File) -> Result { + let status = file + .metadata() + .map_err(|source| DeviceError::device(DeviceOperation::Status, source))?; + if !status.file_type().is_block_device() { + return Err(DeviceError::NotBlockDevice); + } + let (major, minor) = sys::split_device_number(status.rdev()); + let (Ok(major), Ok(minor)) = (u32::try_from(major), u32::try_from(minor)) else { + let source = io::Error::new(io::ErrorKind::InvalidData, "a device number overflows"); + return Err(DeviceError::device(DeviceOperation::Status, source)); + }; + let number = DeviceNumber { major, minor }; + let directory = Path::new(sys::BLOCK_DEVICE_DIRECTORY).join(number.to_string()); + // The disk's own description, read through one descriptor of its sysfs directory, + // confirms that sysfs describes this device, and that the device is a whole disk. + let absent = |path: &Path| DeviceError::sysfs(path, io::ErrorKind::NotFound.into()); + let Some(description) = SysfsDirectory::open(directory.clone())? else { + return Err(absent(&directory)); + }; + match description.device_number(c"dev")? { + Some(described) if described == number => {} + Some(_) => return Err(malformed(&description.path_of(c"dev"))), + None => return Err(absent(&description.path_of(c"dev"))), + } + if description.attribute(c"partition")?.is_some() { + return Err(DeviceError::NotWholeDisk); + } + let logical_block_size = sys::logical_block_size(file.as_fd()).map_err(|errno| { + DeviceError::device(DeviceOperation::LogicalBlockSize, errno.into()) + })?; + let size = sys::block_device_size(file.as_fd()) + .map_err(|errno| DeviceError::device(DeviceOperation::Size, errno.into()))?; + Ok(Self { + file, + number, + logical_block_size, + size, + directory, + }) + } + + /// Returns the device's number. + pub fn number(&self) -> DeviceNumber { + self.number + } + + /// Returns the device's logical block size, in bytes. + pub fn logical_block_size(&self) -> u32 { + self.logical_block_size + } + + /// Returns the device's size, in bytes. + pub fn size(&self) -> u64 { + self.size + } + + /// Reads and checks the disk's partition table through the device, which must also meet + /// `expected`, as [`Disk::read`] describes. + pub fn read_disk(&self, expected: &Expected) -> Result { + Ok(Disk::read( + self.file, + self.logical_block_size, + self.size, + expected, + )?) + } + + /// Asks the kernel to reread the disk's partition table, and to create the disk's partitions + /// again from what the device holds, before it returns. + /// + /// The kernel requires `CAP_SYS_ADMIN`, and fails while any partition of the disk is open. + pub fn reread_partitions(&self) -> Result<(), DeviceError> { + sys::reread_partitions(self.file.as_fd()) + .map_err(|errno| DeviceError::device(DeviceOperation::RereadPartitions, errno.into())) + } + + /// Waits until the kernel's partitions of the device are exactly those of `disk`, which + /// [`BlockDevice::read_disk`] read, and returns their block devices, or fails with the first + /// difference that remains once `timeout` has passed. + /// + /// It compares the partitions once, then again after each pause, which grows from 1 ms to + /// 4 ms, until they match or the timeout passes. It never asks the kernel to reread the + /// table, as [`BlockDevice::validate`] does when the partitions differ. + pub fn await_partitions( + &self, + disk: &Disk, + timeout: Duration, + ) -> Result { + let partitions = + wait_for_partitions(disk, timeout, || observe_partitions(&self.directory))?; + Ok(ValidatedDisk { + disk: disk.clone(), + partitions, + }) + } + + /// Validates the disk, as the module describes: reads and checks its partition table, which + /// must also meet `expected`, and compares the kernel's partitions with it. Only when they + /// differ does it ask the kernel to reread the table, and wait until the partitions match it, + /// for at most `timeout`. + pub fn validate( + &self, + expected: &Expected, + timeout: Duration, + ) -> Result { + let disk = self.read_disk(expected)?; + match self.await_partitions(&disk, Duration::ZERO) { + Err(DeviceError::Partitions(_)) => {} + compared => return compared, + } + self.reread_partitions()?; + self.await_partitions(&disk, timeout) + } +} + +/// A disk whose partition table [`BlockDevice`] checked, and the block devices that the kernel +/// created for its partitions, which matched the table. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedDisk { + /// The disk's partition table. + disk: Disk, + /// The partitions' block devices, in the order of their numbers. + partitions: Vec, +} + +impl ValidatedDisk { + /// Returns the disk's partition table, with which the caller reads the first partition + /// through [`Disk::read_first_partition`]. + pub fn disk(&self) -> &Disk { + &self.disk + } + + /// Returns the block device of every partition, in the order of their numbers, which run + /// from 1 without a gap. + pub fn partitions(&self) -> &[PartitionDevice] { + &self.partitions + } + + /// Returns the block device of the first partition, whose bytes configure the adapter. + pub fn first_partition(&self) -> &PartitionDevice { + &self.partitions[0] + } + + /// Returns the block devices of the partitions after the first, which hold the read-only + /// layers of the root: at least one. + pub fn lowers(&self) -> &[PartitionDevice] { + &self.partitions[1..] + } +} + +/// A partition of a validated disk: its table entry, and the block device that the kernel +/// created for it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartitionDevice { + /// The partition's table entry. + partition: Partition, + /// The kernel's name of the partition's block device. + name: OsString, + /// The device number of the partition's block device. + device_number: DeviceNumber, +} + +impl PartitionDevice { + /// Returns the partition's table entry: its number, GUIDs, and blocks. + pub fn partition(&self) -> &Partition { + &self.partition + } + + /// Returns the kernel's name of the partition's block device, such as `vda2`, under which + /// sysfs lists it and devtmpfs creates its node in `/dev`. + pub fn name(&self) -> &OsStr { + &self.name + } + + /// Returns the device number of the partition's block device. + pub fn device_number(&self) -> DeviceNumber { + self.device_number + } +} + +/// A partition that the kernel created for a disk, as its sysfs directory describes it. +#[derive(Clone, Debug, PartialEq, Eq)] +struct KernelPartition { + /// The partition's number. + number: u8, + /// The kernel's name of the partition's block device. + name: OsString, + /// The device number of the partition's block device. + device_number: DeviceNumber, + /// The partition's first byte on the disk. + start: u64, + /// The partition's size, in bytes. + size: u64, +} + +/// Compares the kernel's partitions of a disk, `kernel`, with the partitions of its table, +/// `disk`, in the order of their numbers, and returns their block devices, or the first +/// difference. +fn compare(disk: &Disk, kernel: &[KernelPartition]) -> Result, Mismatch> { + let table = disk.partitions(); + let highest = table + .iter() + .map(Partition::number) + .chain(kernel.iter().map(|partition| partition.number)) + .max() + .unwrap_or(0); + let mut devices = Vec::with_capacity(table.len()); + for number in 1..=highest { + let entry = table.iter().find(|entry| entry.number() == number); + let observed = kernel.iter().find(|partition| partition.number == number); + match (entry, observed) { + (Some(entry), Some(observed)) => { + if (observed.start, observed.size) != (entry.offset(), entry.size()) { + return Err(Mismatch::Extent { + number, + start: observed.start, + size: observed.size, + expected_start: entry.offset(), + expected_size: entry.size(), + }); + } + devices.push(PartitionDevice { + partition: *entry, + name: observed.name.clone(), + device_number: observed.device_number, + }); + } + (Some(_), None) => return Err(Mismatch::Missing { number }), + (None, Some(_)) => return Err(Mismatch::Unexpected { number }), + (None, None) => {} + } + } + Ok(devices) +} + +/// Observes the kernel's partitions of a disk with `observe`, and compares them with the +/// partitions of the disk's table, `disk`, until they match or `timeout` has passed, as +/// [`BlockDevice::await_partitions`] describes. +/// +/// An observation of `None` means that the partitions changed while they were read, and counts +/// as [`Mismatch::Changed`]. +fn wait_for_partitions( + disk: &Disk, + timeout: Duration, + mut observe: impl FnMut() -> Result>, DeviceError>, +) -> Result, DeviceError> { + let start = Instant::now(); + let mut pause = FIRST_PAUSE; + loop { + let mismatch = match observe()? { + Some(kernel) => match compare(disk, &kernel) { + Ok(devices) => return Ok(devices), + Err(mismatch) => mismatch, + }, + None => Mismatch::Changed, + }; + let remaining = timeout.saturating_sub(start.elapsed()); + if remaining.is_zero() { + return Err(DeviceError::Partitions(mismatch)); + } + thread::sleep(pause.min(remaining)); + pause = pause.saturating_mul(2).min(LONGEST_PAUSE); + } +} + +/// Reads the kernel's partitions of the disk whose sysfs directory is `directory`, in the order +/// of their numbers, or returns `None` if they changed while this read them, as +/// [`confirm_partitions`] describes. +fn observe_partitions(directory: &Path) -> Result>, DeviceError> { + confirm_partitions(directory, read_subdirectories(directory)?) +} + +/// A subdirectory of a disk's sysfs directory, open, with the partition that it described when it +/// was read. +struct Subdirectory { + /// The subdirectory's name. + name: OsString, + /// The open subdirectory. + directory: SysfsDirectory, + /// The partition that the subdirectory described, or `None` if it described none, as the + /// disk's other directories do. + partition: Option, +} + +impl Subdirectory { + /// Opens and reads the subdirectory `name` of the disk's sysfs directory `directory`, or + /// returns `None` if it doesn't exist. + /// + /// It reads the partition's attributes through one descriptor of the subdirectory, so that + /// they all describe one partition, even when the kernel drops the partition and creates + /// another under its name while they're read. A partition that the kernel drops while this + /// reads it describes none, as if the kernel had dropped it before. + fn read(directory: &Path, name: OsString) -> Result, DeviceError> { + let Some(opened) = SysfsDirectory::open(directory.join(&name))? else { + return Ok(None); + }; + let partition = opened.partition(name.clone())?; + Ok(Some(Self { + name, + directory: opened, + partition, + })) + } +} + +/// Opens and reads each subdirectory of the disk's sysfs directory `directory`, in the order of +/// their names, and leaves out any that the kernel dropped before this opened it. +fn read_subdirectories(directory: &Path) -> Result, DeviceError> { + let mut subdirectories = Vec::new(); + for name in list_subdirectories(directory)? { + subdirectories.extend(Subdirectory::read(directory, name)?); + } + Ok(subdirectories) +} + +/// Lists the names of the subdirectories of the disk's sysfs directory `directory`, in order. +fn list_subdirectories(directory: &Path) -> Result, DeviceError> { + let entries = + fs::read_dir(directory).map_err(|source| DeviceError::sysfs(directory, source))?; + let mut names = Vec::new(); + for entry in entries { + let entry = entry.map_err(|source| DeviceError::sysfs(directory, source))?; + // Each partition has a directory, unlike the disk's attributes and links. + let is_directory = entry + .file_type() + .map_err(|source| DeviceError::sysfs(&entry.path(), source))? + .is_dir(); + if is_directory { + names.push(entry.file_name()); + } + } + names.sort(); + Ok(names) +} + +/// Returns the partitions that the subdirectories `first`, which [`read_subdirectories`] read +/// from the disk's sysfs directory `directory`, described, in the order of their numbers, or +/// `None` if the kernel's partitions changed while they were read. +/// +/// It lists the disk's subdirectories again, which must be those of `first`, then reads each +/// subdirectory of `first` again through the descriptor that it holds: a partition's `partition` +/// attribute must give the same number, and another subdirectory must still describe no +/// partition. The kernel never revives a directory that it dropped, even to create another under +/// its name, so each partition that passes existed from its first reading to its last, throughout +/// the second listing, and a partition keeps its number, first byte, and device number while it +/// exists. A reread drops every partition of the disk before it creates any, so the partitions +/// that pass all come from one reading of the table, never some from before a reread and some +/// from after it. The second listing shows that the kernel created no partition under another +/// name since the first reading, and each other subdirectory, that the kernel did not finish +/// creating a partition there. Only a program that deletes, adds, or resizes partitions one at a +/// time could still change them meanwhile, as it can after the comparison anyway. +fn confirm_partitions( + directory: &Path, + first: Vec, +) -> Result>, DeviceError> { + let names = list_subdirectories(directory)?; + if !names + .iter() + .eq(first.iter().map(|subdirectory| &subdirectory.name)) + { + return Ok(None); + } + let mut partitions = Vec::new(); + for subdirectory in first { + let unchanged = match &subdirectory.partition { + Some(partition) => { + subdirectory.directory.number(c"partition")? == Some(partition.number.into()) + } + None => subdirectory + .directory + .partition(subdirectory.name)? + .is_none(), + }; + if !unchanged { + return Ok(None); + } + partitions.extend(subdirectory.partition); + } + partitions.sort_by_key(|partition| partition.number); + Ok(Some(partitions)) +} + +/// The sysfs directory of a disk or of one of its partitions, open, so that every attribute read +/// through it describes the same device: once the kernel drops the device, even to create another +/// under the same name, the directory holds no attributes. +struct SysfsDirectory { + /// The open directory. + descriptor: FileDescriptor, + /// The directory's path, which errors name. + path: PathBuf, +} + +impl SysfsDirectory { + /// Opens the directory at `path`, or returns `None` if it doesn't exist. + fn open(path: PathBuf) -> Result, DeviceError> { + let name = + sys::path_c_string(&path).map_err(|errno| DeviceError::sysfs(&path, errno.into()))?; + let flags = OpenFlags::O_RDONLY | OpenFlags::O_DIRECTORY | OpenFlags::O_CLOEXEC; + match sys::open(&name, flags, Mode::empty()) { + Ok(descriptor) => Ok(Some(Self { descriptor, path })), + Err(Errno::ENOENT | Errno::ENODEV) => Ok(None), + Err(errno) => Err(DeviceError::sysfs(&path, errno.into())), + } + } + + /// Returns the path of the attribute `name`, which errors name. + fn path_of(&self, name: &CStr) -> PathBuf { + self.path.join(OsStr::from_bytes(name.to_bytes())) + } + + /// Reads the partition that the directory describes, named `name`, or returns `None` if the + /// directory describes no partition, as the disk's other directories do, or the kernel + /// dropped it. + fn partition(&self, name: OsString) -> Result, DeviceError> { + let Some(number) = self.number(c"partition")? else { + return Ok(None); + }; + let number = u8::try_from(number) + .ok() + .filter(|number| *number != 0) + .ok_or_else(|| malformed(&self.path_of(c"partition")))?; + let Some(start) = self.bytes(c"start")? else { + return Ok(None); + }; + let Some(size) = self.bytes(c"size")? else { + return Ok(None); + }; + let Some(device_number) = self.device_number(c"dev")? else { + return Ok(None); + }; + Ok(Some(KernelPartition { + number, + name, + device_number, + start, + size, + })) + } + + /// Reads the number of 512-byte sectors that the attribute `name` holds, as sysfs counts a + /// partition's start and size whatever the disk's logical block size, and returns it in + /// bytes, or returns `None` if the attribute doesn't exist. + fn bytes(&self, name: &CStr) -> Result, DeviceError> { + let Some(sectors) = self.number(name)? else { + return Ok(None); + }; + sectors + .checked_mul(sys::SYSFS_SECTOR_SIZE) + .map(Some) + .ok_or_else(|| malformed(&self.path_of(name))) + } + + /// Reads the device number, `MAJOR:MINOR`, that the attribute `name` holds, or returns + /// `None` if the attribute doesn't exist. + fn device_number(&self, name: &CStr) -> Result, DeviceError> { + let Some(text) = self.attribute(name)? else { + return Ok(None); + }; + let number = + |text: &[u8]| parse_decimal(text).and_then(|number| u32::try_from(number).ok()); + let mut parts = text.splitn(2, |byte| *byte == b':'); + match (parts.next().and_then(number), parts.next().and_then(number)) { + (Some(major), Some(minor)) => Ok(Some(DeviceNumber { major, minor })), + _ => Err(malformed(&self.path_of(name))), + } + } + + /// Reads the decimal number that the attribute `name` holds, or returns `None` if the + /// attribute doesn't exist. + fn number(&self, name: &CStr) -> Result, DeviceError> { + let Some(text) = self.attribute(name)? else { + return Ok(None); + }; + parse_decimal(&text) + .map(Some) + .ok_or_else(|| malformed(&self.path_of(name))) + } + + /// Reads the value of the attribute `name`, the one line that the kernel writes, without its + /// newline, or returns `None` if the attribute doesn't exist. + fn attribute(&self, name: &CStr) -> Result>, DeviceError> { + let flags = OpenFlags::O_RDONLY | OpenFlags::O_CLOEXEC; + // Opening an attribute of a device that the kernel dropped fails with `ENOENT`, or with + // `ENODEV` while the kernel drops it, and reading one that it dropped after the open + // fails with `ENODEV`. + let mut file = match sys::open_at(&self.descriptor, name, flags, Mode::empty()) { + Ok(file) => file, + Err(Errno::ENOENT | Errno::ENODEV) => return Ok(None), + Err(errno) => return Err(DeviceError::sysfs(&self.path_of(name), errno.into())), + }; + let mut text = Vec::new(); + match file.read_to_end(&mut text) { + Ok(_) => {} + Err(error) if error.raw_os_error() == Some(Errno::ENODEV as i32) => return Ok(None), + Err(source) => return Err(DeviceError::sysfs(&self.path_of(name), source)), + } + if text.pop() != Some(b'\n') || text.contains(&b'\n') { + return Err(malformed(&self.path_of(name))); + } + Ok(Some(text)) + } +} + +/// Returns the error for the sysfs attribute at `path`, whose text is not in the form that the +/// kernel writes. +fn malformed(path: &Path) -> DeviceError { + let source = io::Error::new(io::ErrorKind::InvalidData, "unexpected text"); + DeviceError::sysfs(path, source) +} diff --git a/nvx-rs/src/guest/gpt/tests.rs b/nvx-rs/src/guest/gpt/tests.rs new file mode 100644 index 000000000..547b445bc --- /dev/null +++ b/nvx-rs/src/guest/gpt/tests.rs @@ -0,0 +1,580 @@ +//! Tests of the comparison of a disk's table with the kernel's partitions, of the wait for them to +//! match, and of the reading and confirmation of fake sysfs descriptions of a disk. + +use std::error::Error; +use std::fs; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::symlink; +use std::time::{Duration, Instant}; + +use super::*; +use crate::gpt::GptError; +use crate::gpt::tests::standard_disk; + +/// Returns the kernel's partition `number` of disk `vda`, 254:0, from byte `start` with `size` +/// bytes. +fn kernel(number: u8, start: u64, size: u64) -> KernelPartition { + KernelPartition { + number, + name: OsString::from(format!("vda{number}")), + device_number: DeviceNumber { + major: 254, + minor: u32::from(number), + }, + start, + size, + } +} + +/// Returns kernel partitions that match the partitions of `disk`. +fn matching(disk: &Disk) -> Vec { + disk.partitions() + .iter() + .map(|entry| kernel(entry.number(), entry.offset(), entry.size())) + .collect() +} + +#[test] +fn matching_partitions_return_their_devices() { + let disk = standard_disk(); + let devices = compare(&disk, &matching(&disk)).unwrap(); + assert_eq!(devices.len(), 3); + for (device, entry) in devices.iter().zip(disk.partitions()) { + assert_eq!(device.partition(), entry); + assert_eq!(device.name(), format!("vda{}", entry.number()).as_str()); + assert_eq!(device.device_number().major(), 254); + assert_eq!(device.device_number().minor(), u32::from(entry.number())); + } +} + +#[test] +fn the_first_difference_in_number_order_fails() { + let disk = standard_disk(); + let all = matching(&disk); + // The standard disk's partitions: partition 1 in LBA 34, partition 2 in 35 through 42, and + // partition 3 in 43 through 58. + assert_eq!( + (all[1].start, all[1].size, all[2].start, all[2].size), + (35 * 512, 8 * 512, 43 * 512, 16 * 512) + ); + assert_eq!(compare(&disk, &[]), Err(Mismatch::Missing { number: 1 })); + let without_second = [all[0].clone(), all[2].clone()]; + assert_eq!( + compare(&disk, &without_second), + Err(Mismatch::Missing { number: 2 }) + ); + assert_eq!( + compare(&disk, &all[..2]), + Err(Mismatch::Missing { number: 3 }) + ); + + let mut extra = all.clone(); + extra.push(kernel(4, 59 * 512, 512)); + assert_eq!( + compare(&disk, &extra), + Err(Mismatch::Unexpected { number: 4 }) + ); + let mut distant = all.clone(); + distant.push(kernel(255, 59 * 512, 512)); + assert_eq!( + compare(&disk, &distant), + Err(Mismatch::Unexpected { number: 255 }) + ); + // A missing partition 2 precedes an extra partition 5. + let mut both = without_second.to_vec(); + both.push(kernel(5, 59 * 512, 512)); + assert_eq!(compare(&disk, &both), Err(Mismatch::Missing { number: 2 })); + + let mut moved = all.clone(); + moved[1].start += 512; + assert_eq!( + compare(&disk, &moved), + Err(Mismatch::Extent { + number: 2, + start: 36 * 512, + size: 8 * 512, + expected_start: 35 * 512, + expected_size: 8 * 512, + }) + ); + let mut truncated = all.clone(); + truncated[2].size -= 512; + assert_eq!( + compare(&disk, &truncated), + Err(Mismatch::Extent { + number: 3, + start: 43 * 512, + size: 15 * 512, + expected_start: 43 * 512, + expected_size: 16 * 512, + }) + ); +} + +#[test] +fn the_wait_returns_once_late_partitions_match() { + let disk = standard_disk(); + let all = matching(&disk); + let mut observations = 0; + let devices = wait_for_partitions(&disk, Duration::from_secs(60), || { + observations += 1; + Ok(Some(match observations { + 1 => Vec::new(), + 2 | 3 => all[..2].to_vec(), + _ => all.clone(), + })) + }) + .unwrap(); + assert_eq!(observations, 4); + assert_eq!(devices, compare(&disk, &all).unwrap()); +} + +#[test] +fn the_wait_fails_with_the_difference_that_remains_at_the_deadline() { + let disk = standard_disk(); + let all = matching(&disk); + let mut extra = all.clone(); + extra.push(kernel(4, 59 * 512, 512)); + let timeout = Duration::from_millis(200); + let start = Instant::now(); + let mut observations = 0; + let error = wait_for_partitions(&disk, timeout, || { + observations += 1; + Ok(Some(if observations == 1 { + all[..2].to_vec() + } else { + extra.clone() + })) + }) + .unwrap_err(); + assert!( + matches!( + error, + DeviceError::Partitions(Mismatch::Unexpected { number: 4 }) + ), + "{error:?}" + ); + assert!(start.elapsed() >= timeout); + // Pauses of at least 1, 2, then 4 ms each: no busy wait, and more than one observation. + let mut most = 1; + let mut paused = Duration::ZERO; + let mut pause = FIRST_PAUSE; + while paused < timeout { + most += 1; + paused += pause; + pause = (pause * 2).min(LONGEST_PAUSE); + } + assert_eq!(most, 53); + assert!((3..=most).contains(&observations), "{observations}"); +} + +#[test] +fn a_zero_timeout_compares_once() { + let disk = standard_disk(); + let mut observations = 0; + let error = wait_for_partitions(&disk, Duration::ZERO, || { + observations += 1; + Ok(Some(Vec::new())) + }) + .unwrap_err(); + assert!(matches!( + error, + DeviceError::Partitions(Mismatch::Missing { number: 1 }) + )); + assert_eq!(observations, 1); + let all = matching(&disk); + assert!(wait_for_partitions(&disk, Duration::ZERO, || Ok(Some(all.clone()))).is_ok()); +} + +#[test] +fn partitions_that_change_while_read_are_read_again() { + let disk = standard_disk(); + let all = matching(&disk); + let mut observations = 0; + let devices = wait_for_partitions(&disk, Duration::from_secs(60), || { + observations += 1; + Ok((observations > 2).then(|| all.clone())) + }) + .unwrap(); + assert_eq!(observations, 3); + assert_eq!(devices, compare(&disk, &all).unwrap()); + // Partitions that are still changing at the deadline fail as such. + let error = wait_for_partitions(&disk, Duration::ZERO, || Ok(None)).unwrap_err(); + assert!( + matches!(error, DeviceError::Partitions(Mismatch::Changed)), + "{error:?}" + ); +} + +#[test] +fn a_failed_observation_ends_the_wait() { + let disk = standard_disk(); + let mut observations = 0; + let error = wait_for_partitions(&disk, Duration::from_secs(60), || { + observations += 1; + Err(malformed(Path::new("/sys/dev/block/254:0/vda1/start"))) + }) + .unwrap_err(); + assert!(matches!(error, DeviceError::Sysfs { .. }), "{error:?}"); + assert_eq!(observations, 1); +} + +/// Writes the attribute `name` of the sysfs directory `directory` with `value`. +fn attribute(directory: &Path, name: &str, value: &str) { + fs::write(directory.join(name), value).unwrap(); +} + +/// Returns a fake sysfs directory of disk `vda`, 254:0, with the attributes, directories, and +/// links of a disk, and no partitions. +fn disk_directory() -> tempfile::TempDir { + let disk = tempfile::tempdir().unwrap(); + attribute(disk.path(), "dev", "254:0\n"); + attribute(disk.path(), "size", "8192\n"); + attribute(disk.path(), "ro", "1\n"); + for directory in ["holders", "mq", "power", "queue", "slaves"] { + fs::create_dir(disk.path().join(directory)).unwrap(); + } + symlink("../../../virtio0", disk.path().join("device")).unwrap(); + symlink("../../../../../class/block", disk.path().join("subsystem")).unwrap(); + disk +} + +/// Writes the sysfs directory of partition `number` of the disk whose sysfs directory is `disk`, +/// `vda{number}`, from sector `start` with `size` sectors, and returns it. +fn partition_directory(disk: &Path, number: u8, start: u64, size: u64) -> PathBuf { + let directory = disk.join(format!("vda{number}")); + fs::create_dir(&directory).unwrap(); + attribute(&directory, "partition", &format!("{number}\n")); + attribute(&directory, "start", &format!("{start}\n")); + attribute(&directory, "size", &format!("{size}\n")); + attribute(&directory, "dev", &format!("254:{number}\n")); + attribute(&directory, "ro", "0\n"); + directory +} + +/// Replaces the partitions of the disk whose sysfs directory is `disk` with `partitions`, each a +/// number, a first sector, and a number of sectors, as the kernel does when it rereads the disk's +/// table: it drops every partition, then creates the table's. +fn reread(disk: &Path, partitions: &[(u8, u64, u64)]) { + for entry in fs::read_dir(disk).unwrap() { + let entry = entry.unwrap(); + if entry.file_name().as_bytes().starts_with(b"vda") { + fs::remove_dir_all(entry.path()).unwrap(); + } + } + for &(number, start, size) in partitions { + partition_directory(disk, number, start, size); + } +} + +/// Observes the partitions of the disk whose sysfs directory is `disk`, which don't change while +/// they're read. +fn observe(disk: &Path) -> Vec { + observe_partitions(disk) + .unwrap() + .expect("the partitions changed while they were read") +} + +#[test] +fn partitions_read_from_their_sysfs_directories_in_number_order() { + let disk = disk_directory(); + assert_eq!(observe(disk.path()), Vec::new()); + for (number, start, size) in [(3, 43, 16), (1, 34, 1), (2, 35, 8)] { + partition_directory(disk.path(), number, start, size); + } + let observed = observe(disk.path()); + assert_eq!( + observed, + [ + kernel(1, 34 * 512, 512), + kernel(2, 35 * 512, 8 * 512), + kernel(3, 43 * 512, 16 * 512), + ] + ); + assert!(compare(&standard_disk(), &observed).is_ok()); +} + +#[test] +fn partitions_that_vanish_while_read_are_left_out() { + let disk = disk_directory(); + partition_directory(disk.path(), 1, 34, 1); + for name in ["partition", "start", "size", "dev"] { + let directory = partition_directory(disk.path(), 2, 35, 8); + fs::remove_file(directory.join(name)).unwrap(); + assert_eq!(observe(disk.path()), [kernel(1, 34 * 512, 512)], "{name}"); + fs::remove_dir_all(directory).unwrap(); + } +} + +#[test] +fn a_partition_that_the_kernel_creates_again_is_never_read_in_parts() { + let disk = disk_directory(); + partition_directory(disk.path(), 1, 34, 1); + partition_directory(disk.path(), 3, 43, 16); + // The table's partition 2 runs from sector 35 for 8 sectors. The kernel's first partition 2 + // starts there but is shorter, and the partition 2 that replaces it has that size but starts + // later, so a start read from the first and a size read from the second would match the + // table. + let path = partition_directory(disk.path(), 2, 35, 4); + let directory = SysfsDirectory::open(path.clone()).unwrap().unwrap(); + assert_eq!(directory.bytes(c"start").unwrap(), Some(35 * 512)); + fs::remove_dir_all(&path).unwrap(); + partition_directory(disk.path(), 2, 36, 8); + assert_eq!(directory.bytes(c"size").unwrap(), None); + assert_eq!(directory.partition(OsString::from("vda2")).unwrap(), None); + // A new observation reads the second partition 2 whole, and it differs from the table. + let observed = observe(disk.path()); + assert_eq!(observed[1], kernel(2, 36 * 512, 8 * 512)); + assert_eq!( + compare(&standard_disk(), &observed).unwrap_err(), + Mismatch::Extent { + number: 2, + start: 36 * 512, + size: 8 * 512, + expected_start: 35 * 512, + expected_size: 8 * 512, + } + ); +} + +/// Reads the subdirectories of the disk whose sysfs directory is `disk` as a reread tears them: the +/// disk's other directories, and partition 1 from an old table, which moved partition 2, then +/// partitions 2 and 3 from a new table, which moves partition 1. Neither table matches the +/// standard disk's, but the partitions read do. +fn read_torn(disk: &Path) -> Vec { + // The standard disk's partitions: partition 1 in LBA 34, partition 2 in 35 through 42, and + // partition 3 in 43 through 58. + reread(disk, &[(1, 34, 1), (2, 36, 8), (3, 43, 16)]); + let mut torn = Vec::new(); + for name in list_subdirectories(disk).unwrap() { + if name == "vda2" { + reread(disk, &[(1, 60, 1), (2, 35, 8), (3, 43, 16)]); + } + torn.extend(Subdirectory::read(disk, name).unwrap()); + } + let partitions: Vec<_> = torn + .iter() + .filter_map(|subdirectory| subdirectory.partition.clone()) + .collect(); + assert!(compare(&standard_disk(), &partitions).is_ok()); + torn +} + +#[test] +fn partitions_from_before_and_after_a_reread_are_never_combined() { + let disk = disk_directory(); + let torn = read_torn(disk.path()); + assert_eq!(confirm_partitions(disk.path(), torn).unwrap(), None); + // The next observation reads the new table's partitions, whose first differs from the table's. + assert_eq!( + compare(&standard_disk(), &observe(disk.path())).unwrap_err(), + Mismatch::Extent { + number: 1, + start: 60 * 512, + size: 512, + expected_start: 34 * 512, + expected_size: 512, + } + ); + // Even when the kernel's partitions come to have the descriptions that the torn reading + // read, as those of another table could, the reading isn't confirmed, since the old table's + // partition 1 no longer exists. + let torn = read_torn(disk.path()); + fs::remove_dir_all(disk.path().join("vda1")).unwrap(); + partition_directory(disk.path(), 1, 34, 1); + assert_eq!(confirm_partitions(disk.path(), torn).unwrap(), None); + assert_eq!(observe(disk.path()), matching(&standard_disk())); +} + +#[test] +fn a_reading_is_confirmed_only_while_the_kernel_keeps_its_partitions() { + let table = [(1, 34, 1), (2, 35, 8), (3, 43, 16)]; + let disk = disk_directory(); + let confirmed_after = |change: &dyn Fn(&Path)| { + reread(disk.path(), &table); + let first = read_subdirectories(disk.path()).unwrap(); + change(disk.path()); + confirm_partitions(disk.path(), first).unwrap() + }; + assert_eq!(confirmed_after(&|_| {}), Some(matching(&standard_disk()))); + // The kernel dropped a partition, or created one. + let dropped = |disk: &Path| fs::remove_dir_all(disk.join("vda3")).unwrap(); + assert_eq!(confirmed_after(&dropped), None); + let created = |disk: &Path| { + partition_directory(disk, 4, 59, 1); + }; + assert_eq!(confirmed_after(&created), None); + // The kernel created the same partitions again, under the same names. + assert_eq!(confirmed_after(&|disk| reread(disk, &table)), None); +} + +#[test] +fn a_partition_that_the_kernel_finishes_creating_after_a_reading_is_not_left_out() { + let disk = disk_directory(); + reread(disk.path(), &[(1, 34, 1), (2, 35, 8), (3, 43, 16)]); + let attributes = [ + ("partition", "4\n"), + ("start", "59\n"), + ("size", "1\n"), + ("dev", "254:4\n"), + ]; + // The kernel creates partition 4's directory, then its attributes, and it is a partition once + // it has them all. + let fourth = disk.path().join("vda4"); + for created in 0..attributes.len() { + fs::create_dir(&fourth).unwrap(); + for (name, value) in &attributes[..created] { + attribute(&fourth, name, value); + } + // A reading that the kernel doesn't change is confirmed without partition 4. + assert_eq!( + observe(disk.path()), + matching(&standard_disk()), + "{created}" + ); + // Once the kernel has created the rest of partition 4's attributes, a reading from before + // isn't confirmed, and the next observation reads partition 4, which the table lacks. + let first = read_subdirectories(disk.path()).unwrap(); + for (name, value) in &attributes[created..] { + attribute(&fourth, name, value); + } + assert_eq!( + confirm_partitions(disk.path(), first).unwrap(), + None, + "{created}" + ); + assert_eq!( + compare(&standard_disk(), &observe(disk.path())).unwrap_err(), + Mismatch::Unexpected { number: 4 }, + "{created}" + ); + fs::remove_dir_all(&fourth).unwrap(); + } +} + +#[test] +fn attributes_in_other_forms_than_the_kernel_writes_fail() { + for (name, value) in [ + ("partition", "0\n"), + ("partition", "256\n"), + ("partition", "+2\n"), + ("start", "35"), + ("start", "35\n\n"), + ("start", " 35\n"), + ("start", "36028797018963968\n"), + ("size", "eight\n"), + ("size", "\n"), + ("dev", "254-2\n"), + ("dev", "254:\n"), + ("dev", "254:2:0\n"), + ("dev", "4294967296:2\n"), + ] { + let disk = disk_directory(); + let directory = partition_directory(disk.path(), 2, 35, 8); + attribute(&directory, name, value); + match observe_partitions(disk.path()) { + Err(DeviceError::Sysfs { path, source }) => { + assert_eq!(path, directory.join(name), "{value:?}"); + assert_eq!(source.kind(), io::ErrorKind::InvalidData, "{value:?}"); + } + other => panic!("{name} {value:?}: {other:?}"), + } + } +} + +#[test] +fn a_disk_without_a_sysfs_directory_fails() { + let disk = disk_directory(); + let absent = disk.path().join("absent"); + match observe_partitions(&absent) { + Err(DeviceError::Sysfs { path, source }) => { + assert_eq!(path, absent); + assert_eq!(source.kind(), io::ErrorKind::NotFound); + } + other => panic!("{other:?}"), + } +} + +#[test] +fn only_block_devices_are_taken() { + let file = tempfile::tempfile().unwrap(); + assert!(matches!( + BlockDevice::new(&file), + Err(DeviceError::NotBlockDevice) + )); + let null = File::open("/dev/null").unwrap(); + assert!(matches!( + BlockDevice::new(&null), + Err(DeviceError::NotBlockDevice) + )); +} + +#[test] +fn errors_describe_the_failure_and_keep_their_cause() { + assert_eq!( + DeviceNumber { + major: 259, + minor: 4 + } + .to_string(), + "259:4" + ); + let error = DeviceError::NotWholeDisk; + assert_eq!( + error.to_string(), + "the block device is a partition, not a whole disk" + ); + assert!(error.source().is_none()); + let busy = io::Error::from_raw_os_error(Errno::EBUSY as i32); + let error = DeviceError::device(DeviceOperation::RereadPartitions, busy); + assert!( + error + .to_string() + .starts_with("cannot reread the device's partition table: "), + "{error}" + ); + assert_eq!( + error + .source() + .and_then(|source| source.downcast_ref::()) + .and_then(io::Error::raw_os_error), + Some(Errno::EBUSY as i32) + ); + let error = malformed(Path::new("/sys/dev/block/254:0/vda2/start")); + assert_eq!( + error.to_string(), + "cannot read /sys/dev/block/254:0/vda2/start: unexpected text" + ); + let error = DeviceError::from(GptError::NoProtectiveRecord); + assert_eq!( + error.to_string(), + "invalid partition table: the MBR has no protective record of type 0xEE, so the disk \ + is not a GPT disk" + ); + assert!(error.source().is_some()); + let error = DeviceError::Partitions(Mismatch::Extent { + number: 2, + start: 36 * 512, + size: 8 * 512, + expected_start: 35 * 512, + expected_size: 8 * 512, + }); + assert_eq!( + error.to_string(), + "the kernel's partitions do not match the table's: the kernel's partition 2 has 4096 \ + bytes from byte 18432, but the table's has 4096 bytes from byte 17920" + ); + assert!(error.source().is_none()); + assert_eq!( + Mismatch::Missing { number: 3 }.to_string(), + "the kernel has no partition 3, which the table has" + ); + assert_eq!( + Mismatch::Unexpected { number: 4 }.to_string(), + "the kernel has a partition 4, which the table lacks" + ); + assert_eq!( + DeviceError::Partitions(Mismatch::Changed).to_string(), + "the kernel's partitions do not match the table's: they changed while they were read" + ); +} diff --git a/nvx-rs/src/guest/mod.rs b/nvx-rs/src/guest/mod.rs index 8d5aad92d..6be5f28a5 100644 --- a/nvx-rs/src/guest/mod.rs +++ b/nvx-rs/src/guest/mod.rs @@ -5,8 +5,9 @@ //! accounts, kills, and settles the processes of its executions; [`mappings`], through which it //! bind-mounts host mappings below the roots that it names; [`exec`], which drives each program //! as one execution, with its launch, its bounded output and input, its ending, and its outcome; -//! and [`root`], through which it bootstraps the guest for a layered root, assembles the root, -//! mounts live shares in it, and tears it down. +//! [`root`], through which it bootstraps the guest for a layered root, assembles the root, mounts +//! live shares in it, and tears it down; and [`gpt`], which validates a GPT disk that the kernel +//! attached as a block device against the partitions that the kernel created from its table. //! //! Each other module holds one program, and the crate's binary of the same name runs its `main`. //! Two of them also offer their steps as typed calls, which any guest init or agent runs in its @@ -83,6 +84,7 @@ pub mod device_io; pub mod exec; #[cfg(target_arch = "x86_64")] pub mod exit; +pub mod gpt; pub mod ingress; pub mod managed_agent; pub mod mappings; diff --git a/nvx-rs/src/guest/sys/block.rs b/nvx-rs/src/guest/sys/block.rs new file mode 100644 index 000000000..3366b9047 --- /dev/null +++ b/nvx-rs/src/guest/sys/block.rs @@ -0,0 +1,116 @@ +//! Block devices: their geometry, the numbers that name them in sysfs, and the rereading of a +//! disk's partition table. + +use std::ffi::c_int; +use std::mem::size_of; +use std::os::fd::{AsRawFd, BorrowedFd}; + +use super::Errno; + +/// The sysfs directory that links each block device's directory under the device's number, +/// `MAJOR:MINOR`. A disk's directory holds a directory for each of its partitions, which holds a +/// `partition` file that the disk's own directory lacks. +pub(crate) const BLOCK_DEVICE_DIRECTORY: &str = "/sys/dev/block"; + +/// The bytes in each of the sectors in which sysfs gives a partition's `start` and `size`, +/// whatever the disk's logical block size. +pub(crate) const SYSFS_SECTOR_SIZE: u64 = 512; + +nix::ioctl_read_bad!( + /// Reads a block device's logical block size: Linux `BLKSSZGET`, `_IO(0x12, 104)`, which + /// writes an `int` although its number declares no argument. + get_logical_block_size, + nix::request_code_none!(0x12, 104), + c_int +); + +nix::ioctl_read_bad!( + /// Reads a block device's size in bytes: Linux `BLKGETSIZE64`, `_IOR(0x12, 114, size_t)`, + /// which writes a 64-bit size whatever the size of `size_t`. + get_size, + nix::request_code_read!(0x12, 114, size_of::()), + u64 +); + +nix::ioctl_none!( + /// Asks the kernel to reread a disk's partition table: Linux `BLKRRPART`, `_IO(0x12, 95)`. + reread_partition_table, + 0x12, + 95 +); + +/// Splits a device number, such as the `st_rdev` of a block device, into its major and minor +/// numbers. +pub(crate) fn split_device_number(device: u64) -> (u64, u64) { + (nix::sys::stat::major(device), nix::sys::stat::minor(device)) +} + +/// Returns the logical block size, in bytes, of the block device that `descriptor` opens. +pub(crate) fn logical_block_size(descriptor: BorrowedFd<'_>) -> Result { + let mut size: c_int = 0; + // SAFETY: `size` is writable for one C `int` and outlives the synchronous `BLKSSZGET`. + unsafe { get_logical_block_size(descriptor.as_raw_fd(), &raw mut size) }?; + u32::try_from(size).map_err(|_| Errno::EINVAL) +} + +/// Returns the size, in bytes, of the block device that `descriptor` opens. +pub(crate) fn block_device_size(descriptor: BorrowedFd<'_>) -> Result { + let mut size = 0; + // SAFETY: `size` is writable for one `u64` and outlives the synchronous `BLKGETSIZE64`. + unsafe { get_size(descriptor.as_raw_fd(), &raw mut size) }?; + Ok(size) +} + +/// Asks the kernel to reread the partition table of the disk that `descriptor` opens, and to +/// create the disk's partitions again from it before it returns. +/// +/// The kernel requires `CAP_SYS_ADMIN`, and fails with `EBUSY` while a partition of the disk is +/// open, with `EINVAL` for a partition or a disk that has none, and with `EIO` when it cannot +/// read the table. +pub(crate) fn reread_partitions(descriptor: BorrowedFd<'_>) -> Result<(), Errno> { + // SAFETY: `BLKRRPART` takes no argument. + unsafe { reread_partition_table(descriptor.as_raw_fd()) }?; + Ok(()) +} + +#[cfg(test)] +mod tests { + //! The numbers of the block-device ioctls, and the ioctls on a file that is not a block + //! device. + + use std::os::fd::AsFd; + + use super::*; + + #[test] + fn block_ioctl_numbers_match_linux() { + assert_eq!(nix::request_code_none!(0x12, 104) as u32, 0x1268); + assert_eq!(nix::request_code_none!(0x12, 95) as u32, 0x125F); + // `_IOR(0x12, 114, size_t)`: the read direction, then the size of `size_t`. + assert_eq!( + nix::request_code_read!(0x12, 114, size_of::()) as u32, + (2 << 30) | ((size_of::() as u32) << 16) | 0x1272 + ); + #[cfg(target_pointer_width = "64")] + assert_eq!( + nix::request_code_read!(0x12, 114, size_of::()) as u32, + 0x8008_1272 + ); + } + + #[test] + fn device_numbers_split_as_linux_encodes_them() { + // Linux's `new_encode_dev`: the minor's low byte, the major, then the minor's high bits. + assert_eq!(split_device_number(0xfe00), (254, 0)); + assert_eq!(split_device_number(0xfe03), (254, 3)); + assert_eq!(split_device_number(0x0101_0323), (259, 0x1023)); + } + + #[test] + fn block_ioctls_fail_on_other_files() { + let file = tempfile::tempfile().unwrap(); + assert_eq!(logical_block_size(file.as_fd()), Err(Errno::ENOTTY)); + assert_eq!(block_device_size(file.as_fd()), Err(Errno::ENOTTY)); + assert_eq!(reread_partitions(file.as_fd()), Err(Errno::ENOTTY)); + } +} diff --git a/nvx-rs/src/guest/sys/mod.rs b/nvx-rs/src/guest/sys/mod.rs index efe97cdb1..da93a2d20 100644 --- a/nvx-rs/src/guest/sys/mod.rs +++ b/nvx-rs/src/guest/sys/mod.rs @@ -40,6 +40,7 @@ //! and the processor's interfaces, such as procfs and sysfs paths, OpenVMM's ports, CPUID //! leaves, and MSRs, are defined here once, so that the programs name them. +mod block; mod clock; mod discipline; mod fd; @@ -70,6 +71,10 @@ pub(crate) use nix::errno::Errno; pub(crate) use nix::sys::signal::Signal; pub(crate) use nix::sys::time::{TimeSpec, TimeValLike}; +pub(crate) use block::{ + BLOCK_DEVICE_DIRECTORY, SYSFS_SECTOR_SIZE, block_device_size, logical_block_size, + reread_partitions, split_device_number, +}; pub(crate) use clock::{Clock, clock_time, monotonic_ms, sleep_once, sleep_until_monotonic}; #[cfg(target_arch = "x86_64")] pub(crate) use clock::{monotonic_ns, syscall_monotonic_nanoseconds}; diff --git a/nvx-rs/src/lib.rs b/nvx-rs/src/lib.rs index 6daa4b547..d9b60bbe0 100644 --- a/nvx-rs/src/lib.rs +++ b/nvx-rs/src/lib.rs @@ -32,11 +32,12 @@ //! each child that it names (`guest::process`), places, kills, and settles the processes of //! its executions in cgroups (`guest::cgroup`), bind-mounts host mappings below the roots that //! it names (`guest::mappings`), drives each program as one execution, with its launch, bounded -//! output, input, ending, and outcome (`guest::exec`), and bootstraps the guest for a layered -//! root, assembles the root, mounts live shares in it, and tears it down (`guest::root`), and -//! the typed calls through which it runs the time ABI's steps (`guest::time`) and reseeds after -//! a restore (`guest::reseed`) in its own process. The crate's `nvx-*` guest binaries run the -//! programs. +//! output, input, ending, and outcome (`guest::exec`), bootstraps the guest for a layered root, +//! assembles the root, mounts live shares in it, and tears it down (`guest::root`), and +//! validates a GPT disk's block device against the partitions that the kernel created from its +//! table (`guest::gpt`), and the typed calls through which it runs the time ABI's steps +//! (`guest::time`) and reseeds after a restore (`guest::reseed`) in its own process. The +//! crate's `nvx-*` guest binaries run the programs. //! //! # Example //! diff --git a/nvx-rs/tests/gpt_device.rs b/nvx-rs/tests/gpt_device.rs new file mode 100644 index 000000000..c30fa052a --- /dev/null +++ b/nvx-rs/tests/gpt_device.rs @@ -0,0 +1,486 @@ +//! Validates GPT disks that a microVM attached as block devices, through `nvx::guest::gpt`, as +//! root in the guest: the `gpt-device` scenario of `nvx.py test-microvm` shares this test's +//! executable into the guest and runs its ignored tests there, one at a time. +//! +//! The scenario attaches two disks of the same size, and shares a directory of disk images, which +//! these environment variables name: +//! +//! - `NVX_GPT_DEVICE_DISK`: the read-only disk, which holds a GPT with a first partition and two +//! layers; +//! - `NVX_GPT_DEVICE_DISK_GUID`: its disk GUID; +//! - `NVX_GPT_DEVICE_PARTITIONS`: the unique GUID and size in bytes of each of its partitions, in +//! the order of their numbers, as `GUID:SIZE` separated by spaces; +//! - `NVX_GPT_DEVICE_FIRST_PARTITION`: the bytes of its first partition without the padding after +//! them, in hexadecimal; +//! - `NVX_GPT_DEVICE_SCRATCH`: the writable disk, over whose start the tests write the images; +//! - `NVX_GPT_DEVICE_IMAGES`: the directory of the images: `three.img`, a GPT with a first +//! partition and two layers, as on the read-only disk; `two.img` and `four.img`, the same +//! without its third partition and with a fourth; `moved.img`, the same with its second +//! partition 512 blocks later; and `mbr.img`, the same with an MBR in place of its protective +//! MBR, whose first partition spans the first two of the GPT and whose second is the third. +//! +//! Every disk has 512-byte logical blocks, a first partition of 64 KiB, and layers that start with +//! `NVX-GPT-LAYER-` and their partition number. +#![cfg(all(target_os = "linux", feature = "guest"))] + +use std::ffi::{OsStr, c_char, c_int, c_void}; +use std::fs::{self, File, OpenOptions}; +use std::io::Read; +use std::mem::size_of; +use std::os::fd::AsRawFd; +use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt}; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::{Duration, Instant}; + +use nix::errno::Errno; +use nvx::gpt::{Expected, ExpectedPartition, GptError}; +use nvx::guest::gpt::{ + BlockDevice, DeviceError, DeviceOperation, Mismatch, PartitionDevice, ValidatedDisk, +}; +use nvx::uuid::Uuid; + +/// How long a test waits for partitions that must match. +const MATCH_TIMEOUT: Duration = Duration::from_secs(10); + +/// How long a test waits for partitions that must not match. +const MISMATCH_TIMEOUT: Duration = Duration::from_millis(500); + +/// How long the late-partition test waits before it adds the partition that it removed. +const LATE: Duration = Duration::from_millis(300); + +/// The size of the first partition of every disk, in bytes. +const FIRST_PARTITION_SIZE: usize = 64 * 1024; + +/// The prefix of the first bytes of every layer, before its partition number. +const LAYER_MARKER: &str = "NVX-GPT-LAYER-"; + +/// Linux's `BLKPG_ADD_PARTITION`, of ``. +const BLKPG_ADD_PARTITION: c_int = 1; + +/// Linux's `BLKPG_DEL_PARTITION`, of ``. +const BLKPG_DEL_PARTITION: c_int = 2; + +/// Linux's `struct blkpg_partition`, of ``. +#[repr(C)] +struct BlkpgPartition { + /// The partition's first byte. + start: i64, + /// The partition's size, in bytes. + length: i64, + /// The partition's number. + pno: c_int, + /// A name that the kernel ignores. + devname: [c_char; 64], + /// A name that the kernel ignores. + volname: [c_char; 64], +} + +/// Linux's `struct blkpg_ioctl_arg`, of ``. +#[repr(C)] +struct BlkpgIoctlArgument { + /// The operation, such as [`BLKPG_ADD_PARTITION`]. + op: c_int, + /// Flags that the kernel ignores. + flags: c_int, + /// The size of the partition that `data` points to. + datalen: c_int, + /// The partition. + data: *mut c_void, +} + +nix::ioctl_write_ptr_bad!( + /// Adds or deletes a partition of a disk: Linux `BLKPG`, `_IO(0x12, 105)`. + blkpg, + nix::request_code_none!(0x12, 105), + BlkpgIoctlArgument +); + +/// Returns the value of the environment variable `name`, which the scenario sets. +fn variable(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| { + panic!("{name} is unset: run the gpt-device scenario of nvx.py test-microvm") + }) +} + +/// Returns the path of the read-only disk. +fn read_only_path() -> PathBuf { + PathBuf::from(variable("NVX_GPT_DEVICE_DISK")) +} + +/// Returns the path of the writable disk. +fn scratch_path() -> PathBuf { + PathBuf::from(variable("NVX_GPT_DEVICE_SCRATCH")) +} + +/// Returns the path of the image `name`. +fn image_path(name: &str) -> PathBuf { + Path::new(&variable("NVX_GPT_DEVICE_IMAGES")).join(name) +} + +/// Returns the values that the read-only disk meets: its disk GUID, and its partitions' unique +/// GUIDs and sizes. +fn expected() -> Expected { + let partitions = variable("NVX_GPT_DEVICE_PARTITIONS") + .split(' ') + .map(|partition| { + let (guid, size) = partition.split_once(':').unwrap(); + ExpectedPartition::new() + .unique_guid(guid.parse().unwrap()) + .size(size.parse().unwrap()) + }) + .collect(); + Expected::new() + .disk_guid(variable("NVX_GPT_DEVICE_DISK_GUID").parse().unwrap()) + .partitions(partitions) +} + +/// Returns the bytes that `text` spells in hexadecimal. +fn decode_hex(text: &str) -> Vec { + assert!(text.len().is_multiple_of(2), "{text}"); + (0..text.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&text[index..index + 2], 16).unwrap()) + .collect() +} + +/// Opens the writable disk for reading and writing, and writes the image `name` over its start. +fn scratch_with(name: &str) -> File { + let disk = OpenOptions::new() + .read(true) + .write(true) + .open(scratch_path()) + .unwrap(); + overwrite(&disk, name); + disk +} + +/// Writes the image `name` over the start of `disk`, and flushes it to the device, from which +/// the kernel rereads the table. +fn overwrite(disk: &File, name: &str) { + let image = fs::read(image_path(name)).unwrap(); + disk.write_all_at(&image, 0).unwrap(); + disk.sync_all().unwrap(); +} + +/// Makes the kernel add partition `number` of `disk`, from byte `start` with `length` bytes, +/// with `operation` [`BLKPG_ADD_PARTITION`], or delete it, with [`BLKPG_DEL_PARTITION`], as a +/// program that manages the disk's partitions may. +fn change_partition(disk: &File, operation: c_int, number: u8, start: u64, length: u64) { + let mut partition = BlkpgPartition { + start: start.try_into().unwrap(), + length: length.try_into().unwrap(), + pno: number.into(), + devname: [0; 64], + volname: [0; 64], + }; + let argument = BlkpgIoctlArgument { + op: operation, + flags: 0, + datalen: size_of::().try_into().unwrap(), + data: (&raw mut partition).cast(), + }; + // SAFETY: `argument`, and the partition that it points to, outlive the synchronous ioctl, + // which only reads them. + unsafe { blkpg(disk.as_raw_fd(), &raw const argument) }.unwrap(); +} + +/// Opens the block device of `partition` in `/dev`, once devtmpfs has created its node. +fn open_partition(partition: &PartitionDevice) -> File { + let path = Path::new("/dev").join(partition.name()); + let deadline = Instant::now() + MATCH_TIMEOUT; + loop { + if let Ok(status) = fs::metadata(&path) + && status.file_type().is_block_device() + { + let number = partition.device_number(); + let rdev = status.rdev(); + assert_eq!( + (nix::sys::stat::major(rdev), nix::sys::stat::minor(rdev)), + (u64::from(number.major()), u64::from(number.minor())), + "{path:?}" + ); + return File::open(&path).unwrap(); + } + assert!(Instant::now() < deadline, "{path:?} never appeared"); + thread::sleep(Duration::from_millis(10)); + } +} + +/// Asserts that `result` failed because the kernel's partitions still differed from the table's +/// by `expected` when the timeout passed. +#[track_caller] +fn assert_mismatch(result: Result, expected: Mismatch) { + match result { + Err(DeviceError::Partitions(mismatch)) => assert_eq!(mismatch, expected), + other => panic!("expected {expected:?}, found {other:?}"), + } +} + +/// Asserts that `result` failed because the kernel refused to reread the table of a disk with an +/// open partition. +#[track_caller] +fn assert_busy(result: Result) { + match result { + Err(DeviceError::Device { + operation: DeviceOperation::RereadPartitions, + source, + }) => assert_eq!(source.raw_os_error(), Some(Errno::EBUSY as i32)), + other => panic!("expected EBUSY from the reread, found {other:?}"), + } +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn a_crafted_read_only_disk_validates() { + let path = read_only_path(); + let file = File::open(&path).unwrap(); + let device = BlockDevice::new(&file).unwrap(); + assert_eq!(device.logical_block_size(), 512); + let validated = device.validate(&expected(), MATCH_TIMEOUT).unwrap(); + let disk = validated.disk(); + assert_eq!(device.size(), disk.size()); + assert_eq!( + validated.first_partition().partition(), + disk.first_partition() + ); + assert_eq!(validated.lowers().len(), 2); + + // The first partition reads as its opaque bytes, without its padding, and only within the + // bound. + let configuration = disk + .read_first_partition(&file, FIRST_PARTITION_SIZE) + .unwrap(); + assert_eq!( + configuration, + decode_hex(&variable("NVX_GPT_DEVICE_FIRST_PARTITION")) + ); + assert!(std::str::from_utf8(&configuration).is_err()); + assert!(matches!( + disk.read_first_partition(&file, FIRST_PARTITION_SIZE - 1), + Err(GptError::FirstPartitionTooLarge { .. }) + )); + + // The partitions are the kernel's devices of the disk, and each layer's holds the layer. + let disk_name = path.file_name().unwrap().to_str().unwrap(); + for partition in validated.partitions() { + let number = partition.partition().number(); + assert_eq!( + partition.name(), + OsStr::new(&format!("{disk_name}{number}")) + ); + let described = fs::read_to_string(format!("/sys/class/block/{disk_name}{number}/dev")); + assert_eq!( + described.unwrap(), + format!("{}\n", partition.device_number()) + ); + } + for layer in validated.lowers() { + let mut marker = vec![0; LAYER_MARKER.len() + 1]; + open_partition(layer).read_exact(&mut marker).unwrap(); + let number = layer.partition().number(); + assert_eq!(marker, format!("{LAYER_MARKER}{number}").as_bytes()); + } + // Nothing that the validation opened stays open: the kernel rereads the table, and the disk + // validates again. + device.reread_partitions().unwrap(); + assert_eq!( + device.validate(&expected(), MATCH_TIMEOUT).unwrap(), + validated + ); +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn a_disk_that_differs_from_the_expected_values_fails() { + let file = File::open(read_only_path()).unwrap(); + let device = BlockDevice::new(&file).unwrap(); + let other_disk = Expected::new().disk_guid(Uuid::from_bytes([0x5A; 16])); + assert!(matches!( + device.validate(&other_disk, MATCH_TIMEOUT), + Err(DeviceError::Table(GptError::DiskGuidMismatch { .. })) + )); + let other_size = Expected::new().partitions(vec![ + ExpectedPartition::new(), + ExpectedPartition::new().size(512), + ExpectedPartition::new(), + ]); + assert!(matches!( + device.validate(&other_size, MATCH_TIMEOUT), + Err(DeviceError::Table(GptError::PartitionSizeMismatch { + number: 2, + .. + })) + )); +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn only_a_whole_disk_validates() { + let mut partition = read_only_path().into_os_string(); + partition.push("2"); + let partition = File::open(partition).unwrap(); + assert!(matches!( + BlockDevice::new(&partition), + Err(DeviceError::NotWholeDisk) + )); + let image = File::open(image_path("three.img")).unwrap(); + assert!(matches!( + BlockDevice::new(&image), + Err(DeviceError::NotBlockDevice) + )); +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn a_written_disk_validates_once_its_partitions_are_reread() { + let disk = scratch_with("two.img"); + let device = BlockDevice::new(&disk).unwrap(); + device.reread_partitions().unwrap(); + // The kernel's partitions are those of the table before, so the validation rereads it. + overwrite(&disk, "three.img"); + let validated = device.validate(&Expected::new(), MATCH_TIMEOUT).unwrap(); + assert_eq!(validated.partitions().len(), 3); + // A device that is open for reading only rereads the table and validates too. + let reader = File::open(scratch_path()).unwrap(); + let reading = BlockDevice::new(&reader).unwrap(); + reading.reread_partitions().unwrap(); + assert_eq!( + reading.validate(&Expected::new(), MATCH_TIMEOUT).unwrap(), + validated + ); +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn late_partitions_are_awaited() { + let disk = scratch_with("three.img"); + let device = BlockDevice::new(&disk).unwrap(); + let validated = device.validate(&Expected::new(), MATCH_TIMEOUT).unwrap(); + let second = *validated.partitions()[1].partition(); + change_partition(&disk, BLKPG_DEL_PARTITION, 2, 0, 0); + let start = Instant::now(); + let awaited = thread::scope(|scope| { + let adder = scope.spawn(|| { + thread::sleep(LATE); + change_partition( + &disk, + BLKPG_ADD_PARTITION, + 2, + second.offset(), + second.size(), + ); + }); + let awaited = device.await_partitions(validated.disk(), MATCH_TIMEOUT); + adder.join().unwrap(); + awaited + }); + assert!(start.elapsed() >= LATE); + assert_eq!(awaited.unwrap(), validated); +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn missing_partitions_fail_when_the_timeout_passes() { + let disk = scratch_with("three.img"); + let device = BlockDevice::new(&disk).unwrap(); + let validated = device.validate(&Expected::new(), MATCH_TIMEOUT).unwrap(); + + // The kernel dropped a partition of the table. + change_partition(&disk, BLKPG_DEL_PARTITION, 3, 0, 0); + let start = Instant::now(); + let awaited = device.await_partitions(validated.disk(), MISMATCH_TIMEOUT); + assert!(start.elapsed() >= MISMATCH_TIMEOUT); + assert_mismatch(awaited, Mismatch::Missing { number: 3 }); + + // The table changed after the validation read it: the kernel rereads one without partition 3. + overwrite(&disk, "two.img"); + device.reread_partitions().unwrap(); + assert_mismatch( + device.await_partitions(validated.disk(), MISMATCH_TIMEOUT), + Mismatch::Missing { number: 3 }, + ); +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn extra_and_moved_partitions_fail() { + let disk = scratch_with("three.img"); + let device = BlockDevice::new(&disk).unwrap(); + let table = device.read_disk(&Expected::new()).unwrap(); + + overwrite(&disk, "four.img"); + device.reread_partitions().unwrap(); + assert_mismatch( + device.await_partitions(&table, MISMATCH_TIMEOUT), + Mismatch::Unexpected { number: 4 }, + ); + + overwrite(&disk, "moved.img"); + device.reread_partitions().unwrap(); + let second = table.lowers()[0]; + assert_mismatch( + device.await_partitions(&table, MISMATCH_TIMEOUT), + Mismatch::Extent { + number: 2, + start: second.offset() + 512 * 512, + size: second.size(), + expected_start: second.offset(), + expected_size: second.size(), + }, + ); +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn an_mbr_that_the_kernel_reads_instead_fails() { + let disk = scratch_with("three.img"); + let device = BlockDevice::new(&disk).unwrap(); + let validated = device.validate(&Expected::new(), MATCH_TIMEOUT).unwrap(); + let table = validated.disk(); + + overwrite(&disk, "mbr.img"); + // Read from the device, the disk is no GPT disk. + assert!(matches!( + device.validate(&Expected::new(), MATCH_TIMEOUT), + Err(DeviceError::Table(GptError::NoProtectiveRecord)) + )); + // The kernel reads the MBR's partitions, which differ from the table that was read before. + device.reread_partitions().unwrap(); + let (first, second) = (table.first_partition(), table.lowers()[0]); + assert_mismatch( + device.await_partitions(table, MISMATCH_TIMEOUT), + Mismatch::Extent { + number: 1, + start: first.offset(), + size: second.offset() + second.size() - first.offset(), + expected_start: first.offset(), + expected_size: first.size(), + }, + ); +} + +#[test] +#[ignore = "runs as root in the gpt-device scenario of nvx.py test-microvm"] +fn an_open_partition_fails_only_a_validation_that_rereads() { + let disk = scratch_with("three.img"); + let device = BlockDevice::new(&disk).unwrap(); + let validated = device.validate(&Expected::new(), MATCH_TIMEOUT).unwrap(); + let layer = open_partition(&validated.lowers()[0]); + // The kernel's partitions still match the table, so the disk validates without the reread, + // which fails while a partition is open. + assert_eq!( + device.validate(&Expected::new(), MATCH_TIMEOUT).unwrap(), + validated + ); + assert_busy(device.reread_partitions()); + // Once the table differs, the validation needs the reread. + overwrite(&disk, "moved.img"); + assert_busy(device.validate(&Expected::new(), MATCH_TIMEOUT)); + drop(layer); + let moved = device.validate(&Expected::new(), MATCH_TIMEOUT).unwrap(); + assert_eq!( + moved.lowers()[0].partition().offset(), + validated.lowers()[0].partition().offset() + 512 * 512 + ); +} diff --git a/scripts/nvx.py b/scripts/nvx.py index 42acb2542..35431fc38 100644 --- a/scripts/nvx.py +++ b/scripts/nvx.py @@ -90,6 +90,8 @@ guest_binary_source_sha256, test_guest_binaries, ) +from nvx_tools.guest_test_programs import OUTPUT_DIRECTORY as GUEST_TESTS_DIRECTORY +from nvx_tools.guest_test_programs import build_guest_test_programs from nvx_tools.guests import GUEST_NAMES, guest_descriptor from nvx_tools.microvm_tests import configure_parser as configure_microvm_test_parser from nvx_tools.microvm_tests import ( @@ -261,6 +263,11 @@ def command_test_guest_binaries(args: argparse.Namespace) -> None: test_guest_binaries(args.binaries) +def command_build_guest_tests(args: argparse.Namespace) -> None: + for program, path in build_guest_test_programs(args.output).items(): + print(f"{program}: {path}") + + def command_build_kernel(args: argparse.Namespace) -> None: build_kernel( KernelBuildConfig.debug_variant() @@ -776,6 +783,18 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: guest_binary_tests.add_argument("--binaries", type=Path, required=True) guest_binary_tests.set_defaults(handler=command_test_guest_binaries) + guest_tests = subparsers.add_parser( + "build-guest-tests", + help="build the test-only Rust programs that microVM scenarios run in a guest", + ) + guest_tests.add_argument( + "--output", + type=Path, + default=GUEST_TESTS_DIRECTORY, + help="directory for the programs (default: build/guest-tests)", + ) + guest_tests.set_defaults(handler=command_build_guest_tests) + kernel = subparsers.add_parser( "build-kernel", help="fetch, patch, and build the pinned kernel natively on Linux", diff --git a/scripts/nvx_tools/ci.py b/scripts/nvx_tools/ci.py index c08015f12..b195d2d7e 100644 --- a/scripts/nvx_tools/ci.py +++ b/scripts/nvx_tools/ci.py @@ -70,6 +70,7 @@ "distro-artifacts": "DISTRO_ARTIFACTS_RESULT", "debug-kernel": "DEBUG_KERNEL_RESULT", "guest-artifact-reuse": "GUEST_ARTIFACT_REUSE_RESULT", + "guest-test-programs": "GUEST_TEST_PROGRAMS_RESULT", "build-openvmm-linux-gnu": "BUILD_LINUX_GNU_RESULT", "build-openvmm-linux-musl": "BUILD_LINUX_MUSL_RESULT", "build-openvmm-windows-msvc": "BUILD_WINDOWS_MSVC_RESULT", @@ -137,6 +138,10 @@ # request, whose artifacts it checks as the dev push that merges it would # before it reuses them; only the required status check waits for it. REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB = "guest-artifact-reuse" +# GitHub-hosted, beside the guest artifacts, under the same condition; only the +# microVM guest and sandbox jobs, which run the test-only guest programs, wait +# for it. +REQUIRED_CI_GUEST_TEST_PROGRAM_JOB = "guest-test-programs" REQUIRED_CI_PLATFORM_JOBS = ( "platform-kvm", "platform-mshv", @@ -205,8 +210,9 @@ r"scripts/nvx\.py$|" r"scripts/nvx_tools/(?!(aci_edge_sandboxes_tests|adversarial|" r"adversarial_broker|adversarial_executor|agent_bundling_tests|" - r"create_linux_source_archive|managed_exec_tests|microvm_shards|" - r"microvm_tests|openvmm_process|sandbox|sandbox_lifecycle_tests)\.py$|" + r"create_linux_source_archive|gpt_image|guest_test_programs|" + r"managed_exec_tests|microvm_shards|microvm_tests|openvmm_process|" + r"sandbox|sandbox_lifecycle_tests)\.py$|" r"microvm_shards\.json$|adversarial_cases/|microvm_test_scripts/))" ) # Inputs of the OpenVMM unit and documentation tests: the pinned revision, the @@ -475,6 +481,9 @@ def required_ci_expected_results( if event_name == "pull_request" and same_repository and run_guest_artifacts else "skipped" ), + REQUIRED_CI_GUEST_TEST_PROGRAM_JOB: ( + "success" if run_guest_artifacts else "skipped" + ), "performance-gate": ( "success" if event_name == "pull_request" diff --git a/scripts/nvx_tools/gpt_image.py b/scripts/nvx_tools/gpt_image.py new file mode 100644 index 000000000..71c89beaf --- /dev/null +++ b/scripts/nvx_tools/gpt_image.py @@ -0,0 +1,130 @@ +"""Write GPT disk images, and MBRs in place of their protective MBRs, for tests.""" + +from __future__ import annotations + +import struct +import uuid +import zlib +from collections.abc import Sequence +from dataclasses import dataclass + +SECTOR_SIZE = 512 +ENTRY_SIZE = 128 +ENTRY_COUNT = 128 +HEADER_SIZE = 92 +REVISION = 0x0001_0000 +SIGNATURE = b"EFI PART" +MBR_SIGNATURE = b"\x55\xaa" +MBR_RECORDS_OFFSET = 446 +MBR_RECORD_SIZE = 16 +PROTECTIVE_TYPE = 0xEE +LINUX_TYPE = 0x83 +# The Linux filesystem data partition type. +LINUX_DATA = uuid.UUID("0fc63daf-8483-4772-8e79-3d69d8477de4") +# The blocks of the entry array, which start at LBA 2, as UEFI's defaults do. +ENTRY_ARRAY_BLOCKS = ENTRY_COUNT * ENTRY_SIZE // SECTOR_SIZE + + +@dataclass(frozen=True) +class Partition: + """A Linux data partition from ``first_lba`` through ``last_lba`` that starts with + ``contents``.""" + + unique_guid: uuid.UUID + first_lba: int + last_lba: int + contents: bytes = b"" + + @property + def offset(self) -> int: + return self.first_lba * SECTOR_SIZE + + @property + def size(self) -> int: + return (self.last_lba - self.first_lba + 1) * SECTOR_SIZE + + +def _header( + blocks: int, + disk_guid: uuid.UUID, + my_lba: int, + alternate_lba: int, + entry_lba: int, + array_crc: int, +) -> bytes: + """Return a GPT header at ``my_lba``, with its CRC-32.""" + fields = struct.pack( + "<8sIIIIQQQQ16sQIII", + SIGNATURE, + REVISION, + HEADER_SIZE, + 0, + 0, + my_lba, + alternate_lba, + 2 + ENTRY_ARRAY_BLOCKS, + blocks - 2 - ENTRY_ARRAY_BLOCKS, + disk_guid.bytes_le, + entry_lba, + ENTRY_COUNT, + ENTRY_SIZE, + array_crc, + ) + return fields[:16] + struct.pack(" bytes: + """Return an MBR partition record of ``type_`` from ``first_lba`` for ``sectors`` + sectors, with the CHS addresses that UEFI gives a protective record.""" + return struct.pack( + " bytes: + """Return a disk of ``blocks`` 512-byte blocks with a protective MBR, a primary GPT + header and entry array of 128 entries, ``partitions`` in that order, and the backup + array and header at the disk's end, as UEFI lays them out.""" + image = bytearray(blocks * SECTOR_SIZE) + image[MBR_RECORDS_OFFSET : MBR_RECORDS_OFFSET + MBR_RECORD_SIZE] = mbr_record( + PROTECTIVE_TYPE, 1, min(blocks - 1, 0xFFFF_FFFF) + ) + image[510:512] = MBR_SIGNATURE + array = bytearray(ENTRY_COUNT * ENTRY_SIZE) + for index, partition in enumerate(partitions): + struct.pack_into( + "<16s16sQQQ", + array, + index * ENTRY_SIZE, + LINUX_DATA.bytes_le, + partition.unique_guid.bytes_le, + partition.first_lba, + partition.last_lba, + 0, + ) + image[partition.offset : partition.offset + len(partition.contents)] = ( + partition.contents + ) + array_crc = zlib.crc32(array) + backup_entry_lba = blocks - 1 - ENTRY_ARRAY_BLOCKS + for my_lba, alternate_lba, entry_lba in ( + (1, blocks - 1, 2), + (blocks - 1, 1, backup_entry_lba), + ): + header = _header(blocks, disk_guid, my_lba, alternate_lba, entry_lba, array_crc) + image[my_lba * SECTOR_SIZE : my_lba * SECTOR_SIZE + HEADER_SIZE] = header + image[entry_lba * SECTOR_SIZE : entry_lba * SECTOR_SIZE + len(array)] = array + return bytes(image) + + +def with_mbr(image: bytes, records: Sequence[bytes]) -> bytes: + """Return ``image`` with an MBR of ``records`` in place of its protective MBR.""" + if len(records) > 4: + raise ValueError("an MBR holds at most four records") + replaced = bytearray(image) + table = b"".join(records).ljust(4 * MBR_RECORD_SIZE, b"\x00") + replaced[MBR_RECORDS_OFFSET : MBR_RECORDS_OFFSET + len(table)] = table + replaced[510:512] = MBR_SIGNATURE + return bytes(replaced) diff --git a/scripts/nvx_tools/guest_test_programs.py b/scripts/nvx_tools/guest_test_programs.py new file mode 100644 index 000000000..4d32a7be7 --- /dev/null +++ b/scripts/nvx_tools/guest_test_programs.py @@ -0,0 +1,108 @@ +"""Build the test-only Rust programs that microVM scenarios share into a guest. + +Unlike the guest binaries, they never ship in an initramfs: a scenario that tests guest +library code shares one into the guest that it boots, and runs it there. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Final, cast + +from .build_constants import BuildConstants, GuestRustBuildConstants +from .common import ScriptError, require_tool +from .elf import check_static_executables +from .guest_binaries import pinned_guest_rust_toolchain + +# Each program and the Cargo integration test of `nvx-rs` that builds it. +GUEST_TEST_PROGRAMS: Final = {"gpt-device": "gpt_device"} +OUTPUT_DIRECTORY: Final = BuildConstants.BUILD_DIR / "guest-tests" + + +def guest_test_program_path(program: str) -> Path: + """Return where ``build-guest-tests`` puts ``program`` by default.""" + return OUTPUT_DIRECTORY / program + + +def _test_executables(messages: str) -> dict[str, Path]: + """Return the executable of each integration test in Cargo's JSON ``messages``.""" + executables: dict[str, Path] = {} + for line in messages.splitlines(): + if not line.startswith("{"): + continue + message = cast(dict[str, object], json.loads(line)) + target = message.get("target") + executable = message.get("executable") + if ( + message.get("reason") == "compiler-artifact" + and isinstance(target, dict) + and cast(dict[str, object], target).get("kind") == ["test"] + and isinstance(executable, str) + ): + name = cast(dict[str, object], target).get("name") + if isinstance(name, str): + executables[name] = Path(executable) + return executables + + +def build_guest_test_programs(output: Path) -> dict[str, Path]: + """Build every test-only guest program for musl into ``output``. + + Each is the static executable of an integration test that Cargo builds with the + ``guest`` feature, and runs its ignored tests when a scenario passes ``--ignored``. + The build keeps the test profile's debug assertions, but strips the debug + information, which would make the programs ten times as large, from the + executables that the guests' memory holds. Return the path of each program. + """ + if sys.platform != "linux": + raise ScriptError("Rust guest test programs can be built only on Linux") + command = [ + require_tool("cargo"), + f"+{pinned_guest_rust_toolchain()}", + "test", + "--locked", + "--no-run", + "--message-format=json-render-diagnostics", + "--config", + 'profile.test.strip="debuginfo"', + "--target", + GuestRustBuildConstants.TARGET, + "--no-default-features", + "--features", + GuestRustBuildConstants.FEATURE, + ] + for test in GUEST_TEST_PROGRAMS.values(): + command.extend(("--test", test)) + try: + built = subprocess.run( + command, + cwd=GuestRustBuildConstants.DIRECTORY, + stdout=subprocess.PIPE, + text=True, + encoding="utf-8", + check=False, + ) + except OSError as error: + raise ScriptError(f"cannot run cargo: {error}") from error + if built.returncode != 0: + raise ScriptError( + f"command failed with exit {built.returncode}: {' '.join(command)}" + ) + executables = _test_executables(built.stdout) + output.mkdir(parents=True, exist_ok=True) + programs: dict[str, Path] = {} + for program, test in GUEST_TEST_PROGRAMS.items(): + source = executables.get(test) + if source is None: + raise ScriptError(f"Cargo built no executable for the {test} test") + destination = output / program + shutil.copyfile(source, destination) + os.chmod(destination, 0o755) + programs[program] = destination + check_static_executables(list(programs.values())) + return programs diff --git a/scripts/nvx_tools/microvm_test_scripts/gpt-device.sh.in b/scripts/nvx_tools/microvm_test_scripts/gpt-device.sh.in new file mode 100644 index 000000000..9162beeb9 --- /dev/null +++ b/scripts/nvx_tools/microvm_test_scripts/gpt-device.sh.in @@ -0,0 +1,34 @@ +set -eu +fail() { + code="$1" + echo "NVX-GPT-DEVICE-FAIL code=$code" + nvx-exit "$code" + exit "$code" +} + +for name in vda vdb; do + tries=0 + while [ ! -b "/dev/$name" ] && [ "$tries" -lt 200 ]; do + sleep 0.05 + tries=$((tries + 1)) + done + [ -b "/dev/$name" ] || fail 20 +done +grep -q ' /mnt/share virtiofs ' /proc/mounts || fail 21 +[ "$(cat /sys/block/vda/ro)" = 1 ] || fail 22 +[ "$(cat /sys/block/vdb/ro)" = 0 ] || fail 23 +# The share need not keep the program's mode, so a copy runs. +cp /mnt/share/gpt-device /tmp/nvx-gpt-device || fail 24 +chmod 0755 /tmp/nvx-gpt-device || fail 25 +if ! NVX_GPT_DEVICE_DISK=/dev/vda \ + NVX_GPT_DEVICE_DISK_GUID=@DISK_GUID@ \ + NVX_GPT_DEVICE_PARTITIONS='@PARTITIONS@' \ + NVX_GPT_DEVICE_FIRST_PARTITION=@FIRST_PARTITION@ \ + NVX_GPT_DEVICE_SCRATCH=/dev/vdb \ + NVX_GPT_DEVICE_IMAGES=/mnt/share \ + /tmp/nvx-gpt-device --ignored --test-threads=1; then + fail 30 +fi +rm -f /tmp/nvx-gpt-device +echo NVX-GPT-DEVICE-OK +nvx-exit 0 diff --git a/scripts/nvx_tools/microvm_tests.py b/scripts/nvx_tools/microvm_tests.py index eaece8403..fd018f351 100644 --- a/scripts/nvx_tools/microvm_tests.py +++ b/scripts/nvx_tools/microvm_tests.py @@ -27,7 +27,7 @@ from stat import S_ISUID from typing import Any, cast -from . import nvx_rs +from . import gpt_image, nvx_rs from .adversarial_oracles import terminate_process_tree from .benchmark import ( CAPTURE_LOG_FILTER, @@ -63,6 +63,7 @@ require_file, sha256_file, ) +from .guest_test_programs import guest_test_program_path from .guests import GUEST_NAMES, GuestDescriptor, guest_descriptor from .managed_exec_tests import run_managed_exec_configuration from .microvm_durations import REPORT_NAME, DurationReport @@ -146,6 +147,10 @@ # the guest-artifact build produces and packages omit, so they run only when # named. PUBLIC_SANDBOX_SCENARIOS = frozenset(("managed-exec-config", "sandbox-lifecycle")) +# The scenarios that test guest library code each run the test-only program of +# their name, which `nvx.py build-guest-tests` builds and packages omit, so +# they are explicit scenarios, which run only when named. +GUEST_TEST_PROGRAM_SCENARIOS = frozenset(("gpt-device",)) # The same-host restore cases also run on the CI debug kernel (doc/ci.md), # whose soft-lockup and hung-task detectors the guest's time ABI watcher # reports. @@ -159,8 +164,9 @@ # Scenarios that run only when named, never in a default suite. The time ABI # hides TSC-deadline on every backend, so `smp` already runs on the one-shot # counting LAPIC; `smp-lapic` repeats it and asserts the counting-LAPIC facts, -# for explicit local use (#286). -MICROVM_EXPLICIT_SCENARIOS = ("smp-lapic",) +# for explicit local use (#286). The guest library scenarios need their +# test-only programs. +MICROVM_EXPLICIT_SCENARIOS = ("smp-lapic", *sorted(GUEST_TEST_PROGRAM_SCENARIOS)) SMP_LAPIC_COUNTING_MARKER = b"NVX-SMP-LAPIC-COUNTING-OK" MICROVM_PROCESSOR_COUNTS = (1, 2, 4, 8) MICROVM_TEST_SCRIPTS_DIR = Path(__file__).with_name("microvm_test_scripts") @@ -244,6 +250,16 @@ PRIVATE_INGRESS_UDP_REQUEST = b"NVX-PRIVATE-INGRESS-UDP-REQUEST\n" PRIVATE_INGRESS_UDP_REPLY = b"NVX-PRIVATE-INGRESS-UDP-REPLY\n" SANDBOX_BLOCK_SIZE = 8 * 1024 * 1024 +GPT_DEVICE_COMPLETION_MARKER = b"NVX-GPT-DEVICE-OK" +# The disks of the gpt-device scenario have 4 MiB of 512-byte blocks: a first +# partition of 64 KiB at LBA 2048, and layers of 512 KiB at LBAs 4096 and 6144. +GPT_DEVICE_BLOCKS = 8192 +GPT_DEVICE_DISK_GUID = uuid.UUID("6a1f3c2e-8b4d-4e5f-9a7b-1c2d3e4f5a00") +GPT_DEVICE_SCRATCH_GUID = uuid.UUID("7b2e4d3f-9c5e-4f60-8b8c-2d3e4f5a6b00") +# The first partition's bytes, which are not UTF-8 and hold a NUL, and the padding +# after them, trailing NUL bytes and ASCII whitespace, which reading it removes. +GPT_DEVICE_FIRST_PARTITION = b"\xff\xfeNVX configuration\x00opaque\x80bytes" +GPT_DEVICE_FIRST_PARTITION_PADDING = b" \t\n\r\x0c\x00" * 4 SNAPSHOT_CORE_CONTINUED_MARKER = b"NVX-SNAPSHOT-CORE-CONTINUED" SNAPSHOT_CORE_COMPLETION_MARKER = b"NVX-SNAPSHOT-CORE-OK" CONSOLE_BINARY_MARKER = b"\0\r\n\x7f\xffNVX-CONSOLE-BINARY" @@ -385,8 +401,8 @@ def configure_parser(parser: argparse.ArgumentParser) -> None: choices=(*MICROVM_TEST_SCENARIOS, *MICROVM_EXPLICIT_SCENARIOS), help=( "scenario to run; repeat to select multiple (default: all except " - "smp-lapic, managed-exec-config, and sandbox-lifecycle, which run " - "only when named)" + "smp-lapic, managed-exec-config, sandbox-lifecycle, and gpt-device, " + "which run only when named)" ), ) parser.add_argument( @@ -4668,6 +4684,153 @@ def run_sandbox_blocks( ) +@dataclass(frozen=True) +class GptDeviceDisks: + """The disks of the gpt-device scenario: the read-only disk's GPT, and the images + that the guest writes over the writable disk, by name.""" + + disk: bytes + disk_guid: uuid.UUID + partitions: tuple[gpt_image.Partition, ...] + images: dict[str, bytes] + + +def _gpt_device_partitions(disk_guid: uuid.UUID) -> list[gpt_image.Partition]: + """Return a first partition and two layers, whose unique GUIDs follow + ``disk_guid``, and which start with the first partition's bytes and padding and + with each layer's marker.""" + return [ + gpt_image.Partition( + uuid.UUID(int=disk_guid.int + 1), + 2048, + 2175, + GPT_DEVICE_FIRST_PARTITION + GPT_DEVICE_FIRST_PARTITION_PADDING, + ), + gpt_image.Partition( + uuid.UUID(int=disk_guid.int + 2), 4096, 5119, b"NVX-GPT-LAYER-2" + ), + gpt_image.Partition( + uuid.UUID(int=disk_guid.int + 3), 6144, 7167, b"NVX-GPT-LAYER-3" + ), + ] + + +def gpt_device_disks() -> GptDeviceDisks: + """Return the disks that ``tests/gpt_device.rs`` of ``nvx-rs`` expects.""" + partitions = _gpt_device_partitions(GPT_DEVICE_DISK_GUID) + partition1, partition2, partition3 = _gpt_device_partitions(GPT_DEVICE_SCRATCH_GUID) + partition4 = gpt_image.Partition( + uuid.UUID(int=GPT_DEVICE_SCRATCH_GUID.int + 4), 7168, 7679, b"NVX-GPT-LAYER-4" + ) + moved = gpt_image.Partition( + partition2.unique_guid, + partition2.first_lba + 512, + partition2.last_lba + 512, + partition2.contents, + ) + + def scratch(*layout: gpt_image.Partition) -> bytes: + return gpt_image.gpt_image(GPT_DEVICE_BLOCKS, GPT_DEVICE_SCRATCH_GUID, layout) + + three = scratch(partition1, partition2, partition3) + # An MBR whose first partition spans the GPT's first two, and whose second is + # the GPT's third. + mbr = gpt_image.with_mbr( + three, + [ + gpt_image.mbr_record( + gpt_image.LINUX_TYPE, + partition1.first_lba, + partition2.last_lba - partition1.first_lba + 1, + ), + gpt_image.mbr_record( + gpt_image.LINUX_TYPE, + partition3.first_lba, + partition3.last_lba - partition3.first_lba + 1, + ), + ], + ) + return GptDeviceDisks( + disk=gpt_image.gpt_image(GPT_DEVICE_BLOCKS, GPT_DEVICE_DISK_GUID, partitions), + disk_guid=GPT_DEVICE_DISK_GUID, + partitions=tuple(partitions), + images={ + "three.img": three, + "two.img": scratch(partition1, partition2), + "four.img": scratch(partition1, partition2, partition3, partition4), + "moved.img": scratch(partition1, moved, partition3), + "mbr.img": mbr, + }, + ) + + +def run_gpt_device( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + *, + memory_mib: int, + timeout: float, + log_path: Path, +) -> None: + """Validate crafted GPT disks in the guest through ``nvx::guest::gpt``. + + The guest runs the ignored tests of the test-only ``gpt-device`` program, which + the scenario shares with the images, as root: a read-only disk with a crafted GPT, + and a writable disk over which the tests write the images, to read tables that + the kernel's partitions then differ from.""" + program = require_file( + guest_test_program_path("gpt-device"), + "gpt-device guest test program (build it with nvx.py build-guest-tests)", + ) + disks = gpt_device_disks() + with tempfile.TemporaryDirectory(prefix="nvx-gpt-device-") as temporary: + root = Path(temporary) + share = root / "share" + share.mkdir() + shutil.copyfile(program, share / "gpt-device") + for name, image in disks.images.items(): + (share / name).write_bytes(image) + disk = root / "disk.img" + disk.write_bytes(disks.disk) + scratch = root / "scratch.img" + with scratch.open("wb") as file: + file.truncate(len(disks.disk)) + command = workload_boot_command( + executable, + backend, + kernel, + initrd, + memory_mib, + "quiet loglevel=0", + mount=f"/mnt/share,{share},ro", + ) + command.extend( + ( + "--microvm-sandbox-block", + f"distro:file:{disk},ro", + "--microvm-sandbox-block", + f"scratch:file:{scratch}", + ) + ) + run_guest_script( + command, + _render_script( + "gpt-device.sh.in", + DISK_GUID=str(disks.disk_guid), + PARTITIONS=" ".join( + f"{partition.unique_guid}:{partition.size}" + for partition in disks.partitions + ), + FIRST_PARTITION=GPT_DEVICE_FIRST_PARTITION.hex(), + ), + GPT_DEVICE_COMPLETION_MARKER, + timeout=timeout, + log_path=log_path, + ) + + def _measure_restore( command: Sequence[str], *, @@ -8041,6 +8204,20 @@ def _run_scenarios( timeout=args.timeout, log_path=output_dir / "sandbox-blocks.log", ) + if "gpt-device" in scenarios: + report.start( + "gpt-device", + f"Running microVM GPT device validation on OpenVMM/{args.backend}", + ) + run_gpt_device( + executable, + kernel, + initrd, + args.backend, + memory_mib=args.memory_mib, + timeout=args.timeout, + log_path=output_dir / "gpt-device.log", + ) if "scratch-snapshot" in scenarios: report.start( "scratch-snapshot", diff --git a/scripts/test_guest_binaries.py b/scripts/test_guest_binaries.py index fd7df2633..f74034eb1 100644 --- a/scripts/test_guest_binaries.py +++ b/scripts/test_guest_binaries.py @@ -6,6 +6,7 @@ import json import os import re +import subprocess import sys import tempfile import unittest @@ -16,7 +17,7 @@ sys.path.insert(0, str(Path(__file__).parent)) import nvx # noqa: E402 -from nvx_tools import guest_binaries # noqa: E402 +from nvx_tools import guest_binaries, guest_test_programs # noqa: E402 from nvx_tools.build_constants import ( # noqa: E402 BuildConstants, GuestRustBuildConstants, @@ -1040,5 +1041,140 @@ def test_source_identity_command_prints_the_recorded_source_digest(self): self.assertEqual(stdout.getvalue(), "c" * 64 + "\n") +def _cargo_messages(executable: Path | None) -> str: + """Return the JSON messages of a ``cargo test --no-run`` that built the library + and, unless ``executable`` is ``None``, the ``gpt_device`` test there.""" + messages: list[dict[str, object]] = [ + { + "reason": "compiler-artifact", + "target": {"name": "nvx", "kind": ["lib"]}, + "executable": None, + }, + { + "reason": "compiler-artifact", + "target": {"name": "nvx", "kind": ["lib"]}, + "profile": {"test": True}, + "executable": "/target/debug/deps/nvx-0123", + }, + ] + if executable is not None: + messages.append( + { + "reason": "compiler-artifact", + "target": {"name": "gpt_device", "kind": ["test"]}, + "executable": str(executable), + } + ) + messages.append({"reason": "build-finished", "success": True}) + return "".join(json.dumps(message) + "\n" for message in messages) + + +class GuestTestProgramTests(unittest.TestCase): + def _build( + self, output: Path, *, stdout: str, returncode: int = 0 + ) -> tuple[dict[str, Path], MagicMock, MagicMock]: + """Build into ``output`` with a Cargo that prints ``stdout``.""" + run = MagicMock( + return_value=subprocess.CompletedProcess([], returncode, stdout=stdout) + ) + check = MagicMock() + with ( + patch.object(guest_test_programs.sys, "platform", "linux"), + patch.object(guest_test_programs, "require_tool", side_effect=_tool), + patch.object(guest_test_programs.subprocess, "run", run), + patch.object(guest_test_programs, "check_static_executables", check), + ): + programs = guest_test_programs.build_guest_test_programs(output) + return programs, run, check + + def test_build_installs_the_test_executable_that_cargo_reports(self): + with tempfile.TemporaryDirectory() as temporary: + built = Path(temporary) / "gpt_device-4e44e7b343e76e7a" + built.write_bytes(b"static test program") + output = Path(temporary) / "guest-tests" + programs, run, check = self._build(output, stdout=_cargo_messages(built)) + program = output / "gpt-device" + self.assertEqual(programs, {"gpt-device": program}) + self.assertEqual(program.read_bytes(), b"static test program") + if os.name != "nt": + self.assertEqual(program.stat().st_mode & 0o777, 0o755) + check.assert_called_once_with([program]) + command = cast(list[str], run.call_args.args[0]) + self.assertEqual( + command, + [ + "cargo", + f"+{GuestRustBuildConstants.TOOLCHAIN}", + "test", + "--locked", + "--no-run", + "--message-format=json-render-diagnostics", + "--config", + 'profile.test.strip="debuginfo"', + "--target", + GuestRustBuildConstants.TARGET, + "--no-default-features", + "--features", + "guest", + "--test", + "gpt_device", + ], + ) + self.assertEqual(run.call_args.kwargs["cwd"], GuestRustBuildConstants.DIRECTORY) + + def test_build_fails_when_cargo_fails_or_builds_no_test(self): + for stdout, returncode, message in ( + ("", 101, "command failed with exit 101"), + (_cargo_messages(None), 0, "no executable for the gpt_device test"), + ): + with ( + self.subTest(message=message), + tempfile.TemporaryDirectory() as temporary, + self.assertRaisesRegex(guest_test_programs.ScriptError, message), + ): + self._build( + Path(temporary) / "guest-tests", + stdout=stdout, + returncode=returncode, + ) + + def test_build_runs_only_on_linux(self): + with ( + patch.object(guest_test_programs.sys, "platform", "win32"), + patch.object(guest_test_programs.subprocess, "run") as run, + self.assertRaisesRegex(guest_test_programs.ScriptError, "only on Linux"), + ): + guest_test_programs.build_guest_test_programs(Path("guest-tests")) + run.assert_not_called() + + def test_build_command_defaults_to_the_directory_that_scenarios_read(self): + default = guest_test_programs.OUTPUT_DIRECTORY + self.assertEqual(default, BuildConstants.BUILD_DIR / "guest-tests") + self.assertEqual( + guest_test_programs.guest_test_program_path("gpt-device"), + default / "gpt-device", + ) + for argv, output in ( + (["build-guest-tests"], default), + (["build-guest-tests", "--output", "tests"], Path("tests")), + ): + with self.subTest(argv=argv): + args = nvx.parse_args(argv) + stdout = io.StringIO() + with ( + patch.object( + nvx, + "build_guest_test_programs", + return_value={"gpt-device": output / "gpt-device"}, + ) as build, + contextlib.redirect_stdout(stdout), + ): + args.handler(args) + build.assert_called_once_with(output) + self.assertEqual( + stdout.getvalue(), f"gpt-device: {output / 'gpt-device'}\n" + ) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 282d9a127..064a12f3b 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -15,12 +15,15 @@ import re import shutil import socket +import struct import subprocess import sys import tempfile import threading import time import unittest +import uuid +import zlib from collections.abc import Callable, Collection from contextlib import ExitStack, nullcontext, redirect_stdout from datetime import datetime @@ -34,6 +37,8 @@ adversarial_executor, benchmark, common, + gpt_image, + guest_test_programs, managed_exec_tests, microvm_durations, microvm_shards, @@ -5890,7 +5895,7 @@ def test_smp_exercises_the_counting_lapic_without_a_duplicate_scenario(self): def test_smp_lapic_remains_an_explicit_scenario(self): # #286 keeps `test-microvm --scenario smp-lapic` for local use. - self.assertEqual(microvm_tests.MICROVM_EXPLICIT_SCENARIOS, ("smp-lapic",)) + self.assertIn("smp-lapic", microvm_tests.MICROVM_EXPLICIT_SCENARIOS) args = nvx.parse_args( ["test-microvm", "--backend", "kvm", "--scenario", "smp-lapic"] ) @@ -8262,6 +8267,273 @@ def test_sandbox_blocks_use_fixed_roles_and_access(self): microvm_tests.SANDBOX_BLOCKS_COMPLETION_MARKER, ) + def test_gpt_device_runs_only_when_named(self): + self.assertIn("gpt-device", microvm_tests.GUEST_TEST_PROGRAM_SCENARIOS) + # Each guest library scenario is an explicit scenario, which no default + # suite, and so no shard, runs, and has a program of its name. + programs = microvm_tests.GUEST_TEST_PROGRAM_SCENARIOS + self.assertLessEqual(programs, set(microvm_tests.MICROVM_EXPLICIT_SCENARIOS)) + self.assertTrue(programs.isdisjoint(microvm_tests.MICROVM_TEST_SCENARIOS)) + self.assertTrue(programs.isdisjoint(microvm_tests.DEBUG_KERNEL_SCENARIOS)) + self.assertTrue(programs.isdisjoint(microvm_tests.default_suite())) + self.assertLessEqual(programs, set(guest_test_programs.GUEST_TEST_PROGRAMS)) + + def require(path: Path, _description: str) -> Path: + return path + + for selection, named in (([], False), (["--scenario", "gpt-device"], True)): + with ( + self.subTest(named=named), + tempfile.TemporaryDirectory() as temporary, + ): + args = nvx.parse_args( + [ + "test-microvm", + "--backend", + "kvm", + "--output-dir", + temporary, + *selection, + ] + ) + with ( + patch.object(microvm_tests, "validate_openvmm_test_backend"), + patch.object( + microvm_tests, "MICROVM_TEST_SCENARIOS", ("guest-boot",) + ), + patch.object(microvm_tests, "require_file", side_effect=require), + patch.object(microvm_tests, "run_guest_boot") as guest_boot, + patch.object(microvm_tests, "run_gpt_device") as gpt_device, + ): + self.assertEqual(microvm_tests.run(args), 0) + self.assertEqual(gpt_device.called, named) + self.assertEqual(guest_boot.called, not named) + if named: + self.assertEqual( + gpt_device.call_args.kwargs["log_path"], + Path(temporary) / "gpt-device.log", + ) + + def test_gpt_device_shares_its_program_and_images_beside_two_disks(self): + disks = microvm_tests.gpt_device_disks() + seen: dict[str, object] = {} + + def run_guest_script( + command: list[str], script: str, marker: bytes, **_kwargs: object + ) -> None: + guest, share, mode = cast(str, boot.call_args.kwargs["mount"]).split(",") + seen["mount"] = (guest, mode) + seen["share"] = sorted(path.name for path in Path(share).iterdir()) + for name, image in disks.images.items(): + self.assertEqual((Path(share) / name).read_bytes(), image, name) + seen["program"] = (Path(share) / "gpt-device").read_bytes() + blocks = [ + command[index + 1] + for index, value in enumerate(command) + if value == "--microvm-sandbox-block" + ] + seen["blocks"] = blocks + disk = Path(blocks[0].removeprefix("distro:file:").removesuffix(",ro")) + scratch = Path(blocks[1].removeprefix("scratch:file:")) + seen["disk"] = disk.read_bytes() + seen["scratch"] = scratch.read_bytes() + seen["script"] = script + seen["marker"] = marker + + with tempfile.TemporaryDirectory() as temporary: + program = Path(temporary) / "gpt-device" + program.write_bytes(b"test program") + with ( + patch.object( + microvm_tests, "guest_test_program_path", return_value=program + ) as program_path, + patch.object( + microvm_tests, + "workload_boot_command", + return_value=["openvmm", "boot"], + ) as boot, + patch.object( + microvm_tests, "run_guest_script", side_effect=run_guest_script + ), + ): + microvm_tests.run_gpt_device( + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "whp", + memory_mib=128, + timeout=60, + log_path=Path("gpt-device.log"), + ) + + # The scenario runs the program of its name (GUEST_TEST_PROGRAM_SCENARIOS). + program_path.assert_called_once_with("gpt-device") + self.assertEqual(seen["mount"], ("/mnt/share", "ro")) + self.assertEqual( + seen["share"], + ["four.img", "gpt-device", "mbr.img", "moved.img", "three.img", "two.img"], + ) + self.assertEqual(seen["program"], b"test program") + blocks = cast(list[str], seen["blocks"]) + self.assertEqual(len(blocks), 2) + self.assertTrue(blocks[0].startswith("distro:file:")) + self.assertTrue(blocks[0].endswith(",ro")) + self.assertTrue(blocks[1].startswith("scratch:file:")) + self.assertFalse(blocks[1].endswith(",ro")) + self.assertEqual(seen["disk"], disks.disk) + self.assertEqual(seen["scratch"], bytes(len(disks.disk))) + script = cast(str, seen["script"]) + self.assertNotIn("@", script) + guid = microvm_tests.GPT_DEVICE_DISK_GUID + self.assertIn(f"NVX_GPT_DEVICE_DISK_GUID={guid}", script) + partitions = " ".join( + f"{uuid.UUID(int=guid.int + number)}:{size}" + for number, size in ((1, 64 * 1024), (2, 512 * 1024), (3, 512 * 1024)) + ) + self.assertIn(f"NVX_GPT_DEVICE_PARTITIONS='{partitions}'", script) + self.assertIn( + f"NVX_GPT_DEVICE_FIRST_PARTITION={microvm_tests.GPT_DEVICE_FIRST_PARTITION.hex()}", + script, + ) + self.assertIn("--ignored --test-threads=1", script) + self.assertTrue(script.endswith("nvx-exit 0\n")) + self.assertEqual(seen["marker"], microvm_tests.GPT_DEVICE_COMPLETION_MARKER) + + def test_gpt_device_needs_its_built_program(self): + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object( + microvm_tests, + "guest_test_program_path", + return_value=Path(temporary) / "gpt-device", + ), + patch.object(microvm_tests, "run_guest_script") as run_guest_script, + self.assertRaisesRegex(common.ScriptError, "nvx.py build-guest-tests"), + ): + microvm_tests.run_gpt_device( + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "whp", + memory_mib=128, + timeout=60, + log_path=Path("gpt-device.log"), + ) + run_guest_script.assert_not_called() + + def test_gpt_device_disks_hold_the_layouts_that_its_guest_test_expects(self): + disks = microvm_tests.gpt_device_disks() + blocks = microvm_tests.GPT_DEVICE_BLOCKS + sector = gpt_image.SECTOR_SIZE + array_size = gpt_image.ENTRY_COUNT * gpt_image.ENTRY_SIZE + + def entries(image: bytes) -> list[tuple[uuid.UUID, int, int]]: + """Check the image's headers and arrays, and return its used entries.""" + self.assertEqual(len(image), blocks * sector) + headers: list[tuple[Any, ...]] = [] + for lba in (1, blocks - 1): + header = image[lba * sector :][: gpt_image.HEADER_SIZE] + fields = struct.unpack("<8sIIIIQQQQ16sQIII", header) + self.assertEqual(fields[:5:2], (b"EFI PART", 92, 0)) + self.assertEqual(fields[1], 0x0001_0000) + self.assertEqual( + zlib.crc32(header[:16] + bytes(4) + header[20:]), fields[3] + ) + headers.append(fields) + primary, backup = headers + self.assertEqual(primary[5:7], (1, blocks - 1)) + self.assertEqual(backup[5:7], (blocks - 1, 1)) + self.assertEqual(primary[7:9], (34, blocks - 34)) + self.assertEqual(primary[7:10], backup[7:10]) + self.assertEqual((primary[10], backup[10]), (2, blocks - 33)) + self.assertEqual(primary[11:13], (128, 128)) + array = image[2 * sector :][:array_size] + self.assertEqual(image[backup[10] * sector :][:array_size], array) + self.assertEqual(zlib.crc32(array), primary[13]) + used: list[tuple[uuid.UUID, int, int]] = [] + for index in range(gpt_image.ENTRY_COUNT): + kind, unique, first, last = struct.unpack_from( + "<16s16sQQ", array, index * gpt_image.ENTRY_SIZE + ) + if kind != bytes(16): + self.assertEqual(uuid.UUID(bytes_le=kind), gpt_image.LINUX_DATA) + used.append((uuid.UUID(bytes_le=unique), first, last)) + return used + + def mbr(image: bytes) -> list[tuple[int, int, int]]: + """Return the type, first LBA, and size of each MBR record.""" + self.assertEqual(image[510:512], b"\x55\xaa") + return [ + struct.unpack_from("<4xB3xII", image, 446 + 16 * index) + for index in range(4) + ] + + def layout( + disk_guid: uuid.UUID, *extents: tuple[int, int] + ) -> list[tuple[uuid.UUID, int, int]]: + return [ + (uuid.UUID(int=disk_guid.int + number), first, last) + for number, (first, last) in enumerate(extents, start=1) + ] + + partition1, partition2, partition3 = (2048, 2175), (4096, 5119), (6144, 7167) + disk_guid = microvm_tests.GPT_DEVICE_DISK_GUID + self.assertEqual( + entries(disks.disk), layout(disk_guid, partition1, partition2, partition3) + ) + self.assertEqual(disks.disk[sector + 56 :][:16], disk_guid.bytes_le) + self.assertEqual( + mbr(disks.disk), [(0xEE, 1, blocks - 1), (0, 0, 0), (0, 0, 0), (0, 0, 0)] + ) + self.assertEqual( + [(p.unique_guid, p.first_lba, p.last_lba) for p in disks.partitions], + entries(disks.disk), + ) + first_lba, last_lba = partition1 + configuration = disks.disk[first_lba * sector :][ + : (last_lba - first_lba + 1) * sector + ] + self.assertNotEqual( + configuration.rstrip(b"\0"), microvm_tests.GPT_DEVICE_FIRST_PARTITION + ) + self.assertEqual( + configuration.rstrip(b"\0 \t\n\r\x0c"), + microvm_tests.GPT_DEVICE_FIRST_PARTITION, + ) + for number, (first, _) in ((2, partition2), (3, partition3)): + marker = f"NVX-GPT-LAYER-{number}".encode() + self.assertEqual(disks.disk[first * sector :][: len(marker)], marker) + + scratch = microvm_tests.GPT_DEVICE_SCRATCH_GUID + images = disks.images + self.assertEqual( + entries(images["three.img"]), + layout(scratch, partition1, partition2, partition3), + ) + self.assertEqual( + entries(images["two.img"]), layout(scratch, partition1, partition2) + ) + self.assertEqual( + entries(images["four.img"]), + layout(scratch, partition1, partition2, partition3, (7168, 7679)), + ) + self.assertEqual( + entries(images["moved.img"]), + layout(scratch, partition1, (4608, 5631), partition3), + ) + # The MBR replaces only the protective MBR, so the GPT behind it remains. + self.assertEqual( + entries(images["mbr.img"]), + layout(scratch, partition1, partition2, partition3), + ) + self.assertEqual( + mbr(images["mbr.img"]), + [(0x83, 2048, 3072), (0x83, 6144, 1024), (0, 0, 0), (0, 0, 0)], + ) + self.assertEqual(images["mbr.img"][sector:], images["three.img"][sector:]) + with self.assertRaisesRegex(ValueError, "at most four records"): + gpt_image.with_mbr(images["three.img"], [bytes(16)] * 5) + def test_smp_snapshot_runs_the_warp_probe_after_each_restore(self): def measure(command: list[str], **kwargs: object) -> None: processors = int(command[command.index("--processors") + 1]) diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index 4466a82d9..2b8225aed 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -54,6 +54,7 @@ elf, guest_artifact_reuse, guest_binaries, + guest_test_programs, guests, microvm_durations, microvm_shards, @@ -76,6 +77,7 @@ ZstdBuildConstants, ) from nvx_tools.microvm_tests import ( # noqa: E402 + GUEST_TEST_PROGRAM_SCENARIOS, MICROVM_PROCESSOR_COUNTS, PUBLIC_SANDBOX_SCENARIOS, default_suite, @@ -6030,19 +6032,23 @@ def load(name: str) -> Any: AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME, }, "guest-debug-kernel": set(DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES), + "guest-test-programs": set(guest_test_programs.GUEST_TEST_PROGRAMS), } guest_files = set[str]().union(*components.values()) - # Every file that the guest builds produce is handed over. + # Every file that the guest builds produce is handed over, and so is + # every test-only guest program. self.assertEqual( guest_files, { *DockerBuildConstants.ALL_GUEST_ARTIFACT_NAMES, *DockerBuildConstants.DEBUG_KERNEL_ARTIFACT_NAMES, scratch, + *guest_test_programs.GUEST_TEST_PROGRAMS, }, ) producers: dict[str, str] = {} + roots: dict[str, str] = {} for job_name, job in jobs.items(): for step in job.get("steps", []): arguments = step.get("with", {}) @@ -6060,6 +6066,11 @@ def load(name: str) -> Any: {PurePosixPath(path).name for path in paths}, components[name] ) self.assertEqual(arguments["if-no-files-found"], "error") + # An artifact holds the files of one directory at its root, + # so a job downloads it into that directory. + directories = {str(PurePosixPath(path).parent) for path in paths} + self.assertEqual(len(directories), 1, paths) + roots[name] = directories.pop() self.assertEqual(set(producers), set(components)) # Each guest artifact job builds exactly the components that it # uploads, and no component builds in two jobs. @@ -6175,6 +6186,8 @@ def reads(step: dict[str, Any], context: Mapping[str, object]) -> set[str]: scenarios = set(re.findall(r"--scenario (\S+)", script)) if scenarios & PUBLIC_SANDBOX_SCENARIOS: files |= workload + # A guest library scenario runs the program of its name. + files |= scenarios & GUEST_TEST_PROGRAM_SCENARIOS else: self.assertIn(command, commands) files |= commands[command] @@ -6220,7 +6233,8 @@ def consume( "guest-" ): self.assertNotIn("pattern", arguments) - self.assertEqual(arguments["path"], "build") + self.assertIn(arguments["name"], roots) + self.assertEqual(arguments["path"], roots[arguments["name"]]) if holds(step.get("if"), context): downloads.add(arguments["name"]) if not downloads: @@ -6537,6 +6551,7 @@ def test_required_ci_result_policy(self): ci.REQUIRED_CI_ARTIFACT_JOB, ci.REQUIRED_CI_DISTRO_ARTIFACT_JOB, ci.REQUIRED_CI_DEBUG_KERNEL_JOB, + ci.REQUIRED_CI_GUEST_TEST_PROGRAM_JOB, } platforms = set(ci.REQUIRED_CI_PLATFORM_JOBS) # A same-repository pull request that builds the guest artifacts also @@ -6803,6 +6818,7 @@ def test_ci_jobs_follow_their_change_detector_output(self): ci.REQUIRED_CI_DISTRO_ARTIFACT_JOB: "run-guest-artifacts", ci.REQUIRED_CI_DEBUG_KERNEL_JOB: "run-guest-artifacts", ci.REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB: "run-guest-artifacts", + ci.REQUIRED_CI_GUEST_TEST_PROGRAM_JOB: "run-guest-artifacts", **dict.fromkeys(ci.REQUIRED_CI_BUILD_JOBS, "run-openvmm-binaries"), **dict.fromkeys(ci.REQUIRED_CI_OPENVMM_TEST_JOBS, "run-openvmm-unit-tests"), **dict.fromkeys( @@ -7602,6 +7618,7 @@ def job_inputs( ci.REQUIRED_CI_ARTIFACT_JOB, ci.REQUIRED_CI_DISTRO_ARTIFACT_JOB, ci.REQUIRED_CI_DEBUG_KERNEL_JOB, + ci.REQUIRED_CI_GUEST_TEST_PROGRAM_JOB, *ci.REQUIRED_CI_BUILD_JOBS, ) | nvx_rs_inputs @@ -8557,10 +8574,14 @@ def test_ci_shares_openvmm_inputs_and_binary_artifacts(self): # Only the jobs that boot the Ubuntu or Azure Linux images wait # for them: the default suites boot the kernel and the Alpine # initramfs alone, and the debug-kernel jobs the debug kernel. + # Of those, only the guest and sandbox jobs run the guest test + # programs, so only they wait for them. if job_name.startswith("nvx-microvm-debug-"): guest_producer = "debug-kernel, " elif job_name.startswith("nvx-microvm-tests-"): guest_producer = "" + elif job_name.startswith("nvx-microvm-guests-"): + guest_producer = "distro-artifacts, guest-test-programs, " else: guest_producer = "distro-artifacts, " cli = ( @@ -8599,10 +8620,11 @@ def test_ci_shares_openvmm_inputs_and_binary_artifacts(self): self.assertEqual(build_workflow.count("overwrite: true"), 2) self.assertEqual(build_workflow.count("retention-days: 1"), 2) # The microVM workflow also downloads nvx-rs, which the benchmarks - # don't run, and the Ubuntu sandbox scratch template and the debug - # kernel, which the platform jobs don't read. + # don't run, and the Ubuntu sandbox scratch template, the debug + # kernel, and the guest test programs, which the platform jobs don't + # read. for consumer_workflow, download_count in ( - (microvm_workflow, 10), + (microvm_workflow, 11), (platform_workflow, 7), ): self.assertIn("path: openvmm/target/release", consumer_workflow) @@ -9233,7 +9255,8 @@ def test_reuse_downloads_the_artifacts_of_each_recorded_component(self): # The artifacts that reuse downloads for each component are those in # which the job that builds it hands its files over, and together they # hold every file of the component; the scratch template, which every - # run formats anew, is in none. + # run formats anew, is in none, and neither are the debug kernel and + # the guest test programs, which no package ships. github = BuildConstants.REPO_ROOT / ".github" workflow = yaml.safe_load( (github / "workflows" / "ci.yml").read_text(encoding="utf-8") @@ -9264,7 +9287,7 @@ def test_reuse_downloads_the_artifacts_of_each_recorded_component(self): ) self.assertEqual( set(uploads) - handed_over, - {"guest-ubuntu-scratch", "guest-debug-kernel"}, + {"guest-ubuntu-scratch", "guest-debug-kernel", "guest-test-programs"}, ) def test_ci_runs_debug_kernel_on_kvm_for_prs_and_all_for_non_prs(self): @@ -9379,10 +9402,10 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): # after its ten-minute default suite. Each backend now runs them in a # second job beside the suite, on the same runner pool and with the # same OpenVMM and nvx-rs builds. Of the two, only that job reads the - # Ubuntu and Azure Linux artifacts, so only it downloads and waits for - # them. Together, the two jobs run every step of the former single job - # exactly once, and the debug-kernel jobs, which never set the - # selector, run as before. + # Ubuntu and Azure Linux artifacts and the guest test programs, so only + # it downloads and waits for them. Together, the two jobs run every + # step of the former single job exactly once, and the debug-kernel + # jobs, which never set the selector, run as before. workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" jobs = yaml.safe_load((workflows / "ci.yml").read_text(encoding="utf-8"))[ "jobs" @@ -9423,9 +9446,11 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): ) self.assertEqual(suite["needs"][0], "artifacts") self.assertNotIn("distro-artifacts", suite["needs"]) + self.assertNotIn("guest-test-programs", suite["needs"]) # The guest and sandbox job also waits for the Ubuntu and Azure - # Linux artifacts, and otherwise equals the suite's job but for - # its name, the selector, and the suite's shards + # Linux artifacts and the guest test programs, and otherwise + # equals the suite's job but for its name, the selector, and the + # suite's shards # (test_ci_runs_each_default_microvm_suite_in_its_shards). conjuncts = _expression_conjuncts(suite["if"]) after = conjuncts.index("needs.artifacts.result == 'success'") + 1 @@ -9442,10 +9467,16 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): unsharded | { "name": f"NVX microVM tests / {name}", - "needs": ["artifacts", "distro-artifacts", *suite["needs"][1:]], + "needs": [ + "artifacts", + "distro-artifacts", + "guest-test-programs", + *suite["needs"][1:], + ], "if": [ *conjuncts[:after], "needs.distro-artifacts.result == 'success'", + "needs.guest-test-programs.result == 'success'", *conjuncts[after:], ], "with": inputs @@ -9537,6 +9568,7 @@ def downloaded(selection: list[str]) -> set[str]: ) for step_name in ( f"Run public managed sandbox acceptance on {system}", + f"Run guest library scenarios on {system}", f"Run Ubuntu guest smoke tests on {system}", f"Run Azure Linux guest smoke tests on {system}", f"Run Ubuntu sandbox layer smoke test on {system}", @@ -11783,6 +11815,76 @@ def test_every_microvm_job_uploads_its_scenario_durations(self): ) self.assertEqual(len(artifacts), 2 * legs) + def test_ci_runs_the_guest_library_scenarios_on_every_backend(self): + # A GitHub-hosted job builds the test-only guest programs beside the + # guest artifacts, under the same condition, and the microVM guest and + # sandbox job of every backend, Windows' too, downloads them into + # build/guest-tests, where test-microvm looks for them + # (test_ci_jobs_download_exactly_the_guest_artifacts_that_they_read), + # and runs every guest library scenario with them. + workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" + workflow = (workflows / "ci.yml").read_text(encoding="utf-8") + microvm_workflow = (workflows / "run-nvx-microvm-tests.yml").read_text( + encoding="utf-8" + ) + producer = ci.REQUIRED_CI_GUEST_TEST_PROGRAM_JOB + job = _workflow_job(workflow, producer) + + def condition(definition: str) -> str: + return definition[ + definition.index(" if: >-\n") : definition.index(" runs-on:") + ] + + self.assertIn(" needs: [openvmm-changes]\n", job) + self.assertIn(" runs-on: ubuntu-latest\n", job) + self.assertEqual( + condition(job), condition(_workflow_job(workflow, "artifacts")) + ) + build = _workflow_step(workflow, producer, "Build guest test programs") + self.assertIn("--target x86_64-unknown-linux-musl", build) + self.assertIn( + "python3 scripts/nvx.py build-guest-tests --output build/guest-tests", build + ) + self.assertEqual( + guest_test_programs.OUTPUT_DIRECTORY, + BuildConstants.BUILD_DIR / "guest-tests", + ) + self.assertIn( + "name: guest-test-programs\n", + _workflow_step(workflow, producer, "Upload guest test programs"), + ) + # The release and the performance persistence accept skipped microVM + # jobs, so they require the build themselves, as they do the debug + # kernel's. + for consumer in ("release", "performance-persist"): + with self.subTest(consumer=consumer): + definition = _workflow_job(workflow, consumer) + self.assertIn(f" - {producer}\n", definition) + self.assertIn(f"needs.{producer}.result == 'success' &&", definition) + for system in ("Linux", "Windows"): + with self.subTest(system=system): + scenarios = _workflow_step( + microvm_workflow, + "test", + f"Run guest library scenarios on {system}", + ) + self.assertIn( + "inputs.guests-and-sandboxes && !inputs.debug-kernel", scenarios + ) + self.assertEqual( + set(re.findall(r"--scenario (\S+)", scenarios)), + GUEST_TEST_PROGRAM_SCENARIOS, + ) + self.assertIn( + '--output-dir "build/test-results/guest-library-' + '${{ inputs.backend }}"', + scenarios, + ) + logs = _workflow_step( + microvm_workflow, "test", "Upload NVX microVM failure logs" + ) + self.assertIn("build/test-results/guest-library-${{ inputs.backend }}", logs) + @unittest.skipUnless(os.name == "nt", "requires Windows PowerShell") def test_windows_cli_validation_stops_at_each_failed_command(self): action = (