diff --git a/.github/workflows/build-packages-self-hosted.yml b/.github/workflows/build-packages-self-hosted.yml index 01f3bb5..dde212b 100644 --- a/.github/workflows/build-packages-self-hosted.yml +++ b/.github/workflows/build-packages-self-hosted.yml @@ -6,12 +6,14 @@ on: build_type: description: "Select build type" required: true - default: "BOTH" + default: "SEV+TDX" type: choice options: - TDX - SNP - - BOTH + - SEV+TDX + - libvirt-ubuntu24 + - libvirt-ubuntu26 jobs: build: @@ -23,7 +25,7 @@ jobs: - name: Setup clean workspace run: | WORK_DIR="/home/gh-runner/builds/run-${{ github.run_number }}" - echo "WORK_DIR=$WORK_DIR" >> $GITHUB_ENV + echo "WORK_DIR=$WORK_DIR" >> "$GITHUB_ENV" mkdir -p /home/gh-runner/builds @@ -35,72 +37,108 @@ jobs: cd "$WORK_DIR" git init . - git remote add origin https://github.com/${{ github.repository }}.git - git fetch --depth 1 origin ${{ github.sha }} - git checkout ${{ github.sha }} + git remote add origin "https://github.com/${{ github.repository }}.git" + git fetch --depth 1 origin "${{ github.sha }}" + git checkout "${{ github.sha }}" - name: Set build type and runner ID working-directory: ${{ env.WORK_DIR }} run: | - echo "BUILD_TYPE=${{ github.event.inputs.build_type }}" >> $GITHUB_ENV - echo "RUNNER_ID=${{ github.run_number }}" >> $GITHUB_ENV + echo "BUILD_TYPE=${{ github.event.inputs.build_type }}" >> "$GITHUB_ENV" + echo "RUNNER_ID=${{ github.run_number }}" >> "$GITHUB_ENV" - name: Run TDX docker build - if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'BOTH' }} + if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'SEV+TDX' }} working-directory: ${{ env.WORK_DIR }} run: | NON_INTERACTIVE=1 FORCE_REBUILD_CONTAINER=1 ./build/build_in_docker.sh tdx - name: Run SNP docker build - if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'BOTH' }} + if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'SEV+TDX' }} working-directory: ${{ env.WORK_DIR }} run: | NON_INTERACTIVE=1 FORCE_REBUILD_CONTAINER=1 ./build/build_in_docker.sh snp + + - name: Build libvirt packages + if: ${{ startsWith(github.event.inputs.build_type, 'libvirt-ubuntu') }} + working-directory: ${{ env.WORK_DIR }} + run: | + case "${BUILD_TYPE}" in + libvirt-ubuntu24) + target="ubuntu24" + output_directory="ubuntu-24.04" + ;; + libvirt-ubuntu26) + target="ubuntu26" + output_directory="ubuntu-26.04" + ;; + *) + echo "Error: unsupported libvirt build type: ${BUILD_TYPE}" >&2 + exit 1 + ;; + esac + + archive_name="${BUILD_TYPE}.tar.gz" + archive_path="${RUNNER_TEMP}/${archive_name}" + + ./build/libvirt/build.sh "${target}" + + tar \ + --create \ + --gzip \ + --file "${archive_path}" \ + --directory "${WORK_DIR}/build/libvirt/out" \ + "${output_directory}" + + tar --list --gzip --file "${archive_path}" >/dev/null + echo "LIBVIRT_ARCHIVE_NAME=${archive_name}" >> "$GITHUB_ENV" + echo "LIBVIRT_ARCHIVE_PATH=${archive_path}" >> "$GITHUB_ENV" - name: Set release name id: release-name run: | - if [[ "${BUILD_TYPE}" == "BOTH" ]]; then - echo "RELEASE_NAME=${RUNNER_ID}-tdx+snp" >> $GITHUB_ENV + if [[ "${BUILD_TYPE}" == "SEV+TDX" ]]; then + echo "RELEASE_NAME=${RUNNER_ID}-sev+tdx" >> "$GITHUB_ENV" elif [[ "${BUILD_TYPE}" == "TDX" ]]; then - echo "RELEASE_NAME=${RUNNER_ID}-tdx" >> $GITHUB_ENV + echo "RELEASE_NAME=${RUNNER_ID}-tdx" >> "$GITHUB_ENV" elif [[ "${BUILD_TYPE}" == "SNP" ]]; then - echo "RELEASE_NAME=${RUNNER_ID}-snp" >> $GITHUB_ENV + echo "RELEASE_NAME=${RUNNER_ID}-snp" >> "$GITHUB_ENV" + elif [[ "${BUILD_TYPE}" == libvirt-ubuntu* ]]; then + echo "RELEASE_NAME=${RUNNER_ID}-${BUILD_TYPE}" >> "$GITHUB_ENV" else echo "Error: Unknown BUILD_TYPE ${BUILD_TYPE}" >&2 exit 1 fi - - name: Create GitHub Release - id: create_release - uses: actions/create-release@v1 + - name: Publish TDX Release Asset + if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'SEV+TDX' }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: "${{ env.RELEASE_NAME }}" - release_name: "Release ${{ env.RELEASE_NAME }}" - draft: false + tag_name: ${{ env.RELEASE_NAME }} + name: Release ${{ env.RELEASE_NAME }} prerelease: true - - - name: Upload TDX Release Asset - if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'BOTH' }} - uses: actions/upload-release-asset@v1 + files: ${{ env.WORK_DIR }}/build/out/tdx/package-tdx.tar.gz + + - name: Publish SNP Release Asset + if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'SEV+TDX' }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ env.WORK_DIR }}/build/out/tdx/package-tdx.tar.gz - asset_name: package-tdx.tar.gz - asset_content_type: application/gzip - - - name: Upload SNP Release Asset - if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'BOTH' }} - uses: actions/upload-release-asset@v1 + tag_name: ${{ env.RELEASE_NAME }} + name: Release ${{ env.RELEASE_NAME }} + prerelease: true + files: ${{ env.WORK_DIR }}/build/out/snp/package-snp.tar.gz + + - name: Publish libvirt Release Asset + if: ${{ startsWith(github.event.inputs.build_type, 'libvirt-ubuntu') }} + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ env.WORK_DIR }}/build/out/snp/package-snp.tar.gz - asset_name: package-snp.tar.gz - asset_content_type: application/gzip + tag_name: ${{ env.RELEASE_NAME }} + name: Release ${{ env.RELEASE_NAME }} + prerelease: true + files: ${{ env.LIBVIRT_ARCHIVE_PATH }} diff --git a/README.md b/README.md index 3fc2b38..43d3755 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Utilities for bootstrapping a Confidential Computing host (Intel **TDX** or AMD | `scripts/bootstrap_tdx.sh` | Turn an Ubuntu host into a TDX-capable hypervisor (kernel, QEMU, OVMF, attestation, GPU passthrough). | | `scripts/bootstrap_snp.sh` | Turn an Ubuntu host into a SEV-SNP-capable hypervisor (firmware, modules, GPU passthrough). | | `scripts/start_super_protocol.sh` | Start a confidential VM (TDX / SEV-SNP / untrusted) from a Super Protocol release image. | +| `scripts/start_super_protocol_libvirt.sh` | Start the same VM as a transient `qemu:///system` domain through libvirt-python (Ubuntu 24.04 or 26.04). | | `scripts/swarm-cluster.sh` | Bring up a 3-node Swarm cluster on a single host. | | `scripts/check_configuration.sh`, `get_super_running_vms.sh` | Auxiliary tooling. | @@ -35,6 +36,49 @@ This is the main path: take a bare Ubuntu host, turn it into a confidential hype For the exact commands to clone the repository, run the bootstrap scripts, and launch a VM, see [docs/swarm.md](docs/swarm.md). +### Libvirt launcher (Ubuntu 24.04 and 26.04) + +`scripts/start_super_protocol_libvirt.sh` reuses the release, disk, provider-config, and VFIO preparation from the direct QEMU launcher, then builds domain XML and starts a transient domain through `libvirt-python`. GPU passthrough uses IOMMUFD. The TDX and SEV-SNP bootstrap scripts install libvirt **12.5.0** when the system version is older, configure AppArmor, grant `passt` the capability required for privileged ports, and validate the daemon and domain capabilities before VFIO devices are bound. + +The command line is the same as for `start_super_protocol.sh`, with an optional domain name: + +```bash +sudo ./scripts/start_super_protocol_libvirt.sh \ + --name super-protocol-3 \ + --provider_config /path/to/provider-configs \ + --mode tdx +``` + +With `--debug false` the command returns after the domain starts. With `--debug true --log_file /path/to/boot.log`, it follows the domain serial log and copies it to the log file; `Ctrl-C` detaches without stopping the VM. The domain always records its console to `/var/log/libvirt/qemu/-serial.log` from the first byte, so a VM that fails early can still be diagnosed. The serial port is a file sink rather than a pty, because a pty nobody reads fills up and stalls the guest inside console output. Use `virsh -c qemu:///system list`, `shutdown`, or `destroy` to manage it. `--gpu none` disables GPU, NVSwitch, and CX7 passthrough for diagnostics. + +#### Libvirt host configuration + +Run the bootstrap matching the host CPU before using the libvirt launcher: + +```bash +sudo ./scripts/bootstrap_tdx.sh +# or +sudo ./scripts/bootstrap_snp.sh +``` + +The bootstrap performs the host-wide work that previously required manual fixes: + +- installs the complete project libvirt 12.5 package set, including `libvirt-dev`, when the installed version is older or any required split package is missing; +- preserves already installed libvirt split drivers during the package transaction; +- enables executable mmap and the libvirt socket in the nested AppArmor `passt` profile; +- permits QEMU to contact TDX QGS through VSOCK; +- applies `CAP_NET_BIND_SERVICE` to every installed `passt` binary; +- prepares `/var/lib/libvirt/images/superprotocol` and validates `qemu:///system`. + +The bootstrap deliberately does not change `net.ipv4.ip_unprivileged_port_start`. File capabilities can be removed when the administrator upgrades or reinstalls `passt`; re-run the same bootstrap to restore them. The launcher detects missing AppArmor rules or capabilities before preparing VM disks and prints the appropriate bootstrap command. + +On Ubuntu 24.04, the bootstrap adapts only the verified temporary copy of the +project `libvirt-daemon-driver-qemu` package to the `systemd-sysusers` syntax +supported by that release. The downloaded release archive itself is not +modified. + +Do not disable AppArmor globally. For diagnostics, inspect recent denials with `journalctl -k --since '-10 min' --no-pager`. + ### 1. Clone the repo Clone the repository onto the target host. See [docs/swarm.md](docs/swarm.md) for the exact command. @@ -48,11 +92,12 @@ Pick the script that matches your CPU vendor. See [docs/swarm.md](docs/swarm.md) What it does: 1. Verifies Ubuntu version and root privileges. -2. Runs `setup_tdx.sh` to install the Canonical TDX 3.3 stack and PCCS attestation host components. +2. Runs `setup_tdx.sh` to install the project-matched TDX kernel/QEMU bundle and PCCS attestation host components. 3. Verifies BIOS/CPU TDX settings (TME, TME-MT, SEAM, TXT, SGX, …). -4. Runs the official `setup-tdx-host.sh` from `canonical/tdx`. +4. Installs the required QGS/PCCS attestation packages directly, without running Canonical's host-setup script or enabling global package downgrades. 5. Updates the Intel TDX-Module to a known-good version. 6. Configures NVIDIA GPUs for Confidential Computing (CC mode + `vfio-pci` binding) and, on B200 systems, sets up ConnectX-7 bridges for VFIO passthrough. +7. Installs and validates libvirt 12.5, AppArmor policy, VSOCK access, and `passt` capabilities before binding devices to the VM stack. > **Note:** Some steps require manual action to take effect. The script may stop and ask you to do something, then need to be re-run — this is expected. Follow the on-screen instructions and re-run to finish. @@ -65,6 +110,7 @@ What it does: 3. Downloads and installs the matching AMD SEV firmware blob to `/lib/firmware/amd/` and reloads `ccp` / `kvm_amd`. 4. Runs SNP status checks (RMP table, SEV / SEV-SNP API versions, ASID allocation, IOMMU groups, hugepages, CPU governor). 5. Configures NVIDIA GPUs for CC mode and binds them to `vfio-pci`. +6. Installs and validates libvirt 12.5, AppArmor policy, and `passt` capabilities before binding devices to the VM stack. > **Ubuntu 24.04 note:** the SNP bootstrap installs a bundled Linux **6.16** kernel. On some systems, network interfaces may be renamed after reboot, which can affect networking and remote SSH access. Make sure you have iKVM or other interactive console access before rebooting, so you can reconfigure networking for the new interface names if needed. @@ -76,13 +122,22 @@ A reboot is required partway through bootstrap. After reboot, re-run the same bo `scripts/check_configuration.sh` prints a hardware overview (CPU, memory, network, disks, RAID/SMART) you can compare against the [Requirements](#requirements). See [docs/swarm.md](docs/swarm.md) for how to run it. +Hardware acceptance remains a manual step because containers cannot validate KVM, IOMMUFD, QGS, VSOCK, or physical GPU assignment. On each prepared host verify: + +- a transient VM starts in release and debug modes; +- TCP/UDP forwarding works on host ports 53, 80, and 443; +- TDX measurement returns a non-empty quote and PKI/gossip become ready; +- SEV-SNP launch security is active; +- `--gpu none` works and an enabled GPU is attached through IOMMUFD; +- the kernel audit log contains no new `passt`, libvirt, or VSOCK AppArmor denial. + ## Running a Swarm cluster There are two ways to run a Super Protocol Swarm cluster. ### Single-host cluster (quick start) -`scripts/swarm-cluster.sh` brings up a **3-node Swarm cluster on a single host** — no multi-machine setup. It creates an isolated bridge network, launches one bootstrap + two join VMs in separate `tmux` sessions, auto-configures provider configs, and sets up ingress via HAProxy. You still need to set `gateway_hostname` in the provider template to point to the machine's public IP. +`scripts/swarm-cluster.sh` brings up a **3-node Swarm cluster on a single host** — no multi-machine setup. It creates an isolated bridge network, launches one bootstrap + two join VMs as transient libvirt domains, auto-configures provider configs, and sets up ingress via HAProxy. In debug mode their serial consoles remain attached in separate `tmux` sessions. You still need to set `gateway_hostname` in the provider template to point to the machine's public IP. Prerequisites: a bootstrapped host (TDX or SEV-SNP), a populated provider config template (see [config.yaml reference](docs/swarm.md#configyaml-reference) for an example), and `tmux` / `nftables` / `curl` installed. @@ -118,7 +173,7 @@ BIOS settings: |---|---| | `CPU PA limit to 46 bits` | Disabled | | `SMT` | Enabled | -| `TXT` | Enabled | +| `TXT` | Optional for TDX; status is reported but does not block setup | | `SGX` | Enabled | | `TME` | Enabled | | `TME-MT (Multi-Tenant)` | Enabled, KeyIDs configured (non-zero key split) | @@ -157,4 +212,4 @@ Planned hardware support. These items are **not yet supported** and are listed f ## License -See [LICENSE](LICENSE). \ No newline at end of file +See [LICENSE](LICENSE). diff --git a/build/libvirt/.dockerignore b/build/libvirt/.dockerignore new file mode 100644 index 0000000..1fcb152 --- /dev/null +++ b/build/libvirt/.dockerignore @@ -0,0 +1 @@ +out diff --git a/build/libvirt/Dockerfile.ubuntu b/build/libvirt/Dockerfile.ubuntu new file mode 100644 index 0000000..427c819 --- /dev/null +++ b/build/libvirt/Dockerfile.ubuntu @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +ARG UBUNTU_VERSION=24.04 +FROM ubuntu:${UBUNTU_VERSION} + +ARG LIBVIRT_VERSION=12.5.0 +ARG DEBIAN_REVISION=1 +ARG PACKAGING_COMMIT=a8f73eb070c24b72f9d6dfbeffc28a334f29e076 + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + devscripts \ + dpkg-dev \ + equivs \ + git \ + libdistro-info-perl \ + && rm -rf /var/lib/apt/lists/* + +COPY docker/prepare-source.sh /usr/local/bin/prepare-libvirt-source +RUN prepare-libvirt-source "${LIBVIRT_VERSION}" "${DEBIAN_REVISION}" "${PACKAGING_COMMIT}" + +COPY docker/build-packages.sh /usr/local/bin/build-libvirt-packages + +ENTRYPOINT ["/usr/local/bin/build-libvirt-packages"] diff --git a/build/libvirt/README.md b/build/libvirt/README.md new file mode 100644 index 0000000..05b890b --- /dev/null +++ b/build/libvirt/README.md @@ -0,0 +1,89 @@ +# Local libvirt packages + +This directory contains a parameterized Docker build environment for Ubuntu +24.04 and Ubuntu 26.04. The build uses the Debian libvirt packaging and resolves +all build dependencies inside the target Ubuntu image. + +The default source is the `debian/12.5.0-1` tag from the Debian libvirt Salsa +repository. Resulting packages have a local version such as: + +```text +12.5.0-1spvm1~ubuntu24.04.1 +12.5.0-1spvm1~ubuntu26.04.1 +``` + +The local suffix sorts after older upstream versions but before a future +`12.5.0-1ubuntu*` package, so an official build of the same upstream release +can replace it normally. + +## Requirements + +- Docker with access to the Docker daemon; +- Internet access for the Ubuntu repositories and Debian Salsa; +- an amd64 host, or a Docker setup capable of building `linux/amd64` images; +- enough free space for two builder images and package artifacts. + +## Manual build + +Run from the repository root: + +```bash +./build/libvirt/build.sh ubuntu24 +./build/libvirt/build.sh ubuntu26 +``` + +Build both targets sequentially: + +```bash +./build/libvirt/build.sh all +``` + +Package tests are enabled by default. For a faster development build that only +compiles and packages libvirt: + +```bash +./build/libvirt/build.sh all --skip-tests +``` + +The script supports alternative upstream/local revisions, for example: + +```bash +./build/libvirt/build.sh ubuntu24 \ + --libvirt-version 12.5.0 \ + --debian-revision 1 \ + --spvm-revision 2 +``` + +Use `./build/libvirt/build.sh --help` to see all options. + +## Artifacts + +Packages are placed in a target- and version-specific directory: + +```text +build/libvirt/out/ubuntu-24.04/12.5.0-1spvm1~ubuntu24.04.1/ +build/libvirt/out/ubuntu-26.04/12.5.0-1spvm1~ubuntu26.04.1/ +``` + +Each directory contains the split libvirt `.deb` packages, debug `.ddeb` +packages, `.changes`, `.buildinfo`, and `SHA256SUMS`. Do not mix packages built +for different Ubuntu releases. + +The build script only creates local, unsigned packages. It does not install +them, create an APT repository, or build QEMU and python3-libvirt. Before using +`scripts/start_super_protocol_libvirt.sh`, run the matching host bootstrap; the +runtime also requires `python3-libvirt`, `passt`, `acl`, and libvirt 12.1 or +newer for GPU passthrough. + +The `Build packages self-hosted` GitHub Actions workflow can build either +Ubuntu target and publish the result to a prerelease. Select +`libvirt-ubuntu24` or `libvirt-ubuntu26` in the workflow dispatch form. The +release contains one compressed tar archive: + +```text +libvirt-ubuntu24.tar.gz +libvirt-ubuntu26.tar.gz +``` + +Each archive preserves the target and package-version directories and contains +the complete package output together with its `SHA256SUMS` file. diff --git a/build/libvirt/build.sh b/build/libvirt/build.sh new file mode 100755 index 0000000..ceeaf74 --- /dev/null +++ b/build/libvirt/build.sh @@ -0,0 +1,213 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + +LIBVIRT_VERSION=12.5.0 +DEBIAN_REVISION=1 +SPVM_REVISION=1 +PACKAGING_COMMIT=a8f73eb070c24b72f9d6dfbeffc28a334f29e076 +OUTPUT_ROOT="${SCRIPT_DIR}/out" +DOCKER_PLATFORM=linux/amd64 +SKIP_TESTS=false +NO_CACHE=false +PULL_BASE=false +TARGET="" + +usage() { + cat <<'EOF' +Build local libvirt Debian packages for Ubuntu 24.04 and/or 26.04. + +Usage: + ./build/libvirt/build.sh [options] + +Options: + --libvirt-version VERSION Upstream libvirt version (default: 12.5.0) + --debian-revision NUMBER Debian packaging revision (default: 1) + --spvm-revision NUMBER Local package revision (default: 1) + --packaging-commit SHA Immutable Debian packaging commit + --output DIR Artifact root (default: build/libvirt/out) + --platform PLATFORM Docker platform (default: linux/amd64) + --skip-tests Set DEB_BUILD_OPTIONS=nocheck + --no-cache Rebuild the Docker image without cache + --pull Pull the latest Ubuntu base image + -h, --help Show this help + +The script only builds local artifacts. It does not install or publish them. +EOF +} + +if [[ $# -eq 0 ]]; then + usage >&2 + exit 2 +fi + +if [[ "$1" == "-h" || "$1" == "--help" ]]; then + usage + exit 0 +fi + +TARGET=$1 +shift + +while [[ $# -gt 0 ]]; do + case "$1" in + --libvirt-version) + [[ $# -ge 2 ]] || { echo "Error: --libvirt-version requires a value" >&2; exit 2; } + LIBVIRT_VERSION=$2 + shift 2 + ;; + --debian-revision) + [[ $# -ge 2 ]] || { echo "Error: --debian-revision requires a value" >&2; exit 2; } + DEBIAN_REVISION=$2 + shift 2 + ;; + --spvm-revision) + [[ $# -ge 2 ]] || { echo "Error: --spvm-revision requires a value" >&2; exit 2; } + SPVM_REVISION=$2 + shift 2 + ;; + --packaging-commit) + [[ $# -ge 2 ]] || { echo "Error: --packaging-commit requires a value" >&2; exit 2; } + PACKAGING_COMMIT=$2 + shift 2 + ;; + --output) + [[ $# -ge 2 ]] || { echo "Error: --output requires a value" >&2; exit 2; } + OUTPUT_ROOT=$2 + shift 2 + ;; + --platform) + [[ $# -ge 2 ]] || { echo "Error: --platform requires a value" >&2; exit 2; } + DOCKER_PLATFORM=$2 + shift 2 + ;; + --skip-tests) + SKIP_TESTS=true + shift + ;; + --no-cache) + NO_CACHE=true + shift + ;; + --pull) + PULL_BASE=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +case "${TARGET}" in + ubuntu24) + targets=(ubuntu24) + ;; + ubuntu26) + targets=(ubuntu26) + ;; + all) + targets=(ubuntu24 ubuntu26) + ;; + *) + echo "Error: target must be ubuntu24, ubuntu26, or all" >&2 + usage >&2 + exit 2 + ;; +esac + +if ! [[ "${LIBVIRT_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: invalid libvirt version: ${LIBVIRT_VERSION}" >&2 + exit 2 +fi +if ! [[ "${DEBIAN_REVISION}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: Debian revision must be a positive integer" >&2 + exit 2 +fi +if ! [[ "${SPVM_REVISION}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: SPVM revision must be a positive integer" >&2 + exit 2 +fi +if ! [[ "${PACKAGING_COMMIT}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Error: packaging commit must be a full lowercase Git SHA" >&2 + exit 2 +fi +if ! command -v docker >/dev/null 2>&1; then + echo "Error: docker is not installed" >&2 + exit 1 +fi +if ! docker info >/dev/null 2>&1; then + echo "Error: cannot connect to the Docker daemon" >&2 + exit 1 +fi + +mkdir -p "${OUTPUT_ROOT}" +OUTPUT_ROOT=$(realpath "${OUTPUT_ROOT}") + +build_target() { + local target=$1 ubuntu_version image_name output_dir package_version + local -a build_args + case "${target}" in + ubuntu24) + ubuntu_version=24.04 + ;; + ubuntu26) + ubuntu_version=26.04 + ;; + esac + + image_name="sp-vm-libvirt-builder:ubuntu${ubuntu_version}-${LIBVIRT_VERSION}-${DEBIAN_REVISION}" + package_version="${LIBVIRT_VERSION}-${DEBIAN_REVISION}spvm${SPVM_REVISION}~ubuntu${ubuntu_version}.1" + output_dir="${OUTPUT_ROOT}/ubuntu-${ubuntu_version}/${package_version}" + rm -rf -- "${output_dir}" + mkdir -p "${output_dir}" + + build_args=( + build + --platform "${DOCKER_PLATFORM}" + --file "${SCRIPT_DIR}/Dockerfile.ubuntu" + --tag "${image_name}" + --build-arg "UBUNTU_VERSION=${ubuntu_version}" + --build-arg "LIBVIRT_VERSION=${LIBVIRT_VERSION}" + --build-arg "DEBIAN_REVISION=${DEBIAN_REVISION}" + --build-arg "PACKAGING_COMMIT=${PACKAGING_COMMIT}" + ) + if [[ "${NO_CACHE}" == "true" ]]; then + build_args+=(--no-cache) + fi + if [[ "${PULL_BASE}" == "true" ]]; then + build_args+=(--pull) + fi + build_args+=("${SCRIPT_DIR}") + + echo "Building Docker image ${image_name}" + docker "${build_args[@]}" + + echo "Building libvirt packages for Ubuntu ${ubuntu_version}" + docker run \ + --rm \ + --platform "${DOCKER_PLATFORM}" \ + --env "LIBVIRT_VERSION=${LIBVIRT_VERSION}" \ + --env "DEBIAN_REVISION=${DEBIAN_REVISION}" \ + --env "SPVM_REVISION=${SPVM_REVISION}" \ + --env "TARGET_UBUNTU_VERSION=${ubuntu_version}" \ + --env "SKIP_TESTS=${SKIP_TESTS}" \ + --env "OUTPUT_UID=$(id -u)" \ + --env "OUTPUT_GID=$(id -g)" \ + --volume "${output_dir}:/out" \ + "${image_name}" + + echo "Artifacts: ${output_dir}" +} + +for target in "${targets[@]}"; do + build_target "${target}" +done diff --git a/build/libvirt/docker/build-packages.sh b/build/libvirt/docker/build-packages.sh new file mode 100755 index 0000000..2fcd216 --- /dev/null +++ b/build/libvirt/docker/build-packages.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +set -euo pipefail + +readonly SOURCE_DIR=/opt/libvirt-source +readonly OUTPUT_DIR=/out + +readonly LIBVIRT_VERSION=${LIBVIRT_VERSION:?LIBVIRT_VERSION is required} +readonly DEBIAN_REVISION=${DEBIAN_REVISION:?DEBIAN_REVISION is required} +readonly SPVM_REVISION=${SPVM_REVISION:-1} +readonly TARGET_UBUNTU_VERSION=${TARGET_UBUNTU_VERSION:?TARGET_UBUNTU_VERSION is required} +readonly SKIP_TESTS=${SKIP_TESTS:-false} +readonly OUTPUT_UID=${OUTPUT_UID:-0} +readonly OUTPUT_GID=${OUTPUT_GID:-0} + +# Provided by every Ubuntu builder image. +# shellcheck disable=SC1091 +source /etc/os-release +if [[ "${ID:-}" != "ubuntu" || "${VERSION_ID:-}" != "${TARGET_UBUNTU_VERSION}" ]]; then + echo "Error: builder is Ubuntu ${VERSION_ID:-unknown}, target is ${TARGET_UBUNTU_VERSION}" >&2 + exit 1 +fi + +case "${TARGET_UBUNTU_VERSION}" in + 24.04) + expected_codename=noble + ;; + 26.04) + expected_codename=resolute + ;; + *) + echo "Error: unsupported Ubuntu target: ${TARGET_UBUNTU_VERSION}" >&2 + exit 1 + ;; +esac + +if [[ "${VERSION_CODENAME:-}" != "${expected_codename}" ]]; then + echo "Error: Ubuntu ${TARGET_UBUNTU_VERSION} has unexpected codename ${VERSION_CODENAME:-unknown}" >&2 + exit 1 +fi + +if [[ ! "${SPVM_REVISION}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: SPVM_REVISION must be a positive integer" >&2 + exit 1 +fi + +readonly PACKAGE_VERSION="${LIBVIRT_VERSION}-${DEBIAN_REVISION}spvm${SPVM_REVISION}~ubuntu${TARGET_UBUNTU_VERSION}.1" + +work_dir=$(mktemp -d /tmp/libvirt-package-build.XXXXXX) +cleanup() { + rm -rf -- "${work_dir}" +} +trap cleanup EXIT + +cp -a "${SOURCE_DIR}" "${work_dir}/libvirt" +cd "${work_dir}/libvirt" + +export DEBFULLNAME="Super Protocol VM Tools" +export DEBEMAIL="devnull@superprotocol.com" +dch \ + --newversion "${PACKAGE_VERSION}" \ + --distribution "${expected_codename}" \ + --force-distribution \ + "Local rebuild for Ubuntu ${TARGET_UBUNTU_VERSION}." + +build_options="" +if [[ "${SKIP_TESTS}" == "true" ]]; then + build_options="nocheck" +elif [[ "${SKIP_TESTS}" != "false" ]]; then + echo "Error: SKIP_TESTS must be true or false" >&2 + exit 1 +fi +export DEB_BUILD_OPTIONS="${build_options}" + +echo "Building libvirt ${PACKAGE_VERSION} on Ubuntu ${TARGET_UBUNTU_VERSION} (${VERSION_CODENAME})" +dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes -jauto + +mkdir -p "${OUTPUT_DIR}" +shopt -s nullglob +artifacts=( + "${work_dir}"/*.deb + "${work_dir}"/*.ddeb + "${work_dir}"/*.changes + "${work_dir}"/*.buildinfo +) +if [[ ${#artifacts[@]} -eq 0 ]]; then + echo "Error: package build produced no artifacts" >&2 + exit 1 +fi + +for artifact in "${artifacts[@]}"; do + install -m 0644 "${artifact}" "${OUTPUT_DIR}/" +done + +( + cd "${OUTPUT_DIR}" + debs=( ./*.deb ./*.ddeb ) + if [[ ${#debs[@]} -eq 0 ]]; then + echo "Error: no .deb or .ddeb packages were produced" >&2 + exit 1 + fi + sha256sum "${debs[@]}" > SHA256SUMS +) + +chown -R "${OUTPUT_UID}:${OUTPUT_GID}" "${OUTPUT_DIR}" + +echo "Built ${#artifacts[@]} artifacts in ${OUTPUT_DIR}" +echo "Package version: ${PACKAGE_VERSION}" diff --git a/build/libvirt/docker/prepare-source.sh b/build/libvirt/docker/prepare-source.sh new file mode 100755 index 0000000..58c43ef --- /dev/null +++ b/build/libvirt/docker/prepare-source.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -euo pipefail + +readonly LIBVIRT_VERSION=${1:?libvirt version is required} +readonly DEBIAN_REVISION=${2:?Debian revision is required} +readonly PACKAGING_COMMIT=${3:?packaging commit is required} +readonly PACKAGING_REF="debian/${LIBVIRT_VERSION}-${DEBIAN_REVISION}" +readonly SOURCE_DIR=/opt/libvirt-source +readonly PACKAGING_REPOSITORY=https://salsa.debian.org/libvirt-team/libvirt.git + +git clone \ + --branch "${PACKAGING_REF}" \ + --depth 1 \ + "${PACKAGING_REPOSITORY}" \ + "${SOURCE_DIR}" + +actual_commit=$(git -C "${SOURCE_DIR}" rev-parse HEAD) +if [[ "${actual_commit}" != "${PACKAGING_COMMIT}" ]]; then + echo "Error: ${PACKAGING_REF} resolves to ${actual_commit}, expected ${PACKAGING_COMMIT}" >&2 + exit 1 +fi + +actual_version=$(dpkg-parsechangelog -l"${SOURCE_DIR}/debian/changelog" -SVersion) +expected_version="${LIBVIRT_VERSION}-${DEBIAN_REVISION}" +if [[ "${actual_version}" != "${expected_version}" ]]; then + echo "Error: ${PACKAGING_REF} contains version ${actual_version}, expected ${expected_version}" >&2 + exit 1 +fi + +rm -rf -- "${SOURCE_DIR}/.git" + +cd "${SOURCE_DIR}" +apt-get update +mk-build-deps \ + --install \ + --remove \ + --tool 'apt-get -y --no-install-recommends' \ + debian/control + +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/build/libvirt/out/.gitignore b/build/libvirt/out/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/build/libvirt/out/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docs/swarm.md b/docs/swarm.md index b49f19d..ea1b6a2 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -68,6 +68,7 @@ ACME_URL: https://acme.zerossl.com/v2/DV90 **You also need:** - A host already bootstrapped for confidential computing (TDX or SEV-SNP) — see the [main README](../README.md). - `tmux`, `nftables`, `curl`, `nc` installed: `apt install tmux nftables curl netcat-openbsd` +- Ubuntu 24.04 or 26.04 with `qemu:///system`, libvirt 12.1+, `python3-libvirt`, `passt`, and `acl` configured as described in the [libvirt launcher section](../README.md#libvirt-launcher-ubuntu-2404-and-2604). > Keep `provider-template/` in its own folder — not inside `sp-vm-tools` and not inside any cache folder. @@ -78,7 +79,7 @@ ACME_URL: https://acme.zerossl.com/v2/DV90 sudo ./scripts/swarm-cluster.sh up --provider-config-template ./provider-template # Check status -./scripts/swarm-cluster.sh status +sudo ./scripts/swarm-cluster.sh status # Stop everything sudo ./scripts/swarm-cluster.sh down @@ -114,13 +115,13 @@ The bootstrap node gets all remaining host resources after subtracting the host 4. Generates per-node provider configs: - **Bootstrap**: `join_addresses: []`, `pki_authority.servers: []`. - **Join nodes**: `join_addresses: ["10.0.0.10:7946"]`, `caBundle` fetched automatically from bootstrap PKI. -5. Starts each VM in its own `tmux` session (`swarm-bootstrap`, `swarm-join-1`, `swarm-join-2`), attached to the bridge via tap interfaces. +5. Starts transient libvirt domains named `swarm-bootstrap`, `swarm-join-1`, and `swarm-join-2`, attached to the bridge via tap interfaces. In debug mode their attached serial consoles run in matching `tmux` sessions. 6. Waits for bootstrap gossip (7946) and PKI (9443) to become ready. 7. Fetches the CA bundle from bootstrap and injects it into join-node configs. 8. Launches join nodes. 9. Sets up HAProxy ingress: `gw.dyn..superprotocol.io` → bootstrap ports 80/443. -Attach to any VM's console with `tmux attach -t swarm-bootstrap` (or `swarm-join-1` / `swarm-join-2`). +Follow a VM's boot output with `tail -f /var/log/libvirt/qemu/swarm-bootstrap-serial.log` (or `swarm-join-1` / `swarm-join-2`); libvirt records it from the first byte, whether or not anything is attached. In debug mode, use the matching `tmux attach -t ` session instead. diff --git a/scripts/bootstrap_snp.sh b/scripts/bootstrap_snp.sh index d84c946..f034484 100755 --- a/scripts/bootstrap_snp.sh +++ b/scripts/bootstrap_snp.sh @@ -4,6 +4,7 @@ set -e source_common() { local script_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" source "${script_dir}/common.sh" + source "${script_dir}/setup_libvirt_host.sh" } get_kernel_log() { @@ -361,6 +362,7 @@ update_snp_firmware() { bootstrap() { check_os_version "24.04" + get_supported_ubuntu_version || return 1 CPU_MODEL=$(lscpu | grep "^Model name:" | sed 's/Model name: *//') @@ -438,6 +440,11 @@ bootstrap() { fi fi + setup_libvirt_host sev-snp || { + echo -e "${RED}ERROR: libvirt host setup failed${NC}" + return 1 + } + print_section_header "Hardware Configuration" if command -v lspci >/dev/null; then echo "Checking NVIDIA GPU configuration..." diff --git a/scripts/bootstrap_tdx.sh b/scripts/bootstrap_tdx.sh index ccb0183..cc60b76 100755 --- a/scripts/bootstrap_tdx.sh +++ b/scripts/bootstrap_tdx.sh @@ -4,10 +4,12 @@ set -e source_common() { local script_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" source "${script_dir}/common.sh" + source "${script_dir}/setup_libvirt_host.sh" } bootstrap() { check_os_version "24.04" + get_supported_ubuntu_version || return 1 # Check if the script is running as root print_section_header "Privilege Check" @@ -16,8 +18,8 @@ bootstrap() { exit 1 fi - # Download and setup official Canonical TDX - print_section_header "Official TDX Setup" + # Install the project TDX kernel/QEMU stack and host attestation runtime. + print_section_header "TDX Host Setup" TMP_DIR=$(mktemp -d) echo "Installing required tools..." @@ -46,6 +48,11 @@ bootstrap() { exit 1 fi + setup_libvirt_host tdx || { + echo -e "${RED}ERROR: libvirt host setup failed${NC}" + return 1 + } + print_section_header "Hardware Configuration" if command -v lspci >/dev/null; then echo "Checking NVIDIA GPU configuration..." @@ -62,9 +69,9 @@ bootstrap() { rm -rf "${TMP_DIR}" print_section_header "Installation Status" - echo "Official TDX installation complete." + echo "TDX host installation complete." echo "System reboot required to activate TDX." - echo "After reboot, use official tools to create and run TDs." + echo "After reboot, re-run this bootstrap to finish validation." } source_common diff --git a/scripts/common.sh b/scripts/common.sh index c811b34..08cf8bd 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -557,6 +557,17 @@ ensure_cmdline_param() { setup_grub() { local new_kernel="$1" local type=$2 + local grub_entry="Advanced options for Ubuntu>Ubuntu, with Linux ${new_kernel}" + local ubuntu_version="" + + if [ -r /etc/os-release ]; then + ubuntu_version=$(. /etc/os-release && printf '%s' "${VERSION_ID:-}") + fi + + if [ -z "$ubuntu_version" ]; then + echo "Unable to determine the Ubuntu version for GRUB setup" >&2 + return 1 + fi if [[ "$type" != "tdx" && "$type" != "snp" ]]; then echo "Invalid type: $type. Must be 'tdx' or 'snp'." >&2 @@ -573,7 +584,8 @@ setup_grub() { cp /etc/default/grub "/etc/default/grub.backup.$(date +%Y%m%d_%H%M%S)" fi - # Directly set the first menuentry as default since it's our new kernel + # Keep the base default release-neutral. The drop-in below selects the + # requested custom kernel only while this Ubuntu release is installed. sed -i '/^GRUB_DEFAULT=/d' /etc/default/grub echo 'GRUB_DEFAULT=0' > /etc/default/grub.new cat /etc/default/grub >> /etc/default/grub.new @@ -603,15 +615,27 @@ setup_grub() { echo 'GRUB_RECORDFAIL_TIMEOUT=5' >> /etc/default/grub fi - # Create a custom configuration file to ensure our kernel is first + # Select the exact kernel instead of assuming it sorts as menu entry zero. + # Limit the override to this Ubuntu release so a future release upgrade + # automatically returns to its newer distro kernel. mkdir -p /etc/default/grub.d - echo "# Custom kernel order configuration" > "/etc/default/grub.d/99-${type}-kernel.cfg" - echo "GRUB_DEFAULT=0" >> "/etc/default/grub.d/99-${type}-kernel.cfg" + { + echo "# Custom kernel selection for Ubuntu ${ubuntu_version}" + echo '[ -r /etc/os-release ] && . /etc/os-release' + echo "if [ \"\${VERSION_ID:-}\" = \"${ubuntu_version}\" ]; then" + echo " GRUB_DEFAULT=\"${grub_entry}\"" + echo 'fi' + } > "/etc/default/grub.d/99-${type}-kernel.cfg" # Force regeneration of grub.cfg and initramfs update-initramfs -u -k "${new_kernel}" update-grub2 || update-grub + if ! grep -Fq "menuentry 'Ubuntu, with Linux ${new_kernel}'" /boot/grub/grub.cfg; then + echo "Failed to find the requested kernel in GRUB: ${new_kernel}" >&2 + return 1 + fi + # For UEFI systems, ensure the boot entry is updated if [ -d /sys/firmware/efi ]; then if command -v efibootmgr >/dev/null 2>&1; then @@ -631,14 +655,14 @@ setup_grub() { fi fi - # Use both grub-set-default and grub-reboot for maximum reliability + # Use the exact submenu entry for both persistent and one-shot selection. if command -v grub-set-default >/dev/null 2>&1; then - grub-set-default 0 + grub-set-default "${grub_entry}" echo "Set default boot entry using grub-set-default" fi if command -v grub-reboot >/dev/null 2>&1; then - grub-reboot 0 + grub-reboot "${grub_entry}" echo "Set next boot entry using grub-reboot" fi diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py new file mode 100755 index 0000000..25c68c5 --- /dev/null +++ b/scripts/libvirt_launcher.py @@ -0,0 +1,869 @@ +#!/usr/bin/env python3 +"""Build and launch a Super Protocol VM through libvirt. + +The XML builder intentionally has no dependency on python-libvirt so it can be +unit tested on development machines. The binding is imported only when a VM +is actually launched. +""" + +from __future__ import annotations + +import argparse +import os +import re +import select +import sys +import time +import termios +import uuid as uuidlib +import tty +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace +from typing import Any, BinaryIO, Iterable, Optional + + +QEMU_NS = "http://libvirt.org/schemas/domain/qemu/1.0" +LIBVIRT_IOMMUFD_VERSION = 12_001_000 +DOMAIN_NAME_RE = re.compile(r"^[A-Za-z0-9_.+:-]+$") +BDF_RE = re.compile( + r"^(?:(?P[0-9A-Fa-f]{4}):)?" + r"(?P[0-9A-Fa-f]{2}):(?P[0-9A-Fa-f]{2})\." + r"(?P[0-7])$" +) +MAC_RE = re.compile(r"^(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$") + + +@dataclass(frozen=True) +class HostDevice: + kind: str + bdf: str + + +@dataclass +class DomainConfig: + name: str + uuid: Optional[str] + mode: str + emulator: str + memory_gib: int + vcpus: int + bios: str + kernel: str + kernel_cmdline: str + rootfs: str + state_disk: str + provider_config_disk: str + guest_cid: int + qgs_socket: str + mac_address: str + netdev_mode: str + debug: bool = False + log_file: Optional[str] = None + cpu_model: Optional[str] = None + phys_bits: Optional[int] = None + cbitpos: Optional[int] = None + bridge: Optional[str] = None + tap_iface: Optional[str] = None + ip_address: str = "0.0.0.0" + ssh_port: Optional[int] = None + wg_port: Optional[int] = None + http_port: Optional[int] = None + https_port: Optional[int] = None + pki_port: Optional[int] = None + pki_vm_measure_port: Optional[int] = None + swarm_db_gossip_port: Optional[int] = None + dns_port: Optional[int] = None + host_devices: list[HostDevice] = field(default_factory=list) + + +def _sub(parent: ET.Element, tag: str, text: Optional[str] = None, **attrs: Any) -> ET.Element: + element = ET.SubElement( + parent, + tag, + {key.rstrip("_"): str(value) for key, value in attrs.items() if value is not None}, + ) + if text is not None: + element.text = str(text) + return element + + +def _parse_bdf(bdf: str) -> dict[str, str]: + match = BDF_RE.fullmatch(bdf) + if not match: + raise ValueError(f"invalid PCI BDF: {bdf!r}") + parts = match.groupdict(default="0000") + return { + "domain": f"0x{parts['domain'].lower()}", + "bus": f"0x{parts['bus'].lower()}", + "slot": f"0x{parts['slot'].lower()}", + "function": f"0x{parts['function'].lower()}", + } + + +def _normalized_bdf(bdf: str) -> str: + match = BDF_RE.fullmatch(bdf) + if not match: + raise ValueError(f"invalid PCI BDF: {bdf!r}") + parts = match.groupdict(default="0000") + return ( + f"{parts['domain']}:{parts['bus']}:{parts['slot']}.{parts['function']}" + ).lower() + + +def _validate_config(config: DomainConfig) -> None: + if not DOMAIN_NAME_RE.fullmatch(config.name): + raise ValueError( + "domain name may contain only letters, digits, '.', '_', '+', ':', and '-'" + ) + if config.uuid is not None: + try: + parsed_uuid = uuidlib.UUID(config.uuid) + except (ValueError, AttributeError) as exc: + raise ValueError(f"invalid domain UUID: {config.uuid!r}") from exc + config.uuid = str(parsed_uuid) + if config.mode not in {"untrusted", "tdx", "sev-snp"}: + raise ValueError(f"unsupported VM mode: {config.mode}") + if config.netdev_mode not in {"user", "tap"}: + raise ValueError(f"unsupported network mode: {config.netdev_mode}") + if config.memory_gib < 1 or config.vcpus < 1: + raise ValueError("memory and vCPU count must be positive") + if config.guest_cid < 3: + raise ValueError("guest CID must be >= 3") + if not MAC_RE.fullmatch(config.mac_address): + raise ValueError(f"invalid MAC address: {config.mac_address!r}") + for path in ( + config.emulator, + config.bios, + config.kernel, + config.rootfs, + config.state_disk, + config.provider_config_disk, + ): + if not Path(path).is_absolute(): + raise ValueError(f"libvirt resource path must be absolute: {path!r}") + if config.mode == "tdx" and not Path(config.qgs_socket).is_absolute(): + raise ValueError( + f"TDX QGS socket path must be absolute: {config.qgs_socket!r}" + ) + ports = ( + config.ssh_port, + config.wg_port, + config.http_port, + config.https_port, + config.pki_port, + config.pki_vm_measure_port, + config.swarm_db_gossip_port, + config.dns_port, + ) + if any(port is not None and not 1 <= port <= 65535 for port in ports): + raise ValueError("network ports must be between 1 and 65535") + if config.netdev_mode == "tap" and (not config.bridge or not config.tap_iface): + raise ValueError("tap mode requires bridge and tap interface") + if config.mode == "sev-snp": + if config.cbitpos is None or config.phys_bits is None or not config.cpu_model: + raise ValueError("SEV-SNP requires cbitpos, phys_bits, and cpu_model") + if config.debug and not config.log_file: + raise ValueError("debug mode requires a log file") + for device in config.host_devices: + if device.kind not in {"gpu", "aux"}: + raise ValueError(f"unsupported host device kind: {device.kind}") + _parse_bdf(device.bdf) + + +def _add_disk( + devices: ET.Element, + path: str, + target: str, + image_format: str, + readonly: bool, +) -> None: + disk = _sub(devices, "disk", type="file", device="disk") + _sub(disk, "driver", name="qemu", type=image_format) + _sub(disk, "source", file=path) + _sub(disk, "target", dev=target, bus="virtio") + if readonly: + _sub(disk, "readonly") + + +def _add_port_forward( + interface: ET.Element, + protocol: str, + host_port: Optional[int], + guest_port: int, + address: Optional[str] = None, +) -> None: + if host_port is None: + return + attrs: dict[str, Any] = {"proto": protocol} + if address and address != "0.0.0.0": + attrs["address"] = address + forward = _sub(interface, "portForward", **attrs) + range_attrs: dict[str, Any] = {"start": host_port} + if host_port != guest_port: + range_attrs["to"] = guest_port + _sub(forward, "range", **range_attrs) + + +def _add_passt_interface( + devices: ET.Element, + config: DomainConfig, + *, + debug_only: bool = False, +) -> ET.Element: + interface = _sub(devices, "interface", type="user") + _sub(interface, "backend", type="passt") + if not debug_only: + _sub(interface, "mac", address=config.mac_address) + _sub(interface, "model", type="virtio") + + if debug_only: + _add_port_forward(interface, "tcp", config.ssh_port, 22, "127.0.0.1") + return interface + + _add_port_forward(interface, "tcp", config.http_port, 80, config.ip_address) + _add_port_forward(interface, "tcp", config.https_port, 443, config.ip_address) + _add_port_forward(interface, "tcp", config.pki_port, 9443, config.ip_address) + _add_port_forward( + interface, + "tcp", + config.pki_vm_measure_port, + 9180, + config.ip_address, + ) + _add_port_forward(interface, "udp", config.wg_port, 51820, config.ip_address) + _add_port_forward( + interface, + "udp", + config.swarm_db_gossip_port, + 7946, + config.ip_address, + ) + _add_port_forward( + interface, + "tcp", + config.swarm_db_gossip_port, + 7946, + config.ip_address, + ) + _add_port_forward(interface, "udp", config.dns_port, 53, config.ip_address) + _add_port_forward(interface, "tcp", config.dns_port, 53, config.ip_address) + _add_port_forward(interface, "tcp", config.ssh_port, 22, "127.0.0.1") + return interface + + +def _add_network(devices: ET.Element, config: DomainConfig) -> None: + if config.netdev_mode == "user": + _add_passt_interface(devices, config) + return + + interface = _sub(devices, "interface", type="ethernet") + _sub(interface, "mac", address=config.mac_address) + _sub(interface, "target", dev=config.tap_iface, managed="no") + _sub(interface, "model", type="virtio") + if config.ssh_port is not None: + _add_passt_interface(devices, config, debug_only=True) + + +def _add_host_devices(devices: ET.Element, config: DomainConfig) -> None: + for index, host_device in enumerate(config.host_devices, start=1): + controller = _sub( + devices, + "controller", + type="pci", + index=index, + model="pcie-root-port", + ) + _sub(controller, "target", chassis=index, port=hex(0x0F + index)) + + hostdev = _sub(devices, "hostdev", mode="subsystem", type="pci", managed="no") + _sub(hostdev, "driver", name="vfio", iommufd="yes") + alias = f"ua-hostdev{index}" + source = _sub(hostdev, "source") + _sub(source, "address", **_parse_bdf(host_device.bdf)) + _sub( + hostdev, + "address", + type="pci", + domain="0x0000", + bus=hex(index), + slot="0x00", + function="0x0", + ) + _sub(hostdev, "alias", name=alias) + + +def _add_cpu_and_features(domain: ET.Element, config: DomainConfig) -> None: + features = _sub(domain, "features") + _sub(features, "acpi") + if config.mode in {"untrusted", "tdx"}: + _sub(features, "ioapic", driver="qemu") + if config.mode == "untrusted": + _sub(features, "pmu", state="off") + if config.mode == "sev-snp": + _sub(features, "vmport", state="off") + + if config.mode == "sev-snp": + cpu = _sub(domain, "cpu", mode="custom", match="exact", check="none") + _sub(cpu, "model", config.cpu_model, fallback="forbid") + _sub(cpu, "maxphysaddr", mode="emulate", bits=config.phys_bits) + else: + cpu = _sub(domain, "cpu", mode="host-passthrough", migratable="off") + if config.mode == "untrusted": + _sub(cpu, "feature", policy="disable", name="kvm-steal-time") + _sub( + cpu, + "topology", + sockets="1", + dies="1", + clusters="1", + cores=config.vcpus, + threads="1", + ) + + +def _qemu_commandline(domain: ET.Element) -> ET.Element: + ET.register_namespace("qemu", QEMU_NS) + commandline = domain.find(f"{{{QEMU_NS}}}commandline") + if commandline is None: + commandline = _sub(domain, f"{{{QEMU_NS}}}commandline") + return commandline + + +def _add_fw_cfg_qemu_args(domain: ET.Element, config: DomainConfig) -> None: + # Libvirt deliberately rejects opt/ovmf/* through native fwcfg XML because + # that namespace is reserved for OVMF. The direct launcher needs this + # existing OVMF knob, so pass it through QEMU's command line namespace. + # + # OVMF reads one knob per PCI root bridge, named X-PciMmio64Mb where N + # is the 1-based root-port index; a bare "X-PciMmio64" is silently ignored. + # Only the ports carrying passthrough devices need the enlarged 64-bit MMIO + # aperture, so emit nothing when no host device is attached. + if not config.host_devices: + return + + commandline = _qemu_commandline(domain) + for index in range(1, len(config.host_devices) + 1): + _sub(commandline, f"{{{QEMU_NS}}}arg", value="-fw_cfg") + _sub( + commandline, + f"{{{QEMU_NS}}}arg", + value=f"name=opt/ovmf/X-PciMmio64Mb{index},string=262144", + ) + + +def build_domain_xml(config: DomainConfig) -> str: + """Return a complete transient libvirt domain definition.""" + _validate_config(config) + + domain = ET.Element("domain", {"type": "kvm"}) + _sub(domain, "name", config.name) + if config.uuid: + _sub(domain, "uuid", config.uuid) + _sub(domain, "memory", config.memory_gib, unit="GiB") + _sub(domain, "currentMemory", config.memory_gib, unit="GiB") + _sub(domain, "vcpu", config.vcpus, placement="static") + if config.host_devices: + # Domain-level IOMMUFD lets libvirt open each assigned device itself + # and hand QEMU the resulting fd, keeping the cdev outside the guest's + # reach. + _sub(domain, "iommufd", enabled="yes") + + os_element = _sub(domain, "os") + _sub(os_element, "type", "hvm", arch="x86_64", machine="q35") + _sub(os_element, "loader", config.bios, readonly="yes", type="rom") + _sub(os_element, "kernel", config.kernel) + _sub(os_element, "cmdline", config.kernel_cmdline) + + _add_cpu_and_features(domain, config) + clock = _sub(domain, "clock", offset="utc") + if config.mode == "tdx": + # TD guests do not emulate the HPET; Intel's and Canonical's reference + # TD definitions both disable it explicitly. + _sub(clock, "timer", name="hpet", present="no") + _sub(domain, "on_poweroff", "destroy") + _sub(domain, "on_reboot", "restart") + _sub(domain, "on_crash", "destroy") + + devices = _sub(domain, "devices") + _sub(devices, "emulator", config.emulator) + _add_disk(devices, config.rootfs, "vda", "raw", True) + _add_disk(devices, config.state_disk, "vdb", "qcow2", False) + _add_disk(devices, config.provider_config_disk, "vdc", "raw", True) + _sub(devices, "controller", type="pci", index="0", model="pcie-root") + _sub(devices, "controller", type="usb", model="none") + _add_network(devices, config) + + # The serial port must drain into a file rather than a bare pty. A pty that + # nothing reads fills up, after which the 16550 line status register never + # reports the transmitter as empty and the guest spins forever inside + # console output -- the boot stops mid-word with vCPU0 burning host CPU on + # port 0x3fd reads. A file sink always accepts writes, so the guest keeps + # running whether or not a console client is attached, and libvirt still + # records everything from the very first byte. + serial = _sub(devices, "serial", type="file") + _sub( + serial, + "source", + path=f"/var/log/libvirt/qemu/{config.name}-serial.log", + append="off", + ) + _sub(serial, "target", type="isa-serial", port="0") + console = _sub(devices, "console", type="file") + _sub( + console, + "source", + path=f"/var/log/libvirt/qemu/{config.name}-serial.log", + append="off", + ) + _sub(console, "target", type="serial", port="0") + video = _sub(devices, "video") + _sub(video, "model", type="none") + _sub(devices, "audio", id="1", type="none") + _sub(devices, "memballoon", model="none") + vsock = _sub(devices, "vsock", model="virtio") + _sub(vsock, "cid", auto="no", address=config.guest_cid) + _add_host_devices(devices, config) + + if config.mode == "sev-snp": + launch_security = _sub( + domain, + "launchSecurity", + type="sev-snp", + kernelHashes="yes", + ) + _sub(launch_security, "cbitpos", config.cbitpos) + _sub(launch_security, "reducedPhysBits", "1") + _sub(launch_security, "policy", "0x30000") + elif config.mode == "tdx": + launch_security = _sub(domain, "launchSecurity", type="tdx") + _sub( + launch_security, + "quoteGenerationService", + path=config.qgs_socket, + ) + + # QEMU namespace extensions must follow native domain elements for the + # libvirt domain schema to accept the document. + _add_fw_cfg_qemu_args(domain, config) + + ET.indent(domain, space=" ") + return ET.tostring(domain, encoding="unicode") + + +def _version_string(version: int) -> str: + return f"{version // 1_000_000}.{(version // 1_000) % 1_000}.{version % 1_000}" + + +def _iommufd_advertised(domain_capabilities: str) -> bool: + try: + root = ET.fromstring(domain_capabilities) + except ET.ParseError: + return False + for enum in root.findall(".//devices/hostdev/enum[@name='iommufd']"): + if any((value.text or "").strip() == "yes" for value in enum.findall("value")): + return True + return False + + +def _tdx_launch_security_advertised(domain_capabilities: str) -> bool: + try: + root = ET.fromstring(domain_capabilities) + except ET.ParseError: + return False + for enum in root.findall(".//features/launchSecurity/enum[@name='sectype']"): + if any((value.text or "").strip() == "tdx" for value in enum.findall("value")): + return True + return False + + +def check_connection_capabilities(conn: Any, config: Any) -> None: + if conn.getType().upper() != "QEMU": + raise RuntimeError(f"qemu:///system returned unexpected driver {conn.getType()!r}") + mode = getattr(config, "mode", None) + if not config.host_devices and mode != "tdx": + return + + if config.host_devices: + version = conn.getLibVersion() + if version < LIBVIRT_IOMMUFD_VERSION: + raise RuntimeError( + "GPU passthrough requires libvirt >= 12.1.0; " + f"the daemon reports {_version_string(version)}" + ) + try: + capabilities = conn.getDomainCapabilities( + config.emulator, + "x86_64", + "q35", + "kvm", + 0, + ) + except Exception as exc: + raise RuntimeError(f"failed to query libvirt domain capabilities: {exc}") from exc + if config.host_devices and not _iommufd_advertised(capabilities): + raise RuntimeError( + "libvirt domain capabilities do not advertise hostdev iommufd support" + ) + if mode == "tdx" and not _tdx_launch_security_advertised(capabilities): + raise RuntimeError( + "libvirt domain capabilities do not advertise native TDX launch security" + ) + + +def ensure_domain_name_available(conn: Any, libvirt_module: Any, name: str) -> None: + try: + domains = conn.listAllDomains(0) + except libvirt_module.libvirtError as exc: + raise RuntimeError(f"failed to check domain name {name!r}: {exc}") from exc + if any(domain.name() == name for domain in domains): + raise RuntimeError( + f"a libvirt domain named {name!r} already exists; stop it or choose --name" + ) + + +def _format_launch_error(exc: BaseException) -> str: + message = str(exc) + if "passt" in message and "fatal signal 11" in message: + return ( + f"libvirt failed to start the domain: {message}\n" + "passt was killed by SIGSEGV. On Ubuntu this commonly means that " + "AppArmor denied passt or its libvirt socket. Check the kernel audit " + "log with: journalctl -k --since '-5 min' --no-pager | " + "grep -E 'apparmor=\"DENIED\".*(passt|libvirt)'" + ) + return f"libvirt failed to start the domain: {message}" + + +def preflight_connection( + emulator: str, name: str, mode: str, require_iommufd: bool +) -> None: + """Check the daemon, domain name, and optional IOMMUFD support without mutation.""" + try: + import libvirt # type: ignore + except ImportError as exc: + raise RuntimeError( + "python3-libvirt is not installed; install the Ubuntu package " + "'python3-libvirt'" + ) from exc + + try: + conn = libvirt.open("qemu:///system") + except libvirt.libvirtError as exc: + raise RuntimeError(f"failed to connect to qemu:///system: {exc}") from exc + if conn is None: + raise RuntimeError("failed to connect to qemu:///system") + try: + probe = SimpleNamespace( + emulator=str(Path(emulator).resolve()), + mode=mode, + host_devices=[object()] if require_iommufd else [], + ) + check_connection_capabilities(conn, probe) + ensure_domain_name_available(conn, libvirt, name) + finally: + conn.close() + + +def _write_console_output(data: bytes, log: BinaryIO) -> None: + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() + log.write(data) + log.flush() + + +def attach_serial_console(conn: Any, domain: Any, libvirt_module: Any, log_path: str) -> None: + """Attach a bidirectional console; Ctrl-C or Ctrl-] only detaches.""" + stream = conn.newStream(getattr(libvirt_module, "VIR_STREAM_NONBLOCK", 1)) + stdin_fd: Optional[int] = None + old_terminal = None + active = False + try: + domain.openConsole(None, stream, 0) + with open(log_path, "ab", buffering=0) as log_file: + stdin_fd = sys.stdin.fileno() + if os.isatty(stdin_fd): + old_terminal = termios.tcgetattr(stdin_fd) + tty.setraw(stdin_fd) + + print( + "\nConnected to serial console. Press Ctrl-C or Ctrl-] to detach; " + "the VM will keep running.\r", + file=sys.stderr, + ) + pending = bytearray() + console_open = True + while console_open: + while True: + chunk = stream.recv(65536) + if chunk == -2: + break + if not chunk: + console_open = False + break + _write_console_output(chunk, log_file) + if not console_open: + break + + if pending: + sent = stream.send(bytes(pending)) + if sent == -2: + sent = 0 + elif sent <= 0: + raise RuntimeError("libvirt console send made no progress") + del pending[:sent] + + readable, _, _ = select.select( + [stdin_fd] if not pending else [], [], [], 0.05 + ) + if not readable: + continue + data = os.read(stdin_fd, 4096) + if not data or b"\x03" in data or b"\x1d" in data: + break + pending.extend(data) + except KeyboardInterrupt: + pass + finally: + try: + if old_terminal is not None and stdin_fd is not None: + termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_terminal) + finally: + try: + stream.abort() + except libvirt_module.libvirtError: + pass + try: + active = bool(domain.isActive()) + except libvirt_module.libvirtError: + active = False + if active: + message = f"Detached from {domain.name()}; VM is still managed by libvirt." + else: + message = "Serial console closed because the VM stopped." + print(f"\n{message}", file=sys.stderr) + + +def follow_serial_log( + domain: Any, libvirt_module: Any, serial_log: str, log_path: str +) -> None: + """Mirror the domain serial log until the VM stops or the user detaches. + + The domain writes its console to a file, so the guest never blocks on a + console nobody reads. Debug mode simply tails that file; Ctrl-C detaches + and leaves the VM running. + """ + print( + "\nFollowing serial console; press Ctrl-C to detach " + "(the VM keeps running).", + file=sys.stderr, + ) + deadline = time.monotonic() + 30.0 + while not Path(serial_log).exists(): + if time.monotonic() > deadline: + print( + f"Serial log did not appear: {serial_log}", + file=sys.stderr, + ) + return + time.sleep(0.2) + + try: + with open(serial_log, "rb") as source, open( + log_path, "ab", buffering=0 + ) as log_file: + while True: + chunk = source.read(65536) + if chunk: + _write_console_output(chunk, log_file) + continue + try: + if not domain.isActive(): + break + except libvirt_module.libvirtError: + break + time.sleep(0.2) + except KeyboardInterrupt: + print( + f"\nDetached from {domain.name()}; VM is still managed by libvirt.", + file=sys.stderr, + ) + return + print("\nSerial console closed because the VM stopped.", file=sys.stderr) + + +def launch(config: DomainConfig) -> None: + try: + import libvirt # type: ignore + except ImportError as exc: + raise RuntimeError( + "python3-libvirt is not installed; install the Ubuntu package " + "'python3-libvirt'" + ) from exc + + try: + conn = libvirt.open("qemu:///system") + except libvirt.libvirtError as exc: + raise RuntimeError(f"failed to connect to qemu:///system: {exc}") from exc + if conn is None: + raise RuntimeError("failed to connect to qemu:///system") + try: + try: + check_connection_capabilities(conn, config) + ensure_domain_name_available(conn, libvirt, config.name) + xml = build_domain_xml(config) + flags = getattr(libvirt, "VIR_DOMAIN_START_VALIDATE", 0) + domain = conn.createXML(xml, flags) + if domain is None: + raise RuntimeError("libvirt did not return a domain after createXML()") + name = domain.name() + uuid = domain.UUIDString() + print(f"Started transient libvirt domain: {name} ({uuid})") + # The console is a write-only file sink, so "virsh console" has + # nothing to attach to; point at the log libvirt actually writes. + print(f" serial log: /var/log/libvirt/qemu/{name}-serial.log") + print(f" shutdown: virsh -c qemu:///system shutdown {name}") + print(f" force stop: virsh -c qemu:///system destroy {name}") + if config.debug: + follow_serial_log( + domain, + libvirt, + f"/var/log/libvirt/qemu/{name}-serial.log", + str(config.log_file), + ) + except libvirt.libvirtError as exc: + raise RuntimeError(_format_launch_error(exc)) from exc + finally: + conn.close() + + +def _host_device(value: str) -> HostDevice: + try: + kind, bdf = value.split(":", 1) + except ValueError as exc: + raise argparse.ArgumentTypeError("hostdev must be KIND:BDF") from exc + try: + _parse_bdf(bdf) + except ValueError as exc: + raise argparse.ArgumentTypeError(str(exc)) from exc + if kind not in {"gpu", "aux"}: + raise argparse.ArgumentTypeError("hostdev kind must be 'gpu' or 'aux'") + return HostDevice(kind, bdf) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--name", required=True) + parser.add_argument("--uuid") + parser.add_argument("--mode", choices=("untrusted", "tdx", "sev-snp"), required=True) + parser.add_argument("--emulator", required=True) + parser.add_argument("--memory-gib", type=int, required=True) + parser.add_argument("--vcpus", type=int, required=True) + parser.add_argument("--bios", required=True) + parser.add_argument("--kernel", required=True) + parser.add_argument("--kernel-cmdline", required=True) + parser.add_argument("--rootfs", required=True) + parser.add_argument("--state-disk", required=True) + parser.add_argument("--provider-config-disk", required=True) + parser.add_argument("--guest-cid", type=int, required=True) + parser.add_argument("--qgs-socket", required=True) + parser.add_argument("--mac-address", required=True) + parser.add_argument("--netdev-mode", choices=("user", "tap"), required=True) + parser.add_argument("--debug", action="store_true") + parser.add_argument("--log-file") + parser.add_argument("--cpu-model") + parser.add_argument("--phys-bits", type=int) + parser.add_argument("--cbitpos", type=int) + parser.add_argument("--bridge") + parser.add_argument("--tap-iface") + parser.add_argument("--ip-address", default="0.0.0.0") + parser.add_argument("--ssh-port", type=int) + parser.add_argument("--wg-port", type=int) + parser.add_argument("--http-port", type=int) + parser.add_argument("--https-port", type=int) + parser.add_argument("--pki-port", type=int) + parser.add_argument("--pki-vm-measure-port", type=int) + parser.add_argument("--swarm-db-gossip-port", type=int) + parser.add_argument("--dns-port", type=int) + parser.add_argument("--hostdev", type=_host_device, action="append", default=[]) + return parser + + +def _preflight_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Check libvirt before preparing VM resources") + parser.add_argument("--emulator", required=True) + parser.add_argument("--name", required=True) + parser.add_argument("--mode", choices=("untrusted", "tdx", "sev-snp"), required=True) + parser.add_argument("--require-iommufd", action="store_true") + return parser + + +def _config_from_args(args: argparse.Namespace) -> DomainConfig: + return DomainConfig( + name=args.name, + uuid=args.uuid, + mode=args.mode, + emulator=str(Path(args.emulator).resolve()), + memory_gib=args.memory_gib, + vcpus=args.vcpus, + bios=str(Path(args.bios).resolve()), + kernel=str(Path(args.kernel).resolve()), + kernel_cmdline=args.kernel_cmdline, + rootfs=str(Path(args.rootfs).resolve()), + state_disk=str(Path(args.state_disk).resolve()), + provider_config_disk=str(Path(args.provider_config_disk).resolve()), + guest_cid=args.guest_cid, + qgs_socket=str(Path(args.qgs_socket).resolve()), + mac_address=args.mac_address, + netdev_mode=args.netdev_mode, + debug=args.debug, + log_file=args.log_file, + cpu_model=args.cpu_model, + phys_bits=args.phys_bits, + cbitpos=args.cbitpos, + bridge=args.bridge, + tap_iface=args.tap_iface, + ip_address=args.ip_address, + ssh_port=args.ssh_port, + wg_port=args.wg_port, + http_port=args.http_port, + https_port=args.https_port, + pki_port=args.pki_port, + pki_vm_measure_port=args.pki_vm_measure_port, + swarm_db_gossip_port=args.swarm_db_gossip_port, + dns_port=args.dns_port, + host_devices=args.hostdev, + ) + + +def main(argv: Optional[Iterable[str]] = None) -> int: + arguments = list(argv) if argv is not None else sys.argv[1:] + if arguments[:1] == ["preflight"]: + args = _preflight_parser().parse_args(arguments[1:]) + try: + if not DOMAIN_NAME_RE.fullmatch(args.name): + raise ValueError( + "domain name may contain only letters, digits, '.', '_', '+', ':', and '-'" + ) + preflight_connection( + args.emulator, args.name, args.mode, args.require_iommufd + ) + except (RuntimeError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + args = _parser().parse_args(arguments) + try: + config = _config_from_args(args) + _validate_config(config) + launch(config) + except (RuntimeError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh new file mode 100755 index 0000000..05d303b --- /dev/null +++ b/scripts/setup_libvirt_host.sh @@ -0,0 +1,866 @@ +#!/bin/bash + +# Shared libvirt host preparation for bootstrap_tdx.sh and bootstrap_snp.sh. +# This file is sourceable for unit tests and can also be executed directly. + +LIBVIRT_REQUIRED_VERSION="12.5.0" +LIBVIRT_RELEASE_REPO="Super-Protocol/sp-vm-tools" +LIBVIRT_URI="qemu:///system" +PASST_UNPRIVILEGED_PORT_START="0" +PASST_APPARMOR_DISCONNECTED_PATH="/att/passt/" + +LIBVIRT_BASE_PACKAGES=( + libvirt0 + libvirt-common + libvirt-clients + libvirt-daemon + libvirt-daemon-common + libvirt-daemon-config-network + libvirt-daemon-config-nwfilter + libvirt-daemon-driver-network + libvirt-daemon-driver-nodedev + libvirt-daemon-driver-nwfilter + libvirt-daemon-driver-qemu + libvirt-daemon-driver-secret + libvirt-daemon-driver-storage + libvirt-daemon-log + libvirt-daemon-lock + libvirt-daemon-plugin-lockd + libvirt-daemon-system + libvirt-daemon-system-systemd + libvirt-dev +) + +LIBVIRT_PACKAGE_PATHS=() +PASST_REAL_BINARIES=() +LIBVIRT_QEMU_CONFIG_CHANGED=0 + +libvirt_host_error() { + echo "ERROR: $*" >&2 + return 1 +} + +select_libvirt_release() { + local ubuntu_version=$1 + case "${ubuntu_version}" in + 24.04) + LIBVIRT_RELEASE_TAG="44-libvirt-ubuntu24" + LIBVIRT_RELEASE_ASSET="libvirt-ubuntu24.tar.gz" + LIBVIRT_RELEASE_SHA256="17a1aa837260e3e584b2ebcdd066ff541d2d89d1b798dc71c626976ff70ae452" + ;; + 26.04) + LIBVIRT_RELEASE_TAG="43-libvirt-ubuntu26" + LIBVIRT_RELEASE_ASSET="libvirt-ubuntu26.tar.gz" + LIBVIRT_RELEASE_SHA256="fd1ba716a9ee722c5fc0d3db72b92ccba3c57954c2464759ff4dc3f70a8bf6ad" + ;; + *) + libvirt_host_error "libvirt bootstrap supports Ubuntu 24.04 and 26.04 only (found ${ubuntu_version:-unknown})" + return 1 + ;; + esac + LIBVIRT_RELEASE_URL="https://github.com/${LIBVIRT_RELEASE_REPO}/releases/download/${LIBVIRT_RELEASE_TAG}/${LIBVIRT_RELEASE_ASSET}" +} + +get_supported_ubuntu_version() { + local os_release=/etc/os-release + if [[ ! -r "${os_release}" ]]; then + libvirt_host_error "cannot read ${os_release}" + return 1 + fi + + local ID="" VERSION_ID="" + # shellcheck disable=SC1090 + source "${os_release}" + if [[ "${ID}" != "ubuntu" ]]; then + libvirt_host_error "libvirt bootstrap requires Ubuntu (found ${ID:-unknown})" + return 1 + fi + select_libvirt_release "${VERSION_ID}" + # Exported result for bootstrap callers. + # shellcheck disable=SC2034 + UBUNTU_VERSION="${VERSION_ID}" +} + +installed_libvirt_version() { + dpkg-query -W -f='${Version}\n' libvirt-daemon 2>/dev/null || true +} + +running_libvirt_version() { + local numeric + numeric=$(python3 -c ' +import libvirt +conn = libvirt.openReadOnly("qemu:///system") +try: + print(conn.getLibVersion()) +finally: + conn.close() +' 2>/dev/null || true) + if [[ ! "${numeric}" =~ ^[0-9]+$ ]]; then + return 1 + fi + printf '%d.%d.%d\n' \ + "$((numeric / 1000000))" \ + "$(((numeric / 1000) % 1000))" \ + "$((numeric % 1000))" +} + +libvirt_upgrade_required() { + local installed_version=${1:-} + [[ -z "${installed_version}" ]] || \ + ! dpkg --compare-versions "${installed_version}" ge "${LIBVIRT_REQUIRED_VERSION}" +} + +missing_required_libvirt_packages() { + local package + for package in "${LIBVIRT_BASE_PACKAGES[@]}"; do + if ! dpkg-query -W -f='${db:Status-Status}' "${package}" 2>/dev/null | grep -qx installed; then + printf '%s\n' "${package}" + fi + done +} + +validate_tar_listing() { + local entry trimmed component + local -a components + while IFS= read -r entry; do + trimmed=${entry#./} + if [[ "${entry}" == /* || -z "${trimmed}" ]]; then + libvirt_host_error "unsafe archive entry: ${entry}" + return 1 + fi + IFS='/' read -r -a components <<< "${trimmed}" + for component in "${components[@]}"; do + if [[ "${component}" == ".." ]]; then + libvirt_host_error "unsafe archive entry: ${entry}" + return 1 + fi + done + done +} + +verify_and_extract_libvirt_archive() { + local archive=$1 destination=$2 actual_sha package_dir sums_file + actual_sha=$(sha256sum "${archive}" | awk '{print $1}') + if [[ "${actual_sha}" != "${LIBVIRT_RELEASE_SHA256}" ]]; then + libvirt_host_error "checksum mismatch for ${archive}: expected ${LIBVIRT_RELEASE_SHA256}, got ${actual_sha}" + return 1 + fi + if ! tar -tzf "${archive}" | validate_tar_listing; then + return 1 + fi + tar -xzf "${archive}" -C "${destination}" + sums_file=$(find "${destination}" -mindepth 2 -maxdepth 3 -type f -name SHA256SUMS -print -quit) + if [[ -z "${sums_file}" ]]; then + libvirt_host_error "archive does not contain SHA256SUMS" + return 1 + fi + package_dir=$(dirname "${sums_file}") + if ! (cd "${package_dir}" && sha256sum -c SHA256SUMS); then + libvirt_host_error "one or more files in the libvirt archive failed checksum validation" + return 1 + fi + LIBVIRT_PACKAGE_DIR="${package_dir}" +} + +prepare_libvirt_package_compatibility() { + local package_dir=$1 ubuntu_version=$2 deb unpacked sysusers_file rebuilt package version + [[ "${ubuntu_version}" == "24.04" ]] || return 0 + deb=$(find_package_deb "${package_dir}" libvirt-daemon-driver-qemu) || { + libvirt_host_error "release archive is missing libvirt-daemon-driver-qemu" + return 1 + } + unpacked=$(mktemp -d) + dpkg-deb --raw-extract "${deb}" "${unpacked}" + sysusers_file="${unpacked}/usr/lib/sysusers.d/libvirt-qemu.conf" + if [[ ! -r "${sysusers_file}" ]]; then + rm -rf "${unpacked}" + libvirt_host_error "libvirt QEMU package does not contain its sysusers configuration" + return 1 + fi + if ! grep -qE '^u![[:space:]]' "${sysusers_file}"; then + rm -rf "${unpacked}" + return 0 + fi + + echo "Adapting libvirt-qemu sysusers syntax for Ubuntu 24.04 systemd 255" + sed -i -E 's/^u!([[:space:]])/u\1/' "${sysusers_file}" + if [[ -f "${unpacked}/DEBIAN/md5sums" ]]; then + local updated_md5 + updated_md5=$(cd "${unpacked}" && md5sum usr/lib/sysusers.d/libvirt-qemu.conf) + sed -i '\| usr/lib/sysusers.d/libvirt-qemu.conf$|d' "${unpacked}/DEBIAN/md5sums" + printf '%s\n' "${updated_md5}" >> "${unpacked}/DEBIAN/md5sums" + fi + rebuilt="${deb}.spvm-rebuilt" + dpkg-deb --build --root-owner-group "${unpacked}" "${rebuilt}" >/dev/null + package=$(dpkg-deb -f "${rebuilt}" Package) + version=$(dpkg-deb -f "${rebuilt}" Version) + if [[ "${package}" != "libvirt-daemon-driver-qemu" || "${version}" != 12.5.0-* ]]; then + rm -rf "${unpacked}" "${rebuilt}" + libvirt_host_error "rebuilt compatibility package has unexpected metadata" + return 1 + fi + mv "${rebuilt}" "${deb}" + rm -rf "${unpacked}" +} + +find_package_deb() { + local package_dir=$1 package=$2 candidate actual_package + while IFS= read -r -d '' candidate; do + actual_package=$(dpkg-deb -f "${candidate}" Package 2>/dev/null || true) + if [[ "${actual_package}" == "${package}" ]]; then + printf '%s\n' "${candidate}" + return 0 + fi + done < <(find "${package_dir}" -maxdepth 1 -type f -name '*.deb' -print0) + return 1 +} + +build_libvirt_package_plan() { + local package_dir=$1 package deb + local -A selected=() + LIBVIRT_PACKAGE_PATHS=() + + for package in "${LIBVIRT_BASE_PACKAGES[@]}"; do + if ! deb=$(find_package_deb "${package_dir}" "${package}"); then + libvirt_host_error "release archive is missing required package ${package}" + return 1 + fi + selected["${package}"]="${deb}" + done + + while IFS= read -r -d '' deb; do + package=$(dpkg-deb -f "${deb}" Package 2>/dev/null || true) + [[ -n "${package}" ]] || continue + if dpkg-query -W -f='${db:Status-Status}' "${package}" 2>/dev/null | grep -qx installed; then + selected["${package}"]="${deb}" + fi + done < <(find "${package_dir}" -maxdepth 1 -type f -name '*.deb' -print0) + + while IFS= read -r package; do + LIBVIRT_PACKAGE_PATHS+=("${selected[${package}]}") + done < <(printf '%s\n' "${!selected[@]}" | sort) +} + +assert_no_running_libvirt_domains() { + command -v virsh >/dev/null 2>&1 || return 0 + local running + running=$(virsh -c "${LIBVIRT_URI}" list --name 2>/dev/null | sed '/^[[:space:]]*$/d' || true) + if [[ -n "${running}" ]]; then + libvirt_host_error "refusing to upgrade libvirt while domains are running: ${running//$'\n'/, }" + return 1 + fi +} + +assert_safe_apt_simulation() { + local simulation=$1 + if grep -qE '^Remv[[:space:]]' <<< "${simulation}"; then + libvirt_host_error "APT would remove packages; refusing the libvirt transaction" + return 1 + fi + if grep -qiE 'DOWNGRADED|downgraded' <<< "${simulation}"; then + libvirt_host_error "APT would downgrade packages; refusing the libvirt transaction" + return 1 + fi +} + +install_project_libvirt() { + local work_dir archive simulation + assert_no_running_libvirt_domains || return 1 + work_dir=$(mktemp -d /var/tmp/sp-vm-libvirt.XXXXXX) || return 1 + chmod 0755 "${work_dir}" || return 1 + archive="${work_dir}/${LIBVIRT_RELEASE_ASSET}" + + echo "Downloading libvirt ${LIBVIRT_REQUIRED_VERSION} from ${LIBVIRT_RELEASE_URL}" + wget --https-only --tries=3 -O "${archive}" "${LIBVIRT_RELEASE_URL}" || return 1 + chmod 0644 "${archive}" || return 1 + verify_and_extract_libvirt_archive "${archive}" "${work_dir}" || return 1 + prepare_libvirt_package_compatibility "${LIBVIRT_PACKAGE_DIR}" "${UBUNTU_VERSION}" || return 1 + find "${work_dir}" -type d -exec chmod a+rx {} + || return 1 + find "${work_dir}" -type f \( -name '*.deb' -o -name '*.ddeb' \) -exec chmod a+r {} + || return 1 + build_libvirt_package_plan "${LIBVIRT_PACKAGE_DIR}" || return 1 + + echo "APT simulation for the libvirt upgrade:" + simulation=$(LC_ALL=C apt-get --simulate --no-install-recommends --no-remove install "${LIBVIRT_PACKAGE_PATHS[@]}") || return 1 + printf '%s\n' "${simulation}" + assert_safe_apt_simulation "${simulation}" || return 1 + + DEBIAN_FRONTEND=noninteractive apt-get \ + --no-install-recommends \ + --no-remove \ + -o Dpkg::Options::=--force-confold \ + install -y "${LIBVIRT_PACKAGE_PATHS[@]}" || return 1 + rm -rf "${work_dir}" || return 1 +} + +passthrough_profile_state() { + local profile=$1 + awk ' + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { + in_passt = 1 + found = 1 + next + } + in_passt && /^[[:space:]]*}/ { in_passt = 0; done = 1 } + in_passt && /\/usr\/bin\/passt[[:space:]]+r,/ { readonly = 1 } + in_passt && /\/usr\/bin\/passt[[:space:]]+rm,/ { mmap = 1 } + in_passt && /^[[:space:]]*capability[[:space:]]+net_bind_service,/ { capability = 1 } + END { + if (!found || !done) print "unknown" + else if (readonly) print "readonly" + else if (mmap && capability) print "ready" + else if (mmap) print "needs-capability" + else print "unknown" + } + ' "${profile}" +} + +passthrough_profile_handles_disconnected_sockets() { + local profile=$1 + awk ' + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { + found = 1 + if ($0 ~ /attach_disconnected/) handled = 1 + } + END { exit !(found && handled) } + ' "${profile}" +} + +patch_passthrough_disconnected_socket_handling() { + local profile=$1 tmp + tmp=$(mktemp) + if ! awk -v path="${PASST_APPARMOR_DISCONNECTED_PATH}" ' + BEGIN { patched = 0 } + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { + if ($0 ~ /attach_disconnected/) { + patched = 1 + } else if ($0 ~ /flags=\(/) { + sub(/flags=\(/, "flags=(attach_disconnected.path=" path " ") + patched = 1 + } else { + sub(/\{[[:space:]]*$/, "flags=(attach_disconnected.path=" path ") {") + patched = 1 + } + } + { print } + END { if (!patched) exit 1 } + ' "${profile}" > "${tmp}"; then + rm -f "${tmp}" + libvirt_host_error "failed to add disconnected socket handling to the nested passt profile" + return 1 + fi + cat "${tmp}" > "${profile}" + rm -f "${tmp}" +} + +patch_libvirt_apparmor_profile() { + local profile=$1 state backup tmp + state=$(passthrough_profile_state "${profile}") + if [[ "${state}" == "unknown" ]]; then + libvirt_host_error "unrecognized nested passt profile in ${profile}; refusing to modify it" + return 1 + fi + backup="${profile}.sp-vm-tools.bak" + if [[ ! -e "${backup}" ]]; then + cp -a "${profile}" "${backup}" + fi + + if [[ "${state}" == "readonly" ]]; then + sed -i \ + '/^[[:space:]]*profile passt[[:space:]]*{/,/^[[:space:]]*}/ s|/usr/bin/passt[[:space:]]\+r,|/usr/bin/passt rm,|' \ + "${profile}" + fi + state=$(passthrough_profile_state "${profile}") + if [[ "${state}" == "needs-capability" ]]; then + tmp=$(mktemp) + awk ' + /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1 } + { print } + in_passt && /\/usr\/bin\/passt[[:space:]]+rm,/ { + print " capability net_bind_service," + } + in_passt && /^[[:space:]]*}/ { in_passt = 0 } + ' "${profile}" > "${tmp}" + cat "${tmp}" > "${profile}" + rm -f "${tmp}" + fi + + # TODO: Remove this workaround after Ubuntu's passt/libvirt AppArmor + # policy handles the listening Unix socket that becomes disconnected when + # passt pivots into its empty sandbox root. AppArmor 5 on Ubuntu 26.04 + # otherwise rejects accept4() with EACCES. A synthetic attachment prefix + # is scoped to the nested passt profile and avoids disabling confinement. + if [[ "${UBUNTU_VERSION:-}" == "26.04" ]] && \ + ! passthrough_profile_handles_disconnected_sockets "${profile}"; then + patch_passthrough_disconnected_socket_handling "${profile}" || return 1 + fi + if [[ "$(passthrough_profile_state "${profile}")" != "ready" ]]; then + libvirt_host_error "failed to make the nested passt AppArmor profile usable" + return 1 + fi + if [[ "${UBUNTU_VERSION:-}" == "26.04" ]] && \ + ! passthrough_profile_handles_disconnected_sockets "${profile}"; then + libvirt_host_error "nested passt AppArmor profile does not handle disconnected Unix sockets" + return 1 + fi +} + +configure_libvirt_apparmor() { + local profile dropin_dir dropin template daemon_profile daemon_local tmp + profile=/etc/apparmor.d/abstractions/libvirt-qemu + dropin_dir=/etc/apparmor.d/abstractions/libvirt-qemu.d + dropin="${dropin_dir}/99-sp-vm-tools-local" + template=/etc/apparmor.d/libvirt/TEMPLATE.qemu + daemon_profile=/etc/apparmor.d/usr.sbin.libvirtd + daemon_local=/etc/apparmor.d/local/usr.sbin.libvirtd + + [[ -r "${profile}" ]] || { + libvirt_host_error "libvirt AppArmor profile is missing: ${profile}" + return 1 + } + patch_libvirt_apparmor_profile "${profile}" || return 1 + install -d -m 0755 "${dropin_dir}" + tmp=$(mktemp) + printf '%s\n' \ + '# Managed by sp-vm-tools bootstrap.' \ + '/usr/local/bin/qemu-system-x86_64 rmix,' \ + '/usr/local/share/qemu/** rk,' \ + '/usr/local/lib{,64}/qemu/*.so mr,' \ + '/usr/local/lib/@{multiarch}/qemu/*.so mr,' \ + 'owner @{run}/libvirt/qemu/passt/* rw,' \ + '@{run}/tdx-qgs/qgs.socket rw,' \ + 'network vsock stream,' > "${tmp}" + install -m 0644 "${tmp}" "${dropin}" + rm -f "${tmp}" + + if [[ -r "${daemon_profile}" ]]; then + if ! grep -qF 'include if exists ' "${daemon_profile}"; then + libvirt_host_error "${daemon_profile} does not include its standard local override; refusing to modify AppArmor" + return 1 + fi + install -d -m 0755 "$(dirname "${daemon_local}")" + touch "${daemon_local}" + chmod 0644 "${daemon_local}" + if ! grep -qF '/usr/local/bin/qemu-system-x86_64 PUx,' "${daemon_local}"; then + printf '%s\n' \ + '# Managed by sp-vm-tools: allow libvirtd capabilities probing.' \ + '/usr/local/bin/qemu-system-x86_64 PUx,' >> "${daemon_local}" + fi + apparmor_parser -Q -r "${daemon_profile}" + fi + + [[ -r "${template}" ]] || { + libvirt_host_error "libvirt AppArmor template is missing: ${template}" + return 1 + } + apparmor_parser -Q -r "${template}" + systemctl reload apparmor +} + +configure_tdx_qgs_access() { + getent group qgsd >/dev/null || { + libvirt_host_error "QGS group qgsd is missing" + return 1 + } + usermod -a -G qgsd libvirt-qemu || return 1 +} + +configure_libvirt_qemu_runtime() { + local config=/etc/libvirt/qemu.conf backup tmp + backup="${config}.sp-vm-tools.bak" + [[ -r "${config}" ]] || { + libvirt_host_error "libvirt QEMU configuration is missing: ${config}" + return 1 + } + tmp=$(mktemp) + awk ' + BEGIN { + user_written = 0 + group_written = 0 + ownership_written = 0 + } + /^[[:space:]]*user[[:space:]]*=/ { + if (!user_written) { + print "user = \"libvirt-qemu\"" + user_written = 1 + } + next + } + /^[[:space:]]*group[[:space:]]*=/ { + if (!group_written) { + print "group = \"libvirt-qemu\"" + group_written = 1 + } + next + } + /^[[:space:]]*dynamic_ownership[[:space:]]*=/ { + if (!ownership_written) { + print "dynamic_ownership = 1" + ownership_written = 1 + } + next + } + { print } + END { + if (!user_written) + print "user = \"libvirt-qemu\"" + if (!group_written) + print "group = \"libvirt-qemu\"" + if (!ownership_written) + print "dynamic_ownership = 1" + } + ' "${config}" > "${tmp}" + + if cmp -s "${tmp}" "${config}"; then + rm -f "${tmp}" + echo "Libvirt QEMU runtime already uses libvirt-qemu." + return + fi + + if ! assert_no_running_libvirt_domains; then + rm -f "${tmp}" + return 1 + fi + if [[ ! -e "${backup}" ]]; then + cp -a "${config}" "${backup}" + fi + cat "${tmp}" > "${config}" + chmod 0600 "${config}" + rm -f "${tmp}" + LIBVIRT_QEMU_CONFIG_CHANGED=1 + echo "Configured libvirt QEMU runtime user/group as libvirt-qemu with dynamic ownership." +} + +collect_passt_binaries() { + local candidate real + local -A seen=() + PASST_REAL_BINARIES=() + if [[ $# -eq 0 ]]; then + for candidate in passt passt.avx2; do + real=$(command -v "${candidate}" 2>/dev/null || true) + [[ -n "${real}" ]] || continue + set -- "$@" "${real}" + done + fi + for candidate in "$@"; do + [[ -x "${candidate}" ]] || continue + real=$(readlink -f -- "${candidate}") + [[ -n "${real}" && -z "${seen[${real}]:-}" ]] || continue + seen["${real}"]=1 + PASST_REAL_BINARIES+=("${real}") + done + if [[ ${#PASST_REAL_BINARIES[@]} -eq 0 ]]; then + libvirt_host_error "no executable passt binary was found" + return 1 + fi +} + +verify_passt_capabilities() { + local binary capabilities + collect_passt_binaries "$@" + for binary in "${PASST_REAL_BINARIES[@]}"; do + capabilities=$(getcap "${binary}" 2>/dev/null || true) + if [[ "${capabilities}" != *cap_net_bind_service* ]]; then + libvirt_host_error "${binary} does not have CAP_NET_BIND_SERVICE" + return 1 + fi + done +} + +# Optional arguments are used by unit tests to exercise symlink handling. +# shellcheck disable=SC2120 +configure_passt_capabilities() { + local binary + collect_passt_binaries "$@" + for binary in "${PASST_REAL_BINARIES[@]}"; do + setcap cap_net_bind_service=ep "${binary}" + if ! verify_passt_capabilities "${binary}"; then + libvirt_host_error "failed to set CAP_NET_BIND_SERVICE on ${binary}; check filesystem xattr support" + return 1 + fi + echo "Configured CAP_NET_BIND_SERVICE on ${binary}" + done +} + +verify_passt_unprivileged_ports() { + local value + value=$(sysctl -n net.ipv4.ip_unprivileged_port_start 2>/dev/null || true) + if [[ "${value}" != "${PASST_UNPRIVILEGED_PORT_START}" ]]; then + libvirt_host_error \ + "net.ipv4.ip_unprivileged_port_start must be ${PASST_UNPRIVILEGED_PORT_START} for passt privileged-port forwarding (found ${value:-unavailable})" + return 1 + fi +} + +configure_passt_unprivileged_ports() { + local sysctl_dir=/etc/sysctl.d + local config="${sysctl_dir}/99-sp-vm-tools-passt.conf" tmp + + install -d -m 0755 "${sysctl_dir}" + tmp=$(mktemp) + printf '%s\n' \ + '# Managed by sp-vm-tools bootstrap.' \ + '# TODO: UNSAFE CONFIGURATION. Temporary workaround for a passt regression.' \ + '# Remove it when passt can bind forwarded low ports before entering its' \ + '# unprivileged user namespace. New passt versions create host listeners' \ + '# after user-namespace isolation,' \ + '# so CAP_NET_BIND_SERVICE on the passt binary no longer authorizes bind()' \ + '# in the host network namespace. This is the only stock, unpatched setup' \ + '# currently found to keep libvirt low-port forwarding working. Setting' \ + '# this value to 0 lets every unprivileged process on the host bind any' \ + '# free TCP or UDP port.' \ + "net.ipv4.ip_unprivileged_port_start = ${PASST_UNPRIVILEGED_PORT_START}" > "${tmp}" + install -m 0644 "${tmp}" "${config}" + rm -f "${tmp}" + + sysctl -w \ + "net.ipv4.ip_unprivileged_port_start=${PASST_UNPRIVILEGED_PORT_START}" >/dev/null + verify_passt_unprivileged_ports || return 1 + echo "Configured net.ipv4.ip_unprivileged_port_start=${PASST_UNPRIVILEGED_PORT_START} for passt port forwarding (unsafe workaround)." +} + +find_bootstrap_qemu() { + local path + for path in \ + /usr/local/bin/qemu-system-x86_64 \ + /usr/bin/qemu-system-x86_64 \ + /bin/qemu-system-x86_64 \ + /usr/local/sbin/qemu-system-x86_64 \ + /usr/sbin/qemu-system-x86_64; do + [[ -x "${path}" ]] && { printf '%s\n' "${path}"; return 0; } + done + return 1 +} + +configure_qemu_binary_permissions() { + local qemu real parent + qemu=$(find_bootstrap_qemu) || { + libvirt_host_error "qemu-system-x86_64 was not found" + return 1 + } + real=$(readlink -f -- "${qemu}") + [[ -n "${real}" && -f "${real}" ]] || { + libvirt_host_error "cannot resolve QEMU binary ${qemu}" + return 1 + } + + if [[ "${real}" == /usr/local/* ]]; then + chmod a+rx "${real}" + parent=$(dirname "${real}") + while [[ "${parent}" == /usr/local/* ]]; do + chmod a+x "${parent}" + parent=$(dirname "${parent}") + done + chmod a+x /usr/local + fi + + if ! runuser -u libvirt-qemu -- test -x "${qemu}"; then + libvirt_host_error "libvirt-qemu cannot execute ${qemu}; check directory permissions and noexec mounts" + return 1 + fi +} + +configure_iommufd() { + local modules_dir=/etc/modules-load.d modules_file tmp + modules_file="${modules_dir}/sp-vm-tools-iommufd.conf" + + install -d -m 0755 "${modules_dir}" + tmp=$(mktemp) + printf '%s\n' \ + '# Managed by sp-vm-tools bootstrap.' \ + 'iommufd' > "${tmp}" + install -m 0644 "${tmp}" "${modules_file}" + rm -f "${tmp}" + + modprobe iommufd + [[ -c /dev/iommu ]] || { + libvirt_host_error "iommufd loaded but /dev/iommu is missing" + return 1 + } +} + +verify_libvirt_host() { + local mode=$1 installed_version missing_packages daemon_version qemu version_line qemu_major capabilities dropin + installed_version=$(installed_libvirt_version) + if [[ -z "${installed_version}" ]] || ! dpkg --compare-versions "${installed_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then + libvirt_host_error "libvirt ${LIBVIRT_REQUIRED_VERSION} or newer is required; installed package is ${installed_version:-missing}" + return 1 + fi + missing_packages=$(missing_required_libvirt_packages) + if [[ -n "${missing_packages}" ]]; then + libvirt_host_error "required libvirt packages are missing: ${missing_packages//$'\n'/, }" + return 1 + fi + python3 -c 'import libvirt' || { + libvirt_host_error "python3-libvirt is not importable" + return 1 + } + id libvirt-qemu >/dev/null || { + libvirt_host_error "the libvirt-qemu user is missing" + return 1 + } + aa-status --enabled >/dev/null || { + libvirt_host_error "AppArmor is not enabled" + return 1 + } + virsh -c "${LIBVIRT_URI}" uri >/dev/null || { + libvirt_host_error "cannot connect to ${LIBVIRT_URI}" + return 1 + } + daemon_version=$(running_libvirt_version || true) + if [[ -z "${daemon_version}" ]] || ! dpkg --compare-versions "${daemon_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then + libvirt_host_error "running libvirt daemon is older than ${LIBVIRT_REQUIRED_VERSION} (${daemon_version:-unknown})" + return 1 + fi + qemu=$(find_bootstrap_qemu) || { + libvirt_host_error "qemu-system-x86_64 was not found" + return 1 + } + version_line=$("${qemu}" --version 2>/dev/null | head -n 1) + qemu_major=$(sed -nE 's/.*version ([0-9]+).*/\1/p' <<< "${version_line}") + if [[ -z "${qemu_major}" || "${qemu_major}" -lt 9 ]]; then + libvirt_host_error "QEMU 9 or newer is required (found ${version_line:-unknown})" + return 1 + fi + if ! capabilities=$(virsh -c "${LIBVIRT_URI}" domcapabilities --emulatorbin "${qemu}" 2>&1); then + libvirt_host_error "libvirt cannot probe ${qemu}: ${capabilities}" + echo "Check recent access denials with: journalctl -k --since '-5 min' --no-pager | grep -E 'apparmor=\"DENIED\"|qemu-system'" >&2 + return 1 + fi + if ! grep -Eq "]*name=['\"]iommufd['\"]" <<< "${capabilities}"; then + libvirt_host_error "libvirt domain capabilities do not advertise IOMMUFD for ${qemu}" + return 1 + fi + # shellcheck disable=SC2119 + verify_passt_capabilities || return 1 + verify_passt_unprivileged_ports || return 1 + + dropin=/etc/apparmor.d/abstractions/libvirt-qemu.d/99-sp-vm-tools-local + grep -qF 'network vsock stream,' "${dropin}" || { + libvirt_host_error "AppArmor VSOCK rule is missing from ${dropin}" + return 1 + } + if [[ "${UBUNTU_VERSION:-}" == "26.04" ]] && \ + ! passthrough_profile_handles_disconnected_sockets \ + /etc/apparmor.d/abstractions/libvirt-qemu; then + libvirt_host_error "nested passt AppArmor profile lacks Ubuntu 26.04 disconnected socket handling; rerun bootstrap" + return 1 + fi + if [[ "${mode}" == "tdx" ]]; then + [[ -c /dev/vhost-vsock ]] || { + libvirt_host_error "/dev/vhost-vsock is missing" + return 1 + } + if grep -Eq '^[[:space:]]*port[[:space:]]*=' /etc/qgs.conf; then + libvirt_host_error "QGS must use its Unix socket; remove the port setting from /etc/qgs.conf" + return 1 + fi + systemctl is-active --quiet qgsd || { + libvirt_host_error "qgsd is not active" + return 1 + } + [[ -S /var/run/tdx-qgs/qgs.socket ]] || { + libvirt_host_error "QGS Unix socket is missing: /var/run/tdx-qgs/qgs.socket" + return 1 + } + id -nG libvirt-qemu | tr ' ' '\n' | grep -qx qgsd || { + libvirt_host_error "libvirt-qemu is not a member of the qgsd group" + return 1 + } + grep -qF '@{run}/tdx-qgs/qgs.socket rw,' "${dropin}" || { + libvirt_host_error "AppArmor QGS socket rule is missing from ${dropin}" + return 1 + } + if ! grep -Eq "]*name=['\"]sectype['\"]" <<< "${capabilities}" || \ + ! grep -Eq 'tdx' <<< "${capabilities}"; then + libvirt_host_error "domain capabilities do not advertise native TDX launch security" + return 1 + fi + elif [[ "${mode}" == "sev-snp" ]]; then + grep -qi 'sev-snp' <<< "${capabilities}" || { + libvirt_host_error "domain capabilities do not advertise SEV-SNP launch security" + return 1 + } + else + libvirt_host_error "invalid confidential VM mode: ${mode}" + return 1 + fi +} + +setup_libvirt_host() { + local mode=$1 installed_version missing_packages + if [[ "${mode}" != "tdx" && "${mode}" != "sev-snp" ]]; then + libvirt_host_error "setup_libvirt_host mode must be tdx or sev-snp" + return 1 + fi + if [[ $(id -u) -ne 0 ]]; then + libvirt_host_error "libvirt host setup must run as root" + return 1 + fi + get_supported_ubuntu_version || return 1 + + print_section_header "Libvirt Host Setup" + installed_version=$(installed_libvirt_version) + missing_packages=$(missing_required_libvirt_packages) + if libvirt_upgrade_required "${installed_version}" || [[ -n "${missing_packages}" ]]; then + assert_no_running_libvirt_domains || return 1 + fi + apt-get update || return 1 + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + acl apparmor-utils ca-certificates libcap2-bin passt procps python3-libvirt \ + qemu-system-x86 qemu-utils wget || return 1 + + installed_version=$(installed_libvirt_version) + missing_packages=$(missing_required_libvirt_packages) + if libvirt_upgrade_required "${installed_version}" || [[ -n "${missing_packages}" ]]; then + if libvirt_upgrade_required "${installed_version}"; then + echo "Installed libvirt ${installed_version:-none} is older than ${LIBVIRT_REQUIRED_VERSION}." + fi + if [[ -n "${missing_packages}" ]]; then + echo "Required libvirt packages are missing: ${missing_packages//$'\n'/, }" + fi + install_project_libvirt || return 1 + else + echo "Installed libvirt ${installed_version} is ${LIBVIRT_REQUIRED_VERSION} or newer and all required packages are present; keeping it." + fi + + if dpkg-query -W -f='${db:Status-Status}' libvirt-daemon-system-systemd 2>/dev/null | grep -qx installed; then + apt-mark manual libvirt-daemon-system-systemd >/dev/null + fi + + configure_libvirt_qemu_runtime || return 1 + if [[ "${mode}" == "tdx" ]]; then + configure_tdx_qgs_access || return 1 + fi + configure_libvirt_apparmor || return 1 + configure_qemu_binary_permissions || return 1 + configure_iommufd || return 1 + # shellcheck disable=SC2119 + configure_passt_capabilities || return 1 + configure_passt_unprivileged_ports || return 1 + systemctl daemon-reload || return 1 + systemctl enable --now libvirtd.service || return 1 + systemctl start virtlogd.socket virtlockd.socket || return 1 + + local daemon_version + daemon_version=$(running_libvirt_version || true) + if [[ "${LIBVIRT_QEMU_CONFIG_CHANGED}" -eq 1 ]] || \ + [[ -z "${daemon_version}" ]] || \ + ! dpkg --compare-versions "${daemon_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then + assert_no_running_libvirt_domains || return 1 + echo "Restarting libvirtd to activate the installed ${LIBVIRT_REQUIRED_VERSION} runtime" + systemctl restart libvirtd.service || return 1 + fi + + install -d -o libvirt-qemu -g libvirt-qemu -m 0750 \ + /var/lib/libvirt/images/superprotocol || return 1 + verify_libvirt_host "${mode}" || return 1 + echo "Libvirt host setup complete." +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + set -euo pipefail + script_dir=$(cd "$(dirname "$0")" && pwd) + # shellcheck disable=SC1091 + source "${script_dir}/common.sh" + setup_libvirt_host "${1:-}" +fi diff --git a/scripts/setup_snp.sh b/scripts/setup_snp.sh index 0136aa6..95493dd 100644 --- a/scripts/setup_snp.sh +++ b/scripts/setup_snp.sh @@ -29,6 +29,24 @@ print_section_header() { echo -e "${BLUE}$(printf '=%.0s' {1..40})${NC}" } +# Early boot messages such as the BIOS-provided RMP range can be evicted from +# the finite kernel ring buffer on long-running or noisy hosts. Prefer the +# persistent journal for the current boot and fall back to dmesg when journald +# is unavailable. +get_kernel_log() { + local log="" + + if command -v journalctl >/dev/null 2>&1; then + log=$(journalctl -k -b --no-pager 2>/dev/null || true) + fi + + if [ -n "$log" ]; then + printf '%s\n' "$log" + else + dmesg 2>/dev/null + fi +} + # --------------------------------------------------------------------------- # Platform detection # --------------------------------------------------------------------------- @@ -158,9 +176,10 @@ check_smee_msr() { # Minimum for SNP: API 1.51 (0x33). # --------------------------------------------------------------------------- check_sev_fw_version() { + local kernel_log="$1" # echoes status lines via the caller's results array is awkward; instead # set globals. - SEV_FW_LINE=$(dmesg | grep -iE "ccp.*SEV-SNP API:" | head -1 || echo "") + SEV_FW_LINE=$(printf '%s\n' "$kernel_log" | grep -im1 -E "ccp.*SEV-SNP API:" || echo "") SEV_FW_OK="unknown" SEV_FW_VER="" if [ -n "$SEV_FW_LINE" ]; then @@ -186,6 +205,9 @@ check_sev_fw_version() { check_all_bios_settings() { local results=() local all_passed=true + local kernel_log + + kernel_log=$(get_kernel_log) print_section_header "BIOS Configuration Check Results" echo "Checking all settings for ${PLATFORM} (${PLATFORM_ZEN})..." @@ -251,7 +273,7 @@ check_all_bios_settings() { # --- SEV-SNP enablement + ASID range (kvm_amd: SEV-SNP enabled (ASIDs..)) results+=("SEV-SNP Initialization:") local snp_enable_line - snp_enable_line=$(dmesg | grep -iE "kvm_amd:.*SEV-SNP enabled" | head -1 || echo "") + snp_enable_line=$(printf '%s\n' "$kernel_log" | grep -im1 -E "kvm_amd:.*SEV-SNP enabled" || echo "") if [ -n "$snp_enable_line" ]; then results+=("${SUCCESS} SEV-SNP enabled${NC}") # e.g. "(ASIDs 1 - 98)" @@ -260,10 +282,11 @@ check_all_bios_settings() { [ -n "$asid_range" ] && results+=(" ${asid_range}") [ -n "$EXPECTED_ASIDS" ] && results+=(" Platform documented total: ${EXPECTED_ASIDS}") else - results+=("${FAILURE} 'SEV-SNP enabled' not found in dmesg${NC}") + results+=("${FAILURE} 'SEV-SNP enabled' not found in the current boot log${NC}") # Try to surface the actual reason rather than guessing BIOS. local snp_err - snp_err=$(dmesg | grep -iE "SEV(-SNP)?:.*(fail|error|disabled)|ccp.*error" | head -3 || echo "") + snp_err=$(printf '%s\n' "$kernel_log" \ + | grep -im3 -E "SEV(-SNP)?:.*(fail|error|disabled)|ccp.*error" || echo "") if [ -n "$snp_err" ]; then results+=(" Reported by kernel:") while IFS= read -r line; do @@ -285,19 +308,19 @@ check_all_bios_settings() { # --- RMP table (SEV-SNP: ... RMP ...) --------------------------------- results+=("RMP Table:") local rmp_line - rmp_line=$(dmesg | grep -iE "SEV-SNP:.*RMP" | head -1 || echo "") + rmp_line=$(printf '%s\n' "$kernel_log" | grep -im1 -E "SEV-SNP:.*RMP" || echo "") if [ -n "$rmp_line" ]; then results+=("${SUCCESS} RMP table present${NC}") results+=(" $(echo "$rmp_line" | sed -E 's/.*SEV-SNP: //')") else - results+=("${FAILURE} RMP table not reported in dmesg${NC}") + results+=("${FAILURE} RMP table not reported in the current boot log${NC}") results+=(" Location: ${PATH_RMP} ${BIOS_NOTE}") all_passed=false fi # --- SEV firmware version (min 1.51 / 0x33) --------------------------- results+=("SEV Firmware (min API 1.51):") - check_sev_fw_version + check_sev_fw_version "$kernel_log" if [ -n "$SEV_FW_VER" ]; then if [ "$SEV_FW_OK" = "yes" ]; then results+=("${SUCCESS} SEV-SNP API ${SEV_FW_VER} (>= 1.51)${NC}") @@ -308,7 +331,7 @@ check_all_bios_settings() { all_passed=false fi else - results+=("${WARNING} Could not read SEV-SNP API version from dmesg${NC}") + results+=("${WARNING} Could not read SEV-SNP API version from the current boot log${NC}") results+=(" If you see 'SEV: failed to INIT error 0x1, rc -5' -> PSP BootLoader too old; update system BIOS") fi diff --git a/scripts/setup_tdx.sh b/scripts/setup_tdx.sh index fd35aa8..0986bcb 100755 --- a/scripts/setup_tdx.sh +++ b/scripts/setup_tdx.sh @@ -17,6 +17,7 @@ NC='\033[0m' # No Color # Modify status indicators: SUCCESS="[${GREEN}✓${NC}]" FAILURE="[${RED}✗${NC}]" +WARNING="[${YELLOW}!${NC}]" print_section_header() { echo -e "\n${BLUE}=== $1 ===${NC}" @@ -146,47 +147,43 @@ check_all_bios_settings() { all_passed=false fi - results+=("TXT Settings:") - + # Intel TXT is useful for TXT/tboot measured-launch workflows, but it is + # not a prerequisite for Intel TDX. Always report its status without + # making the TDX host validation fail. + results+=("TXT Settings (optional for TDX):") + local sinit_base="" - + local senter_en="" + # 0) Does the CPU support SMX/TXT at all? if ! grep -qw smx /proc/cpuinfo; then - results+=("${FAILURE} CPU does not support SMX/TXT${NC}") - all_passed=false + results+=("${WARNING} CPU does not support SMX/TXT${NC}") + results+=(" TXT is not required for TDX; continuing") else # 1) Read SINIT.BASE directly from TXT public config space: # 0xFED30000 + 0x270 (this is what txt-stat used to do) - sinit_base=$(od -An -tx4 -j $((0xFED30270)) -N4 /dev/mem 2>/dev/null | tr -d ' ') + sinit_base=$(od -An -tx4 -j $((0xFED30270)) -N4 /dev/mem 2>/dev/null | tr -d ' ' || true) [ -n "$sinit_base" ] && sinit_base="0x${sinit_base}" - - # 2) Fallback: IA32_FEATURE_CONTROL MSR (0x3A), bit 15 = SENTER global enable. - # Set by BIOS when TXT is enabled. Used when /dev/mem is unavailable - # (e.g. kernel lockdown). - if [ -z "$sinit_base" ] && command -v rdmsr >/dev/null 2>&1; then - modprobe msr 2>/dev/null - local senter_en - senter_en=$(rdmsr -f 15:15 0x3a 2>/dev/null) + + if [ -n "$sinit_base" ] && [ "$sinit_base" != "0x0" ] && \ + [ "$sinit_base" != "0x00000000" ] && [ "$sinit_base" != "0xffffffff" ]; then + results+=("${SUCCESS} TXT enabled (SINIT.BASE = $sinit_base)${NC}") + else + # 2) Fallback: IA32_FEATURE_CONTROL MSR (0x3A), bit 15 = SENTER + # global enable. Check it for every invalid/unavailable + # SINIT.BASE value, not only when /dev/mem returned no output. + modprobe msr 2>/dev/null || true + if command -v rdmsr >/dev/null 2>&1; then + senter_en=$(rdmsr -f 15:15 0x3a 2>/dev/null || true) + fi + if [ "$senter_en" = "1" ]; then results+=("${SUCCESS} TXT enabled (SENTER enabled in IA32_FEATURE_CONTROL)${NC}") else - results+=("${FAILURE} TXT not enabled in BIOS${NC}") - results+=(" Required: Enable TXT in BIOS") - all_passed=false - fi - sinit_base="__msr_checked__" - fi - - if [ "$sinit_base" != "__msr_checked__" ]; then - # 0xffffffff means the chipset does not decode the TXT region => TXT disabled. - # Empty value means we could not read /dev/mem at all. - if [ -n "$sinit_base" ] && [ "$sinit_base" != "0x0" ] && \ - [ "$sinit_base" != "0x00000000" ] && [ "$sinit_base" != "0xffffffff" ]; then - results+=("${SUCCESS} TXT enabled (SINIT.BASE = $sinit_base)${NC}") - else - results+=("${FAILURE} TXT not enabled in BIOS${NC}") - results+=(" Required: Enable TXT in BIOS") - all_passed=false + results+=("${WARNING} TXT not enabled or could not be verified${NC}") + results+=(" SINIT.BASE: ${sinit_base:-unavailable}") + results+=(" IA32_FEATURE_CONTROL.SENTER: ${senter_en:-unavailable}") + results+=(" TXT is not required for TDX; continuing") fi fi fi @@ -226,11 +223,11 @@ check_all_bios_settings() { all_passed=false fi - # Configuration requirements section remains unchanged + # List only settings that can fail the TDX validation. TXT is intentionally + # omitted because it is diagnostic-only for this setup. results+=("${YELLOW}Required BIOS Configuration:${NC}") results+=("• Core Security:") results+=(" - CPU PA: Limit to 46 bits Disable") - results+=(" - TXT: Enable") results+=(" - SGX: Enable") results+=(" - SMT: Enable") results+=("• Memory Protection:") @@ -368,7 +365,7 @@ EOL # package was installed with DEBIAN_FRONTEND=noninteractive, so its interactive # install.sh was skipped. Reproduce here what install.sh would have done: # install the Node.js dependencies and generate the HTTPS SSL keys. The - # Canonical PPA path (< 25.10) does this via setup-attestation-host.sh. + # Ubuntu 24.04 receives the same setting from its attestation packages. if [ "$USE_INTEL_REPO" -eq 1 ]; then # Install PCCS Node.js dependencies. Without node_modules pccs_server.js # fails to start with "Cannot find package 'config'". @@ -399,16 +396,22 @@ EOL chmod -R 750 /opt/intel/sgx-dcap-pccs/ } -# Configure QGS transport. Our stack talks to the Quote Generation Service over -# vsock, but newer tdx-qgs packages (Ubuntu 26.04+) ship /etc/qgs.conf with the -# port commented out (defaulting to a Unix domain socket). Just write the config -# we need: vsock on port 4050. +# Libvirt's native TDX launch security connects QEMU to QGS through the standard +# Unix socket. Leaving "port" unset selects this transport on the QGS packages +# used by both supported Ubuntu releases. Both launchers use the same socket. configure_qgs() { - print_section_header "Configuring QGS (vsock port 4050)..." + print_section_header "Configuring QGS (Unix socket)..." cat > /etc/qgs.conf << EOL -port = 4050 number_threads = 4 EOL + + install -d -m 0755 /etc/systemd/system/qgsd.service.d + cat > /etc/systemd/system/qgsd.service.d/socket.conf << EOL +[Service] +RuntimeDirectory=tdx-qgs +RuntimeDirectoryMode=0755 +EOL + systemctl daemon-reload } # On Ubuntu 24.04 the matched TDX kernel + QEMU are installed from the @@ -527,7 +530,6 @@ install_tdx_release_packages() { } TMP_DIR=$1 -TDX_REF="3.3" check_tdx_os_version() { local min_version="24.04" @@ -561,7 +563,20 @@ check_tdx_os_version() { fi } +cleanup_legacy_canonical_apt_policy() { + # Older bootstrap revisions ran canonical/tdx helpers, which left a global + # priority-4000 pin and enabled unattended package downgrades. Remove those + # settings before any package operation. Repository entries may remain at + # normal APT priority for the attestation packages used below. + rm -f \ + /etc/apt/preferences.d/kobuk-tdx-kobuk-team-tdx-release-pin-4000 \ + /etc/apt/preferences.d/kobuk-tdx-kobuk-team-tdx-attestation-release-pin-4000 \ + /etc/apt/apt.conf.d/99unattended-upgrades-kobuk-tdx-release \ + /etc/apt/apt.conf.d/99unattended-upgrades-kobuk-tdx-attestation-release +} + check_tdx_os_version +cleanup_legacy_canonical_apt_policy # Determine package source based on Ubuntu version UBUNTU_VERSION=$(. /etc/os-release && echo "$VERSION_ID") @@ -573,7 +588,7 @@ if [ "$UBUNTU_NUM" -ge 2510 ]; then echo "Ubuntu ${UBUNTU_VERSION}: using Intel SGX repository" else USE_INTEL_REPO=0 - echo "Ubuntu ${UBUNTU_VERSION}: using Canonical kobuk-team PPA" + echo "Ubuntu ${UBUNTU_VERSION}: using project TDX kernel/QEMU and the Canonical attestation PPA" fi if [ "$USE_INTEL_REPO" -eq 1 ]; then @@ -589,34 +604,10 @@ if [ "$USE_INTEL_REPO" -eq 1 ]; then DEBIAN_FRONTEND=noninteractive apt-get install -y qemu-system-x86 qemu-utils check_error "Failed to install QEMU" else - # Ubuntu < 25.10: TDX host support is not in the stock kernel, so use the - # canonical/tdx host setup (kobuk PPA + -intel kernel). The clone is also - # reused below for attestation (setup-attestation-host.sh). - if [ -d "${TMP_DIR}/tdx-cannonical" ]; then - echo -e "${YELLOW}Directory ${TMP_DIR}/tdx-cannonical already exists${NC}" - echo -e "Removing existing directory..." - rm -rf "${TMP_DIR}/tdx-cannonical" - fi - - git clone https://github.com/canonical/tdx.git "${TMP_DIR}/tdx-cannonical" - if [ $? -ne 0 ]; then - echo "Failed to download the canonical/tdx repository." - exit 1 - fi - SCRIPT_PATH=${TMP_DIR}/tdx-cannonical/setup-tdx-host.sh - - git -C "${TMP_DIR}/tdx-cannonical" checkout --detach "${TDX_REF}" - if [ $? -ne 0 ]; then - echo "Failed to checkout tdx ref ${TDX_REF}." - exit 1 - fi - - print_section_header "Installing hypervisor and kernel..." - echo "Running setup-tdx-host.sh..." - chmod +x "${SCRIPT_PATH}" - "${SCRIPT_PATH}" - - # On 24.04 install our matched custom kernel + sp-qemu-tdx bundle on top. + # Ubuntu 24.04 uses the matched project kernel/QEMU bundle directly. Do not + # run Canonical's setup-tdx-host.sh: it globally pins its PPA at priority + # 4000 and explicitly permits downgrades of QEMU and libvirt. + print_section_header "Installing TDX kernel and QEMU..." install_tdx_release_packages "${TMP_DIR}" fi @@ -647,10 +638,15 @@ if [ "$USE_INTEL_REPO" -eq 1 ]; then deb [signed-by=/etc/apt/keyrings/intel-sgx-keyring.asc arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu ${CODENAME} main EOF else - # Ensure kobuk-team PPA is present - if ! grep -rq "kobuk-team" /etc/apt/sources.list.d/ 2>/dev/null; then - add-apt-repository -y ppa:kobuk-team/tdx-release - check_error "Failed to add kobuk-team PPA" + apt-get install -y software-properties-common + if grep -Rqs 'kobuk-team/tdx-release' /etc/apt/sources.list.d/; then + echo "Removing obsolete kobuk-team/tdx-release PPA" + add-apt-repository -y --remove ppa:kobuk-team/tdx-release + check_error "Failed to remove the obsolete TDX host PPA" + fi + if ! grep -Rqs 'kobuk-team/tdx-attestation-release' /etc/apt/sources.list.d/; then + add-apt-repository -y ppa:kobuk-team/tdx-attestation-release + check_error "Failed to add the TDX attestation PPA" fi fi @@ -873,15 +869,14 @@ if [ "$USE_INTEL_REPO" -eq 1 ]; then sgx-pck-id-retrieval-tool check_error "Failed to install packages" else - # Canonical PPA: install attestation packages via the official script - # from canonical/tdx. - ATTEST_SCRIPT="${TMP_DIR}/tdx-cannonical/attestation/setup-attestation-host.sh" - if [ ! -f "$ATTEST_SCRIPT" ]; then - echo -e "${RED}ERROR: attestation setup script not found at ${ATTEST_SCRIPT}${NC}" - exit 1 - fi - chmod +x "$ATTEST_SCRIPT" - "$ATTEST_SCRIPT" + # Install the same host attestation components used by Canonical, but do it + # directly and without their global PPA pin or --allow-downgrades. + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-remove \ + sgx-dcap-pccs \ + tdx-qgs \ + libsgx-dcap-default-qpl \ + sgx-ra-service \ + sgx-pck-id-retrieval-tool check_error "Failed to install attestation packages" fi @@ -918,7 +913,7 @@ check_error "Failed to register platform" # written config: qgsd re-reads /etc/sgx_default_qcnl.conf, and the one-shot # mpa_registration_tool re-runs the registration flow. print_section_header "Starting remaining services..." -configure_qgs # patch /etc/qgs.conf for vsock before (re)starting qgsd +configure_qgs # select the Unix socket before (re)starting qgsd systemctl restart qgsd wait_for_service qgsd systemctl restart mpa_registration_tool diff --git a/scripts/start_super_protocol.sh b/scripts/start_super_protocol.sh index d973472..4e051ce 100755 --- a/scripts/start_super_protocol.sh +++ b/scripts/start_super_protocol.sh @@ -151,7 +151,7 @@ parse_args() { case $1 in --cores) VM_CPU=$2; shift ;; --mem) VM_RAM=$(echo $2 | sed 's/G//'); shift ;; - --gpu) USED_GPUS+=("$2"); shift ;; + --gpu) USED_GPUS+=("${2#0000:}"); shift ;; --state_disk_path) STATE_DISK_PATH=$2; shift ;; --state_disk_size) STATE_DISK_SIZE=$2; shift ;; --provider_config_disk_path) PROVIDER_CONFIG_DISK_PATH=$2; shift ;; @@ -721,9 +721,11 @@ check_params() { TOTAL_CPUS=$(nproc) TOTAL_RAM=$(free -g | awk '/^Mem:/{print $2}') - # Get list of all NVIDIA GPUs and NVSwitch devices - AVAILABLE_GPUS=($( { lspci -nnk -d 10de: | grep -E '3D controller' | awk '{print $1}'; } || echo)) - AVAILABLE_NVSWITCHES=($( { lspci -mm -n -d 10de:22a3 | cut -d' ' -f1; } || echo)) + # Get list of all NVIDIA GPUs and NVSwitch devices. lspci prints the PCI + # domain on some hosts and omits it on others, and users may pass either + # form, so strip a leading "0000:" everywhere and compare short BDFs. + AVAILABLE_GPUS=($( { lspci -nnk -d 10de: | grep -E '3D controller' | awk '{sub(/^0000:/, "", $1); print $1}'; } || echo)) + AVAILABLE_NVSWITCHES=($( { lspci -mm -n -d 10de:22a3 | awk '{sub(/^0000:/, "", $1); print $1}'; } || echo)) echo "Debug: Found GPUs: ${AVAILABLE_GPUS[@]}" @@ -760,12 +762,20 @@ check_params() { PROVIDER_CONFIG_DISK_PATH="$CACHE/provider_config.img" fi - echo "Removing old state disk..." - rm -f ${STATE_DISK_PATH} - echo "Creating new state disk directory..." - mkdir -p $(dirname ${STATE_DISK_PATH}) - echo "Initializing state disk..." - touch ${STATE_DISK_PATH} + if [[ "${REUSE_DISKS:-false}" == "true" ]]; then + if [[ ! -f "${STATE_DISK_PATH}" || ! -f "${PROVIDER_CONFIG_DISK_PATH}" ]]; then + echo "Error: --reuse-disks requires existing state and provider disks" >&2 + exit 1 + fi + echo "Reusing existing state and provider disks..." + else + echo "Removing old state disk..." + rm -f "${STATE_DISK_PATH}" + echo "Creating new state disk directory..." + mkdir -p "$(dirname "${STATE_DISK_PATH}")" + echo "Initializing state disk..." + touch "${STATE_DISK_PATH}" + fi if [[ -n "$HTTP_PORT" ]]; then @@ -820,7 +830,8 @@ check_params() { fi fi - if [[ "${STATE_DISK_SIZE}" -gt "${MOUNT_SIZE_AVAIL}" ]]; then + if [[ "${REUSE_DISKS:-false}" != "true" && + "${STATE_DISK_SIZE}" -gt "${MOUNT_SIZE_AVAIL}" ]]; then echo "No free space to create virtual disk with ${STATE_DISK_SIZE}Gb" exit 1 fi @@ -1010,7 +1021,7 @@ main() { fi CC_PARAMS+=" -object memory-backend-ram,id=mem0,size=${VM_RAM}G " MACHINE_PARAMS="q35,kernel_irqchip=split,confidential-guest-support=tdx,memory-backend=mem0" - CC_SPECIFIC_PARAMS=" -object '{\"qom-type\":\"tdx-guest\",\"id\":\"tdx\",\"quote-generation-socket\":{\"type\":\"vsock\",\"cid\":\"${BASE_CID}\",\"port\":\"4050\"}}'" + CC_SPECIFIC_PARAMS=" -object '{\"qom-type\":\"tdx-guest\",\"id\":\"tdx\",\"quote-generation-socket\":{\"type\":\"unix\",\"path\":\"/var/run/tdx-qgs/qgs.socket\"}}'" ;; "sev-snp") if [[ ! $SEV_SNP_SUPPORT ]]; then @@ -1157,6 +1168,8 @@ if [[ "${NETDEV_MODE}" == "tap" ]]; then eval $QEMU_COMMAND } -parse_args $@ -detect_cpu_type -main +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + parse_args "$@" + detect_cpu_type + main +fi diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh new file mode 100755 index 0000000..af9e350 --- /dev/null +++ b/scripts/start_super_protocol_libvirt.sh @@ -0,0 +1,601 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +BASE_SCRIPT="${SCRIPT_DIR}/start_super_protocol.sh" +LIBVIRT_LAUNCHER="${SCRIPT_DIR}/libvirt_launcher.py" + +# Reuse the sourceable release, validation, VFIO, and provider-config helpers. +# shellcheck disable=SC1090 +source "${BASE_SCRIPT}" + +# qemu:///system normally runs QEMU as libvirt-qemu, which cannot traverse +# /root. Keep the same --cache option, but use libvirt's image directory as the +# safe default for this launcher. +DEFAULT_CACHE="/var/lib/libvirt/images/superprotocol" +CACHE=${DEFAULT_CACHE} + +LIBVIRT_DOMAIN_NAME="" +LIBVIRT_DOMAIN_UUID="" +REUSE_DISKS=false +LIBVIRT_QEMU_USER="" +BASE_ARGS=() + +usage_libvirt() { + usage + echo "Libvirt-specific options:" + echo " --name Transient domain name (default: super-protocol-)" + echo " --uuid Domain UUID (for Nova-managed instances)" + echo " --reuse-disks Reuse existing state/provider disks on power-on" + echo "" + echo "Runtime behavior:" + echo " --debug false Start in the background and return" + echo " --debug true Attach serial console and tee it to --log_file" +} + +extract_libvirt_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --name) + if [[ $# -lt 2 ]]; then + echo "Error: --name requires a value" >&2 + exit 1 + fi + LIBVIRT_DOMAIN_NAME=$2 + shift 2 + ;; + --uuid) + [[ $# -ge 2 ]] || { echo "Error: --uuid requires a value" >&2; exit 1; } + LIBVIRT_DOMAIN_UUID=$2 + shift 2 + ;; + --reuse-disks) + REUSE_DISKS=true + shift + ;; + --help) + usage_libvirt + exit 0 + ;; + *) + BASE_ARGS+=("$1") + shift + ;; + esac + done +} + +check_target_os() { + local os_id os_version + os_id=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') + os_version=$(sed -n 's/^VERSION_ID=//p' /etc/os-release | tr -d '"') + if [[ "${os_id}" != "ubuntu" ]]; then + echo "Error: this launcher supports Ubuntu 24.04 and 26.04 (found ${os_id:-unknown})." >&2 + exit 1 + fi + if [[ "${os_version}" != "24.04" && "${os_version}" != "26.04" ]]; then + echo "Error: this launcher supports Ubuntu 24.04 and 26.04 (found ${os_version:-unknown})." >&2 + exit 1 + fi +} + +bootstrap_hint() { + if [[ "${VM_MODE}" == "tdx" ]]; then + echo "Fix: run sudo ${SCRIPT_DIR}/bootstrap_tdx.sh to restore the libvirt host configuration." >&2 + elif [[ "${VM_MODE}" == "sev-snp" ]]; then + echo "Fix: run sudo ${SCRIPT_DIR}/bootstrap_snp.sh to restore the libvirt host configuration." >&2 + else + echo "Fix: re-run the host bootstrap with sudo to restore the libvirt host configuration." >&2 + fi +} + +check_libvirt_dependencies() { + local missing=() + command -v python3 >/dev/null 2>&1 || missing+=(python3) + command -v virsh >/dev/null 2>&1 || missing+=(libvirt-clients) + command -v setfacl >/dev/null 2>&1 || missing+=(acl) + command -v runuser >/dev/null 2>&1 || missing+=(util-linux) + if ! python3 -c 'import libvirt' >/dev/null 2>&1; then + missing+=(python3-libvirt) + fi + if [[ "${NETDEV_MODE}" == "user" || -n "${SSH_PORT}" ]]; then + command -v passt >/dev/null 2>&1 || missing+=(passt) + fi + if [[ ${#missing[@]} -gt 0 ]]; then + echo "Error: missing libvirt runtime dependencies: ${missing[*]}" >&2 + echo "Install them with: apt-get install libvirt-daemon-system libvirt-clients python3-libvirt passt acl" >&2 + echo "GPU passthrough additionally requires libvirt >= 12.1.0." >&2 + exit 1 + fi + if [[ ! -x "${LIBVIRT_LAUNCHER}" ]]; then + echo "Error: launcher is not executable: ${LIBVIRT_LAUNCHER}" >&2 + exit 1 + fi +} + +check_passt_apparmor_profile() { + if [[ "${NETDEV_MODE}" != "user" && -z "${SSH_PORT}" ]]; then + return + fi + + local profile=/etc/apparmor.d/abstractions/libvirt-qemu + [[ -r "${profile}" ]] || return 0 + + if awk ' + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { in_passt = 1 } + in_passt && /\/usr\/bin\/passt[[:space:]]+r,/ { incompatible = 1 } + in_passt && /^[[:space:]]*}/ { exit } + END { exit incompatible ? 0 : 1 } + ' "${profile}"; then + echo "Error: the libvirt AppArmor profile permits reading /usr/bin/passt but not mmap." >&2 + echo "Ubuntu AppArmor 5 will kill passt with fatal signal 11." >&2 + bootstrap_hint + exit 1 + fi + + if ! awk ' + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { in_passt = 1 } + in_passt && /^[[:space:]]*capability[[:space:]]+net_bind_service,/ { found = 1 } + in_passt && /^[[:space:]]*}/ { exit } + END { exit found ? 0 : 1 } + ' "${profile}"; then + echo "Error: the nested passt AppArmor profile does not allow CAP_NET_BIND_SERVICE." >&2 + bootstrap_hint + exit 1 + fi +} + +check_tdx_vsock_apparmor_profile() { + [[ "${VM_MODE}" == "tdx" ]] || return 0 + local dropin=/etc/apparmor.d/abstractions/libvirt-qemu.d/99-sp-vm-tools-local + if [[ ! -r "${dropin}" ]] || ! grep -qF 'network vsock stream,' "${dropin}"; then + echo "Error: TDX QGS requires the AppArmor rule 'network vsock stream,'." >&2 + echo "Expected it in ${dropin}." >&2 + bootstrap_hint + exit 1 + fi +} + +check_passt_unprivileged_ports() { + local ports=() + local port port_number minimum=65536 + + if [[ "${NETDEV_MODE}" == "user" ]]; then + ports+=( + "${HTTP_PORT}" "${HTTPS_PORT}" "${PKI_PORT}" + "${PKI_VM_MEASURE_PORT}" "${WG_PORT}" + "${SWARM_DB_GOSSIP_PORT}" "${DNS_PORT}" + ) + fi + if [[ -n "${SSH_PORT}" ]]; then + ports+=("${SSH_PORT}") + fi + + for port in "${ports[@]}"; do + [[ -n "${port}" ]] || continue + port_number=$((10#${port})) + if ((port_number < minimum)); then + minimum=${port_number} + fi + done + + if ((minimum >= 1024)); then + return + fi + + local sysctl_path=/proc/sys/net/ipv4/ip_unprivileged_port_start + local unprivileged_port_start="" + if [[ -r "${sysctl_path}" ]]; then + read -r unprivileged_port_start < "${sysctl_path}" || true + fi + if [[ "${unprivileged_port_start}" != "0" ]]; then + # TODO: UNSAFE CONFIGURATION. Temporary workaround for a passt regression: + # recent versions create forwarded host listeners after entering a user + # namespace, so the binary's CAP_NET_BIND_SERVICE cannot authorize bind() + # in the host network namespace. This is the only stock, unpatched setup + # currently found to work; it makes all host ports unprivileged. + echo "Error: passt must bind host port ${minimum}, but net.ipv4.ip_unprivileged_port_start is ${unprivileged_port_start:-unavailable} (expected 0)." >&2 + echo "This temporary workaround allows every unprivileged process on the host to bind any free TCP/UDP port." >&2 + bootstrap_hint + exit 1 + fi +} + +preflight_libvirt() { + local require_iommufd=false + local gpu + for gpu in "${USED_GPUS[@]}"; do + if [[ "${gpu}" != "none" ]]; then + require_iommufd=true + break + fi + done + # With no --gpu option, check_params will select all available GPUs. + if [[ ${#USED_GPUS[@]} -eq 0 ]] && \ + lspci -nnk -d 10de: 2>/dev/null | grep -qE '3D controller'; then + require_iommufd=true + fi + + local ubuntu_version="" + if [[ -r /etc/os-release ]]; then + ubuntu_version=$( + # shellcheck disable=SC1091 + source /etc/os-release + printf '%s' "${VERSION_ID:-}" + ) + fi + if [[ "${ubuntu_version}" == "26.04" ]]; then + local passt_apparmor=/etc/apparmor.d/abstractions/libvirt-qemu + if [[ ! -r "${passt_apparmor}" ]] || \ + ! grep -Eq '^[[:space:]]*profile[[:space:]]+passt[[:space:]]+flags=\([^)]*attach_disconnected(\.path=[^[:space:])]+)?[^)]*\)[[:space:]]*\{' \ + "${passt_apparmor}"; then + echo "Error: Ubuntu 26.04 AppArmor blocks passt from accepting the libvirt Unix socket." >&2 + bootstrap_hint + exit 1 + fi + fi + + local args=( + preflight + --emulator "${QEMU_PATH}" + --name "${LIBVIRT_DOMAIN_NAME}" + --mode "${VM_MODE}" + ) + if [[ "${require_iommufd}" == "true" ]]; then + args+=(--require-iommufd) + fi + "${LIBVIRT_LAUNCHER}" "${args[@]}" +} + +scan_cx7_bridges() { + local dev_path dev_bdf vpd_file device_info + AVAILABLE_CX7_BRIDGES=() + for dev_path in /sys/bus/pci/devices/*/; do + [[ -e "${dev_path}" ]] || continue + dev_bdf=$(basename "${dev_path}") + vpd_file="${dev_path}vpd" + if [[ -f "${vpd_file}" ]] && grep -q "SW_MNG" "${vpd_file}" 2>/dev/null; then + device_info=$(lspci -s "${dev_bdf}" 2>/dev/null || true) + if [[ "${device_info}" == *"Mellanox"* && "${device_info}" == *"ConnectX-7"* ]]; then + AVAILABLE_CX7_BRIDGES+=("${dev_bdf#0000:}") + fi + fi + done +} + +prepare_selected_host_devices() { + HOSTDEV_ARGS=() + if [[ ${#USED_GPUS[@]} -eq 0 ]]; then + echo "GPU passthrough disabled; NVSwitch and CX7 companion devices will not be attached." + return + fi + + prepare_gpus_for_vfio "${USED_GPUS[@]}" + scan_cx7_bridges + + local device + for device in "${USED_GPUS[@]}"; do + HOSTDEV_ARGS+=(--hostdev "gpu:${device}") + done + for device in "${AVAILABLE_NVSWITCHES[@]}"; do + HOSTDEV_ARGS+=(--hostdev "aux:${device}") + done + for device in "${AVAILABLE_CX7_BRIDGES[@]}"; do + HOSTDEV_ARGS+=(--hostdev "aux:${device}") + done +} + +prepare_mode_parameters() { + SNP_VCPU_ARG="" + PHYS_BITS_ARG="" + CBITPOS_ARG="" + + case "${VM_MODE}" in + tdx) + if [[ -z "${TDX_SUPPORT}" ]]; then + echo "Error: TDX is not supported on this system" >&2 + exit 1 + fi + ;; + sev-snp) + if [[ -z "${SEV_SNP_SUPPORT}" ]]; then + echo "Error: SEV-SNP is not supported on this system" >&2 + exit 1 + fi + get_cbitpos + detect_snp_vCPU + detect_phys_bits + SNP_VCPU_ARG=${SNP_VCPU} + PHYS_BITS_ARG=${PHYS_BITS} + CBITPOS_ARG=${CBITPOS} + ;; + untrusted) + ;; + *) + echo "Error: invalid mode '${VM_MODE}'" >&2 + exit 1 + ;; + esac +} + +prepare_tap_network() { + if [[ "${NETDEV_MODE}" != "tap" ]]; then + return + fi + if [[ -z "${TAP_IFACE}" ]]; then + TAP_IFACE="sw-tap${BASE_NIC}" + fi + if ! ip link show "${BRIDGE}" >/dev/null 2>&1; then + echo "Error: bridge ${BRIDGE} does not exist. Run 'swarm-cluster.sh ensure-network' first." >&2 + exit 1 + fi + if ! ip link show "${TAP_IFACE}" >/dev/null 2>&1; then + ip tuntap add dev "${TAP_IFACE}" mode tap user root + fi + ip link set "${TAP_IFACE}" master "${BRIDGE}" + ip link set "${TAP_IFACE}" up +} + +build_kernel_cmdline() { + local clearcpuid=" " + local snp_additional="" + local rootfs_hash + rootfs_hash=$(<"${ROOTFS_HASH_PATH}") + + if [[ "${VM_MODE}" == "tdx" ]]; then + clearcpuid=" clearcpuid=mtrr " + elif [[ "${VM_MODE}" == "sev-snp" ]]; then + snp_additional=" build=${RELEASE} pci=realloc,nocrs" + fi + + KERNEL_CMD_LINE="root=LABEL=rootfs${clearcpuid}rootfs_verity.scheme=dm-verity rootfs_verity.hash=${rootfs_hash}${snp_additional}" + if [[ "${DEBUG_MODE}" == "true" ]]; then + KERNEL_CMD_LINE+=" console=ttyS0 systemd.log_level=trace systemd.log_target=log" + fi +} + +resolve_libvirt_qemu_user() { + if [[ -n "${LIBVIRT_QEMU_USER}" ]]; then + return + fi + + local qemu_config=/etc/libvirt/qemu.conf + local configured_user="" + if [[ -r "${qemu_config}" ]]; then + configured_user=$(awk ' + /^[[:space:]]*#/ { next } + /^[[:space:]]*user[[:space:]]*=/ { + value = $0 + sub(/^[^=]*=[[:space:]]*/, "", value) + sub(/[[:space:]]*#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + if (value ~ /^"[^"]*"$/) { + sub(/^"/, "", value) + sub(/"$/, "", value) + } + configured = value + } + END { print configured } + ' "${qemu_config}") + fi + configured_user=${configured_user:-libvirt-qemu} + + local passwd_entry="" + if [[ "${configured_user}" =~ ^\+([0-9]+)$ ]]; then + local configured_uid=${BASH_REMATCH[1]} + passwd_entry=$(getent passwd | awk -F: -v uid="${configured_uid}" ' + $3 == uid { print; exit } + ') + else + passwd_entry=$(getent passwd "${configured_user}" || true) + fi + if [[ -z "${passwd_entry}" ]]; then + echo "Error: libvirt QEMU runtime user '${configured_user}' does not exist." >&2 + echo "Check the user setting in ${qemu_config}." >&2 + exit 1 + fi + + LIBVIRT_QEMU_USER=${passwd_entry%%:*} + echo "Libvirt QEMU runtime user: ${LIBVIRT_QEMU_USER}" +} + +grant_libvirt_file_access() { + local label=$1 requested_path=$2 permissions=$3 + resolve_libvirt_qemu_user + local qemu_user=${LIBVIRT_QEMU_USER} + + local path + if ! path=$(realpath -e -- "${requested_path}"); then + echo "Error: cannot resolve ${label} path: ${requested_path}" >&2 + exit 1 + fi + + local directory + directory=$(dirname -- "${path}") + local directories=() + while [[ "${directory}" != "/" ]]; do + directories+=("${directory}") + directory=$(dirname -- "${directory}") + done + + local index + for ((index = ${#directories[@]} - 1; index >= 0; index--)); do + directory=${directories[${index}]} + if ! runuser -u "${qemu_user}" -- test -x "${directory}"; then + if ! setfacl -m "u:${qemu_user}:--x" -- "${directory}"; then + echo "Error: failed to grant ${qemu_user} traversal access to ${directory}" >&2 + exit 1 + fi + fi + done + + if [[ "${permissions}" == "rw-" ]]; then + # The launcher creates mutable disks itself. Giving the QEMU process + # ownership is more reliable than a named ACL on mounted data volumes. + if ! chown -- "${qemu_user}" "${path}" || ! chmod -- u+rw "${path}"; then + echo "Error: failed to assign writable ${label} to ${qemu_user}: ${path}" >&2 + exit 1 + fi + else + if ! setfacl -m "u:${qemu_user}:${permissions}" -- "${path}"; then + echo "Error: failed to grant ${qemu_user} access to ${label}: ${path}" >&2 + exit 1 + fi + fi + + if ! runuser -u "${qemu_user}" -- test -r "${path}"; then + echo "Error: ${qemu_user} still cannot read ${label}: ${path}" >&2 + exit 1 + fi + if [[ "${permissions}" == "rw-" ]] && \ + ! runuser -u "${qemu_user}" -- test -w "${path}"; then + echo "Error: ${qemu_user} still cannot write ${label}: ${path}" >&2 + exit 1 + fi + + echo "Granted ${qemu_user} ${permissions} access to ${label}: ${path}" +} + +grant_static_libvirt_resource_access() { + grant_libvirt_file_access rootfs "${IMAGE_PATH}" r-- + grant_libvirt_file_access kernel "${KERNEL_PATH}" r-- + grant_libvirt_file_access firmware "${BIOS_PATH}" r-- +} + +cleanup_provider_disk() { + local provider_loop=${1:-} provider_mount=${2:-} + if [[ -n "${provider_mount}" ]] && mountpoint -q "${provider_mount}"; then + umount "${provider_mount}" || true + fi + if [[ -n "${provider_loop}" ]]; then + losetup -d "${provider_loop}" 2>/dev/null || true + fi + if [[ -n "${provider_mount}" ]]; then + rmdir "${provider_mount}" 2>/dev/null || true + fi +} + +create_vm_disks() { + local provider_loop="" provider_mount + + if [[ "${REUSE_DISKS}" == "true" && -f "${STATE_DISK_PATH}" && -f "${PROVIDER_CONFIG_DISK_PATH}" ]]; then + if ! qemu-img info --output=json "${STATE_DISK_PATH}" | grep -q '"format": "qcow2"'; then + echo "Error: existing state disk is not a valid qcow2 image: ${STATE_DISK_PATH}" >&2 + exit 1 + fi + if [[ "$(blkid -o value -s TYPE "${PROVIDER_CONFIG_DISK_PATH}" || true)" != "ext4" ]]; then + echo "Error: existing provider config disk is not ext4: ${PROVIDER_CONFIG_DISK_PATH}" >&2 + exit 1 + fi + echo "Validated reusable state and provider disks." + grant_libvirt_file_access state-disk "${STATE_DISK_PATH}" rw- + grant_libvirt_file_access provider-config-disk "${PROVIDER_CONFIG_DISK_PATH}" r-- + return + fi + + rm -f "${STATE_DISK_PATH}" + qemu-img create -f qcow2 "${STATE_DISK_PATH}" "${STATE_DISK_SIZE}G" + + rm -f "${PROVIDER_CONFIG_DISK_PATH}" + dd if=/dev/zero of="${PROVIDER_CONFIG_DISK_PATH}" bs=1M count=1 status=none + mkfs.ext4 -q -O '^has_journal,^huge_file,^meta_bg,^ext_attr' \ + -L provider_config "${PROVIDER_CONFIG_DISK_PATH}" + provider_mount=$(mktemp -d) + trap 'cleanup_provider_disk "${provider_loop:-}" "${provider_mount:-}"' RETURN + provider_loop=$(losetup --find --show --partscan "${PROVIDER_CONFIG_DISK_PATH}") + + mount "${provider_loop}" "${provider_mount}" + cp -a "${PROVIDER_CONFIG}/." "${provider_mount}/" + rm -rf "${provider_mount}/lost+found" + cleanup_provider_disk "${provider_loop}" "${provider_mount}" + trap - RETURN + + grant_libvirt_file_access state-disk "${STATE_DISK_PATH}" rw- + grant_libvirt_file_access provider-config-disk "${PROVIDER_CONFIG_DISK_PATH}" r-- +} + +append_optional_arg() { + local flag=$1 value=$2 + if [[ -n "${value}" ]]; then + LAUNCH_ARGS+=("${flag}" "${value}") + fi +} + +launch_with_libvirt() { + LAUNCH_ARGS=( + "${LIBVIRT_LAUNCHER}" + --name "${LIBVIRT_DOMAIN_NAME}" + --mode "${VM_MODE}" + --emulator "${QEMU_PATH}" + --memory-gib "${VM_RAM}" + --vcpus "${VM_CPU}" + --bios "${BIOS_PATH}" + --kernel "${KERNEL_PATH}" + --kernel-cmdline "${KERNEL_CMD_LINE}" + --rootfs "${IMAGE_PATH}" + --state-disk "${STATE_DISK_PATH}" + --provider-config-disk "${PROVIDER_CONFIG_DISK_PATH}" + --guest-cid "${GUEST_CID}" + --qgs-socket "/var/run/tdx-qgs/qgs.socket" + --mac-address "${MAC_ADDRESS}" + --netdev-mode "${NETDEV_MODE}" + --ip-address "${IP_ADDRESS}" + --ssh-port "${SSH_PORT}" + --wg-port "${WG_PORT}" + --swarm-db-gossip-port "${SWARM_DB_GOSSIP_PORT}" + --dns-port "${DNS_PORT}" + ) + append_optional_arg --uuid "${LIBVIRT_DOMAIN_UUID}" + append_optional_arg --http-port "${HTTP_PORT}" + append_optional_arg --https-port "${HTTPS_PORT}" + append_optional_arg --pki-port "${PKI_PORT}" + append_optional_arg --pki-vm-measure-port "${PKI_VM_MEASURE_PORT}" + append_optional_arg --cpu-model "${SNP_VCPU_ARG}" + append_optional_arg --phys-bits "${PHYS_BITS_ARG}" + append_optional_arg --cbitpos "${CBITPOS_ARG}" + + if [[ "${NETDEV_MODE}" == "tap" ]]; then + LAUNCH_ARGS+=(--bridge "${BRIDGE}" --tap-iface "${TAP_IFACE}") + fi + if [[ "${DEBUG_MODE}" == "true" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" + LAUNCH_ARGS+=(--debug --log-file "${LOG_FILE}") + fi + LAUNCH_ARGS+=("${HOSTDEV_ARGS[@]}") + + echo "Starting ${LIBVIRT_DOMAIN_NAME} through qemu:///system (mode=${VM_MODE}, debug=${DEBUG_MODE})" + "${LAUNCH_ARGS[@]}" +} + +main_libvirt() { + check_target_os + check_packages + check_libvirt_dependencies + check_passt_apparmor_profile + check_tdx_vsock_apparmor_profile + find_qemu_path + check_qemu_version + preflight_libvirt + check_params + check_passt_unprivileged_ports + prepare_selected_host_devices + prepare_mode_parameters + + mkdir -p "${CACHE}" + download_release "${RELEASE}" "${RELEASE_ASSET}" "${CACHE}" "${RELEASE_REPO}" + parse_and_download_release_files "${RELEASE_FILEPATH}" + grant_static_libvirt_resource_access + prepare_tap_network + build_kernel_cmdline + create_vm_disks + launch_with_libvirt +} + +extract_libvirt_args "$@" +parse_args "${BASE_ARGS[@]}" +detect_cpu_type +if [[ -z "${LIBVIRT_DOMAIN_NAME}" ]]; then + LIBVIRT_DOMAIN_NAME="super-protocol-${GUEST_CID}" +fi +main_libvirt diff --git a/scripts/swarm-cluster.sh b/scripts/swarm-cluster.sh index 82a0323..0ef870d 100755 --- a/scripts/swarm-cluster.sh +++ b/scripts/swarm-cluster.sh @@ -8,10 +8,10 @@ # - each VM gets its address from provider_config/swarm/config.yaml spnet # - external ingress (host WAN) DNAT only to bootstrap: 80/443/9443 tcp, 53 tcp+udp # - join nodes fetch PKI (9443) from bootstrap over the LOCAL address 10.0.0.10 (no hairpin) -# - each VM runs in its own tmux session +# - each VM runs as a transient qemu:///system libvirt domain # -# Requires a patched start_super_protocol.sh (see network-tap.patch.md): -# support for --netdev_mode tap --bridge . +# Requires start_super_protocol_libvirt.sh and a working qemu:///system +# connection. tmux is used only for the launcher/serial-console process. # # Usage: # sudo ./swarm-cluster.sh up --provider-config-template ./provider-template [opts] @@ -77,7 +77,8 @@ VM_MODE="" # empty = auto-detect (tdx/sev-snp) in start scri RELEASE="" # empty = latest; pin a working build, e.g. build-358 LOCAL_BUILD_DIR="" # empty = use release; otherwise pass local build dir to start script CACHE="/data/sp-vm/cache" -START_SCRIPT="${SCRIPT_DIR}/start_super_protocol.sh" +START_SCRIPT="${SCRIPT_DIR}/start_super_protocol_libvirt.sh" +LIBVIRT_URI="qemu:///system" PROVIDER_TEMPLATE="" # provider config template dir (--provider-config-template) WORKDIR="/data/sp-vm/cluster" # per-node provider configs are generated here WAN_IFACE="" # empty = auto-detect from ip route @@ -94,9 +95,12 @@ DEBUG_MODE="false" SSH_PORT_BOOTSTRAP=2210 SSH_PORT_JOIN=(2211 2212) -# tmux sessions -TMUX_BOOTSTRAP="swarm-bootstrap" -TMUX_JOIN=("swarm-join-1" "swarm-join-2") +# Domain names are also used as tmux launcher/console session names. +DOMAIN_BOOTSTRAP="swarm-bootstrap" +DOMAIN_JOIN=("swarm-join-1" "swarm-join-2") +CLUSTER_DOMAINS=("${DOMAIN_BOOTSTRAP}" "${DOMAIN_JOIN[@]}") +TMUX_BOOTSTRAP="${DOMAIN_BOOTSTRAP}" +TMUX_JOIN=("${DOMAIN_JOIN[@]}") # ---------------------------------------------------------------------------- # Helpers @@ -109,6 +113,36 @@ require_root() { [[ "$EUID" -eq 0 ]] || die "Must be run as root (use sudo)." } +virsh_cluster() { + LC_ALL=C virsh --connect "${LIBVIRT_URI}" "$@" +} + +require_libvirt() { + command -v virsh >/dev/null 2>&1 || die "virsh is required (install libvirt-clients)." + virsh_cluster list --name >/dev/null 2>&1 \ + || die "Cannot connect to ${LIBVIRT_URI}. Check the libvirt daemon and permissions." +} + +domain_exists() { + local domain="$1" + virsh_cluster dominfo "${domain}" >/dev/null 2>&1 +} + +domain_alive() { + local domain="$1" state + state=$(virsh_cluster domstate "${domain}" 2>/dev/null) || return 1 + [[ "${state}" != "shut off" && "${state}" != "crashed" ]] +} + +ensure_cluster_domains_available() { + local domain + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_exists "${domain}"; then + die "libvirt domain ${domain} already exists. Run 'down' or remove it explicitly." + fi + done +} + detect_wan_iface() { if [[ -n "${WAN_IFACE}" ]]; then echo "${WAN_IFACE}"; return; fi ip route get 8.8.8.8 2>/dev/null | sed -n 's/.* dev \([^ ]*\).*/\1/p' | head -1 @@ -223,7 +257,15 @@ reset_vfio_devices() { local drv="/sys/bus/pci/drivers/vfio-pci" [[ -d "${drv}" ]] || { log "vfio-pci driver not loaded — nothing to reset"; return 0; } - # Refuse to reset devices under a live QEMU + # Refuse to reset devices assigned to a live cluster domain. + local domain + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_alive "${domain}"; then + die "libvirt domain ${domain} is still running — refusing to reset devices. Run 'down' first." + fi + done + + # Also protect unrelated QEMU processes that currently hold VFIO devices. if pgrep -f 'qemu-system-x86_64.*vfio' >/dev/null 2>&1; then die "QEMU with VFIO still running — refusing to reset devices. Run 'down' first." fi @@ -291,18 +333,19 @@ reset_vfio_devices() { } # ---------------------------------------------------------------------------- -# VM liveness check: the runner does `exec qemu | tee`, so if QEMU dies for -# any reason the tmux session collapses. Detect that instead of waiting blind. +# VM liveness is owned by libvirt. In release mode the launcher and its tmux +# session exit immediately after createXML(), while the transient domain keeps +# running, so tmux is not a valid VM health signal. # ---------------------------------------------------------------------------- vm_alive() { - local session="$1" - tmux has-session -t "${session}" 2>/dev/null + local domain="$1" + domain_alive "${domain}" } report_vm_death() { - local session="$1" node_ip="$2" + local domain="$1" node_ip="$2" local logf="${CACHE}/log-${node_ip##*.}.txt" - err "VM session '${session}' has exited — QEMU failed." + err "libvirt domain '${domain}' is not running." err "Last lines of ${logf}:" tail -n 25 "${logf}" 2>/dev/null | sed 's/^/ /' >&2 || true # Common failure hint @@ -619,10 +662,11 @@ inject_spnet_config() { } # ---------------------------------------------------------------------------- -# 3. Start a single VM in tmux +# 3. Start a single libvirt domain through a tmux launcher # ---------------------------------------------------------------------------- start_vm() { - local session="$1" + local domain="$1" + local session="${domain}" local node_ip="$2" local cid="$3" local provider_dir="$4" @@ -634,9 +678,11 @@ start_vm() { local tap_iface="sw-tap-${node_ip##*.}" local mac; mac="$(mac_for_ip "${node_ip}")" + if domain_exists "${domain}"; then + die "libvirt domain ${domain} already exists. Run 'down' or remove it explicitly." + fi if tmux has-session -t "${session}" 2>/dev/null; then - err "tmux session ${session} already exists. Skipping (run 'down' to clean up)." - return 0 + die "stale tmux launcher session ${session} already exists. Run 'down' first." fi local gpu_args=() @@ -655,20 +701,19 @@ start_vm() { local build_args=() [[ -n "${LOCAL_BUILD_DIR}" ]] && build_args=(--build_dir "${LOCAL_BUILD_DIR}") - # Debug mode: verbose boot log + per-node SSH port. start_super_protocol.sh - # requires --log_file when --debug true. NOTE: the script forwards SSH via - # hostfwd (user-mode) only; in tap mode that hostfwd is inactive, so SSH must - # go to the VM's bridge IP (ssh ubuntu@). The main value of debug - # here is the verbose serial/boot log written to the log file. + # Debug mode keeps the libvirt serial console attached in tmux and writes a + # per-node boot log. With tap networking the launcher adds a second passt + # NIC for the requested localhost SSH forwarding. local debug_args=() if [[ "${DEBUG_MODE}" == "true" ]]; then debug_args=(--debug true --log_file "${CACHE}/boot-${node_ip##*.}.log") [[ -n "${ssh_port}" ]] && debug_args+=(--ssh_port "${ssh_port}") fi - # build the patched start-script command line in tap mode + # Build the libvirt start-script command line in tap mode. local cmd=( "${START_SCRIPT}" + --name "${domain}" --netdev_mode tap --bridge "${BRIDGE}" --tap_iface "${tap_iface}" @@ -688,7 +733,7 @@ start_vm() { "${debug_args[@]}" ) - log "Starting ${session}: ip=${node_ip} cid=${cid} tap=${tap_iface} gpu=${with_gpu} cores=${node_cores} mem=${node_mem}GB disk=${node_disk}GB debug=${DEBUG_MODE}" + log "Starting domain ${domain}: ip=${node_ip} cid=${cid} tap=${tap_iface} gpu=${with_gpu} cores=${node_cores} mem=${node_mem}GB disk=${node_disk}GB debug=${DEBUG_MODE}" # Safety net: drop any empty array elements before building the runner. # An empty positional arg would shift the start-script's two-step arg parser @@ -715,17 +760,32 @@ start_vm() { tmux new-session -d -s "${session}" "${runner}" - # Fail fast: if the command dies immediately (bad flags, missing release, etc.), - # the tmux session collapses and we must not proceed into a blind wait. - # The image download alone takes a while, so we only check that the session - # survives the first few seconds — enough to catch instant failures. - sleep 6 - if ! tmux has-session -t "${session}" 2>/dev/null; then - err "Session ${session} exited immediately — startup failed." - err "Last lines of ${CACHE}/log-${node_ip##*.}.txt:" - tail -n 20 "${CACHE}/log-${node_ip##*.}.txt" 2>/dev/null | sed 's/^/ /' >&2 || true - die "Aborting. Fix the error above (often: wrong --release, or start script not patched)." - fi + # The launcher may spend time downloading/preparing images before it creates + # the domain. In release mode it then exits successfully, so wait for libvirt + # rather than requiring the tmux session to remain after startup. + local startup_timeout=1000 waited=0 + while (( waited < startup_timeout )); do + if domain_alive "${domain}"; then + log "Domain ${domain} is running" + return 0 + fi + if ! tmux has-session -t "${session}" 2>/dev/null; then + # The release-mode launcher may exit immediately after createXML(). + if domain_alive "${domain}"; then + log "Domain ${domain} is running" + return 0 + fi + err "Launcher session ${session} exited before the domain started." + err "Last lines of ${CACHE}/log-${node_ip##*.}.txt:" + tail -n 25 "${CACHE}/log-${node_ip##*.}.txt" 2>/dev/null | sed 's/^/ /' >&2 || true + die "Domain ${domain} failed to start." + fi + sleep 2 + waited=$((waited + 2)) + done + + tmux kill-session -t "${session}" 2>/dev/null || true + die "Timed out after ${startup_timeout}s waiting for libvirt domain ${domain}." } # ---------------------------------------------------------------------------- @@ -741,9 +801,9 @@ wait_bootstrap() { while (( waited < timeout )); do # Fail fast: QEMU crashed (vfio bind error, OOM, bad flags, ...) - if ! vm_alive "${TMUX_BOOTSTRAP}"; then + if ! vm_alive "${DOMAIN_BOOTSTRAP}"; then echo >&2 - report_vm_death "${TMUX_BOOTSTRAP}" "${BOOTSTRAP_IP}" + report_vm_death "${DOMAIN_BOOTSTRAP}" "${BOOTSTRAP_IP}" die "Bootstrap VM died while waiting — aborting cluster startup." fi @@ -771,7 +831,7 @@ wait_bootstrap() { sleep 5; waited=$(( waited + 5 )) done echo >&2 - die "Bootstrap did not come up within ${timeout}s. Check: tmux attach -t ${TMUX_BOOTSTRAP}" + die "Bootstrap did not come up within ${timeout}s. Check ${CACHE}/log-${BOOTSTRAP_IP##*.}.txt and /var/log/libvirt/qemu/${DOMAIN_BOOTSTRAP}-serial.log" } # ---------------------------------------------------------------------------- @@ -803,6 +863,8 @@ fetch_ca_bundle() { # ---------------------------------------------------------------------------- cmd_up() { require_root + require_libvirt + ensure_cluster_domains_available [[ -n "${PROVIDER_TEMPLATE}" ]] || die "Specify --provider-config-template " [[ -d "${PROVIDER_TEMPLATE}" ]] || die "Template ${PROVIDER_TEMPLATE} not found" [[ -x "${START_SCRIPT}" ]] || die "start script not found/executable: ${START_SCRIPT}" @@ -822,6 +884,11 @@ cmd_up() { command -v nc &>/dev/null || die "nc is required (apt install netcat-openbsd)" command -v curl &>/dev/null || die "curl is required (apt install curl)" command -v tmux &>/dev/null || die "tmux is required (apt install tmux)" + local session + for session in "${TMUX_BOOTSTRAP}" "${TMUX_JOIN[@]}"; do + tmux has-session -t "${session}" 2>/dev/null \ + && die "stale tmux launcher session ${session} already exists. Run 'down' first." + done if [[ -n "${RELEASE}" && -n "${LOCAL_BUILD_DIR}" ]]; then die "Use either --release or --build-dir, not both." fi @@ -866,7 +933,7 @@ cmd_up() { local boot_gpu=false [[ "${GPU_TARGET}" == "bootstrap" ]] && boot_gpu=true - start_vm "${TMUX_BOOTSTRAP}" "${BOOTSTRAP_IP}" "${CID_BOOTSTRAP}" "${boot_dir}" \ + start_vm "${DOMAIN_BOOTSTRAP}" "${BOOTSTRAP_IP}" "${CID_BOOTSTRAP}" "${boot_dir}" \ "${boot_gpu}" "${BOOTSTRAP_CORES}" "${BOOTSTRAP_MEM}" "${BOOTSTRAP_DISK}" "${SSH_PORT_BOOTSTRAP}" wait_bootstrap 1000 @@ -883,9 +950,9 @@ cmd_up() { join1_dir="$(prepare_config join1 "${JOIN_IPS[0]}" "swarm-join-1" "${BOOTSTRAP_IP}:${GOSSIP_PORT}" "${ca_bundle}" "${network_id}")" join2_dir="$(prepare_config join2 "${JOIN_IPS[1]}" "swarm-join-2" "${BOOTSTRAP_IP}:${GOSSIP_PORT}" "${ca_bundle}" "${network_id}")" - start_vm "${TMUX_JOIN[0]}" "${JOIN_IPS[0]}" "${CID_JOIN[0]}" "${join1_dir}" \ + start_vm "${DOMAIN_JOIN[0]}" "${JOIN_IPS[0]}" "${CID_JOIN[0]}" "${join1_dir}" \ false "${JOIN_CORES}" "${JOIN_MEM}" "${JOIN_DISK}" "${SSH_PORT_JOIN[0]}" - start_vm "${TMUX_JOIN[1]}" "${JOIN_IPS[1]}" "${CID_JOIN[1]}" "${join2_dir}" \ + start_vm "${DOMAIN_JOIN[1]}" "${JOIN_IPS[1]}" "${CID_JOIN[1]}" "${join2_dir}" \ false "${JOIN_CORES}" "${JOIN_MEM}" "${JOIN_DISK}" "${SSH_PORT_JOIN[1]}" # external ingress @@ -898,17 +965,20 @@ cmd_up() { log "Cluster started. Ingress: gw.dyn.${GLOBAL_ID}.${BASE_DOMAIN} -> 80/443" - log "Cluster started. Sessions: tmux ls" - log " bootstrap: tmux attach -t ${TMUX_BOOTSTRAP}" - log " join: tmux attach -t ${TMUX_JOIN[0]} | ${TMUX_JOIN[1]}" + log "Cluster started. Domains: virsh -c ${LIBVIRT_URI} list" + log " bootstrap serial: tail -f /var/log/libvirt/qemu/${DOMAIN_BOOTSTRAP}-serial.log" + log " join serials: /var/log/libvirt/qemu/{${DOMAIN_JOIN[0]},${DOMAIN_JOIN[1]}}-serial.log" + if [[ "${DEBUG_MODE}" == "true" ]]; then + log " attached debug consoles are in tmux: ${TMUX_BOOTSTRAP}, ${TMUX_JOIN[0]}, ${TMUX_JOIN[1]}" + fi # Verify join VMs survived startup (they can hit the same vfio error # if GPU_TARGET is ever changed, or die on bad config / OOM). sleep 10 local i for i in 0 1; do - if ! vm_alive "${TMUX_JOIN[$i]}"; then - report_vm_death "${TMUX_JOIN[$i]}" "${JOIN_IPS[$i]}" + if ! vm_alive "${DOMAIN_JOIN[$i]}"; then + report_vm_death "${DOMAIN_JOIN[$i]}" "${JOIN_IPS[$i]}" die "Join node ${JOIN_IPS[$i]} died right after start — aborting." fi done @@ -924,7 +994,17 @@ cmd_status() { echo " host: ${_c} cores, ${_m}GB" echo " plan: bootstrap=remainder+GPU, join=${JOIN_CORES}c/${JOIN_MEM}g each, reserve=${HOST_RESERVE_CORES}c/${HOST_RESERVE_MEM}g" fi - echo "=== tmux ===" + echo "=== libvirt domains (${LIBVIRT_URI}) ===" + if command -v virsh >/dev/null 2>&1 && virsh_cluster list --name >/dev/null 2>&1; then + local domain state + for domain in "${CLUSTER_DOMAINS[@]}"; do + state=$(virsh_cluster domstate "${domain}" 2>/dev/null || true) + echo " ${domain}: ${state:-absent}" + done + else + echo " unavailable" + fi + echo "=== tmux launcher/console sessions ===" tmux ls 2>/dev/null | grep -E 'swarm-' || echo " no sessions" echo "=== bridge ===" ip -br addr show "${BRIDGE}" 2>/dev/null || echo " no bridge ${BRIDGE}" @@ -943,21 +1023,53 @@ cmd_status() { | sed 's/^/ /' || echo " network script unavailable" } -wait_qemu_gone() { - local timeout="${1:-90}" waited=0 - log "Waiting for QEMU processes to exit (up to ${timeout}s)..." - while pgrep -f 'qemu-system-x86_64.*sw-tap-' >/dev/null 2>&1; do - if (( waited >= timeout )); then - err "QEMU still alive after ${timeout}s, sending SIGKILL" - pkill -9 -f 'qemu-system-x86_64.*sw-tap-' 2>/dev/null || true - timeout=$(( timeout + 120 )) +wait_domains_stopped() { + local timeout="${1:-90}" waited=0 domain active + log "Waiting for libvirt domains to stop (up to ${timeout}s)..." + while (( waited < timeout )); do + active="" + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_alive "${domain}"; then + active="${domain}" + break + fi + done + if [[ -z "${active}" ]]; then + log "All cluster domains stopped (${waited}s)" + return 0 fi - printf '\r[%s] qemu still running... %ss\033[K' "$(date +%H:%M:%S)" "${waited}" >&2 - sleep 2; waited=$(( waited + 2 )) - (( waited >= 300 )) && { echo >&2; die "QEMU did not exit in 300s — check dmesg (stuck unpinning?)"; } + printf '\r[%s] %s still running... %ss\033[K' "$(date +%H:%M:%S)" "${active}" "${waited}" >&2 + sleep 2 + waited=$((waited + 2)) done echo >&2 - log "All QEMU processes gone (${waited}s)" + return 1 +} + +stop_cluster_domains() { + local domain + + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_alive "${domain}"; then + log " shutdown ${domain}" + virsh_cluster shutdown "${domain}" >/dev/null 2>&1 \ + || err "Failed to request shutdown for ${domain}" + fi + done + + if wait_domains_stopped 30; then + return 0 + fi + + err "Graceful shutdown timed out; destroying remaining cluster domains" + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_alive "${domain}"; then + log " destroy ${domain}" + virsh_cluster destroy "${domain}" >/dev/null 2>&1 \ + || err "Failed to destroy ${domain}" + fi + done + wait_domains_stopped 90 || die "Some libvirt domains did not stop; refusing to tear down networking." } wait_vfio_free() { @@ -986,6 +1098,7 @@ wait_vfio_free() { cmd_down() { require_root + require_libvirt local answer while true; do @@ -1005,16 +1118,16 @@ cmd_down() { log "Stopping cluster..." - pkill -TERM -f 'qemu-system-x86_64.*sw-tap-' 2>/dev/null || true - - wait_qemu_gone 90 - - wait_vfio_free 120 || err "GPU may still be busy — next 'up' can fail; check 'fuser -v /dev/vfio/*'" - + # Stop launchers first so a domain still being prepared cannot appear after + # the shutdown pass. Killing an attached debug console only detaches it. for s in "${TMUX_BOOTSTRAP}" "${TMUX_JOIN[@]}"; do tmux kill-session -t "${s}" 2>/dev/null || true done + stop_cluster_domains + + wait_vfio_free 120 || err "GPU may still be busy — next 'up' can fail; check 'fuser -v /dev/vfio/*'" + for ip in "${BOOTSTRAP_IP}" "${JOIN_IPS[@]}"; do local tap="sw-tap-${ip##*.}" ip link show "${tap}" &>/dev/null && { log " del ${tap}"; ip link del "${tap}"; } diff --git a/tests/test_libvirt_launcher.py b/tests/test_libvirt_launcher.py new file mode 100644 index 0000000..cafbc26 --- /dev/null +++ b/tests/test_libvirt_launcher.py @@ -0,0 +1,40 @@ +import unittest +import xml.etree.ElementTree as ET + +from scripts.libvirt_launcher import DomainConfig, build_domain_xml + + +class DomainUuidTest(unittest.TestCase): + def config(self, domain_uuid): + return DomainConfig( + name="instance-00000001", + uuid=domain_uuid, + mode="untrusted", + emulator="/usr/bin/qemu-system-x86_64", + memory_gib=64, + vcpus=16, + bios="/tmp/OVMF.fd", + kernel="/tmp/vmlinuz", + kernel_cmdline="console=ttyS0", + rootfs="/tmp/rootfs.img", + state_disk="/tmp/state.qcow2", + provider_config_disk="/tmp/provider.img", + guest_cid=10, + qgs_socket="/run/tdx-qgs/qgs.socket", + mac_address="52:54:00:77:00:0a", + netdev_mode="user", + ) + + def test_nova_uuid_is_written_to_domain_xml(self): + expected = "ed89b02d-ad43-441b-9923-8f1eb0484f91" + root = ET.fromstring(build_domain_xml(self.config(expected))) + self.assertEqual(expected, root.findtext("uuid")) + self.assertEqual("instance-00000001", root.findtext("name")) + + def test_invalid_uuid_is_rejected(self): + with self.assertRaisesRegex(ValueError, "invalid domain UUID"): + build_domain_xml(self.config("------------------------------------")) + + +if __name__ == "__main__": + unittest.main()