From d39901eac9caa53ad8fcaf18823ea981fdd49f0b Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 4 Aug 2026 06:38:15 -0500 Subject: [PATCH 01/28] draft --- README.md | 20 +- scripts/libvirt_launcher.py | 724 ++++++++++++++++++++++++ scripts/start_super_protocol_libvirt.sh | 339 +++++++++++ 3 files changed, 1082 insertions(+), 1 deletion(-) create mode 100755 scripts/libvirt_launcher.py create mode 100755 scripts/start_super_protocol_libvirt.sh diff --git a/README.md b/README.md index 3fc2b38..7dcd0a5 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Utilities for bootstrapping a Confidential Computing host (Intel **TDX** or AMD | `scripts/bootstrap_tdx.sh` | Turn an Ubuntu host into a TDX-capable hypervisor (kernel, QEMU, OVMF, attestation, GPU passthrough). | | `scripts/bootstrap_snp.sh` | Turn an Ubuntu host into a SEV-SNP-capable hypervisor (firmware, modules, GPU passthrough). | | `scripts/start_super_protocol.sh` | Start a confidential VM (TDX / SEV-SNP / untrusted) from a Super Protocol release image. | +| `scripts/start_super_protocol_libvirt.sh` | Start the same VM as a transient `qemu:///system` domain through libvirt-python (Ubuntu 26.04+). | | `scripts/swarm-cluster.sh` | Bring up a 3-node Swarm cluster on a single host. | | `scripts/check_configuration.sh`, `get_super_running_vms.sh` | Auxiliary tooling. | @@ -35,6 +36,23 @@ This is the main path: take a bare Ubuntu host, turn it into a confidential hype For the exact commands to clone the repository, run the bootstrap scripts, and launch a VM, see [docs/swarm.md](docs/swarm.md). +### Libvirt launcher (Ubuntu 26.04+) + +`scripts/start_super_protocol_libvirt.sh` reuses the release, disk, provider-config, and VFIO preparation from the direct QEMU launcher, then builds domain XML and starts a transient domain through `libvirt-python`. It requires `libvirt-daemon-system`, `libvirt-clients`, `python3-libvirt`, and `passt`. GPU passthrough uses IOMMUFD and therefore requires libvirt **12.1.0 or newer**; the launcher checks the daemon and domain capabilities before binding devices or recreating disks. + +The command line is the same as for `start_super_protocol.sh`, with an optional domain name: + +```bash +sudo ./scripts/start_super_protocol_libvirt.sh \ + --name super-protocol-3 \ + --provider_config /path/to/provider-configs \ + --mode tdx +``` + +The default cache is `/var/lib/libvirt/images/superprotocol`, so the non-root QEMU process used by `qemu:///system` can access the images. A custom `--cache` or `--build_dir` must likewise be traversable by the configured libvirt QEMU user. + +With `--debug false` the command returns after the domain starts. With `--debug true --log_file /path/to/boot.log`, it attaches a bidirectional serial console and copies console output to the log; `Ctrl-C` or `Ctrl-]` detaches without stopping the VM. Use `virsh -c qemu:///system list`, `console`, `shutdown`, or `destroy` to manage it. `--gpu none` disables GPU, NVSwitch, and CX7 passthrough for diagnostics. + ### 1. Clone the repo Clone the repository onto the target host. See [docs/swarm.md](docs/swarm.md) for the exact command. @@ -157,4 +175,4 @@ Planned hardware support. These items are **not yet supported** and are listed f ## License -See [LICENSE](LICENSE). \ No newline at end of file +See [LICENSE](LICENSE). diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py new file mode 100755 index 0000000..071fc7b --- /dev/null +++ b/scripts/libvirt_launcher.py @@ -0,0 +1,724 @@ +#!/usr/bin/env python3 +"""Build and launch a Super Protocol VM through libvirt. + +The XML builder intentionally has no dependency on python-libvirt so it can be +unit tested on development machines. The binding is imported only when a VM +is actually launched. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import select +import sys +import termios +import threading +import tty +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace +from typing import Any, BinaryIO, Iterable, Optional + + +QEMU_NS = "http://libvirt.org/schemas/domain/qemu/1.0" +LIBVIRT_IOMMUFD_VERSION = 12_001_000 +DOMAIN_NAME_RE = re.compile(r"^[A-Za-z0-9_.+:-]+$") +BDF_RE = re.compile( + r"^(?:(?P[0-9A-Fa-f]{4}):)?" + r"(?P[0-9A-Fa-f]{2}):(?P[0-9A-Fa-f]{2})\." + r"(?P[0-7])$" +) +MAC_RE = re.compile(r"^(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}$") + + +@dataclass(frozen=True) +class HostDevice: + kind: str + bdf: str + + +@dataclass +class DomainConfig: + name: str + mode: str + emulator: str + memory_gib: int + vcpus: int + bios: str + kernel: str + kernel_cmdline: str + rootfs: str + state_disk: str + provider_config_disk: str + guest_cid: int + qgs_cid: int + mac_address: str + netdev_mode: str + debug: bool = False + log_file: Optional[str] = None + cpu_model: Optional[str] = None + phys_bits: Optional[int] = None + cbitpos: Optional[int] = None + bridge: Optional[str] = None + tap_iface: Optional[str] = None + ip_address: str = "0.0.0.0" + ssh_port: Optional[int] = None + wg_port: Optional[int] = None + http_port: Optional[int] = None + https_port: Optional[int] = None + pki_port: Optional[int] = None + pki_vm_measure_port: Optional[int] = None + swarm_db_gossip_port: Optional[int] = None + dns_port: Optional[int] = None + host_devices: list[HostDevice] = field(default_factory=list) + + +def _sub(parent: ET.Element, tag: str, text: Optional[str] = None, **attrs: Any) -> ET.Element: + element = ET.SubElement( + parent, + tag, + {key.rstrip("_"): str(value) for key, value in attrs.items() if value is not None}, + ) + if text is not None: + element.text = str(text) + return element + + +def _parse_bdf(bdf: str) -> dict[str, str]: + match = BDF_RE.fullmatch(bdf) + if not match: + raise ValueError(f"invalid PCI BDF: {bdf!r}") + parts = match.groupdict(default="0000") + return { + "domain": f"0x{parts['domain'].lower()}", + "bus": f"0x{parts['bus'].lower()}", + "slot": f"0x{parts['slot'].lower()}", + "function": f"0x{parts['function'].lower()}", + } + + +def _validate_config(config: DomainConfig) -> None: + if not DOMAIN_NAME_RE.fullmatch(config.name): + raise ValueError( + "domain name may contain only letters, digits, '.', '_', '+', ':', and '-'" + ) + if config.mode not in {"untrusted", "tdx", "sev-snp"}: + raise ValueError(f"unsupported VM mode: {config.mode}") + if config.netdev_mode not in {"user", "tap"}: + raise ValueError(f"unsupported network mode: {config.netdev_mode}") + if config.memory_gib < 1 or config.vcpus < 1: + raise ValueError("memory and vCPU count must be positive") + if config.guest_cid < 3 or config.qgs_cid < 2: + raise ValueError("guest CID must be >= 3 and QGS CID must be >= 2") + if not MAC_RE.fullmatch(config.mac_address): + raise ValueError(f"invalid MAC address: {config.mac_address!r}") + for path in ( + config.emulator, + config.bios, + config.kernel, + config.rootfs, + config.state_disk, + config.provider_config_disk, + ): + if not Path(path).is_absolute(): + raise ValueError(f"libvirt resource path must be absolute: {path!r}") + ports = ( + config.ssh_port, + config.wg_port, + config.http_port, + config.https_port, + config.pki_port, + config.pki_vm_measure_port, + config.swarm_db_gossip_port, + config.dns_port, + ) + if any(port is not None and not 1 <= port <= 65535 for port in ports): + raise ValueError("network ports must be between 1 and 65535") + if config.netdev_mode == "tap" and (not config.bridge or not config.tap_iface): + raise ValueError("tap mode requires bridge and tap interface") + if config.mode == "sev-snp": + if config.cbitpos is None or config.phys_bits is None or not config.cpu_model: + raise ValueError("SEV-SNP requires cbitpos, phys_bits, and cpu_model") + if config.debug and not config.log_file: + raise ValueError("debug mode requires a log file") + for device in config.host_devices: + if device.kind not in {"gpu", "aux"}: + raise ValueError(f"unsupported host device kind: {device.kind}") + _parse_bdf(device.bdf) + + +def _add_disk( + devices: ET.Element, + path: str, + target: str, + image_format: str, + readonly: bool, +) -> None: + disk = _sub(devices, "disk", type="file", device="disk") + _sub(disk, "driver", name="qemu", type=image_format) + _sub(disk, "source", file=path) + _sub(disk, "target", dev=target, bus="virtio") + if readonly: + _sub(disk, "readonly") + + +def _add_port_forward( + interface: ET.Element, + protocol: str, + host_port: Optional[int], + guest_port: int, + address: Optional[str] = None, +) -> None: + if host_port is None: + return + attrs: dict[str, Any] = {"proto": protocol} + if address and address != "0.0.0.0": + attrs["address"] = address + forward = _sub(interface, "portForward", **attrs) + range_attrs: dict[str, Any] = {"start": host_port} + if host_port != guest_port: + range_attrs["to"] = guest_port + _sub(forward, "range", **range_attrs) + + +def _add_passt_interface( + devices: ET.Element, + config: DomainConfig, + *, + debug_only: bool = False, +) -> ET.Element: + interface = _sub(devices, "interface", type="user") + _sub(interface, "backend", type="passt") + if not debug_only: + _sub(interface, "mac", address=config.mac_address) + _sub(interface, "model", type="virtio") + + if debug_only: + _add_port_forward(interface, "tcp", config.ssh_port, 22, "127.0.0.1") + return interface + + _add_port_forward(interface, "tcp", config.http_port, 80, config.ip_address) + _add_port_forward(interface, "tcp", config.https_port, 443, config.ip_address) + _add_port_forward(interface, "tcp", config.pki_port, 9443, config.ip_address) + _add_port_forward( + interface, + "tcp", + config.pki_vm_measure_port, + 9180, + config.ip_address, + ) + _add_port_forward(interface, "udp", config.wg_port, 51820, config.ip_address) + _add_port_forward( + interface, + "udp", + config.swarm_db_gossip_port, + 7946, + config.ip_address, + ) + _add_port_forward( + interface, + "tcp", + config.swarm_db_gossip_port, + 7946, + config.ip_address, + ) + _add_port_forward(interface, "udp", config.dns_port, 53, config.ip_address) + _add_port_forward(interface, "tcp", config.dns_port, 53, config.ip_address) + if config.debug: + _add_port_forward(interface, "tcp", config.ssh_port, 22, "127.0.0.1") + return interface + + +def _add_network(devices: ET.Element, config: DomainConfig) -> None: + if config.netdev_mode == "user": + _add_passt_interface(devices, config) + return + + interface = _sub(devices, "interface", type="ethernet") + _sub(interface, "mac", address=config.mac_address) + _sub(interface, "target", dev=config.tap_iface, managed="no") + _sub(interface, "model", type="virtio") + if config.debug: + _add_passt_interface(devices, config, debug_only=True) + + +def _add_host_devices(devices: ET.Element, config: DomainConfig) -> None: + for index, host_device in enumerate(config.host_devices, start=1): + controller = _sub( + devices, + "controller", + type="pci", + index=index, + model="pcie-root-port", + ) + _sub(controller, "target", chassis=index, port=hex(0x0F + index)) + + hostdev = _sub(devices, "hostdev", mode="subsystem", type="pci", managed="no") + _sub(hostdev, "driver", name="vfio", iommufd="yes") + source = _sub(hostdev, "source") + _sub(source, "address", **_parse_bdf(host_device.bdf)) + if host_device.kind == "gpu": + _sub(hostdev, "rom", bar="off") + _sub( + hostdev, + "address", + type="pci", + domain="0x0000", + bus=hex(index), + slot="0x00", + function="0x0", + ) + + +def _add_cpu_and_features(domain: ET.Element, config: DomainConfig) -> None: + features = _sub(domain, "features") + _sub(features, "acpi") + if config.mode in {"untrusted", "tdx"}: + _sub(features, "ioapic", driver="qemu") + if config.mode == "untrusted": + _sub(features, "pmu", state="off") + if config.mode == "sev-snp": + _sub(features, "vmport", state="off") + + if config.mode == "sev-snp": + cpu = _sub(domain, "cpu", mode="custom", match="exact", check="none") + _sub(cpu, "model", config.cpu_model, fallback="forbid") + _sub(cpu, "maxphysaddr", mode="emulate", bits=config.phys_bits) + else: + cpu = _sub(domain, "cpu", mode="host-passthrough", migratable="off") + if config.mode == "untrusted": + _sub(cpu, "feature", policy="disable", name="kvm-steal-time") + _sub( + cpu, + "topology", + sockets="1", + dies="1", + clusters="1", + cores=config.vcpus, + threads="1", + ) + + +def _add_tdx_qemu_args(domain: ET.Element, config: DomainConfig) -> None: + ET.register_namespace("qemu", QEMU_NS) + commandline = _sub(domain, f"{{{QEMU_NS}}}commandline") + _sub(commandline, f"{{{QEMU_NS}}}arg", value="-object") + _sub( + commandline, + f"{{{QEMU_NS}}}arg", + value=f"memory-backend-ram,id=sp-mem,size={config.memory_gib}G", + ) + tdx_object = { + "qom-type": "tdx-guest", + "id": "sp-tdx", + "quote-generation-socket": { + "type": "vsock", + "cid": str(config.qgs_cid), + "port": "4050", + }, + } + _sub(commandline, f"{{{QEMU_NS}}}arg", value="-object") + _sub( + commandline, + f"{{{QEMU_NS}}}arg", + value=json.dumps(tdx_object, separators=(",", ":")), + ) + _sub(commandline, f"{{{QEMU_NS}}}arg", value="-machine") + _sub( + commandline, + f"{{{QEMU_NS}}}arg", + value="confidential-guest-support=sp-tdx,memory-backend=sp-mem", + ) + + +def build_domain_xml(config: DomainConfig) -> str: + """Return a complete transient libvirt domain definition.""" + _validate_config(config) + + domain = ET.Element("domain", {"type": "kvm"}) + _sub(domain, "name", config.name) + _sub(domain, "memory", config.memory_gib, unit="GiB") + _sub(domain, "currentMemory", config.memory_gib, unit="GiB") + _sub(domain, "vcpu", config.vcpus, placement="static") + + os_element = _sub(domain, "os") + _sub(os_element, "type", "hvm", arch="x86_64", machine="q35") + _sub(os_element, "loader", config.bios, readonly="yes", type="rom") + _sub(os_element, "kernel", config.kernel) + _sub(os_element, "cmdline", config.kernel_cmdline) + + _add_cpu_and_features(domain, config) + _sub(domain, "clock", offset="utc") + _sub(domain, "on_poweroff", "destroy") + _sub(domain, "on_reboot", "restart") + _sub(domain, "on_crash", "destroy") + + sysinfo = _sub(domain, "sysinfo", type="fwcfg") + _sub(sysinfo, "entry", "262144", name="opt/ovmf/X-PciMmio64") + + devices = _sub(domain, "devices") + _sub(devices, "emulator", config.emulator) + _add_disk(devices, config.rootfs, "vda", "raw", True) + _add_disk(devices, config.state_disk, "vdb", "qcow2", False) + _add_disk(devices, config.provider_config_disk, "vdc", "raw", True) + _sub(devices, "controller", type="pci", index="0", model="pcie-root") + _sub(devices, "controller", type="usb", model="none") + _add_network(devices, config) + + serial = _sub(devices, "serial", type="pty") + _sub(serial, "target", type="isa-serial", port="0") + console = _sub(devices, "console", type="pty") + _sub(console, "target", type="serial", port="0") + video = _sub(devices, "video") + _sub(video, "model", type="none") + _sub(devices, "audio", id="1", type="none") + _sub(devices, "memballoon", model="none") + vsock = _sub(devices, "vsock", model="virtio") + _sub(vsock, "cid", auto="no", address=config.guest_cid) + _add_host_devices(devices, config) + + if config.mode == "sev-snp": + launch_security = _sub( + domain, + "launchSecurity", + type="sev-snp", + kernelHashes="yes", + ) + _sub(launch_security, "cbitpos", config.cbitpos) + _sub(launch_security, "reducedPhysBits", "1") + _sub(launch_security, "policy", "0x30000") + elif config.mode == "tdx": + _add_tdx_qemu_args(domain, config) + + ET.indent(domain, space=" ") + return ET.tostring(domain, encoding="unicode") + + +def _version_string(version: int) -> str: + return f"{version // 1_000_000}.{(version // 1_000) % 1_000}.{version % 1_000}" + + +def _iommufd_advertised(domain_capabilities: str) -> bool: + try: + root = ET.fromstring(domain_capabilities) + except ET.ParseError: + return False + for enum in root.findall(".//devices/hostdev/enum[@name='iommufd']"): + if any((value.text or "").strip() == "yes" for value in enum.findall("value")): + return True + return False + + +def check_connection_capabilities(conn: Any, config: Any) -> None: + if conn.getType().upper() != "QEMU": + raise RuntimeError(f"qemu:///system returned unexpected driver {conn.getType()!r}") + if not config.host_devices: + return + + version = conn.getLibVersion() + if version < LIBVIRT_IOMMUFD_VERSION: + raise RuntimeError( + "GPU passthrough requires libvirt >= 12.1.0; " + f"the daemon reports {_version_string(version)}" + ) + try: + capabilities = conn.getDomainCapabilities( + config.emulator, + "x86_64", + "q35", + "kvm", + 0, + ) + except Exception as exc: + raise RuntimeError(f"failed to query libvirt domain capabilities: {exc}") from exc + if not _iommufd_advertised(capabilities): + raise RuntimeError( + "libvirt domain capabilities do not advertise hostdev iommufd support" + ) + + +def ensure_domain_name_available(conn: Any, libvirt_module: Any, name: str) -> None: + try: + conn.lookupByName(name) + except libvirt_module.libvirtError as exc: + if exc.get_error_code() == libvirt_module.VIR_ERR_NO_DOMAIN: + return + raise RuntimeError(f"failed to check domain name {name!r}: {exc}") from exc + raise RuntimeError( + f"a libvirt domain named {name!r} already exists; stop it or choose --name" + ) + + +def preflight_connection(emulator: str, name: str, require_iommufd: bool) -> None: + """Check the daemon, domain name, and optional IOMMUFD support without mutation.""" + try: + import libvirt # type: ignore + except ImportError as exc: + raise RuntimeError( + "python3-libvirt is not installed; install the Ubuntu package " + "'python3-libvirt'" + ) from exc + + try: + conn = libvirt.open("qemu:///system") + except libvirt.libvirtError as exc: + raise RuntimeError(f"failed to connect to qemu:///system: {exc}") from exc + if conn is None: + raise RuntimeError("failed to connect to qemu:///system") + try: + probe = SimpleNamespace( + emulator=str(Path(emulator).resolve()), + host_devices=[object()] if require_iommufd else [], + ) + check_connection_capabilities(conn, probe) + ensure_domain_name_available(conn, libvirt, name) + finally: + conn.close() + + +def _write_console_output(data: bytes, log: BinaryIO) -> None: + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() + log.write(data) + log.flush() + + +def attach_serial_console(conn: Any, domain: Any, libvirt_module: Any, log_path: str) -> None: + """Attach a bidirectional console; Ctrl-C or Ctrl-] only detaches.""" + stream = conn.newStream(0) + domain.openConsole(None, stream, 0) + stopped = threading.Event() + receiver_error: list[BaseException] = [] + log_file = open(log_path, "ab", buffering=0) + + def receive() -> None: + try: + while not stopped.is_set(): + chunk = stream.recv(65536) + if not chunk: + break + _write_console_output(chunk, log_file) + except BaseException as exc: # propagated after terminal restoration + if not stopped.is_set(): + receiver_error.append(exc) + finally: + stopped.set() + + receiver = threading.Thread(target=receive, name="libvirt-console-recv", daemon=True) + receiver.start() + + stdin_fd = sys.stdin.fileno() + old_terminal = None + if os.isatty(stdin_fd): + old_terminal = termios.tcgetattr(stdin_fd) + tty.setraw(stdin_fd) + + print( + "\nConnected to serial console. Press Ctrl-C or Ctrl-] to detach; " + "the VM will keep running.\r", + file=sys.stderr, + ) + try: + while not stopped.is_set(): + readable, _, _ = select.select([stdin_fd], [], [], 0.25) + if not readable: + continue + data = os.read(stdin_fd, 4096) + if not data: + break + if b"\x03" in data or b"\x1d" in data: + break + sent = 0 + while sent < len(data): + sent += stream.send(data[sent:]) + except KeyboardInterrupt: + pass + finally: + stopped.set() + if old_terminal is not None: + termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_terminal) + try: + stream.abort() + except libvirt_module.libvirtError: + pass + receiver.join(timeout=1) + log_file.close() + try: + active = bool(domain.isActive()) + except libvirt_module.libvirtError: + active = False + if active: + message = f"Detached from {domain.name()}; VM is still managed by libvirt." + else: + message = "Serial console closed because the VM stopped." + print(f"\n{message}", file=sys.stderr) + if receiver_error and active: + raise RuntimeError(f"serial console failed: {receiver_error[0]}") + + +def launch(config: DomainConfig) -> None: + try: + import libvirt # type: ignore + except ImportError as exc: + raise RuntimeError( + "python3-libvirt is not installed; install the Ubuntu package " + "'python3-libvirt'" + ) from exc + + try: + conn = libvirt.open("qemu:///system") + except libvirt.libvirtError as exc: + raise RuntimeError(f"failed to connect to qemu:///system: {exc}") from exc + if conn is None: + raise RuntimeError("failed to connect to qemu:///system") + try: + try: + check_connection_capabilities(conn, config) + ensure_domain_name_available(conn, libvirt, config.name) + xml = build_domain_xml(config) + flags = getattr(libvirt, "VIR_DOMAIN_START_VALIDATE", 0) + domain = conn.createXML(xml, flags) + if domain is None: + raise RuntimeError("libvirt did not return a domain after createXML()") + name = domain.name() + uuid = domain.UUIDString() + print(f"Started transient libvirt domain: {name} ({uuid})") + print(f" console: virsh -c qemu:///system console {name}") + print(f" shutdown: virsh -c qemu:///system shutdown {name}") + print(f" force stop: virsh -c qemu:///system destroy {name}") + if config.debug: + attach_serial_console(conn, domain, libvirt, str(config.log_file)) + except libvirt.libvirtError as exc: + raise RuntimeError(f"libvirt failed to start the domain: {exc}") from exc + finally: + conn.close() + + +def _host_device(value: str) -> HostDevice: + try: + kind, bdf = value.split(":", 1) + except ValueError as exc: + raise argparse.ArgumentTypeError("hostdev must be KIND:BDF") from exc + try: + _parse_bdf(bdf) + except ValueError as exc: + raise argparse.ArgumentTypeError(str(exc)) from exc + if kind not in {"gpu", "aux"}: + raise argparse.ArgumentTypeError("hostdev kind must be 'gpu' or 'aux'") + return HostDevice(kind, bdf) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--name", required=True) + parser.add_argument("--mode", choices=("untrusted", "tdx", "sev-snp"), required=True) + parser.add_argument("--emulator", required=True) + parser.add_argument("--memory-gib", type=int, required=True) + parser.add_argument("--vcpus", type=int, required=True) + parser.add_argument("--bios", required=True) + parser.add_argument("--kernel", required=True) + parser.add_argument("--kernel-cmdline", required=True) + parser.add_argument("--rootfs", required=True) + parser.add_argument("--state-disk", required=True) + parser.add_argument("--provider-config-disk", required=True) + parser.add_argument("--guest-cid", type=int, required=True) + parser.add_argument("--qgs-cid", type=int, required=True) + parser.add_argument("--mac-address", required=True) + parser.add_argument("--netdev-mode", choices=("user", "tap"), required=True) + parser.add_argument("--debug", action="store_true") + parser.add_argument("--log-file") + parser.add_argument("--cpu-model") + parser.add_argument("--phys-bits", type=int) + parser.add_argument("--cbitpos", type=int) + parser.add_argument("--bridge") + parser.add_argument("--tap-iface") + parser.add_argument("--ip-address", default="0.0.0.0") + parser.add_argument("--ssh-port", type=int) + parser.add_argument("--wg-port", type=int) + parser.add_argument("--http-port", type=int) + parser.add_argument("--https-port", type=int) + parser.add_argument("--pki-port", type=int) + parser.add_argument("--pki-vm-measure-port", type=int) + parser.add_argument("--swarm-db-gossip-port", type=int) + parser.add_argument("--dns-port", type=int) + parser.add_argument("--hostdev", type=_host_device, action="append", default=[]) + return parser + + +def _preflight_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Check libvirt before preparing VM resources") + parser.add_argument("--emulator", required=True) + parser.add_argument("--name", required=True) + parser.add_argument("--require-iommufd", action="store_true") + return parser + + +def _config_from_args(args: argparse.Namespace) -> DomainConfig: + return DomainConfig( + name=args.name, + mode=args.mode, + emulator=str(Path(args.emulator).resolve()), + memory_gib=args.memory_gib, + vcpus=args.vcpus, + bios=str(Path(args.bios).resolve()), + kernel=str(Path(args.kernel).resolve()), + kernel_cmdline=args.kernel_cmdline, + rootfs=str(Path(args.rootfs).resolve()), + state_disk=str(Path(args.state_disk).resolve()), + provider_config_disk=str(Path(args.provider_config_disk).resolve()), + guest_cid=args.guest_cid, + qgs_cid=args.qgs_cid, + mac_address=args.mac_address, + netdev_mode=args.netdev_mode, + debug=args.debug, + log_file=args.log_file, + cpu_model=args.cpu_model, + phys_bits=args.phys_bits, + cbitpos=args.cbitpos, + bridge=args.bridge, + tap_iface=args.tap_iface, + ip_address=args.ip_address, + ssh_port=args.ssh_port, + wg_port=args.wg_port, + http_port=args.http_port, + https_port=args.https_port, + pki_port=args.pki_port, + pki_vm_measure_port=args.pki_vm_measure_port, + swarm_db_gossip_port=args.swarm_db_gossip_port, + dns_port=args.dns_port, + host_devices=args.hostdev, + ) + + +def main(argv: Optional[Iterable[str]] = None) -> int: + arguments = list(argv) if argv is not None else sys.argv[1:] + if arguments[:1] == ["preflight"]: + args = _preflight_parser().parse_args(arguments[1:]) + try: + if not DOMAIN_NAME_RE.fullmatch(args.name): + raise ValueError( + "domain name may contain only letters, digits, '.', '_', '+', ':', and '-'" + ) + preflight_connection(args.emulator, args.name, args.require_iommufd) + except (RuntimeError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + args = _parser().parse_args(arguments) + try: + config = _config_from_args(args) + _validate_config(config) + launch(config) + except (RuntimeError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh new file mode 100755 index 0000000..d98618f --- /dev/null +++ b/scripts/start_super_protocol_libvirt.sh @@ -0,0 +1,339 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +BASE_SCRIPT="${SCRIPT_DIR}/start_super_protocol.sh" +LIBVIRT_LAUNCHER="${SCRIPT_DIR}/libvirt_launcher.py" + +if ! grep -q '^parse_args \$@$' "${BASE_SCRIPT}"; then + echo "Error: could not find the start marker in ${BASE_SCRIPT}" >&2 + exit 1 +fi + +# Reuse the release, validation, VFIO, and provider-config preparation code, +# but deliberately exclude the original entrypoint and direct QEMU execution. +# shellcheck disable=SC1090 +source <(sed '/^parse_args \$@$/,$d' "${BASE_SCRIPT}") + +# qemu:///system normally runs QEMU as libvirt-qemu, which cannot traverse +# /root. Keep the same --cache option, but use libvirt's image directory as the +# safe default for this launcher. +DEFAULT_CACHE="/var/lib/libvirt/images/superprotocol" +CACHE=${DEFAULT_CACHE} + +LIBVIRT_DOMAIN_NAME="" +BASE_ARGS=() + +usage_libvirt() { + usage + echo "Libvirt-specific options:" + echo " --name Transient domain name (default: super-protocol-)" + echo "" + echo "Runtime behavior:" + echo " --debug false Start in the background and return" + echo " --debug true Attach serial console and tee it to --log_file" +} + +extract_libvirt_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --name) + if [[ $# -lt 2 ]]; then + echo "Error: --name requires a value" >&2 + exit 1 + fi + LIBVIRT_DOMAIN_NAME=$2 + shift 2 + ;; + --help) + usage_libvirt + exit 0 + ;; + *) + BASE_ARGS+=("$1") + shift + ;; + esac + done +} + +check_target_os() { + local os_id os_version + os_id=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') + os_version=$(sed -n 's/^VERSION_ID=//p' /etc/os-release | tr -d '"') + if [[ "${os_id}" != "ubuntu" ]]; then + echo "Error: this launcher supports Ubuntu 26.04 or newer (found ${os_id:-unknown})." >&2 + exit 1 + fi + if ! dpkg --compare-versions "${os_version}" ge "26.04"; then + echo "Error: this launcher requires Ubuntu 26.04 or newer (found ${os_version:-unknown})." >&2 + exit 1 + fi +} + +check_libvirt_dependencies() { + local missing=() + command -v python3 >/dev/null 2>&1 || missing+=(python3) + command -v virsh >/dev/null 2>&1 || missing+=(libvirt-clients) + if ! python3 -c 'import libvirt' >/dev/null 2>&1; then + missing+=(python3-libvirt) + fi + if [[ "${NETDEV_MODE}" == "user" || "${DEBUG_MODE}" == "true" ]]; then + command -v passt >/dev/null 2>&1 || missing+=(passt) + fi + if [[ ${#missing[@]} -gt 0 ]]; then + echo "Error: missing libvirt runtime dependencies: ${missing[*]}" >&2 + echo "Install them with: apt-get install libvirt-daemon-system libvirt-clients python3-libvirt passt" >&2 + echo "GPU passthrough additionally requires libvirt >= 12.1.0." >&2 + exit 1 + fi + if [[ ! -x "${LIBVIRT_LAUNCHER}" ]]; then + echo "Error: launcher is not executable: ${LIBVIRT_LAUNCHER}" >&2 + exit 1 + fi +} + +preflight_libvirt() { + local require_iommufd=false + local gpu + for gpu in "${USED_GPUS[@]}"; do + if [[ "${gpu}" != "none" ]]; then + require_iommufd=true + break + fi + done + # With no explicit --gpu option, the base launcher selects all GPUs later. + if [[ ${#USED_GPUS[@]} -eq 0 ]]; then + if lspci -nnk -d 10de: 2>/dev/null | grep -qE '3D controller'; then + require_iommufd=true + fi + fi + + local args=(preflight --emulator "${QEMU_PATH}" --name "${LIBVIRT_DOMAIN_NAME}") + if [[ "${require_iommufd}" == "true" ]]; then + args+=(--require-iommufd) + fi + "${LIBVIRT_LAUNCHER}" "${args[@]}" +} + +scan_cx7_bridges() { + local dev_path dev_bdf vpd_file device_info + AVAILABLE_CX7_BRIDGES=() + for dev_path in /sys/bus/pci/devices/*/; do + [[ -e "${dev_path}" ]] || continue + dev_bdf=$(basename "${dev_path}") + vpd_file="${dev_path}vpd" + if [[ -f "${vpd_file}" ]] && grep -q "SW_MNG" "${vpd_file}" 2>/dev/null; then + device_info=$(lspci -s "${dev_bdf}" 2>/dev/null || true) + if [[ "${device_info}" == *"Mellanox"* && "${device_info}" == *"ConnectX-7"* ]]; then + AVAILABLE_CX7_BRIDGES+=("${dev_bdf#0000:}") + fi + fi + done +} + +prepare_selected_host_devices() { + HOSTDEV_ARGS=() + if [[ ${#USED_GPUS[@]} -eq 0 ]]; then + echo "GPU passthrough disabled; NVSwitch and CX7 companion devices will not be attached." + return + fi + + prepare_gpus_for_vfio "${USED_GPUS[@]}" + scan_cx7_bridges + + local device + for device in "${USED_GPUS[@]}"; do + HOSTDEV_ARGS+=(--hostdev "gpu:${device}") + done + for device in "${AVAILABLE_NVSWITCHES[@]}"; do + HOSTDEV_ARGS+=(--hostdev "aux:${device}") + done + for device in "${AVAILABLE_CX7_BRIDGES[@]}"; do + HOSTDEV_ARGS+=(--hostdev "aux:${device}") + done +} + +prepare_mode_parameters() { + SNP_VCPU_ARG="" + PHYS_BITS_ARG="" + CBITPOS_ARG="" + + case "${VM_MODE}" in + tdx) + if [[ -z "${TDX_SUPPORT}" ]]; then + echo "Error: TDX is not supported on this system" >&2 + exit 1 + fi + ;; + sev-snp) + if [[ -z "${SEV_SNP_SUPPORT}" ]]; then + echo "Error: SEV-SNP is not supported on this system" >&2 + exit 1 + fi + get_cbitpos + detect_snp_vCPU + detect_phys_bits + SNP_VCPU_ARG=${SNP_VCPU} + PHYS_BITS_ARG=${PHYS_BITS} + CBITPOS_ARG=${CBITPOS} + ;; + untrusted) + ;; + *) + echo "Error: invalid mode '${VM_MODE}'" >&2 + exit 1 + ;; + esac +} + +prepare_tap_network() { + if [[ "${NETDEV_MODE}" != "tap" ]]; then + return + fi + if [[ -z "${TAP_IFACE}" ]]; then + TAP_IFACE="sw-tap${BASE_NIC}" + fi + if ! ip link show "${BRIDGE}" >/dev/null 2>&1; then + echo "Error: bridge ${BRIDGE} does not exist. Run 'swarm-cluster.sh ensure-network' first." >&2 + exit 1 + fi + if ! ip link show "${TAP_IFACE}" >/dev/null 2>&1; then + ip tuntap add dev "${TAP_IFACE}" mode tap user root + fi + ip link set "${TAP_IFACE}" master "${BRIDGE}" + ip link set "${TAP_IFACE}" up +} + +build_kernel_cmdline() { + local clearcpuid=" " + local snp_additional="" + local rootfs_hash + rootfs_hash=$(<"${ROOTFS_HASH_PATH}") + + if [[ "${VM_MODE}" == "tdx" ]]; then + clearcpuid=" clearcpuid=mtrr " + elif [[ "${VM_MODE}" == "sev-snp" ]]; then + snp_additional=" build=${RELEASE} pci=realloc,nocrs" + fi + + KERNEL_CMD_LINE="root=LABEL=rootfs${clearcpuid}rootfs_verity.scheme=dm-verity rootfs_verity.hash=${rootfs_hash}${snp_additional}" + if [[ "${DEBUG_MODE}" == "true" ]]; then + KERNEL_CMD_LINE+=" console=ttyS0 systemd.log_level=trace systemd.log_target=log" + fi +} + +create_vm_disks() { + local provider_loop provider_mount + + rm -f "${STATE_DISK_PATH}" + qemu-img create -f qcow2 "${STATE_DISK_PATH}" "${STATE_DISK_SIZE}G" + + rm -f "${PROVIDER_CONFIG_DISK_PATH}" + dd if=/dev/zero of="${PROVIDER_CONFIG_DISK_PATH}" bs=1M count=1 status=none + mkfs.ext4 -q -O '^has_journal,^huge_file,^meta_bg,^ext_attr' \ + -L provider_config "${PROVIDER_CONFIG_DISK_PATH}" + provider_loop=$(losetup --find --show --partscan "${PROVIDER_CONFIG_DISK_PATH}") + provider_mount=$(mktemp -d) + + cleanup_provider_disk() { + if mountpoint -q "${provider_mount}"; then + umount "${provider_mount}" || true + fi + if [[ -n "${provider_loop}" ]]; then + losetup -d "${provider_loop}" 2>/dev/null || true + fi + rmdir "${provider_mount}" 2>/dev/null || true + } + trap cleanup_provider_disk RETURN + + mount "${provider_loop}" "${provider_mount}" + cp -a "${PROVIDER_CONFIG}/." "${provider_mount}/" + rm -rf "${provider_mount}/lost+found" + umount "${provider_mount}" + losetup -d "${provider_loop}" + provider_loop="" + rmdir "${provider_mount}" + trap - RETURN +} + +append_optional_arg() { + local flag=$1 value=$2 + if [[ -n "${value}" ]]; then + LAUNCH_ARGS+=("${flag}" "${value}") + fi +} + +launch_with_libvirt() { + LAUNCH_ARGS=( + "${LIBVIRT_LAUNCHER}" + --name "${LIBVIRT_DOMAIN_NAME}" + --mode "${VM_MODE}" + --emulator "${QEMU_PATH}" + --memory-gib "${VM_RAM}" + --vcpus "${VM_CPU}" + --bios "${BIOS_PATH}" + --kernel "${KERNEL_PATH}" + --kernel-cmdline "${KERNEL_CMD_LINE}" + --rootfs "${IMAGE_PATH}" + --state-disk "${STATE_DISK_PATH}" + --provider-config-disk "${PROVIDER_CONFIG_DISK_PATH}" + --guest-cid "${GUEST_CID}" + --qgs-cid "${BASE_CID}" + --mac-address "${MAC_ADDRESS}" + --netdev-mode "${NETDEV_MODE}" + --ip-address "${IP_ADDRESS}" + --ssh-port "${SSH_PORT}" + --wg-port "${WG_PORT}" + --swarm-db-gossip-port "${SWARM_DB_GOSSIP_PORT}" + --dns-port "${DNS_PORT}" + ) + append_optional_arg --http-port "${HTTP_PORT}" + append_optional_arg --https-port "${HTTPS_PORT}" + append_optional_arg --pki-port "${PKI_PORT}" + append_optional_arg --pki-vm-measure-port "${PKI_VM_MEASURE_PORT}" + append_optional_arg --cpu-model "${SNP_VCPU_ARG}" + append_optional_arg --phys-bits "${PHYS_BITS_ARG}" + append_optional_arg --cbitpos "${CBITPOS_ARG}" + + if [[ "${NETDEV_MODE}" == "tap" ]]; then + LAUNCH_ARGS+=(--bridge "${BRIDGE}" --tap-iface "${TAP_IFACE}") + fi + if [[ "${DEBUG_MODE}" == "true" ]]; then + mkdir -p "$(dirname "${LOG_FILE}")" + LAUNCH_ARGS+=(--debug --log-file "${LOG_FILE}") + fi + LAUNCH_ARGS+=("${HOSTDEV_ARGS[@]}") + + echo "Starting ${LIBVIRT_DOMAIN_NAME} through qemu:///system (mode=${VM_MODE}, debug=${DEBUG_MODE})" + "${LAUNCH_ARGS[@]}" +} + +main_libvirt() { + check_target_os + check_packages + check_libvirt_dependencies + find_qemu_path + check_qemu_version + preflight_libvirt + check_params + prepare_selected_host_devices + prepare_mode_parameters + + mkdir -p "${CACHE}" + download_release "${RELEASE}" "${RELEASE_ASSET}" "${CACHE}" "${RELEASE_REPO}" + parse_and_download_release_files "${RELEASE_FILEPATH}" + prepare_tap_network + build_kernel_cmdline + create_vm_disks + launch_with_libvirt +} + +extract_libvirt_args "$@" +parse_args "${BASE_ARGS[@]}" +detect_cpu_type +if [[ -z "${LIBVIRT_DOMAIN_NAME}" ]]; then + LIBVIRT_DOMAIN_NAME="super-protocol-${GUEST_CID}" +fi +main_libvirt From 797763f99ece98eb7af9608109f225e000f1099c Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 4 Aug 2026 08:26:20 -0500 Subject: [PATCH 02/28] fix firmware name --- scripts/libvirt_launcher.py | 28 +++- tests/test_libvirt_launcher.py | 253 +++++++++++++++++++++++++++++++++ 2 files changed, 276 insertions(+), 5 deletions(-) create mode 100644 tests/test_libvirt_launcher.py diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py index 071fc7b..d949ec7 100755 --- a/scripts/libvirt_launcher.py +++ b/scripts/libvirt_launcher.py @@ -303,9 +303,29 @@ def _add_cpu_and_features(domain: ET.Element, config: DomainConfig) -> None: ) -def _add_tdx_qemu_args(domain: ET.Element, config: DomainConfig) -> None: +def _qemu_commandline(domain: ET.Element) -> ET.Element: ET.register_namespace("qemu", QEMU_NS) - commandline = _sub(domain, f"{{{QEMU_NS}}}commandline") + commandline = domain.find(f"{{{QEMU_NS}}}commandline") + if commandline is None: + commandline = _sub(domain, f"{{{QEMU_NS}}}commandline") + return commandline + + +def _add_fw_cfg_qemu_args(domain: ET.Element) -> None: + # Libvirt deliberately rejects opt/ovmf/* through native fwcfg XML because + # that namespace is reserved for OVMF. The direct launcher needs this + # existing OVMF knob, so pass it through QEMU's command line namespace. + commandline = _qemu_commandline(domain) + _sub(commandline, f"{{{QEMU_NS}}}arg", value="-fw_cfg") + _sub( + commandline, + f"{{{QEMU_NS}}}arg", + value="name=opt/ovmf/X-PciMmio64,string=262144", + ) + + +def _add_tdx_qemu_args(domain: ET.Element, config: DomainConfig) -> None: + commandline = _qemu_commandline(domain) _sub(commandline, f"{{{QEMU_NS}}}arg", value="-object") _sub( commandline, @@ -357,9 +377,6 @@ def build_domain_xml(config: DomainConfig) -> str: _sub(domain, "on_reboot", "restart") _sub(domain, "on_crash", "destroy") - sysinfo = _sub(domain, "sysinfo", type="fwcfg") - _sub(sysinfo, "entry", "262144", name="opt/ovmf/X-PciMmio64") - devices = _sub(domain, "devices") _sub(devices, "emulator", config.emulator) _add_disk(devices, config.rootfs, "vda", "raw", True) @@ -380,6 +397,7 @@ def build_domain_xml(config: DomainConfig) -> str: vsock = _sub(devices, "vsock", model="virtio") _sub(vsock, "cid", auto="no", address=config.guest_cid) _add_host_devices(devices, config) + _add_fw_cfg_qemu_args(domain) if config.mode == "sev-snp": launch_security = _sub( diff --git a/tests/test_libvirt_launcher.py b/tests/test_libvirt_launcher.py new file mode 100644 index 0000000..1e2bcab --- /dev/null +++ b/tests/test_libvirt_launcher.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 + +import re +import sys +import unittest +import xml.etree.ElementTree as ET +from pathlib import Path +from unittest import mock + + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import libvirt_launcher as launcher # noqa: E402 + + +class DomainXMLTests(unittest.TestCase): + def config(self, mode="untrusted", netdev_mode="user", **overrides): + values = dict( + name="super-protocol-3", + mode=mode, + emulator="/usr/bin/qemu-system-x86_64", + memory_gib=16, + vcpus=8, + bios="/var/lib/super protocol/bios.fd", + kernel="/var/lib/super protocol/vmlinuz", + kernel_cmdline="root=LABEL=rootfs hash=a&b", + rootfs="/var/lib/super protocol/rootfs.img", + state_disk="/var/lib/super protocol/state.qcow2", + provider_config_disk="/var/lib/super protocol/provider.img", + guest_cid=3, + qgs_cid=2, + mac_address="52:54:00:12:34:56", + netdev_mode=netdev_mode, + debug=False, + ssh_port=2222, + wg_port=51821, + http_port=8080, + https_port=8443, + pki_port=9443, + pki_vm_measure_port=9181, + swarm_db_gossip_port=17946, + dns_port=1053, + ) + if mode == "sev-snp": + values.update(cpu_model="EPYC-v4", phys_bits=48, cbitpos=51) + values.update(overrides) + return launcher.DomainConfig(**values) + + def root(self, config): + return ET.fromstring(launcher.build_domain_xml(config)) + + def test_untrusted_cpu_and_direct_boot(self): + root = self.root(self.config()) + self.assertEqual(root.findtext("name"), "super-protocol-3") + self.assertEqual(root.findtext("os/kernel"), "/var/lib/super protocol/vmlinuz") + self.assertEqual(root.findtext("os/cmdline"), "root=LABEL=rootfs hash=a&b") + self.assertEqual(root.find("cpu").get("mode"), "host-passthrough") + self.assertIsNotNone(root.find("features/ioapic[@driver='qemu']")) + self.assertIsNotNone(root.find("features/pmu[@state='off']")) + self.assertIsNotNone(root.find("cpu/feature[@name='kvm-steal-time']")) + self.assertIsNone(root.find("launchSecurity")) + + def test_reserved_ovmf_fw_cfg_uses_qemu_commandline(self): + root = self.root(self.config()) + self.assertIsNone(root.find("sysinfo[@type='fwcfg']")) + args = [ + element.get("value") + for element in root.findall( + f"{{{launcher.QEMU_NS}}}commandline/{{{launcher.QEMU_NS}}}arg" + ) + ] + self.assertEqual( + args, + ["-fw_cfg", "name=opt/ovmf/X-PciMmio64,string=262144"], + ) + + def test_sev_snp_launch_security(self): + root = self.root(self.config(mode="sev-snp")) + launch_security = root.find("launchSecurity") + self.assertEqual(launch_security.get("type"), "sev-snp") + self.assertEqual(launch_security.get("kernelHashes"), "yes") + self.assertEqual(launch_security.findtext("cbitpos"), "51") + self.assertEqual(launch_security.findtext("reducedPhysBits"), "1") + self.assertEqual(launch_security.findtext("policy"), "0x30000") + self.assertEqual(root.findtext("cpu/model"), "EPYC-v4") + self.assertEqual(root.find("cpu/maxphysaddr").get("bits"), "48") + self.assertIsNotNone(root.find("features/vmport[@state='off']")) + + def test_tdx_vsock_qgs_uses_qemu_namespace(self): + root = self.root(self.config(mode="tdx")) + args = [ + element.get("value") + for element in root.findall(f"{{{launcher.QEMU_NS}}}commandline/{{{launcher.QEMU_NS}}}arg") + ] + self.assertIn("memory-backend-ram,id=sp-mem,size=16G", args) + tdx_arg = next(value for value in args if '"qom-type":"tdx-guest"' in value) + self.assertIn('"type":"vsock"', tdx_arg) + self.assertIn('"cid":"2"', tdx_arg) + self.assertIn('"port":"4050"', tdx_arg) + self.assertIn( + "confidential-guest-support=sp-tdx,memory-backend=sp-mem", + args, + ) + + def test_user_network_uses_passt_and_all_forwards(self): + root = self.root(self.config(debug=True, log_file="/tmp/serial.log")) + interface = root.find("devices/interface[@type='user']") + self.assertEqual(interface.find("backend").get("type"), "passt") + forwards = { + ( + element.get("proto"), + element.get("address"), + element.find("range").get("start"), + element.find("range").get("to"), + ) + for element in interface.findall("portForward") + } + self.assertIn(("tcp", None, "8080", "80"), forwards) + self.assertIn(("udp", None, "51821", "51820"), forwards) + self.assertIn(("tcp", "127.0.0.1", "2222", "22"), forwards) + self.assertIn(("udp", None, "1053", "53"), forwards) + self.assertIn(("tcp", None, "1053", "53"), forwards) + + def test_tap_debug_has_precreated_tap_and_secondary_passt(self): + config = self.config( + netdev_mode="tap", + bridge="swarmbr0", + tap_iface="sw-tap7", + debug=True, + log_file="/tmp/serial.log", + ) + root = self.root(config) + interfaces = root.findall("devices/interface") + self.assertEqual(len(interfaces), 2) + self.assertEqual(interfaces[0].get("type"), "ethernet") + self.assertEqual(interfaces[0].find("target").get("dev"), "sw-tap7") + self.assertEqual(interfaces[0].find("target").get("managed"), "no") + self.assertEqual(interfaces[1].find("backend").get("type"), "passt") + ssh_range = interfaces[1].find("portForward/range") + self.assertEqual((ssh_range.get("start"), ssh_range.get("to")), ("2222", "22")) + + def test_host_devices_get_iommufd_and_separate_root_ports(self): + config = self.config( + host_devices=[ + launcher.HostDevice("gpu", "65:00.0"), + launcher.HostDevice("aux", "0000:66:00.1"), + ] + ) + root = self.root(config) + hostdevs = root.findall("devices/hostdev") + root_ports = root.findall("devices/controller[@model='pcie-root-port']") + self.assertEqual(len(hostdevs), 2) + self.assertEqual(len(root_ports), 2) + self.assertTrue(all(item.get("managed") == "no" for item in hostdevs)) + self.assertTrue( + all(item.find("driver").get("iommufd") == "yes" for item in hostdevs) + ) + self.assertIsNotNone(hostdevs[0].find("rom[@bar='off']")) + self.assertIsNone(hostdevs[1].find("rom")) + self.assertEqual(hostdevs[1].find("source/address").get("function"), "0x1") + + def test_gpu_none_produces_no_hostdev_or_extra_root_port(self): + root = self.root(self.config(host_devices=[])) + self.assertEqual(root.findall("devices/hostdev"), []) + self.assertEqual(root.findall("devices/controller[@model='pcie-root-port']"), []) + + def test_invalid_bdf_and_debug_without_log_are_rejected(self): + with self.assertRaisesRegex(ValueError, "invalid PCI BDF"): + launcher.build_domain_xml( + self.config(host_devices=[launcher.HostDevice("gpu", "bad")]) + ) + with self.assertRaisesRegex(ValueError, "requires a log file"): + launcher.build_domain_xml(self.config(debug=True)) + with self.assertRaisesRegex(ValueError, "ports must be between"): + launcher.build_domain_xml(self.config(http_port=70000)) + + +class CapabilityTests(unittest.TestCase): + def config(self, host_devices=True): + return DomainXMLTests().config( + host_devices=[launcher.HostDevice("gpu", "65:00.0")] + if host_devices + else [] + ) + + def test_requires_libvirt_12_1_for_host_devices(self): + conn = mock.Mock() + conn.getType.return_value = "QEMU" + conn.getLibVersion.return_value = 12_000_000 + with self.assertRaisesRegex(RuntimeError, ">= 12.1.0"): + launcher.check_connection_capabilities(conn, self.config()) + + def test_requires_iommufd_domain_capability(self): + conn = mock.Mock() + conn.getType.return_value = "QEMU" + conn.getLibVersion.return_value = 12_001_000 + conn.getDomainCapabilities.return_value = "" + with self.assertRaisesRegex(RuntimeError, "do not advertise"): + launcher.check_connection_capabilities(conn, self.config()) + + def test_accepts_advertised_iommufd(self): + conn = mock.Mock() + conn.getType.return_value = "QEMU" + conn.getLibVersion.return_value = 12_001_000 + conn.getDomainCapabilities.return_value = """ + + yesno + + """ + launcher.check_connection_capabilities(conn, self.config()) + + def test_no_hostdev_does_not_require_iommufd(self): + conn = mock.Mock() + conn.getType.return_value = "QEMU" + launcher.check_connection_capabilities(conn, self.config(False)) + conn.getLibVersion.assert_not_called() + + def test_existing_domain_name_is_rejected(self): + conn = mock.Mock() + conn.lookupByName.return_value = object() + libvirt_module = mock.Mock() + with self.assertRaisesRegex(RuntimeError, "already exists"): + launcher.ensure_domain_name_available( + conn, libvirt_module, "super-protocol-3" + ) + + def test_preflight_subcommand_dispatches_without_building_domain(self): + with mock.patch.object(launcher, "preflight_connection") as preflight: + result = launcher.main( + [ + "preflight", + "--emulator", + "/usr/bin/qemu-system-x86_64", + "--name", + "super-protocol-3", + "--require-iommufd", + ] + ) + self.assertEqual(result, 0) + preflight.assert_called_once_with( + "/usr/bin/qemu-system-x86_64", "super-protocol-3", True + ) + + +class SourceSafetyTests(unittest.TestCase): + def test_new_bash_launcher_has_no_eval(self): + source = (REPO_ROOT / "scripts" / "start_super_protocol_libvirt.sh").read_text() + self.assertIsNone(re.search(r"^\s*eval\b", source, re.MULTILINE)) + + +if __name__ == "__main__": + unittest.main() From f5499ef4b02e7b6b88afab7c45396ef73e26e336 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 4 Aug 2026 08:47:21 -0500 Subject: [PATCH 03/28] apparmor --- README.md | 37 +++++++++++++++++++++++++ scripts/libvirt_launcher.py | 26 ++++++++++++----- scripts/start_super_protocol_libvirt.sh | 23 +++++++++++++++ tests/test_libvirt_launcher.py | 19 ++++++++++++- 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7dcd0a5..b29c0dd 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,43 @@ The default cache is `/var/lib/libvirt/images/superprotocol`, so the non-root QE With `--debug false` the command returns after the domain starts. With `--debug true --log_file /path/to/boot.log`, it attaches a bidirectional serial console and copies console output to the log; `Ctrl-C` or `Ctrl-]` detaches without stopping the VM. Use `virsh -c qemu:///system list`, `console`, `shutdown`, or `destroy` to manage it. `--gpu none` disables GPU, NVSwitch, and CX7 passthrough for diagnostics. +#### Ubuntu 26.04 AppArmor and `passt` + +The Ubuntu 26.04 libvirt AppArmor profile may allow `/usr/bin/passt` to be read but not memory-mapped. In that case libvirt reports `passt ... unexpected fatal signal 11`, while the kernel audit log contains a denial similar to: + +```text +apparmor="DENIED" operation="file_mmap" name="/usr/bin/passt" requested_mask="rm" +``` + +Confirm the cause with: + +```bash +sudo journalctl -k --since '-10 min' --no-pager | + grep -E 'apparmor="DENIED".*(passt|libvirt)|comm="passt"' +``` + +Until this host configuration is incorporated into the bootstrap scripts, back up and adjust the nested `passt` profile, then add a local rule allowing QEMU to connect to the libvirt-managed socket: + +```bash +sudo cp -a --update=none \ + /etc/apparmor.d/abstractions/libvirt-qemu \ + /etc/apparmor.d/abstractions/libvirt-qemu.sp-vm-tools.bak + +sudo sed -i \ + '/^[[:space:]]*profile passt[[:space:]]*{/,/^[[:space:]]*}/ s|/usr/bin/passt r,|/usr/bin/passt rm,|' \ + /etc/apparmor.d/abstractions/libvirt-qemu + +sudo install -d -m 0755 \ + /etc/apparmor.d/abstractions/libvirt-qemu.d + +printf '%s\n' 'owner @{run}/libvirt/qemu/passt/* rw,' | \ + sudo tee /etc/apparmor.d/abstractions/libvirt-qemu.d/99-passt-local >/dev/null + +sudo systemctl reload apparmor +``` + +Do not disable AppArmor globally. The launcher detects the incompatible read-only `passt` rule before binding VFIO devices or recreating VM disks. + ### 1. Clone the repo Clone the repository onto the target host. See [docs/swarm.md](docs/swarm.md) for the exact command. diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py index d949ec7..719114a 100755 --- a/scripts/libvirt_launcher.py +++ b/scripts/libvirt_launcher.py @@ -461,14 +461,26 @@ def check_connection_capabilities(conn: Any, config: Any) -> None: def ensure_domain_name_available(conn: Any, libvirt_module: Any, name: str) -> None: try: - conn.lookupByName(name) + domains = conn.listAllDomains(0) except libvirt_module.libvirtError as exc: - if exc.get_error_code() == libvirt_module.VIR_ERR_NO_DOMAIN: - return raise RuntimeError(f"failed to check domain name {name!r}: {exc}") from exc - raise RuntimeError( - f"a libvirt domain named {name!r} already exists; stop it or choose --name" - ) + if any(domain.name() == name for domain in domains): + raise RuntimeError( + f"a libvirt domain named {name!r} already exists; stop it or choose --name" + ) + + +def _format_launch_error(exc: BaseException) -> str: + message = str(exc) + if "passt" in message and "fatal signal 11" in message: + return ( + f"libvirt failed to start the domain: {message}\n" + "passt was killed by SIGSEGV. On Ubuntu this commonly means that " + "AppArmor denied passt or its libvirt socket. Check the kernel audit " + "log with: journalctl -k --since '-5 min' --no-pager | " + "grep -E 'apparmor=\"DENIED\".*(passt|libvirt)'" + ) + return f"libvirt failed to start the domain: {message}" def preflight_connection(emulator: str, name: str, require_iommufd: bool) -> None: @@ -611,7 +623,7 @@ def launch(config: DomainConfig) -> None: if config.debug: attach_serial_console(conn, domain, libvirt, str(config.log_file)) except libvirt.libvirtError as exc: - raise RuntimeError(f"libvirt failed to start the domain: {exc}") from exc + raise RuntimeError(_format_launch_error(exc)) from exc finally: conn.close() diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index d98618f..5136711 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -94,6 +94,28 @@ check_libvirt_dependencies() { fi } +check_passt_apparmor_profile() { + if [[ "${NETDEV_MODE}" != "user" && "${DEBUG_MODE}" != "true" ]]; then + return + fi + + local profile=/etc/apparmor.d/abstractions/libvirt-qemu + [[ -r "${profile}" ]] || return + + if awk ' + /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1 } + in_passt && /\/usr\/bin\/passt[[:space:]]+r,/ { incompatible = 1 } + in_passt && /^[[:space:]]*}/ { exit } + END { exit incompatible ? 0 : 1 } + ' "${profile}"; then + echo "Error: the libvirt AppArmor profile permits reading /usr/bin/passt but not mmap." >&2 + echo "Ubuntu AppArmor 5 will kill passt with fatal signal 11." >&2 + echo "Update the rule inside 'profile passt' from '/usr/bin/passt r,' to '/usr/bin/passt rm,'" >&2 + echo "in ${profile}, reload AppArmor, and retry." >&2 + exit 1 + fi +} + preflight_libvirt() { local require_iommufd=false local gpu @@ -314,6 +336,7 @@ main_libvirt() { check_target_os check_packages check_libvirt_dependencies + check_passt_apparmor_profile find_qemu_path check_qemu_version preflight_libvirt diff --git a/tests/test_libvirt_launcher.py b/tests/test_libvirt_launcher.py index 1e2bcab..d6ee979 100644 --- a/tests/test_libvirt_launcher.py +++ b/tests/test_libvirt_launcher.py @@ -218,13 +218,30 @@ def test_no_hostdev_does_not_require_iommufd(self): def test_existing_domain_name_is_rejected(self): conn = mock.Mock() - conn.lookupByName.return_value = object() + existing = mock.Mock() + existing.name.return_value = "super-protocol-3" + conn.listAllDomains.return_value = [existing] libvirt_module = mock.Mock() with self.assertRaisesRegex(RuntimeError, "already exists"): launcher.ensure_domain_name_available( conn, libvirt_module, "super-protocol-3" ) + def test_available_domain_name_does_not_trigger_libvirt_lookup_error(self): + conn = mock.Mock() + other = mock.Mock() + other.name.return_value = "another-domain" + conn.listAllDomains.return_value = [other] + launcher.ensure_domain_name_available(conn, mock.Mock(), "super-protocol-3") + conn.lookupByName.assert_not_called() + + def test_passt_sigsegv_error_points_to_apparmor_audit_log(self): + message = launcher._format_launch_error( + RuntimeError("Child process (passt --one-off) unexpected fatal signal 11") + ) + self.assertIn("AppArmor", message) + self.assertIn("journalctl -k", message) + def test_preflight_subcommand_dispatches_without_building_domain(self): with mock.patch.object(launcher, "preflight_connection") as preflight: result = launcher.main( From c7f1d9d8c2cdbbf1584ec689288152fc6d2d304c Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 4 Aug 2026 09:13:22 -0500 Subject: [PATCH 04/28] setfacl --- README.md | 29 +++++- scripts/start_super_protocol_libvirt.sh | 125 +++++++++++++++++++++++- 2 files changed, 150 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b29c0dd..80a3fa9 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,6 @@ sudo ./scripts/start_super_protocol_libvirt.sh \ --mode tdx ``` -The default cache is `/var/lib/libvirt/images/superprotocol`, so the non-root QEMU process used by `qemu:///system` can access the images. A custom `--cache` or `--build_dir` must likewise be traversable by the configured libvirt QEMU user. - With `--debug false` the command returns after the domain starts. With `--debug true --log_file /path/to/boot.log`, it attaches a bidirectional serial console and copies console output to the log; `Ctrl-C` or `Ctrl-]` detaches without stopping the VM. Use `virsh -c qemu:///system list`, `console`, `shutdown`, or `destroy` to manage it. `--gpu none` disables GPU, NVSwitch, and CX7 passthrough for diagnostics. #### Ubuntu 26.04 AppArmor and `passt` @@ -90,6 +88,33 @@ sudo systemctl reload apparmor Do not disable AppArmor globally. The launcher detects the incompatible read-only `passt` rule before binding VFIO devices or recreating VM disks. +`passt` runs as the unprivileged libvirt QEMU user. Forwarding a host port below 1024 therefore also requires lowering the host's unprivileged-port boundary to the lowest forwarded port. For the default DNS port, configure it persistently with: + +```bash +printf '%s\n' 'net.ipv4.ip_unprivileged_port_start = 53' | \ + sudo tee /etc/sysctl.d/90-sp-vm-passt.conf >/dev/null + +sudo sysctl --system +``` + +The launcher checks this value before preparing the VM. This sysctl and the AppArmor adjustment are temporary host-preparation steps that should be moved into the Ubuntu 26.04 bootstrap in the future. + +As a narrower alternative to changing the system-wide sysctl, grant `CAP_NET_BIND_SERVICE` only to the installed `passt` binaries: + +```bash +sudo apt-get install libcap2-bin + +for binary in /usr/bin/passt /usr/bin/passt.avx2; do + if [[ -x "${binary}" ]]; then + sudo setcap 'cap_net_bind_service=+ep' "${binary}" + fi +done + +getcap /usr/bin/passt /usr/bin/passt.avx2 2>/dev/null +``` + +The AppArmor profile above already allows this capability, and the launcher recognizes it during preflight. Package upgrades can replace the binaries and remove their file capabilities, in which case the `setcap` step must be repeated. The `passt` project recommends the sysctl method in general, but documents file capabilities as an option on hosts sufficiently constrained by an LSM such as AppArmor. + ### 1. Clone the repo Clone the repository onto the target host. See [docs/swarm.md](docs/swarm.md) for the exact command. diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index 5136711..0a88270 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -76,6 +76,8 @@ check_libvirt_dependencies() { local missing=() command -v python3 >/dev/null 2>&1 || missing+=(python3) command -v virsh >/dev/null 2>&1 || missing+=(libvirt-clients) + command -v setfacl >/dev/null 2>&1 || missing+=(acl) + command -v runuser >/dev/null 2>&1 || missing+=(util-linux) if ! python3 -c 'import libvirt' >/dev/null 2>&1; then missing+=(python3-libvirt) fi @@ -84,7 +86,7 @@ check_libvirt_dependencies() { fi if [[ ${#missing[@]} -gt 0 ]]; then echo "Error: missing libvirt runtime dependencies: ${missing[*]}" >&2 - echo "Install them with: apt-get install libvirt-daemon-system libvirt-clients python3-libvirt passt" >&2 + echo "Install them with: apt-get install libvirt-daemon-system libvirt-clients python3-libvirt passt acl" >&2 echo "GPU passthrough additionally requires libvirt >= 12.1.0." >&2 exit 1 fi @@ -116,6 +118,63 @@ check_passt_apparmor_profile() { fi } +check_passt_privileged_ports() { + local ports=() + local port port_number minimum=65536 + + if [[ "${NETDEV_MODE}" == "user" ]]; then + ports+=( + "${HTTP_PORT}" "${HTTPS_PORT}" "${PKI_PORT}" + "${PKI_VM_MEASURE_PORT}" "${WG_PORT}" + "${SWARM_DB_GOSSIP_PORT}" "${DNS_PORT}" + ) + fi + if [[ "${DEBUG_MODE}" == "true" ]]; then + ports+=("${SSH_PORT}") + fi + + for port in "${ports[@]}"; do + [[ -n "${port}" ]] || continue + port_number=$((10#${port})) + if ((port_number < minimum)); then + minimum=${port_number} + fi + done + + if ((minimum >= 1024)); then + return + fi + + local sysctl_path=/proc/sys/net/ipv4/ip_unprivileged_port_start + [[ -r "${sysctl_path}" ]] || return + local current + current=$(<"${sysctl_path}") + if ((current > minimum)); then + local capability_ok=true found_passt=false binary path capabilities + if ! command -v getcap >/dev/null 2>&1; then + capability_ok=false + else + for binary in passt passt.avx2; do + path=$(command -v "${binary}" 2>/dev/null || true) + [[ -n "${path}" ]] || continue + found_passt=true + capabilities=$(getcap "${path}" 2>/dev/null || true) + if [[ "${capabilities}" != *cap_net_bind_service* ]]; then + capability_ok=false + fi + done + fi + if [[ "${found_passt}" == "true" && "${capability_ok}" == "true" ]]; then + return + fi + + echo "Error: passt must bind host port ${minimum}, but unprivileged ports currently start at ${current}." >&2 + echo "Lower net.ipv4.ip_unprivileged_port_start to ${minimum}, or grant CAP_NET_BIND_SERVICE" >&2 + echo "to every installed passt binary, then retry." >&2 + exit 1 + fi +} + preflight_libvirt() { local require_iommufd=false local gpu @@ -246,6 +305,63 @@ build_kernel_cmdline() { fi } +grant_libvirt_file_access() { + local label=$1 requested_path=$2 permissions=$3 + local qemu_user=libvirt-qemu + if ! id "${qemu_user}" >/dev/null 2>&1; then + echo "Error: the expected Ubuntu libvirt QEMU user '${qemu_user}' does not exist." >&2 + exit 1 + fi + + local path + if ! path=$(realpath -e -- "${requested_path}"); then + echo "Error: cannot resolve ${label} path: ${requested_path}" >&2 + exit 1 + fi + + local directory + directory=$(dirname -- "${path}") + local directories=() + while [[ "${directory}" != "/" ]]; do + directories+=("${directory}") + directory=$(dirname -- "${directory}") + done + + local index + for ((index = ${#directories[@]} - 1; index >= 0; index--)); do + directory=${directories[${index}]} + if ! runuser -u "${qemu_user}" -- test -x "${directory}"; then + if ! setfacl -m "u:${qemu_user}:--x" -- "${directory}"; then + echo "Error: failed to grant ${qemu_user} traversal access to ${directory}" >&2 + exit 1 + fi + fi + done + + if ! setfacl -m "u:${qemu_user}:${permissions}" -- "${path}"; then + echo "Error: failed to grant ${qemu_user} access to ${label}: ${path}" >&2 + exit 1 + fi + + if ! runuser -u "${qemu_user}" -- test -r "${path}"; then + echo "Error: ${qemu_user} still cannot read ${label}: ${path}" >&2 + exit 1 + fi + if [[ "${permissions}" == "rw-" ]] && \ + ! runuser -u "${qemu_user}" -- test -w "${path}"; then + echo "Error: ${qemu_user} still cannot write ${label}: ${path}" >&2 + exit 1 + fi + + echo "Granted ${qemu_user} ${permissions} access to ${label}: ${path}" +} + +grant_static_libvirt_resource_access() { + grant_libvirt_file_access rootfs "${IMAGE_PATH}" r-- + grant_libvirt_file_access kernel "${KERNEL_PATH}" r-- + grant_libvirt_file_access firmware "${BIOS_PATH}" r-- +} + create_vm_disks() { local provider_loop provider_mount @@ -278,6 +394,9 @@ create_vm_disks() { provider_loop="" rmdir "${provider_mount}" trap - RETURN + + grant_libvirt_file_access state-disk "${STATE_DISK_PATH}" rw- + grant_libvirt_file_access provider-config-disk "${PROVIDER_CONFIG_DISK_PATH}" r-- } append_optional_arg() { @@ -339,14 +458,16 @@ main_libvirt() { check_passt_apparmor_profile find_qemu_path check_qemu_version - preflight_libvirt check_params + check_passt_privileged_ports + preflight_libvirt prepare_selected_host_devices prepare_mode_parameters mkdir -p "${CACHE}" download_release "${RELEASE}" "${RELEASE_ASSET}" "${CACHE}" "${RELEASE_REPO}" parse_and_download_release_files "${RELEASE_FILEPATH}" + grant_static_libvirt_resource_access prepare_tap_network build_kernel_cmdline create_vm_disks From 491b4da88f9269753b08efedb8857a24980f0c95 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 4 Aug 2026 09:18:56 -0500 Subject: [PATCH 05/28] check libvirt version --- scripts/start_super_protocol_libvirt.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index 0a88270..0138e76 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -184,11 +184,10 @@ preflight_libvirt() { break fi done - # With no explicit --gpu option, the base launcher selects all GPUs later. - if [[ ${#USED_GPUS[@]} -eq 0 ]]; then - if lspci -nnk -d 10de: 2>/dev/null | grep -qE '3D controller'; then - require_iommufd=true - fi + # With no --gpu option, check_params will select all available GPUs. + if [[ ${#USED_GPUS[@]} -eq 0 ]] && \ + lspci -nnk -d 10de: 2>/dev/null | grep -qE '3D controller'; then + require_iommufd=true fi local args=(preflight --emulator "${QEMU_PATH}" --name "${LIBVIRT_DOMAIN_NAME}") @@ -458,9 +457,9 @@ main_libvirt() { check_passt_apparmor_profile find_qemu_path check_qemu_version + preflight_libvirt check_params check_passt_privileged_ports - preflight_libvirt prepare_selected_host_devices prepare_mode_parameters From 176147f5af7f016af407945b0374f614252f8076 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 4 Aug 2026 09:22:05 -0500 Subject: [PATCH 06/28] state disk access --- scripts/start_super_protocol_libvirt.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index 0138e76..17f8117 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -337,9 +337,18 @@ grant_libvirt_file_access() { fi done - if ! setfacl -m "u:${qemu_user}:${permissions}" -- "${path}"; then - echo "Error: failed to grant ${qemu_user} access to ${label}: ${path}" >&2 - exit 1 + if [[ "${permissions}" == "rw-" ]]; then + # The launcher creates mutable disks itself. Giving the QEMU process + # ownership is more reliable than a named ACL on mounted data volumes. + if ! chown -- "${qemu_user}" "${path}" || ! chmod -- u+rw "${path}"; then + echo "Error: failed to assign writable ${label} to ${qemu_user}: ${path}" >&2 + exit 1 + fi + else + if ! setfacl -m "u:${qemu_user}:${permissions}" -- "${path}"; then + echo "Error: failed to grant ${qemu_user} access to ${label}: ${path}" >&2 + exit 1 + fi fi if ! runuser -u "${qemu_user}" -- test -r "${path}"; then From 5839cad6ce5cd0a67a507a495af05946f9c31890 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Wed, 5 Aug 2026 09:20:27 -0500 Subject: [PATCH 07/28] adapt cluster script --- docs/swarm.md | 7 +- scripts/swarm-cluster.sh | 240 ++++++++++++++++++++++++++++----------- 2 files changed, 178 insertions(+), 69 deletions(-) diff --git a/docs/swarm.md b/docs/swarm.md index b49f19d..473cc03 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -68,6 +68,7 @@ ACME_URL: https://acme.zerossl.com/v2/DV90 **You also need:** - A host already bootstrapped for confidential computing (TDX or SEV-SNP) — see the [main README](../README.md). - `tmux`, `nftables`, `curl`, `nc` installed: `apt install tmux nftables curl netcat-openbsd` +- Ubuntu 26.04+ with `qemu:///system`, libvirt 12.1+, `python3-libvirt`, `passt`, and `acl` configured as described in the [libvirt launcher section](../README.md#libvirt-launcher-ubuntu-2604). > Keep `provider-template/` in its own folder — not inside `sp-vm-tools` and not inside any cache folder. @@ -78,7 +79,7 @@ ACME_URL: https://acme.zerossl.com/v2/DV90 sudo ./scripts/swarm-cluster.sh up --provider-config-template ./provider-template # Check status -./scripts/swarm-cluster.sh status +sudo ./scripts/swarm-cluster.sh status # Stop everything sudo ./scripts/swarm-cluster.sh down @@ -114,13 +115,13 @@ The bootstrap node gets all remaining host resources after subtracting the host 4. Generates per-node provider configs: - **Bootstrap**: `join_addresses: []`, `pki_authority.servers: []`. - **Join nodes**: `join_addresses: ["10.0.0.10:7946"]`, `caBundle` fetched automatically from bootstrap PKI. -5. Starts each VM in its own `tmux` session (`swarm-bootstrap`, `swarm-join-1`, `swarm-join-2`), attached to the bridge via tap interfaces. +5. Starts transient libvirt domains named `swarm-bootstrap`, `swarm-join-1`, and `swarm-join-2`, attached to the bridge via tap interfaces. In debug mode their attached serial consoles run in matching `tmux` sessions. 6. Waits for bootstrap gossip (7946) and PKI (9443) to become ready. 7. Fetches the CA bundle from bootstrap and injects it into join-node configs. 8. Launches join nodes. 9. Sets up HAProxy ingress: `gw.dyn..superprotocol.io` → bootstrap ports 80/443. -Attach to any VM's console with `tmux attach -t swarm-bootstrap` (or `swarm-join-1` / `swarm-join-2`). +Attach to a VM with `virsh -c qemu:///system console swarm-bootstrap` (or `swarm-join-1` / `swarm-join-2`). In debug mode, use the matching `tmux attach -t ` session instead. diff --git a/scripts/swarm-cluster.sh b/scripts/swarm-cluster.sh index 82a0323..a3e05d6 100755 --- a/scripts/swarm-cluster.sh +++ b/scripts/swarm-cluster.sh @@ -8,10 +8,10 @@ # - each VM gets its address from provider_config/swarm/config.yaml spnet # - external ingress (host WAN) DNAT only to bootstrap: 80/443/9443 tcp, 53 tcp+udp # - join nodes fetch PKI (9443) from bootstrap over the LOCAL address 10.0.0.10 (no hairpin) -# - each VM runs in its own tmux session +# - each VM runs as a transient qemu:///system libvirt domain # -# Requires a patched start_super_protocol.sh (see network-tap.patch.md): -# support for --netdev_mode tap --bridge . +# Requires start_super_protocol_libvirt.sh and a working qemu:///system +# connection. tmux is used only for the launcher/serial-console process. # # Usage: # sudo ./swarm-cluster.sh up --provider-config-template ./provider-template [opts] @@ -77,7 +77,8 @@ VM_MODE="" # empty = auto-detect (tdx/sev-snp) in start scri RELEASE="" # empty = latest; pin a working build, e.g. build-358 LOCAL_BUILD_DIR="" # empty = use release; otherwise pass local build dir to start script CACHE="/data/sp-vm/cache" -START_SCRIPT="${SCRIPT_DIR}/start_super_protocol.sh" +START_SCRIPT="${SCRIPT_DIR}/start_super_protocol_libvirt.sh" +LIBVIRT_URI="qemu:///system" PROVIDER_TEMPLATE="" # provider config template dir (--provider-config-template) WORKDIR="/data/sp-vm/cluster" # per-node provider configs are generated here WAN_IFACE="" # empty = auto-detect from ip route @@ -94,9 +95,12 @@ DEBUG_MODE="false" SSH_PORT_BOOTSTRAP=2210 SSH_PORT_JOIN=(2211 2212) -# tmux sessions -TMUX_BOOTSTRAP="swarm-bootstrap" -TMUX_JOIN=("swarm-join-1" "swarm-join-2") +# Domain names are also used as tmux launcher/console session names. +DOMAIN_BOOTSTRAP="swarm-bootstrap" +DOMAIN_JOIN=("swarm-join-1" "swarm-join-2") +CLUSTER_DOMAINS=("${DOMAIN_BOOTSTRAP}" "${DOMAIN_JOIN[@]}") +TMUX_BOOTSTRAP="${DOMAIN_BOOTSTRAP}" +TMUX_JOIN=("${DOMAIN_JOIN[@]}") # ---------------------------------------------------------------------------- # Helpers @@ -109,6 +113,36 @@ require_root() { [[ "$EUID" -eq 0 ]] || die "Must be run as root (use sudo)." } +virsh_cluster() { + LC_ALL=C virsh --connect "${LIBVIRT_URI}" "$@" +} + +require_libvirt() { + command -v virsh >/dev/null 2>&1 || die "virsh is required (install libvirt-clients)." + virsh_cluster list --name >/dev/null 2>&1 \ + || die "Cannot connect to ${LIBVIRT_URI}. Check the libvirt daemon and permissions." +} + +domain_exists() { + local domain="$1" + virsh_cluster dominfo "${domain}" >/dev/null 2>&1 +} + +domain_alive() { + local domain="$1" state + state=$(virsh_cluster domstate "${domain}" 2>/dev/null) || return 1 + [[ "${state}" != "shut off" && "${state}" != "crashed" ]] +} + +ensure_cluster_domains_available() { + local domain + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_exists "${domain}"; then + die "libvirt domain ${domain} already exists. Run 'down' or remove it explicitly." + fi + done +} + detect_wan_iface() { if [[ -n "${WAN_IFACE}" ]]; then echo "${WAN_IFACE}"; return; fi ip route get 8.8.8.8 2>/dev/null | sed -n 's/.* dev \([^ ]*\).*/\1/p' | head -1 @@ -223,7 +257,15 @@ reset_vfio_devices() { local drv="/sys/bus/pci/drivers/vfio-pci" [[ -d "${drv}" ]] || { log "vfio-pci driver not loaded — nothing to reset"; return 0; } - # Refuse to reset devices under a live QEMU + # Refuse to reset devices assigned to a live cluster domain. + local domain + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_alive "${domain}"; then + die "libvirt domain ${domain} is still running — refusing to reset devices. Run 'down' first." + fi + done + + # Also protect unrelated QEMU processes that currently hold VFIO devices. if pgrep -f 'qemu-system-x86_64.*vfio' >/dev/null 2>&1; then die "QEMU with VFIO still running — refusing to reset devices. Run 'down' first." fi @@ -291,18 +333,19 @@ reset_vfio_devices() { } # ---------------------------------------------------------------------------- -# VM liveness check: the runner does `exec qemu | tee`, so if QEMU dies for -# any reason the tmux session collapses. Detect that instead of waiting blind. +# VM liveness is owned by libvirt. In release mode the launcher and its tmux +# session exit immediately after createXML(), while the transient domain keeps +# running, so tmux is not a valid VM health signal. # ---------------------------------------------------------------------------- vm_alive() { - local session="$1" - tmux has-session -t "${session}" 2>/dev/null + local domain="$1" + domain_alive "${domain}" } report_vm_death() { - local session="$1" node_ip="$2" + local domain="$1" node_ip="$2" local logf="${CACHE}/log-${node_ip##*.}.txt" - err "VM session '${session}' has exited — QEMU failed." + err "libvirt domain '${domain}' is not running." err "Last lines of ${logf}:" tail -n 25 "${logf}" 2>/dev/null | sed 's/^/ /' >&2 || true # Common failure hint @@ -619,10 +662,11 @@ inject_spnet_config() { } # ---------------------------------------------------------------------------- -# 3. Start a single VM in tmux +# 3. Start a single libvirt domain through a tmux launcher # ---------------------------------------------------------------------------- start_vm() { - local session="$1" + local domain="$1" + local session="${domain}" local node_ip="$2" local cid="$3" local provider_dir="$4" @@ -634,9 +678,11 @@ start_vm() { local tap_iface="sw-tap-${node_ip##*.}" local mac; mac="$(mac_for_ip "${node_ip}")" + if domain_exists "${domain}"; then + die "libvirt domain ${domain} already exists. Run 'down' or remove it explicitly." + fi if tmux has-session -t "${session}" 2>/dev/null; then - err "tmux session ${session} already exists. Skipping (run 'down' to clean up)." - return 0 + die "stale tmux launcher session ${session} already exists. Run 'down' first." fi local gpu_args=() @@ -655,20 +701,19 @@ start_vm() { local build_args=() [[ -n "${LOCAL_BUILD_DIR}" ]] && build_args=(--build_dir "${LOCAL_BUILD_DIR}") - # Debug mode: verbose boot log + per-node SSH port. start_super_protocol.sh - # requires --log_file when --debug true. NOTE: the script forwards SSH via - # hostfwd (user-mode) only; in tap mode that hostfwd is inactive, so SSH must - # go to the VM's bridge IP (ssh ubuntu@). The main value of debug - # here is the verbose serial/boot log written to the log file. + # Debug mode keeps the libvirt serial console attached in tmux and writes a + # per-node boot log. With tap networking the launcher adds a second passt + # NIC for the requested localhost SSH forwarding. local debug_args=() if [[ "${DEBUG_MODE}" == "true" ]]; then debug_args=(--debug true --log_file "${CACHE}/boot-${node_ip##*.}.log") [[ -n "${ssh_port}" ]] && debug_args+=(--ssh_port "${ssh_port}") fi - # build the patched start-script command line in tap mode + # Build the libvirt start-script command line in tap mode. local cmd=( "${START_SCRIPT}" + --name "${domain}" --netdev_mode tap --bridge "${BRIDGE}" --tap_iface "${tap_iface}" @@ -688,7 +733,7 @@ start_vm() { "${debug_args[@]}" ) - log "Starting ${session}: ip=${node_ip} cid=${cid} tap=${tap_iface} gpu=${with_gpu} cores=${node_cores} mem=${node_mem}GB disk=${node_disk}GB debug=${DEBUG_MODE}" + log "Starting domain ${domain}: ip=${node_ip} cid=${cid} tap=${tap_iface} gpu=${with_gpu} cores=${node_cores} mem=${node_mem}GB disk=${node_disk}GB debug=${DEBUG_MODE}" # Safety net: drop any empty array elements before building the runner. # An empty positional arg would shift the start-script's two-step arg parser @@ -715,17 +760,27 @@ start_vm() { tmux new-session -d -s "${session}" "${runner}" - # Fail fast: if the command dies immediately (bad flags, missing release, etc.), - # the tmux session collapses and we must not proceed into a blind wait. - # The image download alone takes a while, so we only check that the session - # survives the first few seconds — enough to catch instant failures. - sleep 6 - if ! tmux has-session -t "${session}" 2>/dev/null; then - err "Session ${session} exited immediately — startup failed." - err "Last lines of ${CACHE}/log-${node_ip##*.}.txt:" - tail -n 20 "${CACHE}/log-${node_ip##*.}.txt" 2>/dev/null | sed 's/^/ /' >&2 || true - die "Aborting. Fix the error above (often: wrong --release, or start script not patched)." - fi + # The launcher may spend time downloading/preparing images before it creates + # the domain. In release mode it then exits successfully, so wait for libvirt + # rather than requiring the tmux session to remain after startup. + local startup_timeout=1000 waited=0 + while (( waited < startup_timeout )); do + if domain_alive "${domain}"; then + log "Domain ${domain} is running" + return 0 + fi + if ! tmux has-session -t "${session}" 2>/dev/null; then + err "Launcher session ${session} exited before the domain started." + err "Last lines of ${CACHE}/log-${node_ip##*.}.txt:" + tail -n 25 "${CACHE}/log-${node_ip##*.}.txt" 2>/dev/null | sed 's/^/ /' >&2 || true + die "Domain ${domain} failed to start." + fi + sleep 2 + waited=$((waited + 2)) + done + + tmux kill-session -t "${session}" 2>/dev/null || true + die "Timed out after ${startup_timeout}s waiting for libvirt domain ${domain}." } # ---------------------------------------------------------------------------- @@ -741,9 +796,9 @@ wait_bootstrap() { while (( waited < timeout )); do # Fail fast: QEMU crashed (vfio bind error, OOM, bad flags, ...) - if ! vm_alive "${TMUX_BOOTSTRAP}"; then + if ! vm_alive "${DOMAIN_BOOTSTRAP}"; then echo >&2 - report_vm_death "${TMUX_BOOTSTRAP}" "${BOOTSTRAP_IP}" + report_vm_death "${DOMAIN_BOOTSTRAP}" "${BOOTSTRAP_IP}" die "Bootstrap VM died while waiting — aborting cluster startup." fi @@ -771,7 +826,7 @@ wait_bootstrap() { sleep 5; waited=$(( waited + 5 )) done echo >&2 - die "Bootstrap did not come up within ${timeout}s. Check: tmux attach -t ${TMUX_BOOTSTRAP}" + die "Bootstrap did not come up within ${timeout}s. Check ${CACHE}/log-${BOOTSTRAP_IP##*.}.txt and: virsh -c ${LIBVIRT_URI} console ${DOMAIN_BOOTSTRAP}" } # ---------------------------------------------------------------------------- @@ -803,6 +858,8 @@ fetch_ca_bundle() { # ---------------------------------------------------------------------------- cmd_up() { require_root + require_libvirt + ensure_cluster_domains_available [[ -n "${PROVIDER_TEMPLATE}" ]] || die "Specify --provider-config-template " [[ -d "${PROVIDER_TEMPLATE}" ]] || die "Template ${PROVIDER_TEMPLATE} not found" [[ -x "${START_SCRIPT}" ]] || die "start script not found/executable: ${START_SCRIPT}" @@ -822,6 +879,11 @@ cmd_up() { command -v nc &>/dev/null || die "nc is required (apt install netcat-openbsd)" command -v curl &>/dev/null || die "curl is required (apt install curl)" command -v tmux &>/dev/null || die "tmux is required (apt install tmux)" + local session + for session in "${TMUX_BOOTSTRAP}" "${TMUX_JOIN[@]}"; do + tmux has-session -t "${session}" 2>/dev/null \ + && die "stale tmux launcher session ${session} already exists. Run 'down' first." + done if [[ -n "${RELEASE}" && -n "${LOCAL_BUILD_DIR}" ]]; then die "Use either --release or --build-dir, not both." fi @@ -866,7 +928,7 @@ cmd_up() { local boot_gpu=false [[ "${GPU_TARGET}" == "bootstrap" ]] && boot_gpu=true - start_vm "${TMUX_BOOTSTRAP}" "${BOOTSTRAP_IP}" "${CID_BOOTSTRAP}" "${boot_dir}" \ + start_vm "${DOMAIN_BOOTSTRAP}" "${BOOTSTRAP_IP}" "${CID_BOOTSTRAP}" "${boot_dir}" \ "${boot_gpu}" "${BOOTSTRAP_CORES}" "${BOOTSTRAP_MEM}" "${BOOTSTRAP_DISK}" "${SSH_PORT_BOOTSTRAP}" wait_bootstrap 1000 @@ -883,9 +945,9 @@ cmd_up() { join1_dir="$(prepare_config join1 "${JOIN_IPS[0]}" "swarm-join-1" "${BOOTSTRAP_IP}:${GOSSIP_PORT}" "${ca_bundle}" "${network_id}")" join2_dir="$(prepare_config join2 "${JOIN_IPS[1]}" "swarm-join-2" "${BOOTSTRAP_IP}:${GOSSIP_PORT}" "${ca_bundle}" "${network_id}")" - start_vm "${TMUX_JOIN[0]}" "${JOIN_IPS[0]}" "${CID_JOIN[0]}" "${join1_dir}" \ + start_vm "${DOMAIN_JOIN[0]}" "${JOIN_IPS[0]}" "${CID_JOIN[0]}" "${join1_dir}" \ false "${JOIN_CORES}" "${JOIN_MEM}" "${JOIN_DISK}" "${SSH_PORT_JOIN[0]}" - start_vm "${TMUX_JOIN[1]}" "${JOIN_IPS[1]}" "${CID_JOIN[1]}" "${join2_dir}" \ + start_vm "${DOMAIN_JOIN[1]}" "${JOIN_IPS[1]}" "${CID_JOIN[1]}" "${join2_dir}" \ false "${JOIN_CORES}" "${JOIN_MEM}" "${JOIN_DISK}" "${SSH_PORT_JOIN[1]}" # external ingress @@ -898,17 +960,20 @@ cmd_up() { log "Cluster started. Ingress: gw.dyn.${GLOBAL_ID}.${BASE_DOMAIN} -> 80/443" - log "Cluster started. Sessions: tmux ls" - log " bootstrap: tmux attach -t ${TMUX_BOOTSTRAP}" - log " join: tmux attach -t ${TMUX_JOIN[0]} | ${TMUX_JOIN[1]}" + log "Cluster started. Domains: virsh -c ${LIBVIRT_URI} list" + log " bootstrap console: virsh -c ${LIBVIRT_URI} console ${DOMAIN_BOOTSTRAP}" + log " join consoles: ${DOMAIN_JOIN[0]} | ${DOMAIN_JOIN[1]}" + if [[ "${DEBUG_MODE}" == "true" ]]; then + log " attached debug consoles are in tmux: ${TMUX_BOOTSTRAP}, ${TMUX_JOIN[0]}, ${TMUX_JOIN[1]}" + fi # Verify join VMs survived startup (they can hit the same vfio error # if GPU_TARGET is ever changed, or die on bad config / OOM). sleep 10 local i for i in 0 1; do - if ! vm_alive "${TMUX_JOIN[$i]}"; then - report_vm_death "${TMUX_JOIN[$i]}" "${JOIN_IPS[$i]}" + if ! vm_alive "${DOMAIN_JOIN[$i]}"; then + report_vm_death "${DOMAIN_JOIN[$i]}" "${JOIN_IPS[$i]}" die "Join node ${JOIN_IPS[$i]} died right after start — aborting." fi done @@ -924,7 +989,17 @@ cmd_status() { echo " host: ${_c} cores, ${_m}GB" echo " plan: bootstrap=remainder+GPU, join=${JOIN_CORES}c/${JOIN_MEM}g each, reserve=${HOST_RESERVE_CORES}c/${HOST_RESERVE_MEM}g" fi - echo "=== tmux ===" + echo "=== libvirt domains (${LIBVIRT_URI}) ===" + if command -v virsh >/dev/null 2>&1 && virsh_cluster list --name >/dev/null 2>&1; then + local domain state + for domain in "${CLUSTER_DOMAINS[@]}"; do + state=$(virsh_cluster domstate "${domain}" 2>/dev/null || true) + echo " ${domain}: ${state:-absent}" + done + else + echo " unavailable" + fi + echo "=== tmux launcher/console sessions ===" tmux ls 2>/dev/null | grep -E 'swarm-' || echo " no sessions" echo "=== bridge ===" ip -br addr show "${BRIDGE}" 2>/dev/null || echo " no bridge ${BRIDGE}" @@ -943,21 +1018,53 @@ cmd_status() { | sed 's/^/ /' || echo " network script unavailable" } -wait_qemu_gone() { - local timeout="${1:-90}" waited=0 - log "Waiting for QEMU processes to exit (up to ${timeout}s)..." - while pgrep -f 'qemu-system-x86_64.*sw-tap-' >/dev/null 2>&1; do - if (( waited >= timeout )); then - err "QEMU still alive after ${timeout}s, sending SIGKILL" - pkill -9 -f 'qemu-system-x86_64.*sw-tap-' 2>/dev/null || true - timeout=$(( timeout + 120 )) +wait_domains_stopped() { + local timeout="${1:-90}" waited=0 domain active + log "Waiting for libvirt domains to stop (up to ${timeout}s)..." + while (( waited < timeout )); do + active="" + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_alive "${domain}"; then + active="${domain}" + break + fi + done + if [[ -z "${active}" ]]; then + log "All cluster domains stopped (${waited}s)" + return 0 fi - printf '\r[%s] qemu still running... %ss\033[K' "$(date +%H:%M:%S)" "${waited}" >&2 - sleep 2; waited=$(( waited + 2 )) - (( waited >= 300 )) && { echo >&2; die "QEMU did not exit in 300s — check dmesg (stuck unpinning?)"; } + printf '\r[%s] %s still running... %ss\033[K' "$(date +%H:%M:%S)" "${active}" "${waited}" >&2 + sleep 2 + waited=$((waited + 2)) done echo >&2 - log "All QEMU processes gone (${waited}s)" + return 1 +} + +stop_cluster_domains() { + local domain + + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_alive "${domain}"; then + log " shutdown ${domain}" + virsh_cluster shutdown "${domain}" >/dev/null 2>&1 \ + || err "Failed to request shutdown for ${domain}" + fi + done + + if wait_domains_stopped 30; then + return 0 + fi + + err "Graceful shutdown timed out; destroying remaining cluster domains" + for domain in "${CLUSTER_DOMAINS[@]}"; do + if domain_alive "${domain}"; then + log " destroy ${domain}" + virsh_cluster destroy "${domain}" >/dev/null 2>&1 \ + || err "Failed to destroy ${domain}" + fi + done + wait_domains_stopped 90 || die "Some libvirt domains did not stop; refusing to tear down networking." } wait_vfio_free() { @@ -986,6 +1093,7 @@ wait_vfio_free() { cmd_down() { require_root + require_libvirt local answer while true; do @@ -1005,16 +1113,16 @@ cmd_down() { log "Stopping cluster..." - pkill -TERM -f 'qemu-system-x86_64.*sw-tap-' 2>/dev/null || true - - wait_qemu_gone 90 - - wait_vfio_free 120 || err "GPU may still be busy — next 'up' can fail; check 'fuser -v /dev/vfio/*'" - + # Stop launchers first so a domain still being prepared cannot appear after + # the shutdown pass. Killing an attached debug console only detaches it. for s in "${TMUX_BOOTSTRAP}" "${TMUX_JOIN[@]}"; do tmux kill-session -t "${s}" 2>/dev/null || true done + stop_cluster_domains + + wait_vfio_free 120 || err "GPU may still be busy — next 'up' can fail; check 'fuser -v /dev/vfio/*'" + for ip in "${BOOTSTRAP_IP}" "${JOIN_IPS[@]}"; do local tap="sw-tap-${ip##*.}" ip link show "${tap}" &>/dev/null && { log " del ${tap}"; ip link del "${tap}"; } From 8d4d8565a1c616002dd5748adab4b3ab32b28e80 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 6 Aug 2026 04:24:07 -0500 Subject: [PATCH 08/28] libvirt action --- .../workflows/build-packages-self-hosted.yml | 70 +++++- README.md | 2 +- build/libvirt/.dockerignore | 1 + build/libvirt/Dockerfile.ubuntu24 | 26 +++ build/libvirt/Dockerfile.ubuntu26 | 26 +++ build/libvirt/README.md | 86 ++++++++ build/libvirt/build.sh | 201 ++++++++++++++++++ build/libvirt/docker/build-packages.sh | 104 +++++++++ build/libvirt/docker/prepare-source.sh | 33 +++ build/libvirt/out/.gitignore | 2 + 10 files changed, 540 insertions(+), 11 deletions(-) create mode 100644 build/libvirt/.dockerignore create mode 100644 build/libvirt/Dockerfile.ubuntu24 create mode 100644 build/libvirt/Dockerfile.ubuntu26 create mode 100644 build/libvirt/README.md create mode 100755 build/libvirt/build.sh create mode 100755 build/libvirt/docker/build-packages.sh create mode 100755 build/libvirt/docker/prepare-source.sh create mode 100644 build/libvirt/out/.gitignore diff --git a/.github/workflows/build-packages-self-hosted.yml b/.github/workflows/build-packages-self-hosted.yml index 01f3bb5..becbe9a 100644 --- a/.github/workflows/build-packages-self-hosted.yml +++ b/.github/workflows/build-packages-self-hosted.yml @@ -6,12 +6,14 @@ on: build_type: description: "Select build type" required: true - default: "BOTH" + default: "SEV+TDX" type: choice options: - TDX - SNP - - BOTH + - SEV+TDX + - libvirt-ubuntu24 + - libvirt-ubuntu26 jobs: build: @@ -46,26 +48,63 @@ jobs: echo "RUNNER_ID=${{ github.run_number }}" >> $GITHUB_ENV - name: Run TDX docker build - if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'BOTH' }} + if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'SEV+TDX' }} working-directory: ${{ env.WORK_DIR }} run: | NON_INTERACTIVE=1 FORCE_REBUILD_CONTAINER=1 ./build/build_in_docker.sh tdx - name: Run SNP docker build - if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'BOTH' }} + if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'SEV+TDX' }} working-directory: ${{ env.WORK_DIR }} run: | NON_INTERACTIVE=1 FORCE_REBUILD_CONTAINER=1 ./build/build_in_docker.sh snp + + - name: Build libvirt packages + if: ${{ startsWith(github.event.inputs.build_type, 'libvirt-ubuntu') }} + working-directory: ${{ env.WORK_DIR }} + run: | + case "${BUILD_TYPE}" in + libvirt-ubuntu24) + target="ubuntu24" + output_directory="ubuntu-24.04" + ;; + libvirt-ubuntu26) + target="ubuntu26" + output_directory="ubuntu-26.04" + ;; + *) + echo "Error: unsupported libvirt build type: ${BUILD_TYPE}" >&2 + exit 1 + ;; + esac + + archive_name="${BUILD_TYPE}.tar.gz" + archive_path="${WORK_DIR}/build/libvirt/out/${archive_name}" + + ./build/libvirt/build.sh "${target}" + + tar \ + --create \ + --gzip \ + --file "${archive_path}" \ + --directory "${WORK_DIR}/build/libvirt/out" \ + "${output_directory}" + + tar --list --gzip --file "${archive_path}" >/dev/null + echo "LIBVIRT_ARCHIVE_NAME=${archive_name}" >> "$GITHUB_ENV" + echo "LIBVIRT_ARCHIVE_PATH=${archive_path}" >> "$GITHUB_ENV" - name: Set release name id: release-name run: | - if [[ "${BUILD_TYPE}" == "BOTH" ]]; then - echo "RELEASE_NAME=${RUNNER_ID}-tdx+snp" >> $GITHUB_ENV + if [[ "${BUILD_TYPE}" == "SEV+TDX" ]]; then + echo "RELEASE_NAME=${RUNNER_ID}-sev+tdx" >> "$GITHUB_ENV" elif [[ "${BUILD_TYPE}" == "TDX" ]]; then - echo "RELEASE_NAME=${RUNNER_ID}-tdx" >> $GITHUB_ENV + echo "RELEASE_NAME=${RUNNER_ID}-tdx" >> "$GITHUB_ENV" elif [[ "${BUILD_TYPE}" == "SNP" ]]; then - echo "RELEASE_NAME=${RUNNER_ID}-snp" >> $GITHUB_ENV + echo "RELEASE_NAME=${RUNNER_ID}-snp" >> "$GITHUB_ENV" + elif [[ "${BUILD_TYPE}" == libvirt-ubuntu* ]]; then + echo "RELEASE_NAME=${RUNNER_ID}-${BUILD_TYPE}" >> "$GITHUB_ENV" else echo "Error: Unknown BUILD_TYPE ${BUILD_TYPE}" >&2 exit 1 @@ -84,7 +123,7 @@ jobs: prerelease: true - name: Upload TDX Release Asset - if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'BOTH' }} + if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'SEV+TDX' }} uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -95,7 +134,7 @@ jobs: asset_content_type: application/gzip - name: Upload SNP Release Asset - if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'BOTH' }} + if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'SEV+TDX' }} uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -104,3 +143,14 @@ jobs: asset_path: ${{ env.WORK_DIR }}/build/out/snp/package-snp.tar.gz asset_name: package-snp.tar.gz asset_content_type: application/gzip + + - name: Upload libvirt Release Asset + if: ${{ startsWith(github.event.inputs.build_type, 'libvirt-ubuntu') }} + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ env.LIBVIRT_ARCHIVE_PATH }} + asset_name: ${{ env.LIBVIRT_ARCHIVE_NAME }} + asset_content_type: application/gzip diff --git a/README.md b/README.md index 80a3fa9..15d5faf 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ There are two ways to run a Super Protocol Swarm cluster. ### Single-host cluster (quick start) -`scripts/swarm-cluster.sh` brings up a **3-node Swarm cluster on a single host** — no multi-machine setup. It creates an isolated bridge network, launches one bootstrap + two join VMs in separate `tmux` sessions, auto-configures provider configs, and sets up ingress via HAProxy. You still need to set `gateway_hostname` in the provider template to point to the machine's public IP. +`scripts/swarm-cluster.sh` brings up a **3-node Swarm cluster on a single host** — no multi-machine setup. It creates an isolated bridge network, launches one bootstrap + two join VMs as transient libvirt domains, auto-configures provider configs, and sets up ingress via HAProxy. In debug mode their serial consoles remain attached in separate `tmux` sessions. You still need to set `gateway_hostname` in the provider template to point to the machine's public IP. Prerequisites: a bootstrapped host (TDX or SEV-SNP), a populated provider config template (see [config.yaml reference](docs/swarm.md#configyaml-reference) for an example), and `tmux` / `nftables` / `curl` installed. diff --git a/build/libvirt/.dockerignore b/build/libvirt/.dockerignore new file mode 100644 index 0000000..1fcb152 --- /dev/null +++ b/build/libvirt/.dockerignore @@ -0,0 +1 @@ +out diff --git a/build/libvirt/Dockerfile.ubuntu24 b/build/libvirt/Dockerfile.ubuntu24 new file mode 100644 index 0000000..cf3adf0 --- /dev/null +++ b/build/libvirt/Dockerfile.ubuntu24 @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1 + +FROM ubuntu:24.04 + +ARG LIBVIRT_VERSION=12.5.0 +ARG DEBIAN_REVISION=1 + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + devscripts \ + equivs \ + git \ + libdistro-info-perl \ + && rm -rf /var/lib/apt/lists/* + +COPY docker/prepare-source.sh /usr/local/bin/prepare-libvirt-source +RUN prepare-libvirt-source "${LIBVIRT_VERSION}" "${DEBIAN_REVISION}" + +COPY docker/build-packages.sh /usr/local/bin/build-libvirt-packages + +ENTRYPOINT ["/usr/local/bin/build-libvirt-packages"] diff --git a/build/libvirt/Dockerfile.ubuntu26 b/build/libvirt/Dockerfile.ubuntu26 new file mode 100644 index 0000000..0d0c1d4 --- /dev/null +++ b/build/libvirt/Dockerfile.ubuntu26 @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1 + +FROM ubuntu:26.04 + +ARG LIBVIRT_VERSION=12.5.0 +ARG DEBIAN_REVISION=1 + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + devscripts \ + equivs \ + git \ + libdistro-info-perl \ + && rm -rf /var/lib/apt/lists/* + +COPY docker/prepare-source.sh /usr/local/bin/prepare-libvirt-source +RUN prepare-libvirt-source "${LIBVIRT_VERSION}" "${DEBIAN_REVISION}" + +COPY docker/build-packages.sh /usr/local/bin/build-libvirt-packages + +ENTRYPOINT ["/usr/local/bin/build-libvirt-packages"] diff --git a/build/libvirt/README.md b/build/libvirt/README.md new file mode 100644 index 0000000..f87c788 --- /dev/null +++ b/build/libvirt/README.md @@ -0,0 +1,86 @@ +# Local libvirt packages + +This directory contains separate Docker build environments for Ubuntu 24.04 +and Ubuntu 26.04. The build uses the Debian libvirt packaging and resolves all +build dependencies inside the target Ubuntu image. + +The default source is the `debian/12.5.0-1` tag from the Debian libvirt Salsa +repository. Resulting packages have a local version such as: + +```text +12.5.0-1spvm1~ubuntu24.04.1 +12.5.0-1spvm1~ubuntu26.04.1 +``` + +The local suffix sorts after older upstream versions but before a future +`12.5.0-1ubuntu*` package, so an official build of the same upstream release +can replace it normally. + +## Requirements + +- Docker with access to the Docker daemon; +- Internet access for the Ubuntu repositories and Debian Salsa; +- an amd64 host, or a Docker setup capable of building `linux/amd64` images; +- enough free space for two builder images and package artifacts. + +## Manual build + +Run from the repository root: + +```bash +./build/libvirt/build.sh ubuntu24 +./build/libvirt/build.sh ubuntu26 +``` + +Build both targets sequentially: + +```bash +./build/libvirt/build.sh all +``` + +Package tests are enabled by default. For a faster development build that only +compiles and packages libvirt: + +```bash +./build/libvirt/build.sh all --skip-tests +``` + +The script supports alternative upstream/local revisions, for example: + +```bash +./build/libvirt/build.sh ubuntu24 \ + --libvirt-version 12.5.0 \ + --debian-revision 1 \ + --spvm-revision 2 +``` + +Use `./build/libvirt/build.sh --help` to see all options. + +## Artifacts + +Packages are placed in a target- and version-specific directory: + +```text +build/libvirt/out/ubuntu-24.04/12.5.0-1spvm1~ubuntu24.04.1/ +build/libvirt/out/ubuntu-26.04/12.5.0-1spvm1~ubuntu26.04.1/ +``` + +Each directory contains the split libvirt `.deb` packages, debug `.ddeb` +packages, `.changes`, `.buildinfo`, and `SHA256SUMS`. Do not mix packages built +for different Ubuntu releases. + +The build script only creates local, unsigned packages. It does not install +them, create an APT repository, or build QEMU and python3-libvirt. + +The `Build packages self-hosted` GitHub Actions workflow can build either +Ubuntu target and publish the result to a prerelease. Select +`libvirt-ubuntu24` or `libvirt-ubuntu26` in the workflow dispatch form. The +release contains one compressed tar archive: + +```text +libvirt-ubuntu24.tar.gz +libvirt-ubuntu26.tar.gz +``` + +Each archive preserves the target and package-version directories and contains +the complete package output together with its `SHA256SUMS` file. diff --git a/build/libvirt/build.sh b/build/libvirt/build.sh new file mode 100755 index 0000000..fb64fe7 --- /dev/null +++ b/build/libvirt/build.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + +LIBVIRT_VERSION=12.5.0 +DEBIAN_REVISION=1 +SPVM_REVISION=1 +OUTPUT_ROOT="${SCRIPT_DIR}/out" +DOCKER_PLATFORM=linux/amd64 +SKIP_TESTS=false +NO_CACHE=false +PULL_BASE=false +TARGET="" + +usage() { + cat <<'EOF' +Build local libvirt Debian packages for Ubuntu 24.04 and/or 26.04. + +Usage: + ./build/libvirt/build.sh [options] + +Options: + --libvirt-version VERSION Upstream libvirt version (default: 12.5.0) + --debian-revision NUMBER Debian packaging revision (default: 1) + --spvm-revision NUMBER Local package revision (default: 1) + --output DIR Artifact root (default: build/libvirt/out) + --platform PLATFORM Docker platform (default: linux/amd64) + --skip-tests Set DEB_BUILD_OPTIONS=nocheck + --no-cache Rebuild the Docker image without cache + --pull Pull the latest Ubuntu base image + -h, --help Show this help + +The script only builds local artifacts. It does not install or publish them. +EOF +} + +if [[ $# -eq 0 ]]; then + usage >&2 + exit 2 +fi + +if [[ "$1" == "-h" || "$1" == "--help" ]]; then + usage + exit 0 +fi + +TARGET=$1 +shift + +while [[ $# -gt 0 ]]; do + case "$1" in + --libvirt-version) + [[ $# -ge 2 ]] || { echo "Error: --libvirt-version requires a value" >&2; exit 2; } + LIBVIRT_VERSION=$2 + shift 2 + ;; + --debian-revision) + [[ $# -ge 2 ]] || { echo "Error: --debian-revision requires a value" >&2; exit 2; } + DEBIAN_REVISION=$2 + shift 2 + ;; + --spvm-revision) + [[ $# -ge 2 ]] || { echo "Error: --spvm-revision requires a value" >&2; exit 2; } + SPVM_REVISION=$2 + shift 2 + ;; + --output) + [[ $# -ge 2 ]] || { echo "Error: --output requires a value" >&2; exit 2; } + OUTPUT_ROOT=$2 + shift 2 + ;; + --platform) + [[ $# -ge 2 ]] || { echo "Error: --platform requires a value" >&2; exit 2; } + DOCKER_PLATFORM=$2 + shift 2 + ;; + --skip-tests) + SKIP_TESTS=true + shift + ;; + --no-cache) + NO_CACHE=true + shift + ;; + --pull) + PULL_BASE=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +case "${TARGET}" in + ubuntu24) + targets=(ubuntu24) + ;; + ubuntu26) + targets=(ubuntu26) + ;; + all) + targets=(ubuntu24 ubuntu26) + ;; + *) + echo "Error: target must be ubuntu24, ubuntu26, or all" >&2 + usage >&2 + exit 2 + ;; +esac + +if ! [[ "${LIBVIRT_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: invalid libvirt version: ${LIBVIRT_VERSION}" >&2 + exit 2 +fi +if ! [[ "${DEBIAN_REVISION}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: Debian revision must be a positive integer" >&2 + exit 2 +fi +if ! [[ "${SPVM_REVISION}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: SPVM revision must be a positive integer" >&2 + exit 2 +fi +if ! command -v docker >/dev/null 2>&1; then + echo "Error: docker is not installed" >&2 + exit 1 +fi +if ! docker info >/dev/null 2>&1; then + echo "Error: cannot connect to the Docker daemon" >&2 + exit 1 +fi + +mkdir -p "${OUTPUT_ROOT}" +OUTPUT_ROOT=$(realpath "${OUTPUT_ROOT}") + +build_target() { + local target=$1 ubuntu_version dockerfile image_name output_dir package_version + local -a build_args + case "${target}" in + ubuntu24) + ubuntu_version=24.04 + dockerfile=Dockerfile.ubuntu24 + ;; + ubuntu26) + ubuntu_version=26.04 + dockerfile=Dockerfile.ubuntu26 + ;; + esac + + image_name="sp-vm-libvirt-builder:ubuntu${ubuntu_version}-${LIBVIRT_VERSION}-${DEBIAN_REVISION}" + package_version="${LIBVIRT_VERSION}-${DEBIAN_REVISION}spvm${SPVM_REVISION}~ubuntu${ubuntu_version}.1" + output_dir="${OUTPUT_ROOT}/ubuntu-${ubuntu_version}/${package_version}" + mkdir -p "${output_dir}" + + build_args=( + build + --platform "${DOCKER_PLATFORM}" + --file "${SCRIPT_DIR}/${dockerfile}" + --tag "${image_name}" + --build-arg "LIBVIRT_VERSION=${LIBVIRT_VERSION}" + --build-arg "DEBIAN_REVISION=${DEBIAN_REVISION}" + ) + if [[ "${NO_CACHE}" == "true" ]]; then + build_args+=(--no-cache) + fi + if [[ "${PULL_BASE}" == "true" ]]; then + build_args+=(--pull) + fi + build_args+=("${SCRIPT_DIR}") + + echo "Building Docker image ${image_name}" + docker "${build_args[@]}" + + echo "Building libvirt packages for Ubuntu ${ubuntu_version}" + docker run \ + --rm \ + --platform "${DOCKER_PLATFORM}" \ + --env "LIBVIRT_VERSION=${LIBVIRT_VERSION}" \ + --env "DEBIAN_REVISION=${DEBIAN_REVISION}" \ + --env "SPVM_REVISION=${SPVM_REVISION}" \ + --env "TARGET_UBUNTU_VERSION=${ubuntu_version}" \ + --env "SKIP_TESTS=${SKIP_TESTS}" \ + --env "OUTPUT_UID=$(id -u)" \ + --env "OUTPUT_GID=$(id -g)" \ + --volume "${output_dir}:/out" \ + "${image_name}" + + echo "Artifacts: ${output_dir}" +} + +for target in "${targets[@]}"; do + build_target "${target}" +done diff --git a/build/libvirt/docker/build-packages.sh b/build/libvirt/docker/build-packages.sh new file mode 100755 index 0000000..01b8204 --- /dev/null +++ b/build/libvirt/docker/build-packages.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -euo pipefail + +readonly SOURCE_DIR=/opt/libvirt-source +readonly OUTPUT_DIR=/out + +readonly LIBVIRT_VERSION=${LIBVIRT_VERSION:?LIBVIRT_VERSION is required} +readonly DEBIAN_REVISION=${DEBIAN_REVISION:?DEBIAN_REVISION is required} +readonly SPVM_REVISION=${SPVM_REVISION:-1} +readonly TARGET_UBUNTU_VERSION=${TARGET_UBUNTU_VERSION:?TARGET_UBUNTU_VERSION is required} +readonly SKIP_TESTS=${SKIP_TESTS:-false} +readonly OUTPUT_UID=${OUTPUT_UID:-0} +readonly OUTPUT_GID=${OUTPUT_GID:-0} + +# Provided by every Ubuntu builder image. +# shellcheck disable=SC1091 +source /etc/os-release +if [[ "${ID}" != "ubuntu" || "${VERSION_ID}" != "${TARGET_UBUNTU_VERSION}" ]]; then + echo "Error: builder is Ubuntu ${VERSION_ID:-unknown}, target is ${TARGET_UBUNTU_VERSION}" >&2 + exit 1 +fi + +case "${TARGET_UBUNTU_VERSION}" in + 24.04) + expected_codename=noble + ;; + 26.04) + expected_codename=resolute + ;; + *) + echo "Error: unsupported Ubuntu target: ${TARGET_UBUNTU_VERSION}" >&2 + exit 1 + ;; +esac + +if [[ "${VERSION_CODENAME}" != "${expected_codename}" ]]; then + echo "Error: Ubuntu ${TARGET_UBUNTU_VERSION} has unexpected codename ${VERSION_CODENAME}" >&2 + exit 1 +fi + +if [[ ! "${SPVM_REVISION}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: SPVM_REVISION must be a positive integer" >&2 + exit 1 +fi + +readonly PACKAGE_VERSION="${LIBVIRT_VERSION}-${DEBIAN_REVISION}spvm${SPVM_REVISION}~ubuntu${TARGET_UBUNTU_VERSION}.1" + +work_dir=$(mktemp -d /tmp/libvirt-package-build.XXXXXX) +cleanup() { + rm -rf -- "${work_dir}" +} +trap cleanup EXIT + +cp -a "${SOURCE_DIR}" "${work_dir}/libvirt" +cd "${work_dir}/libvirt" + +export DEBFULLNAME="Super Protocol VM Tools" +export DEBEMAIL="devnull@superprotocol.com" +dch \ + --newversion "${PACKAGE_VERSION}" \ + --distribution "${expected_codename}" \ + --force-distribution \ + "Local rebuild for Ubuntu ${TARGET_UBUNTU_VERSION}." + +build_options="parallel=$(nproc)" +if [[ "${SKIP_TESTS}" == "true" ]]; then + build_options="${build_options} nocheck" +elif [[ "${SKIP_TESTS}" != "false" ]]; then + echo "Error: SKIP_TESTS must be true or false" >&2 + exit 1 +fi +export DEB_BUILD_OPTIONS="${build_options}" + +echo "Building libvirt ${PACKAGE_VERSION} on Ubuntu ${TARGET_UBUNTU_VERSION} (${VERSION_CODENAME})" +dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes -jauto + +mkdir -p "${OUTPUT_DIR}" +shopt -s nullglob +artifacts=( + "${work_dir}"/*.deb + "${work_dir}"/*.ddeb + "${work_dir}"/*.changes + "${work_dir}"/*.buildinfo +) +if [[ ${#artifacts[@]} -eq 0 ]]; then + echo "Error: package build produced no artifacts" >&2 + exit 1 +fi + +for artifact in "${artifacts[@]}"; do + install -m 0644 "${artifact}" "${OUTPUT_DIR}/" +done + +( + cd "${OUTPUT_DIR}" + debs=( ./*.deb ./*.ddeb ) + sha256sum "${debs[@]}" > SHA256SUMS +) + +chown "${OUTPUT_UID}:${OUTPUT_GID}" "${OUTPUT_DIR}"/* + +echo "Built ${#artifacts[@]} artifacts in ${OUTPUT_DIR}" +echo "Package version: ${PACKAGE_VERSION}" diff --git a/build/libvirt/docker/prepare-source.sh b/build/libvirt/docker/prepare-source.sh new file mode 100755 index 0000000..3b5ca16 --- /dev/null +++ b/build/libvirt/docker/prepare-source.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -euo pipefail + +readonly LIBVIRT_VERSION=${1:?libvirt version is required} +readonly DEBIAN_REVISION=${2:?Debian revision is required} +readonly PACKAGING_REF="debian/${LIBVIRT_VERSION}-${DEBIAN_REVISION}" +readonly SOURCE_DIR=/opt/libvirt-source +readonly PACKAGING_REPOSITORY=https://salsa.debian.org/libvirt-team/libvirt.git + +git clone \ + --branch "${PACKAGING_REF}" \ + --depth 1 \ + "${PACKAGING_REPOSITORY}" \ + "${SOURCE_DIR}" + +actual_version=$(dpkg-parsechangelog -l"${SOURCE_DIR}/debian/changelog" -SVersion) +expected_version="${LIBVIRT_VERSION}-${DEBIAN_REVISION}" +if [[ "${actual_version}" != "${expected_version}" ]]; then + echo "Error: ${PACKAGING_REF} contains version ${actual_version}, expected ${expected_version}" >&2 + exit 1 +fi + +cd "${SOURCE_DIR}" +apt-get update +mk-build-deps \ + --install \ + --remove \ + --tool 'apt-get -y --no-install-recommends' \ + debian/control + +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/build/libvirt/out/.gitignore b/build/libvirt/out/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/build/libvirt/out/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore From 8750f4ce3fa2a823939fd864aa590c7ddfd04327 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 6 Aug 2026 10:34:49 -0500 Subject: [PATCH 09/28] bootsrap libvirt --- README.md | 89 ++-- scripts/bootstrap_snp.sh | 4 + scripts/bootstrap_tdx.sh | 4 + scripts/setup_libvirt_host.sh | 525 ++++++++++++++++++++++++ scripts/start_super_protocol_libvirt.sh | 91 ++-- 5 files changed, 626 insertions(+), 87 deletions(-) create mode 100755 scripts/setup_libvirt_host.sh diff --git a/README.md b/README.md index 15d5faf..42634b1 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Utilities for bootstrapping a Confidential Computing host (Intel **TDX** or AMD | `scripts/bootstrap_tdx.sh` | Turn an Ubuntu host into a TDX-capable hypervisor (kernel, QEMU, OVMF, attestation, GPU passthrough). | | `scripts/bootstrap_snp.sh` | Turn an Ubuntu host into a SEV-SNP-capable hypervisor (firmware, modules, GPU passthrough). | | `scripts/start_super_protocol.sh` | Start a confidential VM (TDX / SEV-SNP / untrusted) from a Super Protocol release image. | -| `scripts/start_super_protocol_libvirt.sh` | Start the same VM as a transient `qemu:///system` domain through libvirt-python (Ubuntu 26.04+). | +| `scripts/start_super_protocol_libvirt.sh` | Start the same VM as a transient `qemu:///system` domain through libvirt-python (Ubuntu 24.04 or 26.04). | | `scripts/swarm-cluster.sh` | Bring up a 3-node Swarm cluster on a single host. | | `scripts/check_configuration.sh`, `get_super_running_vms.sh` | Auxiliary tooling. | @@ -36,9 +36,9 @@ This is the main path: take a bare Ubuntu host, turn it into a confidential hype For the exact commands to clone the repository, run the bootstrap scripts, and launch a VM, see [docs/swarm.md](docs/swarm.md). -### Libvirt launcher (Ubuntu 26.04+) +### Libvirt launcher (Ubuntu 24.04 and 26.04) -`scripts/start_super_protocol_libvirt.sh` reuses the release, disk, provider-config, and VFIO preparation from the direct QEMU launcher, then builds domain XML and starts a transient domain through `libvirt-python`. It requires `libvirt-daemon-system`, `libvirt-clients`, `python3-libvirt`, and `passt`. GPU passthrough uses IOMMUFD and therefore requires libvirt **12.1.0 or newer**; the launcher checks the daemon and domain capabilities before binding devices or recreating disks. +`scripts/start_super_protocol_libvirt.sh` reuses the release, disk, provider-config, and VFIO preparation from the direct QEMU launcher, then builds domain XML and starts a transient domain through `libvirt-python`. GPU passthrough uses IOMMUFD. The TDX and SEV-SNP bootstrap scripts install libvirt **12.5.0** when the system version is older, configure AppArmor, grant `passt` the capability required for privileged ports, and validate the daemon and domain capabilities before VFIO devices are bound. The command line is the same as for `start_super_protocol.sh`, with an optional domain name: @@ -51,69 +51,33 @@ sudo ./scripts/start_super_protocol_libvirt.sh \ With `--debug false` the command returns after the domain starts. With `--debug true --log_file /path/to/boot.log`, it attaches a bidirectional serial console and copies console output to the log; `Ctrl-C` or `Ctrl-]` detaches without stopping the VM. Use `virsh -c qemu:///system list`, `console`, `shutdown`, or `destroy` to manage it. `--gpu none` disables GPU, NVSwitch, and CX7 passthrough for diagnostics. -#### Ubuntu 26.04 AppArmor and `passt` +#### Libvirt host configuration -The Ubuntu 26.04 libvirt AppArmor profile may allow `/usr/bin/passt` to be read but not memory-mapped. In that case libvirt reports `passt ... unexpected fatal signal 11`, while the kernel audit log contains a denial similar to: - -```text -apparmor="DENIED" operation="file_mmap" name="/usr/bin/passt" requested_mask="rm" -``` - -Confirm the cause with: - -```bash -sudo journalctl -k --since '-10 min' --no-pager | - grep -E 'apparmor="DENIED".*(passt|libvirt)|comm="passt"' -``` - -Until this host configuration is incorporated into the bootstrap scripts, back up and adjust the nested `passt` profile, then add a local rule allowing QEMU to connect to the libvirt-managed socket: - -```bash -sudo cp -a --update=none \ - /etc/apparmor.d/abstractions/libvirt-qemu \ - /etc/apparmor.d/abstractions/libvirt-qemu.sp-vm-tools.bak - -sudo sed -i \ - '/^[[:space:]]*profile passt[[:space:]]*{/,/^[[:space:]]*}/ s|/usr/bin/passt r,|/usr/bin/passt rm,|' \ - /etc/apparmor.d/abstractions/libvirt-qemu - -sudo install -d -m 0755 \ - /etc/apparmor.d/abstractions/libvirt-qemu.d - -printf '%s\n' 'owner @{run}/libvirt/qemu/passt/* rw,' | \ - sudo tee /etc/apparmor.d/abstractions/libvirt-qemu.d/99-passt-local >/dev/null - -sudo systemctl reload apparmor -``` - -Do not disable AppArmor globally. The launcher detects the incompatible read-only `passt` rule before binding VFIO devices or recreating VM disks. - -`passt` runs as the unprivileged libvirt QEMU user. Forwarding a host port below 1024 therefore also requires lowering the host's unprivileged-port boundary to the lowest forwarded port. For the default DNS port, configure it persistently with: +Run the bootstrap matching the host CPU before using the libvirt launcher: ```bash -printf '%s\n' 'net.ipv4.ip_unprivileged_port_start = 53' | \ - sudo tee /etc/sysctl.d/90-sp-vm-passt.conf >/dev/null - -sudo sysctl --system +sudo ./scripts/bootstrap_tdx.sh +# or +sudo ./scripts/bootstrap_snp.sh ``` -The launcher checks this value before preparing the VM. This sysctl and the AppArmor adjustment are temporary host-preparation steps that should be moved into the Ubuntu 26.04 bootstrap in the future. +The bootstrap performs the host-wide work that previously required manual fixes: -As a narrower alternative to changing the system-wide sysctl, grant `CAP_NET_BIND_SERVICE` only to the installed `passt` binaries: +- installs the project libvirt 12.5 packages when the installed version is older; +- preserves already installed libvirt split drivers during the package transaction; +- enables executable mmap and the libvirt socket in the nested AppArmor `passt` profile; +- permits QEMU to contact TDX QGS through VSOCK; +- applies `CAP_NET_BIND_SERVICE` to every installed `passt` binary; +- prepares `/var/lib/libvirt/images/superprotocol` and validates `qemu:///system`. -```bash -sudo apt-get install libcap2-bin - -for binary in /usr/bin/passt /usr/bin/passt.avx2; do - if [[ -x "${binary}" ]]; then - sudo setcap 'cap_net_bind_service=+ep' "${binary}" - fi -done +The bootstrap deliberately does not change `net.ipv4.ip_unprivileged_port_start`. File capabilities can be removed when the administrator upgrades or reinstalls `passt`; re-run the same bootstrap to restore them. The launcher detects missing AppArmor rules or capabilities before preparing VM disks and prints the appropriate bootstrap command. -getcap /usr/bin/passt /usr/bin/passt.avx2 2>/dev/null -``` +On Ubuntu 24.04, the bootstrap adapts only the verified temporary copy of the +project `libvirt-daemon-driver-qemu` package to the `systemd-sysusers` syntax +supported by that release. The downloaded release archive itself is not +modified. -The AppArmor profile above already allows this capability, and the launcher recognizes it during preflight. Package upgrades can replace the binaries and remove their file capabilities, in which case the `setcap` step must be repeated. The `passt` project recommends the sysctl method in general, but documents file capabilities as an option on hosts sufficiently constrained by an LSM such as AppArmor. +Do not disable AppArmor globally. For diagnostics, inspect recent denials with `journalctl -k --since '-10 min' --no-pager`. ### 1. Clone the repo @@ -133,6 +97,7 @@ What it does: 4. Runs the official `setup-tdx-host.sh` from `canonical/tdx`. 5. Updates the Intel TDX-Module to a known-good version. 6. Configures NVIDIA GPUs for Confidential Computing (CC mode + `vfio-pci` binding) and, on B200 systems, sets up ConnectX-7 bridges for VFIO passthrough. +7. Installs and validates libvirt 12.5, AppArmor policy, VSOCK access, and `passt` capabilities before binding devices to the VM stack. > **Note:** Some steps require manual action to take effect. The script may stop and ask you to do something, then need to be re-run — this is expected. Follow the on-screen instructions and re-run to finish. @@ -145,6 +110,7 @@ What it does: 3. Downloads and installs the matching AMD SEV firmware blob to `/lib/firmware/amd/` and reloads `ccp` / `kvm_amd`. 4. Runs SNP status checks (RMP table, SEV / SEV-SNP API versions, ASID allocation, IOMMU groups, hugepages, CPU governor). 5. Configures NVIDIA GPUs for CC mode and binds them to `vfio-pci`. +6. Installs and validates libvirt 12.5, AppArmor policy, and `passt` capabilities before binding devices to the VM stack. > **Ubuntu 24.04 note:** the SNP bootstrap installs a bundled Linux **6.16** kernel. On some systems, network interfaces may be renamed after reboot, which can affect networking and remote SSH access. Make sure you have iKVM or other interactive console access before rebooting, so you can reconfigure networking for the new interface names if needed. @@ -156,6 +122,15 @@ A reboot is required partway through bootstrap. After reboot, re-run the same bo `scripts/check_configuration.sh` prints a hardware overview (CPU, memory, network, disks, RAID/SMART) you can compare against the [Requirements](#requirements). See [docs/swarm.md](docs/swarm.md) for how to run it. +Hardware acceptance remains a manual step because containers cannot validate KVM, IOMMUFD, QGS, VSOCK, or physical GPU assignment. On each prepared host verify: + +- a transient VM starts in release and debug modes; +- TCP/UDP forwarding works on host ports 53, 80, and 443; +- TDX measurement returns a non-empty quote and PKI/gossip become ready; +- SEV-SNP launch security is active; +- `--gpu none` works and an enabled GPU is attached through IOMMUFD; +- the kernel audit log contains no new `passt`, libvirt, or VSOCK AppArmor denial. + ## Running a Swarm cluster There are two ways to run a Super Protocol Swarm cluster. diff --git a/scripts/bootstrap_snp.sh b/scripts/bootstrap_snp.sh index d84c946..f9f9a9a 100755 --- a/scripts/bootstrap_snp.sh +++ b/scripts/bootstrap_snp.sh @@ -4,6 +4,7 @@ set -e source_common() { local script_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" source "${script_dir}/common.sh" + source "${script_dir}/setup_libvirt_host.sh" } get_kernel_log() { @@ -361,6 +362,7 @@ update_snp_firmware() { bootstrap() { check_os_version "24.04" + get_supported_ubuntu_version CPU_MODEL=$(lscpu | grep "^Model name:" | sed 's/Model name: *//') @@ -438,6 +440,8 @@ bootstrap() { fi fi + setup_libvirt_host sev-snp + print_section_header "Hardware Configuration" if command -v lspci >/dev/null; then echo "Checking NVIDIA GPU configuration..." diff --git a/scripts/bootstrap_tdx.sh b/scripts/bootstrap_tdx.sh index ccb0183..33274ac 100755 --- a/scripts/bootstrap_tdx.sh +++ b/scripts/bootstrap_tdx.sh @@ -4,10 +4,12 @@ set -e source_common() { local script_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" source "${script_dir}/common.sh" + source "${script_dir}/setup_libvirt_host.sh" } bootstrap() { check_os_version "24.04" + get_supported_ubuntu_version # Check if the script is running as root print_section_header "Privilege Check" @@ -46,6 +48,8 @@ bootstrap() { exit 1 fi + setup_libvirt_host tdx + print_section_header "Hardware Configuration" if command -v lspci >/dev/null; then echo "Checking NVIDIA GPU configuration..." diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh new file mode 100755 index 0000000..895482a --- /dev/null +++ b/scripts/setup_libvirt_host.sh @@ -0,0 +1,525 @@ +#!/bin/bash + +# Shared libvirt host preparation for bootstrap_tdx.sh and bootstrap_snp.sh. +# This file is sourceable for unit tests and can also be executed directly. + +LIBVIRT_REQUIRED_VERSION="12.5.0" +LIBVIRT_RELEASE_REPO="Super-Protocol/sp-vm-tools" +LIBVIRT_URI="qemu:///system" + +LIBVIRT_BASE_PACKAGES=( + libvirt0 + libvirt-common + libvirt-clients + libvirt-daemon + libvirt-daemon-common + libvirt-daemon-config-network + libvirt-daemon-config-nwfilter + libvirt-daemon-driver-network + libvirt-daemon-driver-nodedev + libvirt-daemon-driver-nwfilter + libvirt-daemon-driver-qemu + libvirt-daemon-driver-secret + libvirt-daemon-driver-storage + libvirt-daemon-log + libvirt-daemon-lock + libvirt-daemon-plugin-lockd + libvirt-daemon-system +) + +LIBVIRT_PACKAGE_PATHS=() +PASST_REAL_BINARIES=() + +libvirt_host_error() { + echo "ERROR: $*" >&2 + return 1 +} + +libvirt_host_path() { + printf '%s%s\n' "${SPVM_TEST_ROOT:-}" "$1" +} + +select_libvirt_release() { + local ubuntu_version=$1 + case "${ubuntu_version}" in + 24.04) + LIBVIRT_RELEASE_TAG="44-libvirt-ubuntu24" + LIBVIRT_RELEASE_ASSET="libvirt-ubuntu24.tar.gz" + LIBVIRT_RELEASE_SHA256="17a1aa837260e3e584b2ebcdd066ff541d2d89d1b798dc71c626976ff70ae452" + ;; + 26.04) + LIBVIRT_RELEASE_TAG="43-libvirt-ubuntu26" + LIBVIRT_RELEASE_ASSET="libvirt-ubuntu26.tar.gz" + LIBVIRT_RELEASE_SHA256="fd1ba716a9ee722c5fc0d3db72b92ccba3c57954c2464759ff4dc3f70a8bf6ad" + ;; + *) + libvirt_host_error "libvirt bootstrap supports Ubuntu 24.04 and 26.04 only (found ${ubuntu_version:-unknown})" + return 1 + ;; + esac + LIBVIRT_RELEASE_URL="https://github.com/${LIBVIRT_RELEASE_REPO}/releases/download/${LIBVIRT_RELEASE_TAG}/${LIBVIRT_RELEASE_ASSET}" +} + +get_supported_ubuntu_version() { + local os_release + os_release=$(libvirt_host_path /etc/os-release) + if [[ ! -r "${os_release}" ]]; then + libvirt_host_error "cannot read ${os_release}" + return 1 + fi + + local ID="" VERSION_ID="" + # shellcheck disable=SC1090 + source "${os_release}" + if [[ "${ID}" != "ubuntu" ]]; then + libvirt_host_error "libvirt bootstrap requires Ubuntu (found ${ID:-unknown})" + return 1 + fi + select_libvirt_release "${VERSION_ID}" + # Exported result for bootstrap callers. + # shellcheck disable=SC2034 + UBUNTU_VERSION="${VERSION_ID}" +} + +installed_libvirt_version() { + dpkg-query -W -f='${Version}\n' libvirt-daemon 2>/dev/null || true +} + +libvirt_upgrade_required() { + local installed_version=${1:-} + [[ -z "${installed_version}" ]] || \ + ! dpkg --compare-versions "${installed_version}" ge "${LIBVIRT_REQUIRED_VERSION}" +} + +validate_tar_listing() { + local entry trimmed component + local -a components + while IFS= read -r entry; do + trimmed=${entry#./} + if [[ "${entry}" == /* || -z "${trimmed}" ]]; then + libvirt_host_error "unsafe archive entry: ${entry}" + return 1 + fi + IFS='/' read -r -a components <<< "${trimmed}" + for component in "${components[@]}"; do + if [[ "${component}" == ".." ]]; then + libvirt_host_error "unsafe archive entry: ${entry}" + return 1 + fi + done + done +} + +verify_and_extract_libvirt_archive() { + local archive=$1 destination=$2 actual_sha package_dir sums_file + actual_sha=$(sha256sum "${archive}" | awk '{print $1}') + if [[ "${actual_sha}" != "${LIBVIRT_RELEASE_SHA256}" ]]; then + libvirt_host_error "checksum mismatch for ${archive}: expected ${LIBVIRT_RELEASE_SHA256}, got ${actual_sha}" + return 1 + fi + if ! tar -tzf "${archive}" | validate_tar_listing; then + return 1 + fi + tar -xzf "${archive}" -C "${destination}" + sums_file=$(find "${destination}" -mindepth 2 -maxdepth 3 -type f -name SHA256SUMS -print -quit) + if [[ -z "${sums_file}" ]]; then + libvirt_host_error "archive does not contain SHA256SUMS" + return 1 + fi + package_dir=$(dirname "${sums_file}") + if ! (cd "${package_dir}" && sha256sum -c SHA256SUMS); then + libvirt_host_error "one or more files in the libvirt archive failed checksum validation" + return 1 + fi + LIBVIRT_PACKAGE_DIR="${package_dir}" +} + +prepare_libvirt_package_compatibility() { + local package_dir=$1 ubuntu_version=$2 deb unpacked sysusers_file rebuilt package version + [[ "${ubuntu_version}" == "24.04" ]] || return 0 + deb=$(find_package_deb "${package_dir}" libvirt-daemon-driver-qemu) || { + libvirt_host_error "release archive is missing libvirt-daemon-driver-qemu" + return 1 + } + unpacked=$(mktemp -d) + dpkg-deb --raw-extract "${deb}" "${unpacked}" + sysusers_file="${unpacked}/usr/lib/sysusers.d/libvirt-qemu.conf" + if [[ ! -r "${sysusers_file}" ]]; then + rm -rf "${unpacked}" + libvirt_host_error "libvirt QEMU package does not contain its sysusers configuration" + return 1 + fi + if ! grep -qE '^u![[:space:]]' "${sysusers_file}"; then + rm -rf "${unpacked}" + return 0 + fi + + echo "Adapting libvirt-qemu sysusers syntax for Ubuntu 24.04 systemd 255" + sed -i -E 's/^u!([[:space:]])/u\1/' "${sysusers_file}" + if [[ -f "${unpacked}/DEBIAN/md5sums" ]]; then + local updated_md5 + updated_md5=$(cd "${unpacked}" && md5sum usr/lib/sysusers.d/libvirt-qemu.conf) + sed -i '\| usr/lib/sysusers.d/libvirt-qemu.conf$|d' "${unpacked}/DEBIAN/md5sums" + printf '%s\n' "${updated_md5}" >> "${unpacked}/DEBIAN/md5sums" + fi + rebuilt="${deb}.spvm-rebuilt" + dpkg-deb --build --root-owner-group "${unpacked}" "${rebuilt}" >/dev/null + package=$(dpkg-deb -f "${rebuilt}" Package) + version=$(dpkg-deb -f "${rebuilt}" Version) + if [[ "${package}" != "libvirt-daemon-driver-qemu" || "${version}" != 12.5.0-* ]]; then + rm -rf "${unpacked}" "${rebuilt}" + libvirt_host_error "rebuilt compatibility package has unexpected metadata" + return 1 + fi + mv "${rebuilt}" "${deb}" + rm -rf "${unpacked}" +} + +find_package_deb() { + local package_dir=$1 package=$2 candidate actual_package + while IFS= read -r -d '' candidate; do + actual_package=$(dpkg-deb -f "${candidate}" Package 2>/dev/null || true) + if [[ "${actual_package}" == "${package}" ]]; then + printf '%s\n' "${candidate}" + return 0 + fi + done < <(find "${package_dir}" -maxdepth 1 -type f -name '*.deb' -print0) + return 1 +} + +build_libvirt_package_plan() { + local package_dir=$1 package deb + local -A selected=() + LIBVIRT_PACKAGE_PATHS=() + + for package in "${LIBVIRT_BASE_PACKAGES[@]}"; do + if ! deb=$(find_package_deb "${package_dir}" "${package}"); then + libvirt_host_error "release archive is missing required package ${package}" + return 1 + fi + selected["${package}"]="${deb}" + done + + while IFS= read -r -d '' deb; do + package=$(dpkg-deb -f "${deb}" Package 2>/dev/null || true) + [[ -n "${package}" ]] || continue + if dpkg-query -W -f='${db:Status-Status}' "${package}" 2>/dev/null | grep -qx installed; then + selected["${package}"]="${deb}" + fi + done < <(find "${package_dir}" -maxdepth 1 -type f -name '*.deb' -print0) + + while IFS= read -r package; do + LIBVIRT_PACKAGE_PATHS+=("${selected[${package}]}") + done < <(printf '%s\n' "${!selected[@]}" | sort) +} + +assert_no_running_libvirt_domains() { + command -v virsh >/dev/null 2>&1 || return 0 + local running + running=$(virsh -c "${LIBVIRT_URI}" list --name 2>/dev/null | sed '/^[[:space:]]*$/d' || true) + if [[ -n "${running}" ]]; then + libvirt_host_error "refusing to upgrade libvirt while domains are running: ${running//$'\n'/, }" + return 1 + fi +} + +assert_safe_apt_simulation() { + local simulation=$1 + if grep -qE '^Remv[[:space:]]' <<< "${simulation}"; then + libvirt_host_error "APT would remove packages; refusing the libvirt transaction" + return 1 + fi + if grep -qiE 'DOWNGRADED|downgraded' <<< "${simulation}"; then + libvirt_host_error "APT would downgrade packages; refusing the libvirt transaction" + return 1 + fi +} + +install_project_libvirt() { + local work_dir archive simulation + assert_no_running_libvirt_domains + work_dir=$(mktemp -d /var/tmp/sp-vm-libvirt.XXXXXX) + chmod 0755 "${work_dir}" + archive="${work_dir}/${LIBVIRT_RELEASE_ASSET}" + + echo "Downloading libvirt ${LIBVIRT_REQUIRED_VERSION} from ${LIBVIRT_RELEASE_URL}" + wget --https-only --tries=3 -O "${archive}" "${LIBVIRT_RELEASE_URL}" + chmod 0644 "${archive}" + verify_and_extract_libvirt_archive "${archive}" "${work_dir}" + prepare_libvirt_package_compatibility "${LIBVIRT_PACKAGE_DIR}" "${UBUNTU_VERSION}" + find "${work_dir}" -type d -exec chmod a+rx {} + + find "${work_dir}" -type f \( -name '*.deb' -o -name '*.ddeb' \) -exec chmod a+r {} + + build_libvirt_package_plan "${LIBVIRT_PACKAGE_DIR}" + + echo "APT simulation for the libvirt upgrade:" + simulation=$(apt-get --simulate --no-install-recommends --no-remove install "${LIBVIRT_PACKAGE_PATHS[@]}") + printf '%s\n' "${simulation}" + assert_safe_apt_simulation "${simulation}" + + DEBIAN_FRONTEND=noninteractive apt-get \ + --no-install-recommends \ + --no-remove \ + -o Dpkg::Options::=--force-confold \ + install -y "${LIBVIRT_PACKAGE_PATHS[@]}" + rm -rf "${work_dir}" +} + +passthrough_profile_state() { + local profile=$1 + awk ' + /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1; found = 1; next } + in_passt && /^[[:space:]]*}/ { in_passt = 0; done = 1 } + in_passt && /\/usr\/bin\/passt[[:space:]]+r,/ { readonly = 1 } + in_passt && /\/usr\/bin\/passt[[:space:]]+rm,/ { mmap = 1 } + in_passt && /^[[:space:]]*capability[[:space:]]+net_bind_service,/ { capability = 1 } + END { + if (!found || !done) print "unknown" + else if (readonly) print "readonly" + else if (mmap && capability) print "ready" + else if (mmap) print "needs-capability" + else print "unknown" + } + ' "${profile}" +} + +patch_libvirt_apparmor_profile() { + local profile=$1 state backup tmp + state=$(passthrough_profile_state "${profile}") + if [[ "${state}" == "unknown" ]]; then + libvirt_host_error "unrecognized nested passt profile in ${profile}; refusing to modify it" + return 1 + fi + backup="${profile}.sp-vm-tools.bak" + if [[ ! -e "${backup}" ]]; then + cp -a "${profile}" "${backup}" + fi + + if [[ "${state}" == "readonly" ]]; then + sed -i \ + '/^[[:space:]]*profile passt[[:space:]]*{/,/^[[:space:]]*}/ s|/usr/bin/passt[[:space:]]\+r,|/usr/bin/passt rm,|' \ + "${profile}" + fi + state=$(passthrough_profile_state "${profile}") + if [[ "${state}" == "needs-capability" ]]; then + tmp=$(mktemp) + awk ' + /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1 } + { print } + in_passt && /\/usr\/bin\/passt[[:space:]]+rm,/ { + print " capability net_bind_service," + } + in_passt && /^[[:space:]]*}/ { in_passt = 0 } + ' "${profile}" > "${tmp}" + cat "${tmp}" > "${profile}" + rm -f "${tmp}" + fi + if [[ "$(passthrough_profile_state "${profile}")" != "ready" ]]; then + libvirt_host_error "failed to make the nested passt AppArmor profile usable" + return 1 + fi +} + +configure_libvirt_apparmor() { + local profile dropin_dir dropin template tmp + profile=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu) + dropin_dir=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu.d) + dropin="${dropin_dir}/99-sp-vm-tools-local" + template=$(libvirt_host_path /etc/apparmor.d/libvirt/TEMPLATE.qemu) + + [[ -r "${profile}" ]] || { + libvirt_host_error "libvirt AppArmor profile is missing: ${profile}" + return 1 + } + patch_libvirt_apparmor_profile "${profile}" + install -d -m 0755 "${dropin_dir}" + tmp=$(mktemp) + printf '%s\n' \ + '# Managed by sp-vm-tools bootstrap.' \ + 'owner @{run}/libvirt/qemu/passt/* rw,' \ + 'network vsock stream,' > "${tmp}" + install -m 0644 "${tmp}" "${dropin}" + rm -f "${tmp}" + + [[ -r "${template}" ]] || { + libvirt_host_error "libvirt AppArmor template is missing: ${template}" + return 1 + } + apparmor_parser -Q -r "${template}" + if [[ -z "${SPVM_TEST_ROOT:-}" ]]; then + systemctl reload apparmor + fi +} + +collect_passt_binaries() { + local candidate real + local -A seen=() + PASST_REAL_BINARIES=() + if [[ $# -eq 0 ]]; then + for candidate in passt passt.avx2; do + real=$(command -v "${candidate}" 2>/dev/null || true) + [[ -n "${real}" ]] || continue + set -- "$@" "${real}" + done + fi + for candidate in "$@"; do + [[ -x "${candidate}" ]] || continue + real=$(readlink -f -- "${candidate}") + [[ -n "${real}" && -z "${seen[${real}]:-}" ]] || continue + seen["${real}"]=1 + PASST_REAL_BINARIES+=("${real}") + done + if [[ ${#PASST_REAL_BINARIES[@]} -eq 0 ]]; then + libvirt_host_error "no executable passt binary was found" + return 1 + fi +} + +verify_passt_capabilities() { + local binary capabilities + collect_passt_binaries "$@" + for binary in "${PASST_REAL_BINARIES[@]}"; do + capabilities=$(getcap "${binary}" 2>/dev/null || true) + if [[ "${capabilities}" != *cap_net_bind_service* ]]; then + libvirt_host_error "${binary} does not have CAP_NET_BIND_SERVICE" + return 1 + fi + done +} + +# Optional arguments are used by unit tests to exercise symlink handling. +# shellcheck disable=SC2120 +configure_passt_capabilities() { + local binary + collect_passt_binaries "$@" + for binary in "${PASST_REAL_BINARIES[@]}"; do + setcap cap_net_bind_service=ep "${binary}" + if ! verify_passt_capabilities "${binary}"; then + libvirt_host_error "failed to set CAP_NET_BIND_SERVICE on ${binary}; check filesystem xattr support" + return 1 + fi + echo "Configured CAP_NET_BIND_SERVICE on ${binary}" + done +} + +find_bootstrap_qemu() { + local path + for path in \ + /usr/local/bin/qemu-system-x86_64 \ + /usr/bin/qemu-system-x86_64 \ + /bin/qemu-system-x86_64 \ + /usr/local/sbin/qemu-system-x86_64 \ + /usr/sbin/qemu-system-x86_64; do + [[ -x "${path}" ]] && { printf '%s\n' "${path}"; return 0; } + done + return 1 +} + +verify_libvirt_host() { + local mode=$1 installed_version daemon_version qemu version_line qemu_major capabilities dropin + installed_version=$(installed_libvirt_version) + if [[ -z "${installed_version}" ]] || ! dpkg --compare-versions "${installed_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then + libvirt_host_error "libvirt ${LIBVIRT_REQUIRED_VERSION} or newer is required; installed package is ${installed_version:-missing}" + return 1 + fi + python3 -c 'import libvirt' + id libvirt-qemu >/dev/null + aa-status --enabled >/dev/null + virsh -c "${LIBVIRT_URI}" uri >/dev/null + daemon_version=$(virsh -c "${LIBVIRT_URI}" version --daemon 2>/dev/null | sed -nE 's/.*daemon:[[:space:]]*([0-9.]+).*/\1/p' | tail -n 1) + if [[ -z "${daemon_version}" ]] || ! dpkg --compare-versions "${daemon_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then + libvirt_host_error "running libvirt daemon is older than ${LIBVIRT_REQUIRED_VERSION} (${daemon_version:-unknown})" + return 1 + fi + qemu=$(find_bootstrap_qemu) || { + libvirt_host_error "qemu-system-x86_64 was not found" + return 1 + } + version_line=$("${qemu}" --version 2>/dev/null | head -n 1) + qemu_major=$(sed -nE 's/.*version ([0-9]+).*/\1/p' <<< "${version_line}") + if [[ -z "${qemu_major}" || "${qemu_major}" -lt 9 ]]; then + libvirt_host_error "QEMU 9 or newer is required (found ${version_line:-unknown})" + return 1 + fi + capabilities=$(virsh -c "${LIBVIRT_URI}" domcapabilities --emulatorbin "${qemu}") + if ! grep -Eq "]*name=['\"]iommufd['\"]" <<< "${capabilities}"; then + libvirt_host_error "libvirt domain capabilities do not advertise IOMMUFD for ${qemu}" + return 1 + fi + # shellcheck disable=SC2119 + verify_passt_capabilities + + dropin=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu.d/99-sp-vm-tools-local) + grep -qF 'network vsock stream,' "${dropin}" || { + libvirt_host_error "AppArmor VSOCK rule is missing from ${dropin}" + return 1 + } + if [[ "${mode}" == "tdx" ]]; then + [[ -c /dev/vhost-vsock ]] || { + libvirt_host_error "/dev/vhost-vsock is missing" + return 1 + } + grep -Eq '^[[:space:]]*port[[:space:]]*=[[:space:]]*4050([[:space:]]|$)' /etc/qgs.conf || { + libvirt_host_error "QGS is not configured for VSOCK port 4050" + return 1 + } + systemctl is-active --quiet qgsd + elif [[ "${mode}" == "sev-snp" ]]; then + grep -qi 'sev-snp' <<< "${capabilities}" || { + libvirt_host_error "domain capabilities do not advertise SEV-SNP launch security" + return 1 + } + else + libvirt_host_error "invalid confidential VM mode: ${mode}" + return 1 + fi +} + +setup_libvirt_host() { + local mode=$1 installed_version + if [[ "${mode}" != "tdx" && "${mode}" != "sev-snp" ]]; then + libvirt_host_error "setup_libvirt_host mode must be tdx or sev-snp" + return 1 + fi + if [[ $(id -u) -ne 0 ]]; then + libvirt_host_error "libvirt host setup must run as root" + return 1 + fi + get_supported_ubuntu_version + + print_section_header "Libvirt Host Setup" + installed_version=$(installed_libvirt_version) + if libvirt_upgrade_required "${installed_version}"; then + assert_no_running_libvirt_domains + fi + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + acl apparmor-utils ca-certificates libcap2-bin passt python3-libvirt \ + qemu-system-x86 qemu-utils wget + + installed_version=$(installed_libvirt_version) + if libvirt_upgrade_required "${installed_version}"; then + echo "Installed libvirt ${installed_version:-none} is older than ${LIBVIRT_REQUIRED_VERSION}." + install_project_libvirt + else + echo "Installed libvirt ${installed_version} is ${LIBVIRT_REQUIRED_VERSION} or newer; keeping it." + fi + + configure_libvirt_apparmor + # shellcheck disable=SC2119 + configure_passt_capabilities + systemctl daemon-reload + systemctl enable --now libvirtd.service + systemctl start virtlogd.socket virtlockd.socket + install -d -o libvirt-qemu -g libvirt-qemu -m 0750 \ + /var/lib/libvirt/images/superprotocol + verify_libvirt_host "${mode}" + echo "Libvirt host setup complete." +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + set -euo pipefail + script_dir=$(cd "$(dirname "$0")" && pwd) + # shellcheck disable=SC1091 + source "${script_dir}/common.sh" + setup_libvirt_host "${1:-}" +fi diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index 17f8117..e79af6e 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -63,15 +63,25 @@ check_target_os() { os_id=$(sed -n 's/^ID=//p' /etc/os-release | tr -d '"') os_version=$(sed -n 's/^VERSION_ID=//p' /etc/os-release | tr -d '"') if [[ "${os_id}" != "ubuntu" ]]; then - echo "Error: this launcher supports Ubuntu 26.04 or newer (found ${os_id:-unknown})." >&2 + echo "Error: this launcher supports Ubuntu 24.04 and 26.04 (found ${os_id:-unknown})." >&2 exit 1 fi - if ! dpkg --compare-versions "${os_version}" ge "26.04"; then - echo "Error: this launcher requires Ubuntu 26.04 or newer (found ${os_version:-unknown})." >&2 + if [[ "${os_version}" != "24.04" && "${os_version}" != "26.04" ]]; then + echo "Error: this launcher supports Ubuntu 24.04 and 26.04 (found ${os_version:-unknown})." >&2 exit 1 fi } +bootstrap_hint() { + if [[ "${VM_MODE}" == "tdx" ]]; then + echo "Re-run scripts/bootstrap_tdx.sh to restore the libvirt host configuration." >&2 + elif [[ "${VM_MODE}" == "sev-snp" ]]; then + echo "Re-run scripts/bootstrap_snp.sh to restore the libvirt host configuration." >&2 + else + echo "Re-run the host bootstrap to restore the libvirt host configuration." >&2 + fi +} + check_libvirt_dependencies() { local missing=() command -v python3 >/dev/null 2>&1 || missing+=(python3) @@ -112,8 +122,29 @@ check_passt_apparmor_profile() { ' "${profile}"; then echo "Error: the libvirt AppArmor profile permits reading /usr/bin/passt but not mmap." >&2 echo "Ubuntu AppArmor 5 will kill passt with fatal signal 11." >&2 - echo "Update the rule inside 'profile passt' from '/usr/bin/passt r,' to '/usr/bin/passt rm,'" >&2 - echo "in ${profile}, reload AppArmor, and retry." >&2 + bootstrap_hint + exit 1 + fi + + if ! awk ' + /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1 } + in_passt && /^[[:space:]]*capability[[:space:]]+net_bind_service,/ { found = 1 } + in_passt && /^[[:space:]]*}/ { exit } + END { exit found ? 0 : 1 } + ' "${profile}"; then + echo "Error: the nested passt AppArmor profile does not allow CAP_NET_BIND_SERVICE." >&2 + bootstrap_hint + exit 1 + fi +} + +check_tdx_vsock_apparmor_profile() { + [[ "${VM_MODE}" == "tdx" ]] || return + local dropin=/etc/apparmor.d/abstractions/libvirt-qemu.d/99-sp-vm-tools-local + if [[ ! -r "${dropin}" ]] || ! grep -qF 'network vsock stream,' "${dropin}"; then + echo "Error: TDX QGS requires the AppArmor rule 'network vsock stream,'." >&2 + echo "Expected it in ${dropin}." >&2 + bootstrap_hint exit 1 fi } @@ -145,32 +176,31 @@ check_passt_privileged_ports() { return fi - local sysctl_path=/proc/sys/net/ipv4/ip_unprivileged_port_start - [[ -r "${sysctl_path}" ]] || return - local current - current=$(<"${sysctl_path}") - if ((current > minimum)); then - local capability_ok=true found_passt=false binary path capabilities - if ! command -v getcap >/dev/null 2>&1; then - capability_ok=false - else - for binary in passt passt.avx2; do - path=$(command -v "${binary}" 2>/dev/null || true) - [[ -n "${path}" ]] || continue - found_passt=true - capabilities=$(getcap "${path}" 2>/dev/null || true) - if [[ "${capabilities}" != *cap_net_bind_service* ]]; then - capability_ok=false - fi - done - fi - if [[ "${found_passt}" == "true" && "${capability_ok}" == "true" ]]; then - return - fi + command -v getcap >/dev/null 2>&1 || { + echo "Error: getcap is required to verify privileged passt port ${minimum}." >&2 + bootstrap_hint + exit 1 + } - echo "Error: passt must bind host port ${minimum}, but unprivileged ports currently start at ${current}." >&2 - echo "Lower net.ipv4.ip_unprivileged_port_start to ${minimum}, or grant CAP_NET_BIND_SERVICE" >&2 - echo "to every installed passt binary, then retry." >&2 + local binary path real capabilities found_passt=false + local -A checked=() + for binary in passt passt.avx2; do + path=$(command -v "${binary}" 2>/dev/null || true) + [[ -n "${path}" ]] || continue + real=$(readlink -f -- "${path}") + [[ -n "${real}" && -z "${checked[${real}]:-}" ]] || continue + checked["${real}"]=1 + found_passt=true + capabilities=$(getcap "${real}" 2>/dev/null || true) + if [[ "${capabilities}" != *cap_net_bind_service* ]]; then + echo "Error: passt must bind host port ${minimum}, but ${real} lacks CAP_NET_BIND_SERVICE." >&2 + bootstrap_hint + exit 1 + fi + done + if [[ "${found_passt}" != "true" ]]; then + echo "Error: no passt binary was found for privileged host port ${minimum}." >&2 + bootstrap_hint exit 1 fi } @@ -464,6 +494,7 @@ main_libvirt() { check_packages check_libvirt_dependencies check_passt_apparmor_profile + check_tdx_vsock_apparmor_profile find_qemu_path check_qemu_version preflight_libvirt From 47650a327e9e60b03a64569ea8afb4fddadf7de2 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 6 Aug 2026 13:08:23 -0500 Subject: [PATCH 10/28] fix installation on ubuntu 24 --- scripts/setup_libvirt_host.sh | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index 895482a..18969cc 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -85,6 +85,25 @@ installed_libvirt_version() { dpkg-query -W -f='${Version}\n' libvirt-daemon 2>/dev/null || true } +running_libvirt_version() { + local numeric + numeric=$(python3 -c ' +import libvirt +conn = libvirt.openReadOnly("qemu:///system") +try: + print(conn.getLibVersion()) +finally: + conn.close() +' 2>/dev/null || true) + if [[ ! "${numeric}" =~ ^[0-9]+$ ]]; then + return 1 + fi + printf '%d.%d.%d\n' \ + "$((numeric / 1000000))" \ + "$(((numeric / 1000) % 1000))" \ + "$((numeric % 1000))" +} + libvirt_upgrade_required() { local installed_version=${1:-} [[ -z "${installed_version}" ]] || \ @@ -425,7 +444,7 @@ verify_libvirt_host() { id libvirt-qemu >/dev/null aa-status --enabled >/dev/null virsh -c "${LIBVIRT_URI}" uri >/dev/null - daemon_version=$(virsh -c "${LIBVIRT_URI}" version --daemon 2>/dev/null | sed -nE 's/.*daemon:[[:space:]]*([0-9.]+).*/\1/p' | tail -n 1) + daemon_version=$(running_libvirt_version || true) if [[ -z "${daemon_version}" ]] || ! dpkg --compare-versions "${daemon_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then libvirt_host_error "running libvirt daemon is older than ${LIBVIRT_REQUIRED_VERSION} (${daemon_version:-unknown})" return 1 @@ -510,6 +529,16 @@ setup_libvirt_host() { systemctl daemon-reload systemctl enable --now libvirtd.service systemctl start virtlogd.socket virtlockd.socket + + local daemon_version + daemon_version=$(running_libvirt_version || true) + if [[ -z "${daemon_version}" ]] || \ + ! dpkg --compare-versions "${daemon_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then + assert_no_running_libvirt_domains + echo "Restarting libvirtd to activate the installed ${LIBVIRT_REQUIRED_VERSION} runtime" + systemctl restart libvirtd.service + fi + install -d -o libvirt-qemu -g libvirt-qemu -m 0750 \ /var/lib/libvirt/images/superprotocol verify_libvirt_host "${mode}" From d408564485e566a06cf44ef3098a028eb5d06c1b Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 6 Aug 2026 13:37:08 -0500 Subject: [PATCH 11/28] improved ubuntu 24 tdx setup --- scripts/bootstrap_tdx.sh | 8 ++-- scripts/setup_tdx.sh | 80 ++++++++++++++++++---------------------- 2 files changed, 40 insertions(+), 48 deletions(-) diff --git a/scripts/bootstrap_tdx.sh b/scripts/bootstrap_tdx.sh index 33274ac..5029adf 100755 --- a/scripts/bootstrap_tdx.sh +++ b/scripts/bootstrap_tdx.sh @@ -18,8 +18,8 @@ bootstrap() { exit 1 fi - # Download and setup official Canonical TDX - print_section_header "Official TDX Setup" + # Install the project TDX kernel/QEMU stack and host attestation runtime. + print_section_header "TDX Host Setup" TMP_DIR=$(mktemp -d) echo "Installing required tools..." @@ -66,9 +66,9 @@ bootstrap() { rm -rf "${TMP_DIR}" print_section_header "Installation Status" - echo "Official TDX installation complete." + echo "TDX host installation complete." echo "System reboot required to activate TDX." - echo "After reboot, use official tools to create and run TDs." + echo "After reboot, re-run this bootstrap to finish validation." } source_common diff --git a/scripts/setup_tdx.sh b/scripts/setup_tdx.sh index fd35aa8..39eac2a 100755 --- a/scripts/setup_tdx.sh +++ b/scripts/setup_tdx.sh @@ -368,7 +368,7 @@ EOL # package was installed with DEBIAN_FRONTEND=noninteractive, so its interactive # install.sh was skipped. Reproduce here what install.sh would have done: # install the Node.js dependencies and generate the HTTPS SSL keys. The - # Canonical PPA path (< 25.10) does this via setup-attestation-host.sh. + # Ubuntu 24.04 receives the same setting from its attestation packages. if [ "$USE_INTEL_REPO" -eq 1 ]; then # Install PCCS Node.js dependencies. Without node_modules pccs_server.js # fails to start with "Cannot find package 'config'". @@ -527,7 +527,6 @@ install_tdx_release_packages() { } TMP_DIR=$1 -TDX_REF="3.3" check_tdx_os_version() { local min_version="24.04" @@ -561,7 +560,20 @@ check_tdx_os_version() { fi } +cleanup_legacy_canonical_apt_policy() { + # Older bootstrap revisions ran canonical/tdx helpers, which left a global + # priority-4000 pin and enabled unattended package downgrades. Remove those + # settings before any package operation. Repository entries may remain at + # normal APT priority for the attestation packages used below. + rm -f \ + /etc/apt/preferences.d/kobuk-tdx-kobuk-team-tdx-release-pin-4000 \ + /etc/apt/preferences.d/kobuk-tdx-kobuk-team-tdx-attestation-release-pin-4000 \ + /etc/apt/apt.conf.d/99unattended-upgrades-kobuk-tdx-release \ + /etc/apt/apt.conf.d/99unattended-upgrades-kobuk-tdx-attestation-release +} + check_tdx_os_version +cleanup_legacy_canonical_apt_policy # Determine package source based on Ubuntu version UBUNTU_VERSION=$(. /etc/os-release && echo "$VERSION_ID") @@ -573,7 +585,7 @@ if [ "$UBUNTU_NUM" -ge 2510 ]; then echo "Ubuntu ${UBUNTU_VERSION}: using Intel SGX repository" else USE_INTEL_REPO=0 - echo "Ubuntu ${UBUNTU_VERSION}: using Canonical kobuk-team PPA" + echo "Ubuntu ${UBUNTU_VERSION}: using project TDX kernel/QEMU and the Canonical attestation PPA" fi if [ "$USE_INTEL_REPO" -eq 1 ]; then @@ -589,34 +601,10 @@ if [ "$USE_INTEL_REPO" -eq 1 ]; then DEBIAN_FRONTEND=noninteractive apt-get install -y qemu-system-x86 qemu-utils check_error "Failed to install QEMU" else - # Ubuntu < 25.10: TDX host support is not in the stock kernel, so use the - # canonical/tdx host setup (kobuk PPA + -intel kernel). The clone is also - # reused below for attestation (setup-attestation-host.sh). - if [ -d "${TMP_DIR}/tdx-cannonical" ]; then - echo -e "${YELLOW}Directory ${TMP_DIR}/tdx-cannonical already exists${NC}" - echo -e "Removing existing directory..." - rm -rf "${TMP_DIR}/tdx-cannonical" - fi - - git clone https://github.com/canonical/tdx.git "${TMP_DIR}/tdx-cannonical" - if [ $? -ne 0 ]; then - echo "Failed to download the canonical/tdx repository." - exit 1 - fi - SCRIPT_PATH=${TMP_DIR}/tdx-cannonical/setup-tdx-host.sh - - git -C "${TMP_DIR}/tdx-cannonical" checkout --detach "${TDX_REF}" - if [ $? -ne 0 ]; then - echo "Failed to checkout tdx ref ${TDX_REF}." - exit 1 - fi - - print_section_header "Installing hypervisor and kernel..." - echo "Running setup-tdx-host.sh..." - chmod +x "${SCRIPT_PATH}" - "${SCRIPT_PATH}" - - # On 24.04 install our matched custom kernel + sp-qemu-tdx bundle on top. + # Ubuntu 24.04 uses the matched project kernel/QEMU bundle directly. Do not + # run Canonical's setup-tdx-host.sh: it globally pins its PPA at priority + # 4000 and explicitly permits downgrades of QEMU and libvirt. + print_section_header "Installing TDX kernel and QEMU..." install_tdx_release_packages "${TMP_DIR}" fi @@ -647,10 +635,15 @@ if [ "$USE_INTEL_REPO" -eq 1 ]; then deb [signed-by=/etc/apt/keyrings/intel-sgx-keyring.asc arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu ${CODENAME} main EOF else - # Ensure kobuk-team PPA is present - if ! grep -rq "kobuk-team" /etc/apt/sources.list.d/ 2>/dev/null; then - add-apt-repository -y ppa:kobuk-team/tdx-release - check_error "Failed to add kobuk-team PPA" + apt-get install -y software-properties-common + if grep -Rqs 'kobuk-team/tdx-release' /etc/apt/sources.list.d/; then + echo "Removing obsolete kobuk-team/tdx-release PPA" + add-apt-repository -y --remove ppa:kobuk-team/tdx-release + check_error "Failed to remove the obsolete TDX host PPA" + fi + if ! grep -Rqs 'kobuk-team/tdx-attestation-release' /etc/apt/sources.list.d/; then + add-apt-repository -y ppa:kobuk-team/tdx-attestation-release + check_error "Failed to add the TDX attestation PPA" fi fi @@ -873,15 +866,14 @@ if [ "$USE_INTEL_REPO" -eq 1 ]; then sgx-pck-id-retrieval-tool check_error "Failed to install packages" else - # Canonical PPA: install attestation packages via the official script - # from canonical/tdx. - ATTEST_SCRIPT="${TMP_DIR}/tdx-cannonical/attestation/setup-attestation-host.sh" - if [ ! -f "$ATTEST_SCRIPT" ]; then - echo -e "${RED}ERROR: attestation setup script not found at ${ATTEST_SCRIPT}${NC}" - exit 1 - fi - chmod +x "$ATTEST_SCRIPT" - "$ATTEST_SCRIPT" + # Install the same host attestation components used by Canonical, but do it + # directly and without their global PPA pin or --allow-downgrades. + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-remove \ + sgx-dcap-pccs \ + tdx-qgs \ + libsgx-dcap-default-qpl \ + sgx-ra-service \ + sgx-pck-id-retrieval-tool check_error "Failed to install attestation packages" fi From a9e8f628e38762a5cce98e5704a4ab5a58e634e1 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 6 Aug 2026 13:39:34 -0500 Subject: [PATCH 12/28] readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 42634b1..e804618 100644 --- a/README.md +++ b/README.md @@ -92,9 +92,9 @@ Pick the script that matches your CPU vendor. See [docs/swarm.md](docs/swarm.md) What it does: 1. Verifies Ubuntu version and root privileges. -2. Runs `setup_tdx.sh` to install the Canonical TDX 3.3 stack and PCCS attestation host components. +2. Runs `setup_tdx.sh` to install the project-matched TDX kernel/QEMU bundle and PCCS attestation host components. 3. Verifies BIOS/CPU TDX settings (TME, TME-MT, SEAM, TXT, SGX, …). -4. Runs the official `setup-tdx-host.sh` from `canonical/tdx`. +4. Installs the required QGS/PCCS attestation packages directly, without running Canonical's host-setup script or enabling global package downgrades. 5. Updates the Intel TDX-Module to a known-good version. 6. Configures NVIDIA GPUs for Confidential Computing (CC mode + `vfio-pci` binding) and, on B200 systems, sets up ConnectX-7 bridges for VFIO passthrough. 7. Installs and validates libvirt 12.5, AppArmor policy, VSOCK access, and `passt` capabilities before binding devices to the VM stack. From a25724cf644b70630ffa7eda1c2b127d720ca1a4 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 6 Aug 2026 14:31:02 -0500 Subject: [PATCH 13/28] qemu apparmor --- scripts/setup_libvirt_host.sh | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index 18969cc..ab64ffe 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -25,6 +25,7 @@ LIBVIRT_BASE_PACKAGES=( libvirt-daemon-lock libvirt-daemon-plugin-lockd libvirt-daemon-system + libvirt-daemon-system-systemd ) LIBVIRT_PACKAGE_PATHS=() @@ -354,6 +355,10 @@ configure_libvirt_apparmor() { tmp=$(mktemp) printf '%s\n' \ '# Managed by sp-vm-tools bootstrap.' \ + '/usr/local/bin/qemu-system-x86_64 rmix,' \ + '/usr/local/share/qemu/** rk,' \ + '/usr/local/lib{,64}/qemu/*.so mr,' \ + '/usr/local/lib/@{multiarch}/qemu/*.so mr,' \ 'owner @{run}/libvirt/qemu/passt/* rw,' \ 'network vsock stream,' > "${tmp}" install -m 0644 "${tmp}" "${dropin}" @@ -433,6 +438,34 @@ find_bootstrap_qemu() { return 1 } +configure_qemu_binary_permissions() { + local qemu real parent + qemu=$(find_bootstrap_qemu) || { + libvirt_host_error "qemu-system-x86_64 was not found" + return 1 + } + real=$(readlink -f -- "${qemu}") + [[ -n "${real}" && -f "${real}" ]] || { + libvirt_host_error "cannot resolve QEMU binary ${qemu}" + return 1 + } + + if [[ "${real}" == /usr/local/* ]]; then + chmod a+rx "${real}" + parent=$(dirname "${real}") + while [[ "${parent}" == /usr/local/* ]]; do + chmod a+x "${parent}" + parent=$(dirname "${parent}") + done + chmod a+x /usr/local + fi + + if ! runuser -u libvirt-qemu -- test -x "${qemu}"; then + libvirt_host_error "libvirt-qemu cannot execute ${qemu}; check directory permissions and noexec mounts" + return 1 + fi +} + verify_libvirt_host() { local mode=$1 installed_version daemon_version qemu version_line qemu_major capabilities dropin installed_version=$(installed_libvirt_version) @@ -459,7 +492,11 @@ verify_libvirt_host() { libvirt_host_error "QEMU 9 or newer is required (found ${version_line:-unknown})" return 1 fi - capabilities=$(virsh -c "${LIBVIRT_URI}" domcapabilities --emulatorbin "${qemu}") + if ! capabilities=$(virsh -c "${LIBVIRT_URI}" domcapabilities --emulatorbin "${qemu}" 2>&1); then + libvirt_host_error "libvirt cannot probe ${qemu}: ${capabilities}" + echo "Check recent access denials with: journalctl -k --since '-5 min' --no-pager | grep -E 'apparmor=\"DENIED\"|qemu-system'" >&2 + return 1 + fi if ! grep -Eq "]*name=['\"]iommufd['\"]" <<< "${capabilities}"; then libvirt_host_error "libvirt domain capabilities do not advertise IOMMUFD for ${qemu}" return 1 @@ -523,7 +560,12 @@ setup_libvirt_host() { echo "Installed libvirt ${installed_version} is ${LIBVIRT_REQUIRED_VERSION} or newer; keeping it." fi + if dpkg-query -W -f='${db:Status-Status}' libvirt-daemon-system-systemd 2>/dev/null | grep -qx installed; then + apt-mark manual libvirt-daemon-system-systemd >/dev/null + fi + configure_libvirt_apparmor + configure_qemu_binary_permissions # shellcheck disable=SC2119 configure_passt_capabilities systemctl daemon-reload From a9c2d03c6d626091d185815fbfc353efa8e77981 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 6 Aug 2026 14:39:09 -0500 Subject: [PATCH 14/28] one more attempt --- scripts/setup_libvirt_host.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index ab64ffe..2ed2f94 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -340,11 +340,13 @@ patch_libvirt_apparmor_profile() { } configure_libvirt_apparmor() { - local profile dropin_dir dropin template tmp + local profile dropin_dir dropin template daemon_profile daemon_local tmp profile=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu) dropin_dir=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu.d) dropin="${dropin_dir}/99-sp-vm-tools-local" template=$(libvirt_host_path /etc/apparmor.d/libvirt/TEMPLATE.qemu) + daemon_profile=$(libvirt_host_path /etc/apparmor.d/usr.sbin.libvirtd) + daemon_local=$(libvirt_host_path /etc/apparmor.d/local/usr.sbin.libvirtd) [[ -r "${profile}" ]] || { libvirt_host_error "libvirt AppArmor profile is missing: ${profile}" @@ -364,6 +366,22 @@ configure_libvirt_apparmor() { install -m 0644 "${tmp}" "${dropin}" rm -f "${tmp}" + if [[ -r "${daemon_profile}" ]]; then + if ! grep -qF 'include if exists ' "${daemon_profile}"; then + libvirt_host_error "${daemon_profile} does not include its standard local override; refusing to modify AppArmor" + return 1 + fi + install -d -m 0755 "$(dirname "${daemon_local}")" + touch "${daemon_local}" + chmod 0644 "${daemon_local}" + if ! grep -qF '/usr/local/bin/qemu-system-x86_64 PUx,' "${daemon_local}"; then + printf '%s\n' \ + '# Managed by sp-vm-tools: allow libvirtd capabilities probing.' \ + '/usr/local/bin/qemu-system-x86_64 PUx,' >> "${daemon_local}" + fi + apparmor_parser -Q -r "${daemon_profile}" + fi + [[ -r "${template}" ]] || { libvirt_host_error "libvirt AppArmor template is missing: ${template}" return 1 From f07045b47ddbe090544eda49b582f7b14301f923 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 6 Aug 2026 15:03:27 -0500 Subject: [PATCH 15/28] one more fix --- scripts/setup_libvirt_host.sh | 74 ++++++++++++++++++++++++- scripts/start_super_protocol_libvirt.sh | 53 ++++++++++++++++-- 2 files changed, 121 insertions(+), 6 deletions(-) diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index 2ed2f94..2054b47 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -30,6 +30,7 @@ LIBVIRT_BASE_PACKAGES=( LIBVIRT_PACKAGE_PATHS=() PASST_REAL_BINARIES=() +LIBVIRT_QEMU_CONFIG_CHANGED=0 libvirt_host_error() { echo "ERROR: $*" >&2 @@ -392,6 +393,75 @@ configure_libvirt_apparmor() { fi } +configure_libvirt_qemu_runtime() { + local config backup tmp + config=$(libvirt_host_path /etc/libvirt/qemu.conf) + backup="${config}.sp-vm-tools.bak" + [[ -r "${config}" ]] || { + libvirt_host_error "libvirt QEMU configuration is missing: ${config}" + return 1 + } + tmp=$(mktemp) + awk ' + BEGIN { + user_written = 0 + group_written = 0 + ownership_written = 0 + } + /^[[:space:]]*user[[:space:]]*=/ { + if (!user_written) { + print "user = \"libvirt-qemu\"" + user_written = 1 + } + next + } + /^[[:space:]]*group[[:space:]]*=/ { + if (!group_written) { + print "group = \"libvirt-qemu\"" + group_written = 1 + } + next + } + /^[[:space:]]*dynamic_ownership[[:space:]]*=/ { + if (!ownership_written) { + print "dynamic_ownership = 1" + ownership_written = 1 + } + next + } + { print } + END { + if (!user_written) + print "user = \"libvirt-qemu\"" + if (!group_written) + print "group = \"libvirt-qemu\"" + if (!ownership_written) + print "dynamic_ownership = 1" + } + ' "${config}" > "${tmp}" + + if cmp -s "${tmp}" "${config}"; then + rm -f "${tmp}" + echo "Libvirt QEMU runtime already uses libvirt-qemu." + return + fi + + if [[ -z "${SPVM_TEST_ROOT:-}" ]]; then + if ! assert_no_running_libvirt_domains; then + rm -f "${tmp}" + return 1 + fi + fi + if [[ ! -e "${backup}" ]]; then + cp -a "${config}" "${backup}" + fi + cat "${tmp}" > "${config}" + chmod 0600 "${config}" + rm -f "${tmp}" + LIBVIRT_QEMU_CONFIG_CHANGED=1 + echo "Configured libvirt QEMU runtime user/group as libvirt-qemu with dynamic ownership." +} + collect_passt_binaries() { local candidate real local -A seen=() @@ -582,6 +652,7 @@ setup_libvirt_host() { apt-mark manual libvirt-daemon-system-systemd >/dev/null fi + configure_libvirt_qemu_runtime configure_libvirt_apparmor configure_qemu_binary_permissions # shellcheck disable=SC2119 @@ -592,7 +663,8 @@ setup_libvirt_host() { local daemon_version daemon_version=$(running_libvirt_version || true) - if [[ -z "${daemon_version}" ]] || \ + if [[ "${LIBVIRT_QEMU_CONFIG_CHANGED}" -eq 1 ]] || \ + [[ -z "${daemon_version}" ]] || \ ! dpkg --compare-versions "${daemon_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then assert_no_running_libvirt_domains echo "Restarting libvirtd to activate the installed ${LIBVIRT_REQUIRED_VERSION} runtime" diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index e79af6e..2c05950 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -23,6 +23,7 @@ DEFAULT_CACHE="/var/lib/libvirt/images/superprotocol" CACHE=${DEFAULT_CACHE} LIBVIRT_DOMAIN_NAME="" +LIBVIRT_QEMU_USER="" BASE_ARGS=() usage_libvirt() { @@ -334,14 +335,56 @@ build_kernel_cmdline() { fi } -grant_libvirt_file_access() { - local label=$1 requested_path=$2 permissions=$3 - local qemu_user=libvirt-qemu - if ! id "${qemu_user}" >/dev/null 2>&1; then - echo "Error: the expected Ubuntu libvirt QEMU user '${qemu_user}' does not exist." >&2 +resolve_libvirt_qemu_user() { + if [[ -n "${LIBVIRT_QEMU_USER}" ]]; then + return + fi + + local qemu_config=/etc/libvirt/qemu.conf + local configured_user="" + if [[ -r "${qemu_config}" ]]; then + configured_user=$(awk ' + /^[[:space:]]*#/ { next } + /^[[:space:]]*user[[:space:]]*=/ { + value = $0 + sub(/^[^=]*=[[:space:]]*/, "", value) + sub(/[[:space:]]*#.*/, "", value) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + if (value ~ /^"[^"]*"$/) { + sub(/^"/, "", value) + sub(/"$/, "", value) + } + configured = value + } + END { print configured } + ' "${qemu_config}") + fi + configured_user=${configured_user:-libvirt-qemu} + + local passwd_entry="" + if [[ "${configured_user}" =~ ^\+([0-9]+)$ ]]; then + local configured_uid=${BASH_REMATCH[1]} + passwd_entry=$(getent passwd | awk -F: -v uid="${configured_uid}" ' + $3 == uid { print; exit } + ') + else + passwd_entry=$(getent passwd "${configured_user}" || true) + fi + if [[ -z "${passwd_entry}" ]]; then + echo "Error: libvirt QEMU runtime user '${configured_user}' does not exist." >&2 + echo "Check the user setting in ${qemu_config}." >&2 exit 1 fi + LIBVIRT_QEMU_USER=${passwd_entry%%:*} + echo "Libvirt QEMU runtime user: ${LIBVIRT_QEMU_USER}" +} + +grant_libvirt_file_access() { + local label=$1 requested_path=$2 permissions=$3 + resolve_libvirt_qemu_user + local qemu_user=${LIBVIRT_QEMU_USER} + local path if ! path=$(realpath -e -- "${requested_path}"); then echo "Error: cannot resolve ${label} path: ${requested_path}" >&2 From 5ff8a47a7d8b46953ca55c481a293a653936cd99 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Mon, 10 Aug 2026 13:04:38 -0500 Subject: [PATCH 16/28] improving update kerne --- scripts/common.sh | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/scripts/common.sh b/scripts/common.sh index c811b34..08cf8bd 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -557,6 +557,17 @@ ensure_cmdline_param() { setup_grub() { local new_kernel="$1" local type=$2 + local grub_entry="Advanced options for Ubuntu>Ubuntu, with Linux ${new_kernel}" + local ubuntu_version="" + + if [ -r /etc/os-release ]; then + ubuntu_version=$(. /etc/os-release && printf '%s' "${VERSION_ID:-}") + fi + + if [ -z "$ubuntu_version" ]; then + echo "Unable to determine the Ubuntu version for GRUB setup" >&2 + return 1 + fi if [[ "$type" != "tdx" && "$type" != "snp" ]]; then echo "Invalid type: $type. Must be 'tdx' or 'snp'." >&2 @@ -573,7 +584,8 @@ setup_grub() { cp /etc/default/grub "/etc/default/grub.backup.$(date +%Y%m%d_%H%M%S)" fi - # Directly set the first menuentry as default since it's our new kernel + # Keep the base default release-neutral. The drop-in below selects the + # requested custom kernel only while this Ubuntu release is installed. sed -i '/^GRUB_DEFAULT=/d' /etc/default/grub echo 'GRUB_DEFAULT=0' > /etc/default/grub.new cat /etc/default/grub >> /etc/default/grub.new @@ -603,15 +615,27 @@ setup_grub() { echo 'GRUB_RECORDFAIL_TIMEOUT=5' >> /etc/default/grub fi - # Create a custom configuration file to ensure our kernel is first + # Select the exact kernel instead of assuming it sorts as menu entry zero. + # Limit the override to this Ubuntu release so a future release upgrade + # automatically returns to its newer distro kernel. mkdir -p /etc/default/grub.d - echo "# Custom kernel order configuration" > "/etc/default/grub.d/99-${type}-kernel.cfg" - echo "GRUB_DEFAULT=0" >> "/etc/default/grub.d/99-${type}-kernel.cfg" + { + echo "# Custom kernel selection for Ubuntu ${ubuntu_version}" + echo '[ -r /etc/os-release ] && . /etc/os-release' + echo "if [ \"\${VERSION_ID:-}\" = \"${ubuntu_version}\" ]; then" + echo " GRUB_DEFAULT=\"${grub_entry}\"" + echo 'fi' + } > "/etc/default/grub.d/99-${type}-kernel.cfg" # Force regeneration of grub.cfg and initramfs update-initramfs -u -k "${new_kernel}" update-grub2 || update-grub + if ! grep -Fq "menuentry 'Ubuntu, with Linux ${new_kernel}'" /boot/grub/grub.cfg; then + echo "Failed to find the requested kernel in GRUB: ${new_kernel}" >&2 + return 1 + fi + # For UEFI systems, ensure the boot entry is updated if [ -d /sys/firmware/efi ]; then if command -v efibootmgr >/dev/null 2>&1; then @@ -631,14 +655,14 @@ setup_grub() { fi fi - # Use both grub-set-default and grub-reboot for maximum reliability + # Use the exact submenu entry for both persistent and one-shot selection. if command -v grub-set-default >/dev/null 2>&1; then - grub-set-default 0 + grub-set-default "${grub_entry}" echo "Set default boot entry using grub-set-default" fi if command -v grub-reboot >/dev/null 2>&1; then - grub-reboot 0 + grub-reboot "${grub_entry}" echo "Set next boot entry using grub-reboot" fi From 17c0c04c3c3d6d475c48b9dcf19ed5099a91ab3c Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 11 Aug 2026 04:17:12 -0500 Subject: [PATCH 17/28] check kernel log --- scripts/setup_snp.sh | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/scripts/setup_snp.sh b/scripts/setup_snp.sh index 0136aa6..95493dd 100644 --- a/scripts/setup_snp.sh +++ b/scripts/setup_snp.sh @@ -29,6 +29,24 @@ print_section_header() { echo -e "${BLUE}$(printf '=%.0s' {1..40})${NC}" } +# Early boot messages such as the BIOS-provided RMP range can be evicted from +# the finite kernel ring buffer on long-running or noisy hosts. Prefer the +# persistent journal for the current boot and fall back to dmesg when journald +# is unavailable. +get_kernel_log() { + local log="" + + if command -v journalctl >/dev/null 2>&1; then + log=$(journalctl -k -b --no-pager 2>/dev/null || true) + fi + + if [ -n "$log" ]; then + printf '%s\n' "$log" + else + dmesg 2>/dev/null + fi +} + # --------------------------------------------------------------------------- # Platform detection # --------------------------------------------------------------------------- @@ -158,9 +176,10 @@ check_smee_msr() { # Minimum for SNP: API 1.51 (0x33). # --------------------------------------------------------------------------- check_sev_fw_version() { + local kernel_log="$1" # echoes status lines via the caller's results array is awkward; instead # set globals. - SEV_FW_LINE=$(dmesg | grep -iE "ccp.*SEV-SNP API:" | head -1 || echo "") + SEV_FW_LINE=$(printf '%s\n' "$kernel_log" | grep -im1 -E "ccp.*SEV-SNP API:" || echo "") SEV_FW_OK="unknown" SEV_FW_VER="" if [ -n "$SEV_FW_LINE" ]; then @@ -186,6 +205,9 @@ check_sev_fw_version() { check_all_bios_settings() { local results=() local all_passed=true + local kernel_log + + kernel_log=$(get_kernel_log) print_section_header "BIOS Configuration Check Results" echo "Checking all settings for ${PLATFORM} (${PLATFORM_ZEN})..." @@ -251,7 +273,7 @@ check_all_bios_settings() { # --- SEV-SNP enablement + ASID range (kvm_amd: SEV-SNP enabled (ASIDs..)) results+=("SEV-SNP Initialization:") local snp_enable_line - snp_enable_line=$(dmesg | grep -iE "kvm_amd:.*SEV-SNP enabled" | head -1 || echo "") + snp_enable_line=$(printf '%s\n' "$kernel_log" | grep -im1 -E "kvm_amd:.*SEV-SNP enabled" || echo "") if [ -n "$snp_enable_line" ]; then results+=("${SUCCESS} SEV-SNP enabled${NC}") # e.g. "(ASIDs 1 - 98)" @@ -260,10 +282,11 @@ check_all_bios_settings() { [ -n "$asid_range" ] && results+=(" ${asid_range}") [ -n "$EXPECTED_ASIDS" ] && results+=(" Platform documented total: ${EXPECTED_ASIDS}") else - results+=("${FAILURE} 'SEV-SNP enabled' not found in dmesg${NC}") + results+=("${FAILURE} 'SEV-SNP enabled' not found in the current boot log${NC}") # Try to surface the actual reason rather than guessing BIOS. local snp_err - snp_err=$(dmesg | grep -iE "SEV(-SNP)?:.*(fail|error|disabled)|ccp.*error" | head -3 || echo "") + snp_err=$(printf '%s\n' "$kernel_log" \ + | grep -im3 -E "SEV(-SNP)?:.*(fail|error|disabled)|ccp.*error" || echo "") if [ -n "$snp_err" ]; then results+=(" Reported by kernel:") while IFS= read -r line; do @@ -285,19 +308,19 @@ check_all_bios_settings() { # --- RMP table (SEV-SNP: ... RMP ...) --------------------------------- results+=("RMP Table:") local rmp_line - rmp_line=$(dmesg | grep -iE "SEV-SNP:.*RMP" | head -1 || echo "") + rmp_line=$(printf '%s\n' "$kernel_log" | grep -im1 -E "SEV-SNP:.*RMP" || echo "") if [ -n "$rmp_line" ]; then results+=("${SUCCESS} RMP table present${NC}") results+=(" $(echo "$rmp_line" | sed -E 's/.*SEV-SNP: //')") else - results+=("${FAILURE} RMP table not reported in dmesg${NC}") + results+=("${FAILURE} RMP table not reported in the current boot log${NC}") results+=(" Location: ${PATH_RMP} ${BIOS_NOTE}") all_passed=false fi # --- SEV firmware version (min 1.51 / 0x33) --------------------------- results+=("SEV Firmware (min API 1.51):") - check_sev_fw_version + check_sev_fw_version "$kernel_log" if [ -n "$SEV_FW_VER" ]; then if [ "$SEV_FW_OK" = "yes" ]; then results+=("${SUCCESS} SEV-SNP API ${SEV_FW_VER} (>= 1.51)${NC}") @@ -308,7 +331,7 @@ check_all_bios_settings() { all_passed=false fi else - results+=("${WARNING} Could not read SEV-SNP API version from dmesg${NC}") + results+=("${WARNING} Could not read SEV-SNP API version from the current boot log${NC}") results+=(" If you see 'SEV: failed to INIT error 0x1, rc -5' -> PSP BootLoader too old; update system BIOS") fi From 15badfe4273f7c53d09f0ac063982eed631c4f7e Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 11 Aug 2026 04:31:30 -0500 Subject: [PATCH 18/28] fix iommu --- scripts/setup_libvirt_host.sh | 23 +++++++++++++++++++++++ scripts/start_super_protocol_libvirt.sh | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index 2054b47..70848b5 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -554,6 +554,28 @@ configure_qemu_binary_permissions() { fi } +configure_iommufd() { + local modules_dir modules_file tmp + modules_dir=$(libvirt_host_path /etc/modules-load.d) + modules_file="${modules_dir}/sp-vm-tools-iommufd.conf" + + install -d -m 0755 "${modules_dir}" + tmp=$(mktemp) + printf '%s\n' \ + '# Managed by sp-vm-tools bootstrap.' \ + 'iommufd' > "${tmp}" + install -m 0644 "${tmp}" "${modules_file}" + rm -f "${tmp}" + + if [[ -z "${SPVM_TEST_ROOT:-}" ]]; then + modprobe iommufd + [[ -c /dev/iommu ]] || { + libvirt_host_error "iommufd loaded but /dev/iommu is missing" + return 1 + } + fi +} + verify_libvirt_host() { local mode=$1 installed_version daemon_version qemu version_line qemu_major capabilities dropin installed_version=$(installed_libvirt_version) @@ -655,6 +677,7 @@ setup_libvirt_host() { configure_libvirt_qemu_runtime configure_libvirt_apparmor configure_qemu_binary_permissions + configure_iommufd # shellcheck disable=SC2119 configure_passt_capabilities systemctl daemon-reload diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index 2c05950..f4992e0 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -113,7 +113,7 @@ check_passt_apparmor_profile() { fi local profile=/etc/apparmor.d/abstractions/libvirt-qemu - [[ -r "${profile}" ]] || return + [[ -r "${profile}" ]] || return 0 if awk ' /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1 } @@ -140,7 +140,7 @@ check_passt_apparmor_profile() { } check_tdx_vsock_apparmor_profile() { - [[ "${VM_MODE}" == "tdx" ]] || return + [[ "${VM_MODE}" == "tdx" ]] || return 0 local dropin=/etc/apparmor.d/abstractions/libvirt-qemu.d/99-sp-vm-tools-local if [[ ! -r "${dropin}" ]] || ! grep -qF 'network vsock stream,' "${dropin}"; then echo "Error: TDX QGS requires the AppArmor rule 'network vsock stream,'." >&2 From 7d0c8d71fc961d688d4cad603f791bf63760f456 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 11 Aug 2026 05:43:21 -0500 Subject: [PATCH 19/28] coderabbit fixes --- .../workflows/build-packages-self-hosted.yml | 62 +++++------ ...{Dockerfile.ubuntu24 => Dockerfile.ubuntu} | 7 +- build/libvirt/Dockerfile.ubuntu26 | 26 ----- build/libvirt/README.md | 11 +- build/libvirt/build.sh | 20 +++- build/libvirt/docker/build-packages.sh | 16 ++- build/libvirt/docker/prepare-source.sh | 9 ++ docs/swarm.md | 2 +- scripts/bootstrap_snp.sh | 7 +- scripts/bootstrap_tdx.sh | 7 +- scripts/libvirt_launcher.py | 104 +++++++++--------- scripts/setup_libvirt_host.sh | 91 ++++++++------- scripts/start_super_protocol.sh | 8 +- scripts/start_super_protocol_libvirt.sh | 44 ++++---- scripts/swarm-cluster.sh | 5 + 15 files changed, 216 insertions(+), 203 deletions(-) rename build/libvirt/{Dockerfile.ubuntu24 => Dockerfile.ubuntu} (79%) delete mode 100644 build/libvirt/Dockerfile.ubuntu26 diff --git a/.github/workflows/build-packages-self-hosted.yml b/.github/workflows/build-packages-self-hosted.yml index becbe9a..dde212b 100644 --- a/.github/workflows/build-packages-self-hosted.yml +++ b/.github/workflows/build-packages-self-hosted.yml @@ -25,7 +25,7 @@ jobs: - name: Setup clean workspace run: | WORK_DIR="/home/gh-runner/builds/run-${{ github.run_number }}" - echo "WORK_DIR=$WORK_DIR" >> $GITHUB_ENV + echo "WORK_DIR=$WORK_DIR" >> "$GITHUB_ENV" mkdir -p /home/gh-runner/builds @@ -37,15 +37,15 @@ jobs: cd "$WORK_DIR" git init . - git remote add origin https://github.com/${{ github.repository }}.git - git fetch --depth 1 origin ${{ github.sha }} - git checkout ${{ github.sha }} + git remote add origin "https://github.com/${{ github.repository }}.git" + git fetch --depth 1 origin "${{ github.sha }}" + git checkout "${{ github.sha }}" - name: Set build type and runner ID working-directory: ${{ env.WORK_DIR }} run: | - echo "BUILD_TYPE=${{ github.event.inputs.build_type }}" >> $GITHUB_ENV - echo "RUNNER_ID=${{ github.run_number }}" >> $GITHUB_ENV + echo "BUILD_TYPE=${{ github.event.inputs.build_type }}" >> "$GITHUB_ENV" + echo "RUNNER_ID=${{ github.run_number }}" >> "$GITHUB_ENV" - name: Run TDX docker build if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'SEV+TDX' }} @@ -79,7 +79,7 @@ jobs: esac archive_name="${BUILD_TYPE}.tar.gz" - archive_path="${WORK_DIR}/build/libvirt/out/${archive_name}" + archive_path="${RUNNER_TEMP}/${archive_name}" ./build/libvirt/build.sh "${target}" @@ -110,47 +110,35 @@ jobs: exit 1 fi - - name: Create GitHub Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - with: - tag_name: "${{ env.RELEASE_NAME }}" - release_name: "Release ${{ env.RELEASE_NAME }}" - draft: false - prerelease: true - - - name: Upload TDX Release Asset + - name: Publish TDX Release Asset if: ${{ github.event.inputs.build_type == 'TDX' || github.event.inputs.build_type == 'SEV+TDX' }} - uses: actions/upload-release-asset@v1 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ env.WORK_DIR }}/build/out/tdx/package-tdx.tar.gz - asset_name: package-tdx.tar.gz - asset_content_type: application/gzip + tag_name: ${{ env.RELEASE_NAME }} + name: Release ${{ env.RELEASE_NAME }} + prerelease: true + files: ${{ env.WORK_DIR }}/build/out/tdx/package-tdx.tar.gz - - name: Upload SNP Release Asset + - name: Publish SNP Release Asset if: ${{ github.event.inputs.build_type == 'SNP' || github.event.inputs.build_type == 'SEV+TDX' }} - uses: actions/upload-release-asset@v1 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ env.WORK_DIR }}/build/out/snp/package-snp.tar.gz - asset_name: package-snp.tar.gz - asset_content_type: application/gzip + tag_name: ${{ env.RELEASE_NAME }} + name: Release ${{ env.RELEASE_NAME }} + prerelease: true + files: ${{ env.WORK_DIR }}/build/out/snp/package-snp.tar.gz - - name: Upload libvirt Release Asset + - name: Publish libvirt Release Asset if: ${{ startsWith(github.event.inputs.build_type, 'libvirt-ubuntu') }} - uses: actions/upload-release-asset@v1 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ env.LIBVIRT_ARCHIVE_PATH }} - asset_name: ${{ env.LIBVIRT_ARCHIVE_NAME }} - asset_content_type: application/gzip + tag_name: ${{ env.RELEASE_NAME }} + name: Release ${{ env.RELEASE_NAME }} + prerelease: true + files: ${{ env.LIBVIRT_ARCHIVE_PATH }} diff --git a/build/libvirt/Dockerfile.ubuntu24 b/build/libvirt/Dockerfile.ubuntu similarity index 79% rename from build/libvirt/Dockerfile.ubuntu24 rename to build/libvirt/Dockerfile.ubuntu index cf3adf0..427c819 100644 --- a/build/libvirt/Dockerfile.ubuntu24 +++ b/build/libvirt/Dockerfile.ubuntu @@ -1,9 +1,11 @@ # syntax=docker/dockerfile:1 -FROM ubuntu:24.04 +ARG UBUNTU_VERSION=24.04 +FROM ubuntu:${UBUNTU_VERSION} ARG LIBVIRT_VERSION=12.5.0 ARG DEBIAN_REVISION=1 +ARG PACKAGING_COMMIT=a8f73eb070c24b72f9d6dfbeffc28a334f29e076 ENV DEBIAN_FRONTEND=noninteractive \ LANG=C.UTF-8 \ @@ -13,13 +15,14 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ devscripts \ + dpkg-dev \ equivs \ git \ libdistro-info-perl \ && rm -rf /var/lib/apt/lists/* COPY docker/prepare-source.sh /usr/local/bin/prepare-libvirt-source -RUN prepare-libvirt-source "${LIBVIRT_VERSION}" "${DEBIAN_REVISION}" +RUN prepare-libvirt-source "${LIBVIRT_VERSION}" "${DEBIAN_REVISION}" "${PACKAGING_COMMIT}" COPY docker/build-packages.sh /usr/local/bin/build-libvirt-packages diff --git a/build/libvirt/Dockerfile.ubuntu26 b/build/libvirt/Dockerfile.ubuntu26 deleted file mode 100644 index 0d0c1d4..0000000 --- a/build/libvirt/Dockerfile.ubuntu26 +++ /dev/null @@ -1,26 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM ubuntu:26.04 - -ARG LIBVIRT_VERSION=12.5.0 -ARG DEBIAN_REVISION=1 - -ENV DEBIAN_FRONTEND=noninteractive \ - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - ca-certificates \ - devscripts \ - equivs \ - git \ - libdistro-info-perl \ - && rm -rf /var/lib/apt/lists/* - -COPY docker/prepare-source.sh /usr/local/bin/prepare-libvirt-source -RUN prepare-libvirt-source "${LIBVIRT_VERSION}" "${DEBIAN_REVISION}" - -COPY docker/build-packages.sh /usr/local/bin/build-libvirt-packages - -ENTRYPOINT ["/usr/local/bin/build-libvirt-packages"] diff --git a/build/libvirt/README.md b/build/libvirt/README.md index f87c788..05b890b 100644 --- a/build/libvirt/README.md +++ b/build/libvirt/README.md @@ -1,8 +1,8 @@ # Local libvirt packages -This directory contains separate Docker build environments for Ubuntu 24.04 -and Ubuntu 26.04. The build uses the Debian libvirt packaging and resolves all -build dependencies inside the target Ubuntu image. +This directory contains a parameterized Docker build environment for Ubuntu +24.04 and Ubuntu 26.04. The build uses the Debian libvirt packaging and resolves +all build dependencies inside the target Ubuntu image. The default source is the `debian/12.5.0-1` tag from the Debian libvirt Salsa repository. Resulting packages have a local version such as: @@ -70,7 +70,10 @@ packages, `.changes`, `.buildinfo`, and `SHA256SUMS`. Do not mix packages built for different Ubuntu releases. The build script only creates local, unsigned packages. It does not install -them, create an APT repository, or build QEMU and python3-libvirt. +them, create an APT repository, or build QEMU and python3-libvirt. Before using +`scripts/start_super_protocol_libvirt.sh`, run the matching host bootstrap; the +runtime also requires `python3-libvirt`, `passt`, `acl`, and libvirt 12.1 or +newer for GPU passthrough. The `Build packages self-hosted` GitHub Actions workflow can build either Ubuntu target and publish the result to a prerelease. Select diff --git a/build/libvirt/build.sh b/build/libvirt/build.sh index fb64fe7..ceeaf74 100755 --- a/build/libvirt/build.sh +++ b/build/libvirt/build.sh @@ -7,6 +7,7 @@ SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) LIBVIRT_VERSION=12.5.0 DEBIAN_REVISION=1 SPVM_REVISION=1 +PACKAGING_COMMIT=a8f73eb070c24b72f9d6dfbeffc28a334f29e076 OUTPUT_ROOT="${SCRIPT_DIR}/out" DOCKER_PLATFORM=linux/amd64 SKIP_TESTS=false @@ -25,6 +26,7 @@ Options: --libvirt-version VERSION Upstream libvirt version (default: 12.5.0) --debian-revision NUMBER Debian packaging revision (default: 1) --spvm-revision NUMBER Local package revision (default: 1) + --packaging-commit SHA Immutable Debian packaging commit --output DIR Artifact root (default: build/libvirt/out) --platform PLATFORM Docker platform (default: linux/amd64) --skip-tests Set DEB_BUILD_OPTIONS=nocheck @@ -66,6 +68,11 @@ while [[ $# -gt 0 ]]; do SPVM_REVISION=$2 shift 2 ;; + --packaging-commit) + [[ $# -ge 2 ]] || { echo "Error: --packaging-commit requires a value" >&2; exit 2; } + PACKAGING_COMMIT=$2 + shift 2 + ;; --output) [[ $# -ge 2 ]] || { echo "Error: --output requires a value" >&2; exit 2; } OUTPUT_ROOT=$2 @@ -129,6 +136,10 @@ if ! [[ "${SPVM_REVISION}" =~ ^[1-9][0-9]*$ ]]; then echo "Error: SPVM revision must be a positive integer" >&2 exit 2 fi +if ! [[ "${PACKAGING_COMMIT}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Error: packaging commit must be a full lowercase Git SHA" >&2 + exit 2 +fi if ! command -v docker >/dev/null 2>&1; then echo "Error: docker is not installed" >&2 exit 1 @@ -142,31 +153,32 @@ mkdir -p "${OUTPUT_ROOT}" OUTPUT_ROOT=$(realpath "${OUTPUT_ROOT}") build_target() { - local target=$1 ubuntu_version dockerfile image_name output_dir package_version + local target=$1 ubuntu_version image_name output_dir package_version local -a build_args case "${target}" in ubuntu24) ubuntu_version=24.04 - dockerfile=Dockerfile.ubuntu24 ;; ubuntu26) ubuntu_version=26.04 - dockerfile=Dockerfile.ubuntu26 ;; esac image_name="sp-vm-libvirt-builder:ubuntu${ubuntu_version}-${LIBVIRT_VERSION}-${DEBIAN_REVISION}" package_version="${LIBVIRT_VERSION}-${DEBIAN_REVISION}spvm${SPVM_REVISION}~ubuntu${ubuntu_version}.1" output_dir="${OUTPUT_ROOT}/ubuntu-${ubuntu_version}/${package_version}" + rm -rf -- "${output_dir}" mkdir -p "${output_dir}" build_args=( build --platform "${DOCKER_PLATFORM}" - --file "${SCRIPT_DIR}/${dockerfile}" + --file "${SCRIPT_DIR}/Dockerfile.ubuntu" --tag "${image_name}" + --build-arg "UBUNTU_VERSION=${ubuntu_version}" --build-arg "LIBVIRT_VERSION=${LIBVIRT_VERSION}" --build-arg "DEBIAN_REVISION=${DEBIAN_REVISION}" + --build-arg "PACKAGING_COMMIT=${PACKAGING_COMMIT}" ) if [[ "${NO_CACHE}" == "true" ]]; then build_args+=(--no-cache) diff --git a/build/libvirt/docker/build-packages.sh b/build/libvirt/docker/build-packages.sh index 01b8204..2fcd216 100755 --- a/build/libvirt/docker/build-packages.sh +++ b/build/libvirt/docker/build-packages.sh @@ -16,7 +16,7 @@ readonly OUTPUT_GID=${OUTPUT_GID:-0} # Provided by every Ubuntu builder image. # shellcheck disable=SC1091 source /etc/os-release -if [[ "${ID}" != "ubuntu" || "${VERSION_ID}" != "${TARGET_UBUNTU_VERSION}" ]]; then +if [[ "${ID:-}" != "ubuntu" || "${VERSION_ID:-}" != "${TARGET_UBUNTU_VERSION}" ]]; then echo "Error: builder is Ubuntu ${VERSION_ID:-unknown}, target is ${TARGET_UBUNTU_VERSION}" >&2 exit 1 fi @@ -34,8 +34,8 @@ case "${TARGET_UBUNTU_VERSION}" in ;; esac -if [[ "${VERSION_CODENAME}" != "${expected_codename}" ]]; then - echo "Error: Ubuntu ${TARGET_UBUNTU_VERSION} has unexpected codename ${VERSION_CODENAME}" >&2 +if [[ "${VERSION_CODENAME:-}" != "${expected_codename}" ]]; then + echo "Error: Ubuntu ${TARGET_UBUNTU_VERSION} has unexpected codename ${VERSION_CODENAME:-unknown}" >&2 exit 1 fi @@ -63,9 +63,9 @@ dch \ --force-distribution \ "Local rebuild for Ubuntu ${TARGET_UBUNTU_VERSION}." -build_options="parallel=$(nproc)" +build_options="" if [[ "${SKIP_TESTS}" == "true" ]]; then - build_options="${build_options} nocheck" + build_options="nocheck" elif [[ "${SKIP_TESTS}" != "false" ]]; then echo "Error: SKIP_TESTS must be true or false" >&2 exit 1 @@ -95,10 +95,14 @@ done ( cd "${OUTPUT_DIR}" debs=( ./*.deb ./*.ddeb ) + if [[ ${#debs[@]} -eq 0 ]]; then + echo "Error: no .deb or .ddeb packages were produced" >&2 + exit 1 + fi sha256sum "${debs[@]}" > SHA256SUMS ) -chown "${OUTPUT_UID}:${OUTPUT_GID}" "${OUTPUT_DIR}"/* +chown -R "${OUTPUT_UID}:${OUTPUT_GID}" "${OUTPUT_DIR}" echo "Built ${#artifacts[@]} artifacts in ${OUTPUT_DIR}" echo "Package version: ${PACKAGE_VERSION}" diff --git a/build/libvirt/docker/prepare-source.sh b/build/libvirt/docker/prepare-source.sh index 3b5ca16..58c43ef 100755 --- a/build/libvirt/docker/prepare-source.sh +++ b/build/libvirt/docker/prepare-source.sh @@ -4,6 +4,7 @@ set -euo pipefail readonly LIBVIRT_VERSION=${1:?libvirt version is required} readonly DEBIAN_REVISION=${2:?Debian revision is required} +readonly PACKAGING_COMMIT=${3:?packaging commit is required} readonly PACKAGING_REF="debian/${LIBVIRT_VERSION}-${DEBIAN_REVISION}" readonly SOURCE_DIR=/opt/libvirt-source readonly PACKAGING_REPOSITORY=https://salsa.debian.org/libvirt-team/libvirt.git @@ -14,6 +15,12 @@ git clone \ "${PACKAGING_REPOSITORY}" \ "${SOURCE_DIR}" +actual_commit=$(git -C "${SOURCE_DIR}" rev-parse HEAD) +if [[ "${actual_commit}" != "${PACKAGING_COMMIT}" ]]; then + echo "Error: ${PACKAGING_REF} resolves to ${actual_commit}, expected ${PACKAGING_COMMIT}" >&2 + exit 1 +fi + actual_version=$(dpkg-parsechangelog -l"${SOURCE_DIR}/debian/changelog" -SVersion) expected_version="${LIBVIRT_VERSION}-${DEBIAN_REVISION}" if [[ "${actual_version}" != "${expected_version}" ]]; then @@ -21,6 +28,8 @@ if [[ "${actual_version}" != "${expected_version}" ]]; then exit 1 fi +rm -rf -- "${SOURCE_DIR}/.git" + cd "${SOURCE_DIR}" apt-get update mk-build-deps \ diff --git a/docs/swarm.md b/docs/swarm.md index 473cc03..7ab58e1 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -68,7 +68,7 @@ ACME_URL: https://acme.zerossl.com/v2/DV90 **You also need:** - A host already bootstrapped for confidential computing (TDX or SEV-SNP) — see the [main README](../README.md). - `tmux`, `nftables`, `curl`, `nc` installed: `apt install tmux nftables curl netcat-openbsd` -- Ubuntu 26.04+ with `qemu:///system`, libvirt 12.1+, `python3-libvirt`, `passt`, and `acl` configured as described in the [libvirt launcher section](../README.md#libvirt-launcher-ubuntu-2604). +- Ubuntu 24.04 or 26.04 with `qemu:///system`, libvirt 12.1+, `python3-libvirt`, `passt`, and `acl` configured as described in the [libvirt launcher section](../README.md#libvirt-launcher-ubuntu-2404-and-2604). > Keep `provider-template/` in its own folder — not inside `sp-vm-tools` and not inside any cache folder. diff --git a/scripts/bootstrap_snp.sh b/scripts/bootstrap_snp.sh index f9f9a9a..f034484 100755 --- a/scripts/bootstrap_snp.sh +++ b/scripts/bootstrap_snp.sh @@ -362,7 +362,7 @@ update_snp_firmware() { bootstrap() { check_os_version "24.04" - get_supported_ubuntu_version + get_supported_ubuntu_version || return 1 CPU_MODEL=$(lscpu | grep "^Model name:" | sed 's/Model name: *//') @@ -440,7 +440,10 @@ bootstrap() { fi fi - setup_libvirt_host sev-snp + setup_libvirt_host sev-snp || { + echo -e "${RED}ERROR: libvirt host setup failed${NC}" + return 1 + } print_section_header "Hardware Configuration" if command -v lspci >/dev/null; then diff --git a/scripts/bootstrap_tdx.sh b/scripts/bootstrap_tdx.sh index 5029adf..cc60b76 100755 --- a/scripts/bootstrap_tdx.sh +++ b/scripts/bootstrap_tdx.sh @@ -9,7 +9,7 @@ source_common() { bootstrap() { check_os_version "24.04" - get_supported_ubuntu_version + get_supported_ubuntu_version || return 1 # Check if the script is running as root print_section_header "Privilege Check" @@ -48,7 +48,10 @@ bootstrap() { exit 1 fi - setup_libvirt_host tdx + setup_libvirt_host tdx || { + echo -e "${RED}ERROR: libvirt host setup failed${NC}" + return 1 + } print_section_header "Hardware Configuration" if command -v lspci >/dev/null; then diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py index 719114a..cfcecc5 100755 --- a/scripts/libvirt_launcher.py +++ b/scripts/libvirt_launcher.py @@ -15,7 +15,6 @@ import select import sys import termios -import threading import tty import xml.etree.ElementTree as ET from dataclasses import dataclass, field @@ -519,64 +518,65 @@ def _write_console_output(data: bytes, log: BinaryIO) -> None: def attach_serial_console(conn: Any, domain: Any, libvirt_module: Any, log_path: str) -> None: """Attach a bidirectional console; Ctrl-C or Ctrl-] only detaches.""" - stream = conn.newStream(0) + stream = conn.newStream(getattr(libvirt_module, "VIR_STREAM_NONBLOCK", 1)) domain.openConsole(None, stream, 0) - stopped = threading.Event() - receiver_error: list[BaseException] = [] - log_file = open(log_path, "ab", buffering=0) - - def receive() -> None: - try: - while not stopped.is_set(): - chunk = stream.recv(65536) - if not chunk: + stdin_fd: Optional[int] = None + old_terminal = None + active = False + try: + with open(log_path, "ab", buffering=0) as log_file: + stdin_fd = sys.stdin.fileno() + if os.isatty(stdin_fd): + old_terminal = termios.tcgetattr(stdin_fd) + tty.setraw(stdin_fd) + + print( + "\nConnected to serial console. Press Ctrl-C or Ctrl-] to detach; " + "the VM will keep running.\r", + file=sys.stderr, + ) + pending = bytearray() + console_open = True + while console_open: + while True: + chunk = stream.recv(65536) + if chunk == -2: + break + if not chunk: + console_open = False + break + _write_console_output(chunk, log_file) + if not console_open: break - _write_console_output(chunk, log_file) - except BaseException as exc: # propagated after terminal restoration - if not stopped.is_set(): - receiver_error.append(exc) - finally: - stopped.set() - receiver = threading.Thread(target=receive, name="libvirt-console-recv", daemon=True) - receiver.start() + if pending: + sent = stream.send(bytes(pending)) + if sent == -2: + sent = 0 + elif sent <= 0: + raise RuntimeError("libvirt console send made no progress") + del pending[:sent] - stdin_fd = sys.stdin.fileno() - old_terminal = None - if os.isatty(stdin_fd): - old_terminal = termios.tcgetattr(stdin_fd) - tty.setraw(stdin_fd) - - print( - "\nConnected to serial console. Press Ctrl-C or Ctrl-] to detach; " - "the VM will keep running.\r", - file=sys.stderr, - ) - try: - while not stopped.is_set(): - readable, _, _ = select.select([stdin_fd], [], [], 0.25) - if not readable: - continue - data = os.read(stdin_fd, 4096) - if not data: - break - if b"\x03" in data or b"\x1d" in data: - break - sent = 0 - while sent < len(data): - sent += stream.send(data[sent:]) + readable, _, _ = select.select( + [stdin_fd] if not pending else [], [], [], 0.05 + ) + if not readable: + continue + data = os.read(stdin_fd, 4096) + if not data or b"\x03" in data or b"\x1d" in data: + break + pending.extend(data) except KeyboardInterrupt: pass finally: - stopped.set() - if old_terminal is not None: - termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_terminal) try: - stream.abort() - except libvirt_module.libvirtError: - pass - receiver.join(timeout=1) - log_file.close() + if old_terminal is not None and stdin_fd is not None: + termios.tcsetattr(stdin_fd, termios.TCSADRAIN, old_terminal) + finally: + try: + stream.abort() + except libvirt_module.libvirtError: + pass try: active = bool(domain.isActive()) except libvirt_module.libvirtError: @@ -586,8 +586,6 @@ def receive() -> None: else: message = "Serial console closed because the VM stopped." print(f"\n{message}", file=sys.stderr) - if receiver_error and active: - raise RuntimeError(f"serial console failed: {receiver_error[0]}") def launch(config: DomainConfig) -> None: diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index 70848b5..b4c2169 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -258,31 +258,31 @@ assert_safe_apt_simulation() { install_project_libvirt() { local work_dir archive simulation - assert_no_running_libvirt_domains - work_dir=$(mktemp -d /var/tmp/sp-vm-libvirt.XXXXXX) - chmod 0755 "${work_dir}" + assert_no_running_libvirt_domains || return 1 + work_dir=$(mktemp -d /var/tmp/sp-vm-libvirt.XXXXXX) || return 1 + chmod 0755 "${work_dir}" || return 1 archive="${work_dir}/${LIBVIRT_RELEASE_ASSET}" echo "Downloading libvirt ${LIBVIRT_REQUIRED_VERSION} from ${LIBVIRT_RELEASE_URL}" - wget --https-only --tries=3 -O "${archive}" "${LIBVIRT_RELEASE_URL}" - chmod 0644 "${archive}" - verify_and_extract_libvirt_archive "${archive}" "${work_dir}" - prepare_libvirt_package_compatibility "${LIBVIRT_PACKAGE_DIR}" "${UBUNTU_VERSION}" - find "${work_dir}" -type d -exec chmod a+rx {} + - find "${work_dir}" -type f \( -name '*.deb' -o -name '*.ddeb' \) -exec chmod a+r {} + - build_libvirt_package_plan "${LIBVIRT_PACKAGE_DIR}" + wget --https-only --tries=3 -O "${archive}" "${LIBVIRT_RELEASE_URL}" || return 1 + chmod 0644 "${archive}" || return 1 + verify_and_extract_libvirt_archive "${archive}" "${work_dir}" || return 1 + prepare_libvirt_package_compatibility "${LIBVIRT_PACKAGE_DIR}" "${UBUNTU_VERSION}" || return 1 + find "${work_dir}" -type d -exec chmod a+rx {} + || return 1 + find "${work_dir}" -type f \( -name '*.deb' -o -name '*.ddeb' \) -exec chmod a+r {} + || return 1 + build_libvirt_package_plan "${LIBVIRT_PACKAGE_DIR}" || return 1 echo "APT simulation for the libvirt upgrade:" - simulation=$(apt-get --simulate --no-install-recommends --no-remove install "${LIBVIRT_PACKAGE_PATHS[@]}") + simulation=$(LC_ALL=C apt-get --simulate --no-install-recommends --no-remove install "${LIBVIRT_PACKAGE_PATHS[@]}") || return 1 printf '%s\n' "${simulation}" - assert_safe_apt_simulation "${simulation}" + assert_safe_apt_simulation "${simulation}" || return 1 DEBIAN_FRONTEND=noninteractive apt-get \ --no-install-recommends \ --no-remove \ -o Dpkg::Options::=--force-confold \ - install -y "${LIBVIRT_PACKAGE_PATHS[@]}" - rm -rf "${work_dir}" + install -y "${LIBVIRT_PACKAGE_PATHS[@]}" || return 1 + rm -rf "${work_dir}" || return 1 } passthrough_profile_state() { @@ -353,7 +353,7 @@ configure_libvirt_apparmor() { libvirt_host_error "libvirt AppArmor profile is missing: ${profile}" return 1 } - patch_libvirt_apparmor_profile "${profile}" + patch_libvirt_apparmor_profile "${profile}" || return 1 install -d -m 0755 "${dropin_dir}" tmp=$(mktemp) printf '%s\n' \ @@ -583,10 +583,22 @@ verify_libvirt_host() { libvirt_host_error "libvirt ${LIBVIRT_REQUIRED_VERSION} or newer is required; installed package is ${installed_version:-missing}" return 1 fi - python3 -c 'import libvirt' - id libvirt-qemu >/dev/null - aa-status --enabled >/dev/null - virsh -c "${LIBVIRT_URI}" uri >/dev/null + python3 -c 'import libvirt' || { + libvirt_host_error "python3-libvirt is not importable" + return 1 + } + id libvirt-qemu >/dev/null || { + libvirt_host_error "the libvirt-qemu user is missing" + return 1 + } + aa-status --enabled >/dev/null || { + libvirt_host_error "AppArmor is not enabled" + return 1 + } + virsh -c "${LIBVIRT_URI}" uri >/dev/null || { + libvirt_host_error "cannot connect to ${LIBVIRT_URI}" + return 1 + } daemon_version=$(running_libvirt_version || true) if [[ -z "${daemon_version}" ]] || ! dpkg --compare-versions "${daemon_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then libvirt_host_error "running libvirt daemon is older than ${LIBVIRT_REQUIRED_VERSION} (${daemon_version:-unknown})" @@ -612,7 +624,7 @@ verify_libvirt_host() { return 1 fi # shellcheck disable=SC2119 - verify_passt_capabilities + verify_passt_capabilities || return 1 dropin=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu.d/99-sp-vm-tools-local) grep -qF 'network vsock stream,' "${dropin}" || { @@ -628,7 +640,10 @@ verify_libvirt_host() { libvirt_host_error "QGS is not configured for VSOCK port 4050" return 1 } - systemctl is-active --quiet qgsd + systemctl is-active --quiet qgsd || { + libvirt_host_error "qgsd is not active" + return 1 + } elif [[ "${mode}" == "sev-snp" ]]; then grep -qi 'sev-snp' <<< "${capabilities}" || { libvirt_host_error "domain capabilities do not advertise SEV-SNP launch security" @@ -650,22 +665,22 @@ setup_libvirt_host() { libvirt_host_error "libvirt host setup must run as root" return 1 fi - get_supported_ubuntu_version + get_supported_ubuntu_version || return 1 print_section_header "Libvirt Host Setup" installed_version=$(installed_libvirt_version) if libvirt_upgrade_required "${installed_version}"; then - assert_no_running_libvirt_domains + assert_no_running_libvirt_domains || return 1 fi - apt-get update + apt-get update || return 1 DEBIAN_FRONTEND=noninteractive apt-get install -y \ acl apparmor-utils ca-certificates libcap2-bin passt python3-libvirt \ - qemu-system-x86 qemu-utils wget + qemu-system-x86 qemu-utils wget || return 1 installed_version=$(installed_libvirt_version) if libvirt_upgrade_required "${installed_version}"; then echo "Installed libvirt ${installed_version:-none} is older than ${LIBVIRT_REQUIRED_VERSION}." - install_project_libvirt + install_project_libvirt || return 1 else echo "Installed libvirt ${installed_version} is ${LIBVIRT_REQUIRED_VERSION} or newer; keeping it." fi @@ -674,29 +689,29 @@ setup_libvirt_host() { apt-mark manual libvirt-daemon-system-systemd >/dev/null fi - configure_libvirt_qemu_runtime - configure_libvirt_apparmor - configure_qemu_binary_permissions - configure_iommufd + configure_libvirt_qemu_runtime || return 1 + configure_libvirt_apparmor || return 1 + configure_qemu_binary_permissions || return 1 + configure_iommufd || return 1 # shellcheck disable=SC2119 - configure_passt_capabilities - systemctl daemon-reload - systemctl enable --now libvirtd.service - systemctl start virtlogd.socket virtlockd.socket + configure_passt_capabilities || return 1 + systemctl daemon-reload || return 1 + systemctl enable --now libvirtd.service || return 1 + systemctl start virtlogd.socket virtlockd.socket || return 1 local daemon_version daemon_version=$(running_libvirt_version || true) if [[ "${LIBVIRT_QEMU_CONFIG_CHANGED}" -eq 1 ]] || \ [[ -z "${daemon_version}" ]] || \ ! dpkg --compare-versions "${daemon_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then - assert_no_running_libvirt_domains + assert_no_running_libvirt_domains || return 1 echo "Restarting libvirtd to activate the installed ${LIBVIRT_REQUIRED_VERSION} runtime" - systemctl restart libvirtd.service + systemctl restart libvirtd.service || return 1 fi install -d -o libvirt-qemu -g libvirt-qemu -m 0750 \ - /var/lib/libvirt/images/superprotocol - verify_libvirt_host "${mode}" + /var/lib/libvirt/images/superprotocol || return 1 + verify_libvirt_host "${mode}" || return 1 echo "Libvirt host setup complete." } diff --git a/scripts/start_super_protocol.sh b/scripts/start_super_protocol.sh index d973472..558a4e9 100755 --- a/scripts/start_super_protocol.sh +++ b/scripts/start_super_protocol.sh @@ -1157,6 +1157,8 @@ if [[ "${NETDEV_MODE}" == "tap" ]]; then eval $QEMU_COMMAND } -parse_args $@ -detect_cpu_type -main +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + parse_args "$@" + detect_cpu_type + main +fi diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index f4992e0..30d94a2 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -6,15 +6,9 @@ SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) BASE_SCRIPT="${SCRIPT_DIR}/start_super_protocol.sh" LIBVIRT_LAUNCHER="${SCRIPT_DIR}/libvirt_launcher.py" -if ! grep -q '^parse_args \$@$' "${BASE_SCRIPT}"; then - echo "Error: could not find the start marker in ${BASE_SCRIPT}" >&2 - exit 1 -fi - -# Reuse the release, validation, VFIO, and provider-config preparation code, -# but deliberately exclude the original entrypoint and direct QEMU execution. +# Reuse the sourceable release, validation, VFIO, and provider-config helpers. # shellcheck disable=SC1090 -source <(sed '/^parse_args \$@$/,$d' "${BASE_SCRIPT}") +source "${BASE_SCRIPT}" # qemu:///system normally runs QEMU as libvirt-qemu, which cannot traverse # /root. Keep the same --cache option, but use libvirt's image directory as the @@ -443,8 +437,21 @@ grant_static_libvirt_resource_access() { grant_libvirt_file_access firmware "${BIOS_PATH}" r-- } +cleanup_provider_disk() { + local provider_loop=${1:-} provider_mount=${2:-} + if [[ -n "${provider_mount}" ]] && mountpoint -q "${provider_mount}"; then + umount "${provider_mount}" || true + fi + if [[ -n "${provider_loop}" ]]; then + losetup -d "${provider_loop}" 2>/dev/null || true + fi + if [[ -n "${provider_mount}" ]]; then + rmdir "${provider_mount}" 2>/dev/null || true + fi +} + create_vm_disks() { - local provider_loop provider_mount + local provider_loop="" provider_mount rm -f "${STATE_DISK_PATH}" qemu-img create -f qcow2 "${STATE_DISK_PATH}" "${STATE_DISK_SIZE}G" @@ -453,27 +460,14 @@ create_vm_disks() { dd if=/dev/zero of="${PROVIDER_CONFIG_DISK_PATH}" bs=1M count=1 status=none mkfs.ext4 -q -O '^has_journal,^huge_file,^meta_bg,^ext_attr' \ -L provider_config "${PROVIDER_CONFIG_DISK_PATH}" - provider_loop=$(losetup --find --show --partscan "${PROVIDER_CONFIG_DISK_PATH}") provider_mount=$(mktemp -d) - - cleanup_provider_disk() { - if mountpoint -q "${provider_mount}"; then - umount "${provider_mount}" || true - fi - if [[ -n "${provider_loop}" ]]; then - losetup -d "${provider_loop}" 2>/dev/null || true - fi - rmdir "${provider_mount}" 2>/dev/null || true - } - trap cleanup_provider_disk RETURN + trap 'cleanup_provider_disk "${provider_loop:-}" "${provider_mount:-}"' RETURN + provider_loop=$(losetup --find --show --partscan "${PROVIDER_CONFIG_DISK_PATH}") mount "${provider_loop}" "${provider_mount}" cp -a "${PROVIDER_CONFIG}/." "${provider_mount}/" rm -rf "${provider_mount}/lost+found" - umount "${provider_mount}" - losetup -d "${provider_loop}" - provider_loop="" - rmdir "${provider_mount}" + cleanup_provider_disk "${provider_loop}" "${provider_mount}" trap - RETURN grant_libvirt_file_access state-disk "${STATE_DISK_PATH}" rw- diff --git a/scripts/swarm-cluster.sh b/scripts/swarm-cluster.sh index a3e05d6..b3f28c1 100755 --- a/scripts/swarm-cluster.sh +++ b/scripts/swarm-cluster.sh @@ -770,6 +770,11 @@ start_vm() { return 0 fi if ! tmux has-session -t "${session}" 2>/dev/null; then + # The release-mode launcher may exit immediately after createXML(). + if domain_alive "${domain}"; then + log "Domain ${domain} is running" + return 0 + fi err "Launcher session ${session} exited before the domain started." err "Last lines of ${CACHE}/log-${node_ip##*.}.txt:" tail -n 25 "${CACHE}/log-${node_ip##*.}.txt" 2>/dev/null | sed 's/^/ /' >&2 || true From 1892efd894bbc642a477e3e3c93e525f985a79aa Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 11 Aug 2026 09:25:23 -0500 Subject: [PATCH 20/28] one more coderabbit fix --- scripts/libvirt_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py index cfcecc5..e3cba89 100755 --- a/scripts/libvirt_launcher.py +++ b/scripts/libvirt_launcher.py @@ -519,11 +519,11 @@ def _write_console_output(data: bytes, log: BinaryIO) -> None: def attach_serial_console(conn: Any, domain: Any, libvirt_module: Any, log_path: str) -> None: """Attach a bidirectional console; Ctrl-C or Ctrl-] only detaches.""" stream = conn.newStream(getattr(libvirt_module, "VIR_STREAM_NONBLOCK", 1)) - domain.openConsole(None, stream, 0) stdin_fd: Optional[int] = None old_terminal = None active = False try: + domain.openConsole(None, stream, 0) with open(log_path, "ab", buffering=0) as log_file: stdin_fd = sys.stdin.fileno() if os.isatty(stdin_fd): From 890753d4c9d0cf445a391059a9995387c7ba047a Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Fri, 14 Aug 2026 12:58:54 -0500 Subject: [PATCH 21/28] libvirt-dev --- README.md | 2 +- scripts/setup_libvirt_host.sh | 34 ++++++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e804618..3469513 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ sudo ./scripts/bootstrap_snp.sh The bootstrap performs the host-wide work that previously required manual fixes: -- installs the project libvirt 12.5 packages when the installed version is older; +- installs the complete project libvirt 12.5 package set, including `libvirt-dev`, when the installed version is older or any required split package is missing; - preserves already installed libvirt split drivers during the package transaction; - enables executable mmap and the libvirt socket in the nested AppArmor `passt` profile; - permits QEMU to contact TDX QGS through VSOCK; diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index b4c2169..1fe161e 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -26,6 +26,7 @@ LIBVIRT_BASE_PACKAGES=( libvirt-daemon-plugin-lockd libvirt-daemon-system libvirt-daemon-system-systemd + libvirt-dev ) LIBVIRT_PACKAGE_PATHS=() @@ -112,6 +113,15 @@ libvirt_upgrade_required() { ! dpkg --compare-versions "${installed_version}" ge "${LIBVIRT_REQUIRED_VERSION}" } +missing_required_libvirt_packages() { + local package + for package in "${LIBVIRT_BASE_PACKAGES[@]}"; do + if ! dpkg-query -W -f='${db:Status-Status}' "${package}" 2>/dev/null | grep -qx installed; then + printf '%s\n' "${package}" + fi + done +} + validate_tar_listing() { local entry trimmed component local -a components @@ -577,12 +587,17 @@ configure_iommufd() { } verify_libvirt_host() { - local mode=$1 installed_version daemon_version qemu version_line qemu_major capabilities dropin + local mode=$1 installed_version missing_packages daemon_version qemu version_line qemu_major capabilities dropin installed_version=$(installed_libvirt_version) if [[ -z "${installed_version}" ]] || ! dpkg --compare-versions "${installed_version}" ge "${LIBVIRT_REQUIRED_VERSION}"; then libvirt_host_error "libvirt ${LIBVIRT_REQUIRED_VERSION} or newer is required; installed package is ${installed_version:-missing}" return 1 fi + missing_packages=$(missing_required_libvirt_packages) + if [[ -n "${missing_packages}" ]]; then + libvirt_host_error "required libvirt packages are missing: ${missing_packages//$'\n'/, }" + return 1 + fi python3 -c 'import libvirt' || { libvirt_host_error "python3-libvirt is not importable" return 1 @@ -656,7 +671,7 @@ verify_libvirt_host() { } setup_libvirt_host() { - local mode=$1 installed_version + local mode=$1 installed_version missing_packages if [[ "${mode}" != "tdx" && "${mode}" != "sev-snp" ]]; then libvirt_host_error "setup_libvirt_host mode must be tdx or sev-snp" return 1 @@ -669,7 +684,8 @@ setup_libvirt_host() { print_section_header "Libvirt Host Setup" installed_version=$(installed_libvirt_version) - if libvirt_upgrade_required "${installed_version}"; then + missing_packages=$(missing_required_libvirt_packages) + if libvirt_upgrade_required "${installed_version}" || [[ -n "${missing_packages}" ]]; then assert_no_running_libvirt_domains || return 1 fi apt-get update || return 1 @@ -678,11 +694,17 @@ setup_libvirt_host() { qemu-system-x86 qemu-utils wget || return 1 installed_version=$(installed_libvirt_version) - if libvirt_upgrade_required "${installed_version}"; then - echo "Installed libvirt ${installed_version:-none} is older than ${LIBVIRT_REQUIRED_VERSION}." + missing_packages=$(missing_required_libvirt_packages) + if libvirt_upgrade_required "${installed_version}" || [[ -n "${missing_packages}" ]]; then + if libvirt_upgrade_required "${installed_version}"; then + echo "Installed libvirt ${installed_version:-none} is older than ${LIBVIRT_REQUIRED_VERSION}." + fi + if [[ -n "${missing_packages}" ]]; then + echo "Required libvirt packages are missing: ${missing_packages//$'\n'/, }" + fi install_project_libvirt || return 1 else - echo "Installed libvirt ${installed_version} is ${LIBVIRT_REQUIRED_VERSION} or newer; keeping it." + echo "Installed libvirt ${installed_version} is ${LIBVIRT_REQUIRED_VERSION} or newer and all required packages are present; keeping it." fi if dpkg-query -W -f='${db:Status-Status}' libvirt-daemon-system-systemd 2>/dev/null | grep -qx installed; then From b0b4c1fb5f1a53de006673c97b2000bd8ce32e45 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Sun, 16 Aug 2026 10:10:45 -0500 Subject: [PATCH 22/28] txt is not mandatory --- README.md | 2 +- scripts/setup_tdx.sh | 63 +++++++++++++++++++++----------------------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 3469513..c2088eb 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ BIOS settings: |---|---| | `CPU PA limit to 46 bits` | Disabled | | `SMT` | Enabled | -| `TXT` | Enabled | +| `TXT` | Optional for TDX; status is reported but does not block setup | | `SGX` | Enabled | | `TME` | Enabled | | `TME-MT (Multi-Tenant)` | Enabled, KeyIDs configured (non-zero key split) | diff --git a/scripts/setup_tdx.sh b/scripts/setup_tdx.sh index 39eac2a..2b641e0 100755 --- a/scripts/setup_tdx.sh +++ b/scripts/setup_tdx.sh @@ -17,6 +17,7 @@ NC='\033[0m' # No Color # Modify status indicators: SUCCESS="[${GREEN}✓${NC}]" FAILURE="[${RED}✗${NC}]" +WARNING="[${YELLOW}!${NC}]" print_section_header() { echo -e "\n${BLUE}=== $1 ===${NC}" @@ -146,47 +147,43 @@ check_all_bios_settings() { all_passed=false fi - results+=("TXT Settings:") - + # Intel TXT is useful for TXT/tboot measured-launch workflows, but it is + # not a prerequisite for Intel TDX. Always report its status without + # making the TDX host validation fail. + results+=("TXT Settings (optional for TDX):") + local sinit_base="" - + local senter_en="" + # 0) Does the CPU support SMX/TXT at all? if ! grep -qw smx /proc/cpuinfo; then - results+=("${FAILURE} CPU does not support SMX/TXT${NC}") - all_passed=false + results+=("${WARNING} CPU does not support SMX/TXT${NC}") + results+=(" TXT is not required for TDX; continuing") else # 1) Read SINIT.BASE directly from TXT public config space: # 0xFED30000 + 0x270 (this is what txt-stat used to do) - sinit_base=$(od -An -tx4 -j $((0xFED30270)) -N4 /dev/mem 2>/dev/null | tr -d ' ') + sinit_base=$(od -An -tx4 -j $((0xFED30270)) -N4 /dev/mem 2>/dev/null | tr -d ' ' || true) [ -n "$sinit_base" ] && sinit_base="0x${sinit_base}" - - # 2) Fallback: IA32_FEATURE_CONTROL MSR (0x3A), bit 15 = SENTER global enable. - # Set by BIOS when TXT is enabled. Used when /dev/mem is unavailable - # (e.g. kernel lockdown). - if [ -z "$sinit_base" ] && command -v rdmsr >/dev/null 2>&1; then - modprobe msr 2>/dev/null - local senter_en - senter_en=$(rdmsr -f 15:15 0x3a 2>/dev/null) + + if [ -n "$sinit_base" ] && [ "$sinit_base" != "0x0" ] && \ + [ "$sinit_base" != "0x00000000" ] && [ "$sinit_base" != "0xffffffff" ]; then + results+=("${SUCCESS} TXT enabled (SINIT.BASE = $sinit_base)${NC}") + else + # 2) Fallback: IA32_FEATURE_CONTROL MSR (0x3A), bit 15 = SENTER + # global enable. Check it for every invalid/unavailable + # SINIT.BASE value, not only when /dev/mem returned no output. + modprobe msr 2>/dev/null || true + if command -v rdmsr >/dev/null 2>&1; then + senter_en=$(rdmsr -f 15:15 0x3a 2>/dev/null || true) + fi + if [ "$senter_en" = "1" ]; then results+=("${SUCCESS} TXT enabled (SENTER enabled in IA32_FEATURE_CONTROL)${NC}") else - results+=("${FAILURE} TXT not enabled in BIOS${NC}") - results+=(" Required: Enable TXT in BIOS") - all_passed=false - fi - sinit_base="__msr_checked__" - fi - - if [ "$sinit_base" != "__msr_checked__" ]; then - # 0xffffffff means the chipset does not decode the TXT region => TXT disabled. - # Empty value means we could not read /dev/mem at all. - if [ -n "$sinit_base" ] && [ "$sinit_base" != "0x0" ] && \ - [ "$sinit_base" != "0x00000000" ] && [ "$sinit_base" != "0xffffffff" ]; then - results+=("${SUCCESS} TXT enabled (SINIT.BASE = $sinit_base)${NC}") - else - results+=("${FAILURE} TXT not enabled in BIOS${NC}") - results+=(" Required: Enable TXT in BIOS") - all_passed=false + results+=("${WARNING} TXT not enabled or could not be verified${NC}") + results+=(" SINIT.BASE: ${sinit_base:-unavailable}") + results+=(" IA32_FEATURE_CONTROL.SENTER: ${senter_en:-unavailable}") + results+=(" TXT is not required for TDX; continuing") fi fi fi @@ -226,11 +223,11 @@ check_all_bios_settings() { all_passed=false fi - # Configuration requirements section remains unchanged + # List only settings that can fail the TDX validation. TXT is intentionally + # omitted because it is diagnostic-only for this setup. results+=("${YELLOW}Required BIOS Configuration:${NC}") results+=("• Core Security:") results+=(" - CPU PA: Limit to 46 bits Disable") - results+=(" - TXT: Enable") results+=(" - SGX: Enable") results+=(" - SMT: Enable") results+=("• Memory Protection:") From 812026073568cd96493a5b6b99f16023fd7be387 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Mon, 17 Aug 2026 13:18:56 -0500 Subject: [PATCH 23/28] passt ports --- scripts/setup_libvirt_host.sh | 91 ++++++++++++++++--------- scripts/start_super_protocol_libvirt.sh | 47 +++++-------- 2 files changed, 77 insertions(+), 61 deletions(-) diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index 1fe161e..a2a0543 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -6,6 +6,7 @@ LIBVIRT_REQUIRED_VERSION="12.5.0" LIBVIRT_RELEASE_REPO="Super-Protocol/sp-vm-tools" LIBVIRT_URI="qemu:///system" +PASST_UNPRIVILEGED_PORT_START="0" LIBVIRT_BASE_PACKAGES=( libvirt0 @@ -38,10 +39,6 @@ libvirt_host_error() { return 1 } -libvirt_host_path() { - printf '%s%s\n' "${SPVM_TEST_ROOT:-}" "$1" -} - select_libvirt_release() { local ubuntu_version=$1 case "${ubuntu_version}" in @@ -64,8 +61,7 @@ select_libvirt_release() { } get_supported_ubuntu_version() { - local os_release - os_release=$(libvirt_host_path /etc/os-release) + local os_release=/etc/os-release if [[ ! -r "${os_release}" ]]; then libvirt_host_error "cannot read ${os_release}" return 1 @@ -352,12 +348,12 @@ patch_libvirt_apparmor_profile() { configure_libvirt_apparmor() { local profile dropin_dir dropin template daemon_profile daemon_local tmp - profile=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu) - dropin_dir=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu.d) + profile=/etc/apparmor.d/abstractions/libvirt-qemu + dropin_dir=/etc/apparmor.d/abstractions/libvirt-qemu.d dropin="${dropin_dir}/99-sp-vm-tools-local" - template=$(libvirt_host_path /etc/apparmor.d/libvirt/TEMPLATE.qemu) - daemon_profile=$(libvirt_host_path /etc/apparmor.d/usr.sbin.libvirtd) - daemon_local=$(libvirt_host_path /etc/apparmor.d/local/usr.sbin.libvirtd) + template=/etc/apparmor.d/libvirt/TEMPLATE.qemu + daemon_profile=/etc/apparmor.d/usr.sbin.libvirtd + daemon_local=/etc/apparmor.d/local/usr.sbin.libvirtd [[ -r "${profile}" ]] || { libvirt_host_error "libvirt AppArmor profile is missing: ${profile}" @@ -398,14 +394,11 @@ configure_libvirt_apparmor() { return 1 } apparmor_parser -Q -r "${template}" - if [[ -z "${SPVM_TEST_ROOT:-}" ]]; then - systemctl reload apparmor - fi + systemctl reload apparmor } configure_libvirt_qemu_runtime() { - local config backup tmp - config=$(libvirt_host_path /etc/libvirt/qemu.conf) + local config=/etc/libvirt/qemu.conf backup tmp backup="${config}.sp-vm-tools.bak" [[ -r "${config}" ]] || { libvirt_host_error "libvirt QEMU configuration is missing: ${config}" @@ -456,11 +449,9 @@ configure_libvirt_qemu_runtime() { return fi - if [[ -z "${SPVM_TEST_ROOT:-}" ]]; then - if ! assert_no_running_libvirt_domains; then - rm -f "${tmp}" - return 1 - fi + if ! assert_no_running_libvirt_domains; then + rm -f "${tmp}" + return 1 fi if [[ ! -e "${backup}" ]]; then cp -a "${config}" "${backup}" @@ -523,6 +514,43 @@ configure_passt_capabilities() { done } +verify_passt_unprivileged_ports() { + local value + value=$(sysctl -n net.ipv4.ip_unprivileged_port_start 2>/dev/null || true) + if [[ "${value}" != "${PASST_UNPRIVILEGED_PORT_START}" ]]; then + libvirt_host_error \ + "net.ipv4.ip_unprivileged_port_start must be ${PASST_UNPRIVILEGED_PORT_START} for passt privileged-port forwarding (found ${value:-unavailable})" + return 1 + fi +} + +configure_passt_unprivileged_ports() { + local sysctl_dir=/etc/sysctl.d + local config="${sysctl_dir}/99-sp-vm-tools-passt.conf" tmp + + install -d -m 0755 "${sysctl_dir}" + tmp=$(mktemp) + printf '%s\n' \ + '# Managed by sp-vm-tools bootstrap.' \ + '# TODO: UNSAFE CONFIGURATION. Temporary workaround for a passt regression.' \ + '# Remove it when passt can bind forwarded low ports before entering its' \ + '# unprivileged user namespace. New passt versions create host listeners' \ + '# after user-namespace isolation,' \ + '# so CAP_NET_BIND_SERVICE on the passt binary no longer authorizes bind()' \ + '# in the host network namespace. This is the only stock, unpatched setup' \ + '# currently found to keep libvirt low-port forwarding working. Setting' \ + '# this value to 0 lets every unprivileged process on the host bind any' \ + '# free TCP or UDP port.' \ + "net.ipv4.ip_unprivileged_port_start = ${PASST_UNPRIVILEGED_PORT_START}" > "${tmp}" + install -m 0644 "${tmp}" "${config}" + rm -f "${tmp}" + + sysctl -w \ + "net.ipv4.ip_unprivileged_port_start=${PASST_UNPRIVILEGED_PORT_START}" >/dev/null + verify_passt_unprivileged_ports || return 1 + echo "Configured net.ipv4.ip_unprivileged_port_start=${PASST_UNPRIVILEGED_PORT_START} for passt port forwarding (unsafe workaround)." +} + find_bootstrap_qemu() { local path for path in \ @@ -565,8 +593,7 @@ configure_qemu_binary_permissions() { } configure_iommufd() { - local modules_dir modules_file tmp - modules_dir=$(libvirt_host_path /etc/modules-load.d) + local modules_dir=/etc/modules-load.d modules_file tmp modules_file="${modules_dir}/sp-vm-tools-iommufd.conf" install -d -m 0755 "${modules_dir}" @@ -577,13 +604,11 @@ configure_iommufd() { install -m 0644 "${tmp}" "${modules_file}" rm -f "${tmp}" - if [[ -z "${SPVM_TEST_ROOT:-}" ]]; then - modprobe iommufd - [[ -c /dev/iommu ]] || { - libvirt_host_error "iommufd loaded but /dev/iommu is missing" - return 1 - } - fi + modprobe iommufd + [[ -c /dev/iommu ]] || { + libvirt_host_error "iommufd loaded but /dev/iommu is missing" + return 1 + } } verify_libvirt_host() { @@ -640,8 +665,9 @@ verify_libvirt_host() { fi # shellcheck disable=SC2119 verify_passt_capabilities || return 1 + verify_passt_unprivileged_ports || return 1 - dropin=$(libvirt_host_path /etc/apparmor.d/abstractions/libvirt-qemu.d/99-sp-vm-tools-local) + dropin=/etc/apparmor.d/abstractions/libvirt-qemu.d/99-sp-vm-tools-local grep -qF 'network vsock stream,' "${dropin}" || { libvirt_host_error "AppArmor VSOCK rule is missing from ${dropin}" return 1 @@ -690,7 +716,7 @@ setup_libvirt_host() { fi apt-get update || return 1 DEBIAN_FRONTEND=noninteractive apt-get install -y \ - acl apparmor-utils ca-certificates libcap2-bin passt python3-libvirt \ + acl apparmor-utils ca-certificates libcap2-bin passt procps python3-libvirt \ qemu-system-x86 qemu-utils wget || return 1 installed_version=$(installed_libvirt_version) @@ -717,6 +743,7 @@ setup_libvirt_host() { configure_iommufd || return 1 # shellcheck disable=SC2119 configure_passt_capabilities || return 1 + configure_passt_unprivileged_ports || return 1 systemctl daemon-reload || return 1 systemctl enable --now libvirtd.service || return 1 systemctl start virtlogd.socket virtlockd.socket || return 1 diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index 30d94a2..e1e127b 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -69,11 +69,11 @@ check_target_os() { bootstrap_hint() { if [[ "${VM_MODE}" == "tdx" ]]; then - echo "Re-run scripts/bootstrap_tdx.sh to restore the libvirt host configuration." >&2 + echo "Fix: run sudo ${SCRIPT_DIR}/bootstrap_tdx.sh to restore the libvirt host configuration." >&2 elif [[ "${VM_MODE}" == "sev-snp" ]]; then - echo "Re-run scripts/bootstrap_snp.sh to restore the libvirt host configuration." >&2 + echo "Fix: run sudo ${SCRIPT_DIR}/bootstrap_snp.sh to restore the libvirt host configuration." >&2 else - echo "Re-run the host bootstrap to restore the libvirt host configuration." >&2 + echo "Fix: re-run the host bootstrap with sudo to restore the libvirt host configuration." >&2 fi } @@ -144,7 +144,7 @@ check_tdx_vsock_apparmor_profile() { fi } -check_passt_privileged_ports() { +check_passt_unprivileged_ports() { local ports=() local port port_number minimum=65536 @@ -171,30 +171,19 @@ check_passt_privileged_ports() { return fi - command -v getcap >/dev/null 2>&1 || { - echo "Error: getcap is required to verify privileged passt port ${minimum}." >&2 - bootstrap_hint - exit 1 - } - - local binary path real capabilities found_passt=false - local -A checked=() - for binary in passt passt.avx2; do - path=$(command -v "${binary}" 2>/dev/null || true) - [[ -n "${path}" ]] || continue - real=$(readlink -f -- "${path}") - [[ -n "${real}" && -z "${checked[${real}]:-}" ]] || continue - checked["${real}"]=1 - found_passt=true - capabilities=$(getcap "${real}" 2>/dev/null || true) - if [[ "${capabilities}" != *cap_net_bind_service* ]]; then - echo "Error: passt must bind host port ${minimum}, but ${real} lacks CAP_NET_BIND_SERVICE." >&2 - bootstrap_hint - exit 1 - fi - done - if [[ "${found_passt}" != "true" ]]; then - echo "Error: no passt binary was found for privileged host port ${minimum}." >&2 + local sysctl_path=/proc/sys/net/ipv4/ip_unprivileged_port_start + local unprivileged_port_start="" + if [[ -r "${sysctl_path}" ]]; then + read -r unprivileged_port_start < "${sysctl_path}" || true + fi + if [[ "${unprivileged_port_start}" != "0" ]]; then + # TODO: UNSAFE CONFIGURATION. Temporary workaround for a passt regression: + # recent versions create forwarded host listeners after entering a user + # namespace, so the binary's CAP_NET_BIND_SERVICE cannot authorize bind() + # in the host network namespace. This is the only stock, unpatched setup + # currently found to work; it makes all host ports unprivileged. + echo "Error: passt must bind host port ${minimum}, but net.ipv4.ip_unprivileged_port_start is ${unprivileged_port_start:-unavailable} (expected 0)." >&2 + echo "This temporary workaround allows every unprivileged process on the host to bind any free TCP/UDP port." >&2 bootstrap_hint exit 1 fi @@ -536,7 +525,7 @@ main_libvirt() { check_qemu_version preflight_libvirt check_params - check_passt_privileged_ports + check_passt_unprivileged_ports prepare_selected_host_devices prepare_mode_parameters From 77b9639b749bc6bcd46346f6e3350ac173f86f21 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 18 Aug 2026 10:19:22 -0500 Subject: [PATCH 24/28] fixed libvirt start --- scripts/libvirt_launcher.py | 230 ++++++++++++++------ scripts/setup_libvirt_host.sh | 101 ++++++++- scripts/setup_tdx.sh | 20 +- scripts/start_super_protocol.sh | 10 +- scripts/start_super_protocol_libvirt.sh | 32 ++- scripts/swarm-cluster.sh | 6 +- tests/test_libvirt_launcher.py | 270 ------------------------ 7 files changed, 318 insertions(+), 351 deletions(-) delete mode 100644 tests/test_libvirt_launcher.py diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py index e3cba89..38d4ba7 100755 --- a/scripts/libvirt_launcher.py +++ b/scripts/libvirt_launcher.py @@ -9,11 +9,11 @@ from __future__ import annotations import argparse -import json import os import re import select import sys +import time import termios import tty import xml.etree.ElementTree as ET @@ -54,7 +54,7 @@ class DomainConfig: state_disk: str provider_config_disk: str guest_cid: int - qgs_cid: int + qgs_socket: str mac_address: str netdev_mode: str debug: bool = False @@ -100,6 +100,16 @@ def _parse_bdf(bdf: str) -> dict[str, str]: } +def _normalized_bdf(bdf: str) -> str: + match = BDF_RE.fullmatch(bdf) + if not match: + raise ValueError(f"invalid PCI BDF: {bdf!r}") + parts = match.groupdict(default="0000") + return ( + f"{parts['domain']}:{parts['bus']}:{parts['slot']}.{parts['function']}" + ).lower() + + def _validate_config(config: DomainConfig) -> None: if not DOMAIN_NAME_RE.fullmatch(config.name): raise ValueError( @@ -111,8 +121,8 @@ def _validate_config(config: DomainConfig) -> None: raise ValueError(f"unsupported network mode: {config.netdev_mode}") if config.memory_gib < 1 or config.vcpus < 1: raise ValueError("memory and vCPU count must be positive") - if config.guest_cid < 3 or config.qgs_cid < 2: - raise ValueError("guest CID must be >= 3 and QGS CID must be >= 2") + if config.guest_cid < 3: + raise ValueError("guest CID must be >= 3") if not MAC_RE.fullmatch(config.mac_address): raise ValueError(f"invalid MAC address: {config.mac_address!r}") for path in ( @@ -125,6 +135,10 @@ def _validate_config(config: DomainConfig) -> None: ): if not Path(path).is_absolute(): raise ValueError(f"libvirt resource path must be absolute: {path!r}") + if config.mode == "tdx" and not Path(config.qgs_socket).is_absolute(): + raise ValueError( + f"TDX QGS socket path must be absolute: {config.qgs_socket!r}" + ) ports = ( config.ssh_port, config.wg_port, @@ -258,10 +272,9 @@ def _add_host_devices(devices: ET.Element, config: DomainConfig) -> None: hostdev = _sub(devices, "hostdev", mode="subsystem", type="pci", managed="no") _sub(hostdev, "driver", name="vfio", iommufd="yes") + alias = f"ua-hostdev{index}" source = _sub(hostdev, "source") _sub(source, "address", **_parse_bdf(host_device.bdf)) - if host_device.kind == "gpu": - _sub(hostdev, "rom", bar="off") _sub( hostdev, "address", @@ -271,6 +284,7 @@ def _add_host_devices(devices: ET.Element, config: DomainConfig) -> None: slot="0x00", function="0x0", ) + _sub(hostdev, "alias", name=alias) def _add_cpu_and_features(domain: ET.Element, config: DomainConfig) -> None: @@ -310,48 +324,26 @@ def _qemu_commandline(domain: ET.Element) -> ET.Element: return commandline -def _add_fw_cfg_qemu_args(domain: ET.Element) -> None: +def _add_fw_cfg_qemu_args(domain: ET.Element, config: DomainConfig) -> None: # Libvirt deliberately rejects opt/ovmf/* through native fwcfg XML because # that namespace is reserved for OVMF. The direct launcher needs this # existing OVMF knob, so pass it through QEMU's command line namespace. - commandline = _qemu_commandline(domain) - _sub(commandline, f"{{{QEMU_NS}}}arg", value="-fw_cfg") - _sub( - commandline, - f"{{{QEMU_NS}}}arg", - value="name=opt/ovmf/X-PciMmio64,string=262144", - ) - + # + # OVMF reads one knob per PCI root bridge, named X-PciMmio64Mb where N + # is the 1-based root-port index; a bare "X-PciMmio64" is silently ignored. + # Only the ports carrying passthrough devices need the enlarged 64-bit MMIO + # aperture, so emit nothing when no host device is attached. + if not config.host_devices: + return -def _add_tdx_qemu_args(domain: ET.Element, config: DomainConfig) -> None: commandline = _qemu_commandline(domain) - _sub(commandline, f"{{{QEMU_NS}}}arg", value="-object") - _sub( - commandline, - f"{{{QEMU_NS}}}arg", - value=f"memory-backend-ram,id=sp-mem,size={config.memory_gib}G", - ) - tdx_object = { - "qom-type": "tdx-guest", - "id": "sp-tdx", - "quote-generation-socket": { - "type": "vsock", - "cid": str(config.qgs_cid), - "port": "4050", - }, - } - _sub(commandline, f"{{{QEMU_NS}}}arg", value="-object") - _sub( - commandline, - f"{{{QEMU_NS}}}arg", - value=json.dumps(tdx_object, separators=(",", ":")), - ) - _sub(commandline, f"{{{QEMU_NS}}}arg", value="-machine") - _sub( - commandline, - f"{{{QEMU_NS}}}arg", - value="confidential-guest-support=sp-tdx,memory-backend=sp-mem", - ) + for index in range(1, len(config.host_devices) + 1): + _sub(commandline, f"{{{QEMU_NS}}}arg", value="-fw_cfg") + _sub( + commandline, + f"{{{QEMU_NS}}}arg", + value=f"name=opt/ovmf/X-PciMmio64Mb{index},string=262144", + ) def build_domain_xml(config: DomainConfig) -> str: @@ -363,6 +355,11 @@ def build_domain_xml(config: DomainConfig) -> str: _sub(domain, "memory", config.memory_gib, unit="GiB") _sub(domain, "currentMemory", config.memory_gib, unit="GiB") _sub(domain, "vcpu", config.vcpus, placement="static") + if config.host_devices: + # Domain-level IOMMUFD lets libvirt open each assigned device itself + # and hand QEMU the resulting fd, keeping the cdev outside the guest's + # reach. + _sub(domain, "iommufd", enabled="yes") os_element = _sub(domain, "os") _sub(os_element, "type", "hvm", arch="x86_64", machine="q35") @@ -371,7 +368,11 @@ def build_domain_xml(config: DomainConfig) -> str: _sub(os_element, "cmdline", config.kernel_cmdline) _add_cpu_and_features(domain, config) - _sub(domain, "clock", offset="utc") + clock = _sub(domain, "clock", offset="utc") + if config.mode == "tdx": + # TD guests do not emulate the HPET; Intel's and Canonical's reference + # TD definitions both disable it explicitly. + _sub(clock, "timer", name="hpet", present="no") _sub(domain, "on_poweroff", "destroy") _sub(domain, "on_reboot", "restart") _sub(domain, "on_crash", "destroy") @@ -385,9 +386,28 @@ def build_domain_xml(config: DomainConfig) -> str: _sub(devices, "controller", type="usb", model="none") _add_network(devices, config) - serial = _sub(devices, "serial", type="pty") + # The serial port must drain into a file rather than a bare pty. A pty that + # nothing reads fills up, after which the 16550 line status register never + # reports the transmitter as empty and the guest spins forever inside + # console output -- the boot stops mid-word with vCPU0 burning host CPU on + # port 0x3fd reads. A file sink always accepts writes, so the guest keeps + # running whether or not a console client is attached, and libvirt still + # records everything from the very first byte. + serial = _sub(devices, "serial", type="file") + _sub( + serial, + "source", + path=f"/var/log/libvirt/qemu/{config.name}-serial.log", + append="off", + ) _sub(serial, "target", type="isa-serial", port="0") - console = _sub(devices, "console", type="pty") + console = _sub(devices, "console", type="file") + _sub( + console, + "source", + path=f"/var/log/libvirt/qemu/{config.name}-serial.log", + append="off", + ) _sub(console, "target", type="serial", port="0") video = _sub(devices, "video") _sub(video, "model", type="none") @@ -396,7 +416,6 @@ def build_domain_xml(config: DomainConfig) -> str: vsock = _sub(devices, "vsock", model="virtio") _sub(vsock, "cid", auto="no", address=config.guest_cid) _add_host_devices(devices, config) - _add_fw_cfg_qemu_args(domain) if config.mode == "sev-snp": launch_security = _sub( @@ -409,7 +428,16 @@ def build_domain_xml(config: DomainConfig) -> str: _sub(launch_security, "reducedPhysBits", "1") _sub(launch_security, "policy", "0x30000") elif config.mode == "tdx": - _add_tdx_qemu_args(domain, config) + launch_security = _sub(domain, "launchSecurity", type="tdx") + _sub( + launch_security, + "quoteGenerationService", + path=config.qgs_socket, + ) + + # QEMU namespace extensions must follow native domain elements for the + # libvirt domain schema to accept the document. + _add_fw_cfg_qemu_args(domain, config) ET.indent(domain, space=" ") return ET.tostring(domain, encoding="unicode") @@ -430,18 +458,31 @@ def _iommufd_advertised(domain_capabilities: str) -> bool: return False +def _tdx_launch_security_advertised(domain_capabilities: str) -> bool: + try: + root = ET.fromstring(domain_capabilities) + except ET.ParseError: + return False + for enum in root.findall(".//features/launchSecurity/enum[@name='sectype']"): + if any((value.text or "").strip() == "tdx" for value in enum.findall("value")): + return True + return False + + def check_connection_capabilities(conn: Any, config: Any) -> None: if conn.getType().upper() != "QEMU": raise RuntimeError(f"qemu:///system returned unexpected driver {conn.getType()!r}") - if not config.host_devices: + mode = getattr(config, "mode", None) + if not config.host_devices and mode != "tdx": return - version = conn.getLibVersion() - if version < LIBVIRT_IOMMUFD_VERSION: - raise RuntimeError( - "GPU passthrough requires libvirt >= 12.1.0; " - f"the daemon reports {_version_string(version)}" - ) + if config.host_devices: + version = conn.getLibVersion() + if version < LIBVIRT_IOMMUFD_VERSION: + raise RuntimeError( + "GPU passthrough requires libvirt >= 12.1.0; " + f"the daemon reports {_version_string(version)}" + ) try: capabilities = conn.getDomainCapabilities( config.emulator, @@ -452,10 +493,14 @@ def check_connection_capabilities(conn: Any, config: Any) -> None: ) except Exception as exc: raise RuntimeError(f"failed to query libvirt domain capabilities: {exc}") from exc - if not _iommufd_advertised(capabilities): + if config.host_devices and not _iommufd_advertised(capabilities): raise RuntimeError( "libvirt domain capabilities do not advertise hostdev iommufd support" ) + if mode == "tdx" and not _tdx_launch_security_advertised(capabilities): + raise RuntimeError( + "libvirt domain capabilities do not advertise native TDX launch security" + ) def ensure_domain_name_available(conn: Any, libvirt_module: Any, name: str) -> None: @@ -482,7 +527,9 @@ def _format_launch_error(exc: BaseException) -> str: return f"libvirt failed to start the domain: {message}" -def preflight_connection(emulator: str, name: str, require_iommufd: bool) -> None: +def preflight_connection( + emulator: str, name: str, mode: str, require_iommufd: bool +) -> None: """Check the daemon, domain name, and optional IOMMUFD support without mutation.""" try: import libvirt # type: ignore @@ -501,6 +548,7 @@ def preflight_connection(emulator: str, name: str, require_iommufd: bool) -> Non try: probe = SimpleNamespace( emulator=str(Path(emulator).resolve()), + mode=mode, host_devices=[object()] if require_iommufd else [], ) check_connection_capabilities(conn, probe) @@ -588,6 +636,54 @@ def attach_serial_console(conn: Any, domain: Any, libvirt_module: Any, log_path: print(f"\n{message}", file=sys.stderr) +def follow_serial_log( + domain: Any, libvirt_module: Any, serial_log: str, log_path: str +) -> None: + """Mirror the domain serial log until the VM stops or the user detaches. + + The domain writes its console to a file, so the guest never blocks on a + console nobody reads. Debug mode simply tails that file; Ctrl-C detaches + and leaves the VM running. + """ + print( + "\nFollowing serial console; press Ctrl-C to detach " + "(the VM keeps running).", + file=sys.stderr, + ) + deadline = time.monotonic() + 30.0 + while not Path(serial_log).exists(): + if time.monotonic() > deadline: + print( + f"Serial log did not appear: {serial_log}", + file=sys.stderr, + ) + return + time.sleep(0.2) + + try: + with open(serial_log, "rb") as source, open( + log_path, "ab", buffering=0 + ) as log_file: + while True: + chunk = source.read(65536) + if chunk: + _write_console_output(chunk, log_file) + continue + try: + if not domain.isActive(): + break + except libvirt_module.libvirtError: + break + time.sleep(0.2) + except KeyboardInterrupt: + print( + f"\nDetached from {domain.name()}; VM is still managed by libvirt.", + file=sys.stderr, + ) + return + print("\nSerial console closed because the VM stopped.", file=sys.stderr) + + def launch(config: DomainConfig) -> None: try: import libvirt # type: ignore @@ -615,11 +711,18 @@ def launch(config: DomainConfig) -> None: name = domain.name() uuid = domain.UUIDString() print(f"Started transient libvirt domain: {name} ({uuid})") - print(f" console: virsh -c qemu:///system console {name}") + # The console is a write-only file sink, so "virsh console" has + # nothing to attach to; point at the log libvirt actually writes. + print(f" serial log: /var/log/libvirt/qemu/{name}-serial.log") print(f" shutdown: virsh -c qemu:///system shutdown {name}") print(f" force stop: virsh -c qemu:///system destroy {name}") if config.debug: - attach_serial_console(conn, domain, libvirt, str(config.log_file)) + follow_serial_log( + domain, + libvirt, + f"/var/log/libvirt/qemu/{name}-serial.log", + str(config.log_file), + ) except libvirt.libvirtError as exc: raise RuntimeError(_format_launch_error(exc)) from exc finally: @@ -654,7 +757,7 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--state-disk", required=True) parser.add_argument("--provider-config-disk", required=True) parser.add_argument("--guest-cid", type=int, required=True) - parser.add_argument("--qgs-cid", type=int, required=True) + parser.add_argument("--qgs-socket", required=True) parser.add_argument("--mac-address", required=True) parser.add_argument("--netdev-mode", choices=("user", "tap"), required=True) parser.add_argument("--debug", action="store_true") @@ -681,6 +784,7 @@ def _preflight_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Check libvirt before preparing VM resources") parser.add_argument("--emulator", required=True) parser.add_argument("--name", required=True) + parser.add_argument("--mode", choices=("untrusted", "tdx", "sev-snp"), required=True) parser.add_argument("--require-iommufd", action="store_true") return parser @@ -699,7 +803,7 @@ def _config_from_args(args: argparse.Namespace) -> DomainConfig: state_disk=str(Path(args.state_disk).resolve()), provider_config_disk=str(Path(args.provider_config_disk).resolve()), guest_cid=args.guest_cid, - qgs_cid=args.qgs_cid, + qgs_socket=str(Path(args.qgs_socket).resolve()), mac_address=args.mac_address, netdev_mode=args.netdev_mode, debug=args.debug, @@ -731,7 +835,9 @@ def main(argv: Optional[Iterable[str]] = None) -> int: raise ValueError( "domain name may contain only letters, digits, '.', '_', '+', ':', and '-'" ) - preflight_connection(args.emulator, args.name, args.require_iommufd) + preflight_connection( + args.emulator, args.name, args.mode, args.require_iommufd + ) except (RuntimeError, ValueError) as exc: print(f"Error: {exc}", file=sys.stderr) return 1 diff --git a/scripts/setup_libvirt_host.sh b/scripts/setup_libvirt_host.sh index a2a0543..05d303b 100755 --- a/scripts/setup_libvirt_host.sh +++ b/scripts/setup_libvirt_host.sh @@ -7,6 +7,7 @@ LIBVIRT_REQUIRED_VERSION="12.5.0" LIBVIRT_RELEASE_REPO="Super-Protocol/sp-vm-tools" LIBVIRT_URI="qemu:///system" PASST_UNPRIVILEGED_PORT_START="0" +PASST_APPARMOR_DISCONNECTED_PATH="/att/passt/" LIBVIRT_BASE_PACKAGES=( libvirt0 @@ -294,7 +295,11 @@ install_project_libvirt() { passthrough_profile_state() { local profile=$1 awk ' - /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1; found = 1; next } + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { + in_passt = 1 + found = 1 + next + } in_passt && /^[[:space:]]*}/ { in_passt = 0; done = 1 } in_passt && /\/usr\/bin\/passt[[:space:]]+r,/ { readonly = 1 } in_passt && /\/usr\/bin\/passt[[:space:]]+rm,/ { mmap = 1 } @@ -309,6 +314,44 @@ passthrough_profile_state() { ' "${profile}" } +passthrough_profile_handles_disconnected_sockets() { + local profile=$1 + awk ' + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { + found = 1 + if ($0 ~ /attach_disconnected/) handled = 1 + } + END { exit !(found && handled) } + ' "${profile}" +} + +patch_passthrough_disconnected_socket_handling() { + local profile=$1 tmp + tmp=$(mktemp) + if ! awk -v path="${PASST_APPARMOR_DISCONNECTED_PATH}" ' + BEGIN { patched = 0 } + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { + if ($0 ~ /attach_disconnected/) { + patched = 1 + } else if ($0 ~ /flags=\(/) { + sub(/flags=\(/, "flags=(attach_disconnected.path=" path " ") + patched = 1 + } else { + sub(/\{[[:space:]]*$/, "flags=(attach_disconnected.path=" path ") {") + patched = 1 + } + } + { print } + END { if (!patched) exit 1 } + ' "${profile}" > "${tmp}"; then + rm -f "${tmp}" + libvirt_host_error "failed to add disconnected socket handling to the nested passt profile" + return 1 + fi + cat "${tmp}" > "${profile}" + rm -f "${tmp}" +} + patch_libvirt_apparmor_profile() { local profile=$1 state backup tmp state=$(passthrough_profile_state "${profile}") @@ -340,10 +383,25 @@ patch_libvirt_apparmor_profile() { cat "${tmp}" > "${profile}" rm -f "${tmp}" fi + + # TODO: Remove this workaround after Ubuntu's passt/libvirt AppArmor + # policy handles the listening Unix socket that becomes disconnected when + # passt pivots into its empty sandbox root. AppArmor 5 on Ubuntu 26.04 + # otherwise rejects accept4() with EACCES. A synthetic attachment prefix + # is scoped to the nested passt profile and avoids disabling confinement. + if [[ "${UBUNTU_VERSION:-}" == "26.04" ]] && \ + ! passthrough_profile_handles_disconnected_sockets "${profile}"; then + patch_passthrough_disconnected_socket_handling "${profile}" || return 1 + fi if [[ "$(passthrough_profile_state "${profile}")" != "ready" ]]; then libvirt_host_error "failed to make the nested passt AppArmor profile usable" return 1 fi + if [[ "${UBUNTU_VERSION:-}" == "26.04" ]] && \ + ! passthrough_profile_handles_disconnected_sockets "${profile}"; then + libvirt_host_error "nested passt AppArmor profile does not handle disconnected Unix sockets" + return 1 + fi } configure_libvirt_apparmor() { @@ -369,6 +427,7 @@ configure_libvirt_apparmor() { '/usr/local/lib{,64}/qemu/*.so mr,' \ '/usr/local/lib/@{multiarch}/qemu/*.so mr,' \ 'owner @{run}/libvirt/qemu/passt/* rw,' \ + '@{run}/tdx-qgs/qgs.socket rw,' \ 'network vsock stream,' > "${tmp}" install -m 0644 "${tmp}" "${dropin}" rm -f "${tmp}" @@ -397,6 +456,14 @@ configure_libvirt_apparmor() { systemctl reload apparmor } +configure_tdx_qgs_access() { + getent group qgsd >/dev/null || { + libvirt_host_error "QGS group qgsd is missing" + return 1 + } + usermod -a -G qgsd libvirt-qemu || return 1 +} + configure_libvirt_qemu_runtime() { local config=/etc/libvirt/qemu.conf backup tmp backup="${config}.sp-vm-tools.bak" @@ -672,19 +739,42 @@ verify_libvirt_host() { libvirt_host_error "AppArmor VSOCK rule is missing from ${dropin}" return 1 } + if [[ "${UBUNTU_VERSION:-}" == "26.04" ]] && \ + ! passthrough_profile_handles_disconnected_sockets \ + /etc/apparmor.d/abstractions/libvirt-qemu; then + libvirt_host_error "nested passt AppArmor profile lacks Ubuntu 26.04 disconnected socket handling; rerun bootstrap" + return 1 + fi if [[ "${mode}" == "tdx" ]]; then [[ -c /dev/vhost-vsock ]] || { libvirt_host_error "/dev/vhost-vsock is missing" return 1 } - grep -Eq '^[[:space:]]*port[[:space:]]*=[[:space:]]*4050([[:space:]]|$)' /etc/qgs.conf || { - libvirt_host_error "QGS is not configured for VSOCK port 4050" + if grep -Eq '^[[:space:]]*port[[:space:]]*=' /etc/qgs.conf; then + libvirt_host_error "QGS must use its Unix socket; remove the port setting from /etc/qgs.conf" return 1 - } + fi systemctl is-active --quiet qgsd || { libvirt_host_error "qgsd is not active" return 1 } + [[ -S /var/run/tdx-qgs/qgs.socket ]] || { + libvirt_host_error "QGS Unix socket is missing: /var/run/tdx-qgs/qgs.socket" + return 1 + } + id -nG libvirt-qemu | tr ' ' '\n' | grep -qx qgsd || { + libvirt_host_error "libvirt-qemu is not a member of the qgsd group" + return 1 + } + grep -qF '@{run}/tdx-qgs/qgs.socket rw,' "${dropin}" || { + libvirt_host_error "AppArmor QGS socket rule is missing from ${dropin}" + return 1 + } + if ! grep -Eq "]*name=['\"]sectype['\"]" <<< "${capabilities}" || \ + ! grep -Eq 'tdx' <<< "${capabilities}"; then + libvirt_host_error "domain capabilities do not advertise native TDX launch security" + return 1 + fi elif [[ "${mode}" == "sev-snp" ]]; then grep -qi 'sev-snp' <<< "${capabilities}" || { libvirt_host_error "domain capabilities do not advertise SEV-SNP launch security" @@ -738,6 +828,9 @@ setup_libvirt_host() { fi configure_libvirt_qemu_runtime || return 1 + if [[ "${mode}" == "tdx" ]]; then + configure_tdx_qgs_access || return 1 + fi configure_libvirt_apparmor || return 1 configure_qemu_binary_permissions || return 1 configure_iommufd || return 1 diff --git a/scripts/setup_tdx.sh b/scripts/setup_tdx.sh index 2b641e0..0986bcb 100755 --- a/scripts/setup_tdx.sh +++ b/scripts/setup_tdx.sh @@ -396,16 +396,22 @@ EOL chmod -R 750 /opt/intel/sgx-dcap-pccs/ } -# Configure QGS transport. Our stack talks to the Quote Generation Service over -# vsock, but newer tdx-qgs packages (Ubuntu 26.04+) ship /etc/qgs.conf with the -# port commented out (defaulting to a Unix domain socket). Just write the config -# we need: vsock on port 4050. +# Libvirt's native TDX launch security connects QEMU to QGS through the standard +# Unix socket. Leaving "port" unset selects this transport on the QGS packages +# used by both supported Ubuntu releases. Both launchers use the same socket. configure_qgs() { - print_section_header "Configuring QGS (vsock port 4050)..." + print_section_header "Configuring QGS (Unix socket)..." cat > /etc/qgs.conf << EOL -port = 4050 number_threads = 4 EOL + + install -d -m 0755 /etc/systemd/system/qgsd.service.d + cat > /etc/systemd/system/qgsd.service.d/socket.conf << EOL +[Service] +RuntimeDirectory=tdx-qgs +RuntimeDirectoryMode=0755 +EOL + systemctl daemon-reload } # On Ubuntu 24.04 the matched TDX kernel + QEMU are installed from the @@ -907,7 +913,7 @@ check_error "Failed to register platform" # written config: qgsd re-reads /etc/sgx_default_qcnl.conf, and the one-shot # mpa_registration_tool re-runs the registration flow. print_section_header "Starting remaining services..." -configure_qgs # patch /etc/qgs.conf for vsock before (re)starting qgsd +configure_qgs # select the Unix socket before (re)starting qgsd systemctl restart qgsd wait_for_service qgsd systemctl restart mpa_registration_tool diff --git a/scripts/start_super_protocol.sh b/scripts/start_super_protocol.sh index 558a4e9..134677d 100755 --- a/scripts/start_super_protocol.sh +++ b/scripts/start_super_protocol.sh @@ -734,6 +734,14 @@ check_params() { USED_GPUS=("${AVAILABLE_GPUS[@]}") fi + # lspci reports BDFs without the PCI domain, so accept the fully qualified + # form too and compare everything in the short form. + local -a NORMALIZED_GPUS=() + for GPU in "${USED_GPUS[@]}"; do + NORMALIZED_GPUS+=("${GPU#0000:}") + done + USED_GPUS=("${NORMALIZED_GPUS[@]}") + # Remove duplicates efficiently declare -A UNIQUE_GPUS for GPU in "${USED_GPUS[@]}"; do @@ -1010,7 +1018,7 @@ main() { fi CC_PARAMS+=" -object memory-backend-ram,id=mem0,size=${VM_RAM}G " MACHINE_PARAMS="q35,kernel_irqchip=split,confidential-guest-support=tdx,memory-backend=mem0" - CC_SPECIFIC_PARAMS=" -object '{\"qom-type\":\"tdx-guest\",\"id\":\"tdx\",\"quote-generation-socket\":{\"type\":\"vsock\",\"cid\":\"${BASE_CID}\",\"port\":\"4050\"}}'" + CC_SPECIFIC_PARAMS=" -object '{\"qom-type\":\"tdx-guest\",\"id\":\"tdx\",\"quote-generation-socket\":{\"type\":\"unix\",\"path\":\"/var/run/tdx-qgs/qgs.socket\"}}'" ;; "sev-snp") if [[ ! $SEV_SNP_SUPPORT ]]; then diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index e1e127b..d431665 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -110,7 +110,7 @@ check_passt_apparmor_profile() { [[ -r "${profile}" ]] || return 0 if awk ' - /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1 } + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { in_passt = 1 } in_passt && /\/usr\/bin\/passt[[:space:]]+r,/ { incompatible = 1 } in_passt && /^[[:space:]]*}/ { exit } END { exit incompatible ? 0 : 1 } @@ -122,7 +122,7 @@ check_passt_apparmor_profile() { fi if ! awk ' - /^[[:space:]]*profile passt[[:space:]]*\{/ { in_passt = 1 } + /^[[:space:]]*profile passt([[:space:]]+flags=\([^)]*\))?[[:space:]]*\{/ { in_passt = 1 } in_passt && /^[[:space:]]*capability[[:space:]]+net_bind_service,/ { found = 1 } in_passt && /^[[:space:]]*}/ { exit } END { exit found ? 0 : 1 } @@ -204,7 +204,31 @@ preflight_libvirt() { require_iommufd=true fi - local args=(preflight --emulator "${QEMU_PATH}" --name "${LIBVIRT_DOMAIN_NAME}") + local ubuntu_version="" + if [[ -r /etc/os-release ]]; then + ubuntu_version=$( + # shellcheck disable=SC1091 + source /etc/os-release + printf '%s' "${VERSION_ID:-}" + ) + fi + if [[ "${ubuntu_version}" == "26.04" ]]; then + local passt_apparmor=/etc/apparmor.d/abstractions/libvirt-qemu + if [[ ! -r "${passt_apparmor}" ]] || \ + ! grep -Eq '^[[:space:]]*profile[[:space:]]+passt[[:space:]]+flags=\([^)]*attach_disconnected(\.path=[^[:space:])]+)?[^)]*\)[[:space:]]*\{' \ + "${passt_apparmor}"; then + echo "Error: Ubuntu 26.04 AppArmor blocks passt from accepting the libvirt Unix socket." >&2 + bootstrap_hint + exit 1 + fi + fi + + local args=( + preflight + --emulator "${QEMU_PATH}" + --name "${LIBVIRT_DOMAIN_NAME}" + --mode "${VM_MODE}" + ) if [[ "${require_iommufd}" == "true" ]]; then args+=(--require-iommufd) fi @@ -485,7 +509,7 @@ launch_with_libvirt() { --state-disk "${STATE_DISK_PATH}" --provider-config-disk "${PROVIDER_CONFIG_DISK_PATH}" --guest-cid "${GUEST_CID}" - --qgs-cid "${BASE_CID}" + --qgs-socket "/var/run/tdx-qgs/qgs.socket" --mac-address "${MAC_ADDRESS}" --netdev-mode "${NETDEV_MODE}" --ip-address "${IP_ADDRESS}" diff --git a/scripts/swarm-cluster.sh b/scripts/swarm-cluster.sh index b3f28c1..0ef870d 100755 --- a/scripts/swarm-cluster.sh +++ b/scripts/swarm-cluster.sh @@ -831,7 +831,7 @@ wait_bootstrap() { sleep 5; waited=$(( waited + 5 )) done echo >&2 - die "Bootstrap did not come up within ${timeout}s. Check ${CACHE}/log-${BOOTSTRAP_IP##*.}.txt and: virsh -c ${LIBVIRT_URI} console ${DOMAIN_BOOTSTRAP}" + die "Bootstrap did not come up within ${timeout}s. Check ${CACHE}/log-${BOOTSTRAP_IP##*.}.txt and /var/log/libvirt/qemu/${DOMAIN_BOOTSTRAP}-serial.log" } # ---------------------------------------------------------------------------- @@ -966,8 +966,8 @@ cmd_up() { log "Cluster started. Ingress: gw.dyn.${GLOBAL_ID}.${BASE_DOMAIN} -> 80/443" log "Cluster started. Domains: virsh -c ${LIBVIRT_URI} list" - log " bootstrap console: virsh -c ${LIBVIRT_URI} console ${DOMAIN_BOOTSTRAP}" - log " join consoles: ${DOMAIN_JOIN[0]} | ${DOMAIN_JOIN[1]}" + log " bootstrap serial: tail -f /var/log/libvirt/qemu/${DOMAIN_BOOTSTRAP}-serial.log" + log " join serials: /var/log/libvirt/qemu/{${DOMAIN_JOIN[0]},${DOMAIN_JOIN[1]}}-serial.log" if [[ "${DEBUG_MODE}" == "true" ]]; then log " attached debug consoles are in tmux: ${TMUX_BOOTSTRAP}, ${TMUX_JOIN[0]}, ${TMUX_JOIN[1]}" fi diff --git a/tests/test_libvirt_launcher.py b/tests/test_libvirt_launcher.py deleted file mode 100644 index d6ee979..0000000 --- a/tests/test_libvirt_launcher.py +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys -import unittest -import xml.etree.ElementTree as ET -from pathlib import Path -from unittest import mock - - -REPO_ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(REPO_ROOT / "scripts")) - -import libvirt_launcher as launcher # noqa: E402 - - -class DomainXMLTests(unittest.TestCase): - def config(self, mode="untrusted", netdev_mode="user", **overrides): - values = dict( - name="super-protocol-3", - mode=mode, - emulator="/usr/bin/qemu-system-x86_64", - memory_gib=16, - vcpus=8, - bios="/var/lib/super protocol/bios.fd", - kernel="/var/lib/super protocol/vmlinuz", - kernel_cmdline="root=LABEL=rootfs hash=a&b", - rootfs="/var/lib/super protocol/rootfs.img", - state_disk="/var/lib/super protocol/state.qcow2", - provider_config_disk="/var/lib/super protocol/provider.img", - guest_cid=3, - qgs_cid=2, - mac_address="52:54:00:12:34:56", - netdev_mode=netdev_mode, - debug=False, - ssh_port=2222, - wg_port=51821, - http_port=8080, - https_port=8443, - pki_port=9443, - pki_vm_measure_port=9181, - swarm_db_gossip_port=17946, - dns_port=1053, - ) - if mode == "sev-snp": - values.update(cpu_model="EPYC-v4", phys_bits=48, cbitpos=51) - values.update(overrides) - return launcher.DomainConfig(**values) - - def root(self, config): - return ET.fromstring(launcher.build_domain_xml(config)) - - def test_untrusted_cpu_and_direct_boot(self): - root = self.root(self.config()) - self.assertEqual(root.findtext("name"), "super-protocol-3") - self.assertEqual(root.findtext("os/kernel"), "/var/lib/super protocol/vmlinuz") - self.assertEqual(root.findtext("os/cmdline"), "root=LABEL=rootfs hash=a&b") - self.assertEqual(root.find("cpu").get("mode"), "host-passthrough") - self.assertIsNotNone(root.find("features/ioapic[@driver='qemu']")) - self.assertIsNotNone(root.find("features/pmu[@state='off']")) - self.assertIsNotNone(root.find("cpu/feature[@name='kvm-steal-time']")) - self.assertIsNone(root.find("launchSecurity")) - - def test_reserved_ovmf_fw_cfg_uses_qemu_commandline(self): - root = self.root(self.config()) - self.assertIsNone(root.find("sysinfo[@type='fwcfg']")) - args = [ - element.get("value") - for element in root.findall( - f"{{{launcher.QEMU_NS}}}commandline/{{{launcher.QEMU_NS}}}arg" - ) - ] - self.assertEqual( - args, - ["-fw_cfg", "name=opt/ovmf/X-PciMmio64,string=262144"], - ) - - def test_sev_snp_launch_security(self): - root = self.root(self.config(mode="sev-snp")) - launch_security = root.find("launchSecurity") - self.assertEqual(launch_security.get("type"), "sev-snp") - self.assertEqual(launch_security.get("kernelHashes"), "yes") - self.assertEqual(launch_security.findtext("cbitpos"), "51") - self.assertEqual(launch_security.findtext("reducedPhysBits"), "1") - self.assertEqual(launch_security.findtext("policy"), "0x30000") - self.assertEqual(root.findtext("cpu/model"), "EPYC-v4") - self.assertEqual(root.find("cpu/maxphysaddr").get("bits"), "48") - self.assertIsNotNone(root.find("features/vmport[@state='off']")) - - def test_tdx_vsock_qgs_uses_qemu_namespace(self): - root = self.root(self.config(mode="tdx")) - args = [ - element.get("value") - for element in root.findall(f"{{{launcher.QEMU_NS}}}commandline/{{{launcher.QEMU_NS}}}arg") - ] - self.assertIn("memory-backend-ram,id=sp-mem,size=16G", args) - tdx_arg = next(value for value in args if '"qom-type":"tdx-guest"' in value) - self.assertIn('"type":"vsock"', tdx_arg) - self.assertIn('"cid":"2"', tdx_arg) - self.assertIn('"port":"4050"', tdx_arg) - self.assertIn( - "confidential-guest-support=sp-tdx,memory-backend=sp-mem", - args, - ) - - def test_user_network_uses_passt_and_all_forwards(self): - root = self.root(self.config(debug=True, log_file="/tmp/serial.log")) - interface = root.find("devices/interface[@type='user']") - self.assertEqual(interface.find("backend").get("type"), "passt") - forwards = { - ( - element.get("proto"), - element.get("address"), - element.find("range").get("start"), - element.find("range").get("to"), - ) - for element in interface.findall("portForward") - } - self.assertIn(("tcp", None, "8080", "80"), forwards) - self.assertIn(("udp", None, "51821", "51820"), forwards) - self.assertIn(("tcp", "127.0.0.1", "2222", "22"), forwards) - self.assertIn(("udp", None, "1053", "53"), forwards) - self.assertIn(("tcp", None, "1053", "53"), forwards) - - def test_tap_debug_has_precreated_tap_and_secondary_passt(self): - config = self.config( - netdev_mode="tap", - bridge="swarmbr0", - tap_iface="sw-tap7", - debug=True, - log_file="/tmp/serial.log", - ) - root = self.root(config) - interfaces = root.findall("devices/interface") - self.assertEqual(len(interfaces), 2) - self.assertEqual(interfaces[0].get("type"), "ethernet") - self.assertEqual(interfaces[0].find("target").get("dev"), "sw-tap7") - self.assertEqual(interfaces[0].find("target").get("managed"), "no") - self.assertEqual(interfaces[1].find("backend").get("type"), "passt") - ssh_range = interfaces[1].find("portForward/range") - self.assertEqual((ssh_range.get("start"), ssh_range.get("to")), ("2222", "22")) - - def test_host_devices_get_iommufd_and_separate_root_ports(self): - config = self.config( - host_devices=[ - launcher.HostDevice("gpu", "65:00.0"), - launcher.HostDevice("aux", "0000:66:00.1"), - ] - ) - root = self.root(config) - hostdevs = root.findall("devices/hostdev") - root_ports = root.findall("devices/controller[@model='pcie-root-port']") - self.assertEqual(len(hostdevs), 2) - self.assertEqual(len(root_ports), 2) - self.assertTrue(all(item.get("managed") == "no" for item in hostdevs)) - self.assertTrue( - all(item.find("driver").get("iommufd") == "yes" for item in hostdevs) - ) - self.assertIsNotNone(hostdevs[0].find("rom[@bar='off']")) - self.assertIsNone(hostdevs[1].find("rom")) - self.assertEqual(hostdevs[1].find("source/address").get("function"), "0x1") - - def test_gpu_none_produces_no_hostdev_or_extra_root_port(self): - root = self.root(self.config(host_devices=[])) - self.assertEqual(root.findall("devices/hostdev"), []) - self.assertEqual(root.findall("devices/controller[@model='pcie-root-port']"), []) - - def test_invalid_bdf_and_debug_without_log_are_rejected(self): - with self.assertRaisesRegex(ValueError, "invalid PCI BDF"): - launcher.build_domain_xml( - self.config(host_devices=[launcher.HostDevice("gpu", "bad")]) - ) - with self.assertRaisesRegex(ValueError, "requires a log file"): - launcher.build_domain_xml(self.config(debug=True)) - with self.assertRaisesRegex(ValueError, "ports must be between"): - launcher.build_domain_xml(self.config(http_port=70000)) - - -class CapabilityTests(unittest.TestCase): - def config(self, host_devices=True): - return DomainXMLTests().config( - host_devices=[launcher.HostDevice("gpu", "65:00.0")] - if host_devices - else [] - ) - - def test_requires_libvirt_12_1_for_host_devices(self): - conn = mock.Mock() - conn.getType.return_value = "QEMU" - conn.getLibVersion.return_value = 12_000_000 - with self.assertRaisesRegex(RuntimeError, ">= 12.1.0"): - launcher.check_connection_capabilities(conn, self.config()) - - def test_requires_iommufd_domain_capability(self): - conn = mock.Mock() - conn.getType.return_value = "QEMU" - conn.getLibVersion.return_value = 12_001_000 - conn.getDomainCapabilities.return_value = "" - with self.assertRaisesRegex(RuntimeError, "do not advertise"): - launcher.check_connection_capabilities(conn, self.config()) - - def test_accepts_advertised_iommufd(self): - conn = mock.Mock() - conn.getType.return_value = "QEMU" - conn.getLibVersion.return_value = 12_001_000 - conn.getDomainCapabilities.return_value = """ - - yesno - - """ - launcher.check_connection_capabilities(conn, self.config()) - - def test_no_hostdev_does_not_require_iommufd(self): - conn = mock.Mock() - conn.getType.return_value = "QEMU" - launcher.check_connection_capabilities(conn, self.config(False)) - conn.getLibVersion.assert_not_called() - - def test_existing_domain_name_is_rejected(self): - conn = mock.Mock() - existing = mock.Mock() - existing.name.return_value = "super-protocol-3" - conn.listAllDomains.return_value = [existing] - libvirt_module = mock.Mock() - with self.assertRaisesRegex(RuntimeError, "already exists"): - launcher.ensure_domain_name_available( - conn, libvirt_module, "super-protocol-3" - ) - - def test_available_domain_name_does_not_trigger_libvirt_lookup_error(self): - conn = mock.Mock() - other = mock.Mock() - other.name.return_value = "another-domain" - conn.listAllDomains.return_value = [other] - launcher.ensure_domain_name_available(conn, mock.Mock(), "super-protocol-3") - conn.lookupByName.assert_not_called() - - def test_passt_sigsegv_error_points_to_apparmor_audit_log(self): - message = launcher._format_launch_error( - RuntimeError("Child process (passt --one-off) unexpected fatal signal 11") - ) - self.assertIn("AppArmor", message) - self.assertIn("journalctl -k", message) - - def test_preflight_subcommand_dispatches_without_building_domain(self): - with mock.patch.object(launcher, "preflight_connection") as preflight: - result = launcher.main( - [ - "preflight", - "--emulator", - "/usr/bin/qemu-system-x86_64", - "--name", - "super-protocol-3", - "--require-iommufd", - ] - ) - self.assertEqual(result, 0) - preflight.assert_called_once_with( - "/usr/bin/qemu-system-x86_64", "super-protocol-3", True - ) - - -class SourceSafetyTests(unittest.TestCase): - def test_new_bash_launcher_has_no_eval(self): - source = (REPO_ROOT / "scripts" / "start_super_protocol_libvirt.sh").read_text() - self.assertIsNone(re.search(r"^\s*eval\b", source, re.MULTILINE)) - - -if __name__ == "__main__": - unittest.main() From 3ec74b667fe7799b4408953f69d751809ac52230 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 18 Aug 2026 10:44:33 -0500 Subject: [PATCH 25/28] fix passing gpu --- scripts/start_super_protocol.sh | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/scripts/start_super_protocol.sh b/scripts/start_super_protocol.sh index 134677d..1b5b126 100755 --- a/scripts/start_super_protocol.sh +++ b/scripts/start_super_protocol.sh @@ -151,7 +151,7 @@ parse_args() { case $1 in --cores) VM_CPU=$2; shift ;; --mem) VM_RAM=$(echo $2 | sed 's/G//'); shift ;; - --gpu) USED_GPUS+=("$2"); shift ;; + --gpu) USED_GPUS+=("${2#0000:}"); shift ;; --state_disk_path) STATE_DISK_PATH=$2; shift ;; --state_disk_size) STATE_DISK_SIZE=$2; shift ;; --provider_config_disk_path) PROVIDER_CONFIG_DISK_PATH=$2; shift ;; @@ -721,9 +721,11 @@ check_params() { TOTAL_CPUS=$(nproc) TOTAL_RAM=$(free -g | awk '/^Mem:/{print $2}') - # Get list of all NVIDIA GPUs and NVSwitch devices - AVAILABLE_GPUS=($( { lspci -nnk -d 10de: | grep -E '3D controller' | awk '{print $1}'; } || echo)) - AVAILABLE_NVSWITCHES=($( { lspci -mm -n -d 10de:22a3 | cut -d' ' -f1; } || echo)) + # Get list of all NVIDIA GPUs and NVSwitch devices. lspci prints the PCI + # domain on some hosts and omits it on others, and users may pass either + # form, so strip a leading "0000:" everywhere and compare short BDFs. + AVAILABLE_GPUS=($( { lspci -nnk -d 10de: | grep -E '3D controller' | awk '{sub(/^0000:/, "", $1); print $1}'; } || echo)) + AVAILABLE_NVSWITCHES=($( { lspci -mm -n -d 10de:22a3 | awk '{sub(/^0000:/, "", $1); print $1}'; } || echo)) echo "Debug: Found GPUs: ${AVAILABLE_GPUS[@]}" @@ -734,14 +736,6 @@ check_params() { USED_GPUS=("${AVAILABLE_GPUS[@]}") fi - # lspci reports BDFs without the PCI domain, so accept the fully qualified - # form too and compare everything in the short form. - local -a NORMALIZED_GPUS=() - for GPU in "${USED_GPUS[@]}"; do - NORMALIZED_GPUS+=("${GPU#0000:}") - done - USED_GPUS=("${NORMALIZED_GPUS[@]}") - # Remove duplicates efficiently declare -A UNIQUE_GPUS for GPU in "${USED_GPUS[@]}"; do From 2f93c85eb95d922cae1bc4888d44d7471b5add52 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 25 Aug 2026 17:51:01 +0000 Subject: [PATCH 26/28] feat(libvirt): allow SSH forwarding without debug --- scripts/libvirt_launcher.py | 5 ++--- scripts/start_super_protocol_libvirt.sh | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py index 38d4ba7..220c052 100755 --- a/scripts/libvirt_launcher.py +++ b/scripts/libvirt_launcher.py @@ -241,8 +241,7 @@ def _add_passt_interface( ) _add_port_forward(interface, "udp", config.dns_port, 53, config.ip_address) _add_port_forward(interface, "tcp", config.dns_port, 53, config.ip_address) - if config.debug: - _add_port_forward(interface, "tcp", config.ssh_port, 22, "127.0.0.1") + _add_port_forward(interface, "tcp", config.ssh_port, 22, "127.0.0.1") return interface @@ -255,7 +254,7 @@ def _add_network(devices: ET.Element, config: DomainConfig) -> None: _sub(interface, "mac", address=config.mac_address) _sub(interface, "target", dev=config.tap_iface, managed="no") _sub(interface, "model", type="virtio") - if config.debug: + if config.ssh_port is not None: _add_passt_interface(devices, config, debug_only=True) diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index d431665..1cac61d 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -86,7 +86,7 @@ check_libvirt_dependencies() { if ! python3 -c 'import libvirt' >/dev/null 2>&1; then missing+=(python3-libvirt) fi - if [[ "${NETDEV_MODE}" == "user" || "${DEBUG_MODE}" == "true" ]]; then + if [[ "${NETDEV_MODE}" == "user" || -n "${SSH_PORT}" ]]; then command -v passt >/dev/null 2>&1 || missing+=(passt) fi if [[ ${#missing[@]} -gt 0 ]]; then @@ -102,7 +102,7 @@ check_libvirt_dependencies() { } check_passt_apparmor_profile() { - if [[ "${NETDEV_MODE}" != "user" && "${DEBUG_MODE}" != "true" ]]; then + if [[ "${NETDEV_MODE}" != "user" && -z "${SSH_PORT}" ]]; then return fi @@ -155,7 +155,7 @@ check_passt_unprivileged_ports() { "${SWARM_DB_GOSSIP_PORT}" "${DNS_PORT}" ) fi - if [[ "${DEBUG_MODE}" == "true" ]]; then + if [[ -n "${SSH_PORT}" ]]; then ports+=("${SSH_PORT}") fi From d9582bc3fe7dbce5915c04af4da093f2d2d68677 Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Tue, 25 Aug 2026 19:49:18 +0000 Subject: [PATCH 27/28] fix(libvirt): preserve Nova-managed SPVM state --- scripts/libvirt_launcher.py | 12 ++++++++ scripts/start_super_protocol.sh | 23 +++++++++----- scripts/start_super_protocol_libvirt.sh | 29 ++++++++++++++++++ tests/test_libvirt_launcher.py | 40 +++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 tests/test_libvirt_launcher.py diff --git a/scripts/libvirt_launcher.py b/scripts/libvirt_launcher.py index 220c052..25c68c5 100755 --- a/scripts/libvirt_launcher.py +++ b/scripts/libvirt_launcher.py @@ -15,6 +15,7 @@ import sys import time import termios +import uuid as uuidlib import tty import xml.etree.ElementTree as ET from dataclasses import dataclass, field @@ -43,6 +44,7 @@ class HostDevice: @dataclass class DomainConfig: name: str + uuid: Optional[str] mode: str emulator: str memory_gib: int @@ -115,6 +117,12 @@ def _validate_config(config: DomainConfig) -> None: raise ValueError( "domain name may contain only letters, digits, '.', '_', '+', ':', and '-'" ) + if config.uuid is not None: + try: + parsed_uuid = uuidlib.UUID(config.uuid) + except (ValueError, AttributeError) as exc: + raise ValueError(f"invalid domain UUID: {config.uuid!r}") from exc + config.uuid = str(parsed_uuid) if config.mode not in {"untrusted", "tdx", "sev-snp"}: raise ValueError(f"unsupported VM mode: {config.mode}") if config.netdev_mode not in {"user", "tap"}: @@ -351,6 +359,8 @@ def build_domain_xml(config: DomainConfig) -> str: domain = ET.Element("domain", {"type": "kvm"}) _sub(domain, "name", config.name) + if config.uuid: + _sub(domain, "uuid", config.uuid) _sub(domain, "memory", config.memory_gib, unit="GiB") _sub(domain, "currentMemory", config.memory_gib, unit="GiB") _sub(domain, "vcpu", config.vcpus, placement="static") @@ -745,6 +755,7 @@ def _host_device(value: str) -> HostDevice: def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--name", required=True) + parser.add_argument("--uuid") parser.add_argument("--mode", choices=("untrusted", "tdx", "sev-snp"), required=True) parser.add_argument("--emulator", required=True) parser.add_argument("--memory-gib", type=int, required=True) @@ -791,6 +802,7 @@ def _preflight_parser() -> argparse.ArgumentParser: def _config_from_args(args: argparse.Namespace) -> DomainConfig: return DomainConfig( name=args.name, + uuid=args.uuid, mode=args.mode, emulator=str(Path(args.emulator).resolve()), memory_gib=args.memory_gib, diff --git a/scripts/start_super_protocol.sh b/scripts/start_super_protocol.sh index 1b5b126..4e051ce 100755 --- a/scripts/start_super_protocol.sh +++ b/scripts/start_super_protocol.sh @@ -762,12 +762,20 @@ check_params() { PROVIDER_CONFIG_DISK_PATH="$CACHE/provider_config.img" fi - echo "Removing old state disk..." - rm -f ${STATE_DISK_PATH} - echo "Creating new state disk directory..." - mkdir -p $(dirname ${STATE_DISK_PATH}) - echo "Initializing state disk..." - touch ${STATE_DISK_PATH} + if [[ "${REUSE_DISKS:-false}" == "true" ]]; then + if [[ ! -f "${STATE_DISK_PATH}" || ! -f "${PROVIDER_CONFIG_DISK_PATH}" ]]; then + echo "Error: --reuse-disks requires existing state and provider disks" >&2 + exit 1 + fi + echo "Reusing existing state and provider disks..." + else + echo "Removing old state disk..." + rm -f "${STATE_DISK_PATH}" + echo "Creating new state disk directory..." + mkdir -p "$(dirname "${STATE_DISK_PATH}")" + echo "Initializing state disk..." + touch "${STATE_DISK_PATH}" + fi if [[ -n "$HTTP_PORT" ]]; then @@ -822,7 +830,8 @@ check_params() { fi fi - if [[ "${STATE_DISK_SIZE}" -gt "${MOUNT_SIZE_AVAIL}" ]]; then + if [[ "${REUSE_DISKS:-false}" != "true" && + "${STATE_DISK_SIZE}" -gt "${MOUNT_SIZE_AVAIL}" ]]; then echo "No free space to create virtual disk with ${STATE_DISK_SIZE}Gb" exit 1 fi diff --git a/scripts/start_super_protocol_libvirt.sh b/scripts/start_super_protocol_libvirt.sh index 1cac61d..af9e350 100755 --- a/scripts/start_super_protocol_libvirt.sh +++ b/scripts/start_super_protocol_libvirt.sh @@ -17,6 +17,8 @@ DEFAULT_CACHE="/var/lib/libvirt/images/superprotocol" CACHE=${DEFAULT_CACHE} LIBVIRT_DOMAIN_NAME="" +LIBVIRT_DOMAIN_UUID="" +REUSE_DISKS=false LIBVIRT_QEMU_USER="" BASE_ARGS=() @@ -24,6 +26,8 @@ usage_libvirt() { usage echo "Libvirt-specific options:" echo " --name Transient domain name (default: super-protocol-)" + echo " --uuid Domain UUID (for Nova-managed instances)" + echo " --reuse-disks Reuse existing state/provider disks on power-on" echo "" echo "Runtime behavior:" echo " --debug false Start in the background and return" @@ -41,6 +45,15 @@ extract_libvirt_args() { LIBVIRT_DOMAIN_NAME=$2 shift 2 ;; + --uuid) + [[ $# -ge 2 ]] || { echo "Error: --uuid requires a value" >&2; exit 1; } + LIBVIRT_DOMAIN_UUID=$2 + shift 2 + ;; + --reuse-disks) + REUSE_DISKS=true + shift + ;; --help) usage_libvirt exit 0 @@ -466,6 +479,21 @@ cleanup_provider_disk() { create_vm_disks() { local provider_loop="" provider_mount + if [[ "${REUSE_DISKS}" == "true" && -f "${STATE_DISK_PATH}" && -f "${PROVIDER_CONFIG_DISK_PATH}" ]]; then + if ! qemu-img info --output=json "${STATE_DISK_PATH}" | grep -q '"format": "qcow2"'; then + echo "Error: existing state disk is not a valid qcow2 image: ${STATE_DISK_PATH}" >&2 + exit 1 + fi + if [[ "$(blkid -o value -s TYPE "${PROVIDER_CONFIG_DISK_PATH}" || true)" != "ext4" ]]; then + echo "Error: existing provider config disk is not ext4: ${PROVIDER_CONFIG_DISK_PATH}" >&2 + exit 1 + fi + echo "Validated reusable state and provider disks." + grant_libvirt_file_access state-disk "${STATE_DISK_PATH}" rw- + grant_libvirt_file_access provider-config-disk "${PROVIDER_CONFIG_DISK_PATH}" r-- + return + fi + rm -f "${STATE_DISK_PATH}" qemu-img create -f qcow2 "${STATE_DISK_PATH}" "${STATE_DISK_SIZE}G" @@ -518,6 +546,7 @@ launch_with_libvirt() { --swarm-db-gossip-port "${SWARM_DB_GOSSIP_PORT}" --dns-port "${DNS_PORT}" ) + append_optional_arg --uuid "${LIBVIRT_DOMAIN_UUID}" append_optional_arg --http-port "${HTTP_PORT}" append_optional_arg --https-port "${HTTPS_PORT}" append_optional_arg --pki-port "${PKI_PORT}" diff --git a/tests/test_libvirt_launcher.py b/tests/test_libvirt_launcher.py new file mode 100644 index 0000000..cafbc26 --- /dev/null +++ b/tests/test_libvirt_launcher.py @@ -0,0 +1,40 @@ +import unittest +import xml.etree.ElementTree as ET + +from scripts.libvirt_launcher import DomainConfig, build_domain_xml + + +class DomainUuidTest(unittest.TestCase): + def config(self, domain_uuid): + return DomainConfig( + name="instance-00000001", + uuid=domain_uuid, + mode="untrusted", + emulator="/usr/bin/qemu-system-x86_64", + memory_gib=64, + vcpus=16, + bios="/tmp/OVMF.fd", + kernel="/tmp/vmlinuz", + kernel_cmdline="console=ttyS0", + rootfs="/tmp/rootfs.img", + state_disk="/tmp/state.qcow2", + provider_config_disk="/tmp/provider.img", + guest_cid=10, + qgs_socket="/run/tdx-qgs/qgs.socket", + mac_address="52:54:00:77:00:0a", + netdev_mode="user", + ) + + def test_nova_uuid_is_written_to_domain_xml(self): + expected = "ed89b02d-ad43-441b-9923-8f1eb0484f91" + root = ET.fromstring(build_domain_xml(self.config(expected))) + self.assertEqual(expected, root.findtext("uuid")) + self.assertEqual("instance-00000001", root.findtext("name")) + + def test_invalid_uuid_is_rejected(self): + with self.assertRaisesRegex(ValueError, "invalid domain UUID"): + build_domain_xml(self.config("------------------------------------")) + + +if __name__ == "__main__": + unittest.main() From 732046e7b70f1ef9f9d5dd7c65399c15bbf9854b Mon Sep 17 00:00:00 2001 From: Petr Evstifeev Date: Thu, 27 Aug 2026 18:26:41 +0000 Subject: [PATCH 28/28] update docs --- README.md | 2 +- docs/swarm.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c2088eb..43d3755 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ sudo ./scripts/start_super_protocol_libvirt.sh \ --mode tdx ``` -With `--debug false` the command returns after the domain starts. With `--debug true --log_file /path/to/boot.log`, it attaches a bidirectional serial console and copies console output to the log; `Ctrl-C` or `Ctrl-]` detaches without stopping the VM. Use `virsh -c qemu:///system list`, `console`, `shutdown`, or `destroy` to manage it. `--gpu none` disables GPU, NVSwitch, and CX7 passthrough for diagnostics. +With `--debug false` the command returns after the domain starts. With `--debug true --log_file /path/to/boot.log`, it follows the domain serial log and copies it to the log file; `Ctrl-C` detaches without stopping the VM. The domain always records its console to `/var/log/libvirt/qemu/-serial.log` from the first byte, so a VM that fails early can still be diagnosed. The serial port is a file sink rather than a pty, because a pty nobody reads fills up and stalls the guest inside console output. Use `virsh -c qemu:///system list`, `shutdown`, or `destroy` to manage it. `--gpu none` disables GPU, NVSwitch, and CX7 passthrough for diagnostics. #### Libvirt host configuration diff --git a/docs/swarm.md b/docs/swarm.md index 7ab58e1..ea1b6a2 100644 --- a/docs/swarm.md +++ b/docs/swarm.md @@ -121,7 +121,7 @@ The bootstrap node gets all remaining host resources after subtracting the host 8. Launches join nodes. 9. Sets up HAProxy ingress: `gw.dyn..superprotocol.io` → bootstrap ports 80/443. -Attach to a VM with `virsh -c qemu:///system console swarm-bootstrap` (or `swarm-join-1` / `swarm-join-2`). In debug mode, use the matching `tmux attach -t ` session instead. +Follow a VM's boot output with `tail -f /var/log/libvirt/qemu/swarm-bootstrap-serial.log` (or `swarm-join-1` / `swarm-join-2`); libvirt records it from the first byte, whether or not anything is attached. In debug mode, use the matching `tmux attach -t ` session instead.