From 315a78886fcc313841c79874e801fba12a31acda Mon Sep 17 00:00:00 2001 From: castrojo Date: Mon, 5 Oct 2026 23:08:08 -0400 Subject: [PATCH] feat(argo): add reusable Dakota VM boot test pod template and Just recipe Add dakota-vm-boot-test WorkflowTemplate for direct QEMU/KVM kernel boot verification of Dakota images inside an ephemeral privileged pod. The template unpacks image layers directly into an ext4 raw disk via mkfs.ext4 -d (plain-root approach, avoiding loop devices and bootc install splitstream failures), injects root SSH credentials, boots QEMU/KVM with host CPU passthrough, and executes in-VM evidence commands over SSH. Add dakota-vm-boot semaphore to workflow-semaphores, run-dakota-vm-boot recipe to Justfile, unit tests, and documentation. Assisted-by: Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 2 +- Justfile | 8 + .../dakota-container-qa-pipeline.yaml | 7 +- .../dakota-vm-boot-test.yaml | 164 ++++++++++++++++++ docs/ops/RUNBOOK.md | 11 ++ docs/reference/workflow-reference.md | 19 ++ docs/skills/argo-workflows/patterns.md | 36 ++-- docs/skills/dakota-pr-review/SKILL.md | 4 +- docs/skills/kubevirt-vms/SKILL.md | 38 ++++ manifests/workflow-semaphores.yaml | 6 + tests/unit/test_dakota_vm_boot_test.py | 71 ++++++++ 11 files changed, 346 insertions(+), 20 deletions(-) create mode 100644 argo/workflow-templates/dakota-vm-boot-test.yaml create mode 100644 tests/unit/test_dakota_vm_boot_test.py diff --git a/AGENTS.md b/AGENTS.md index 8c95869000..5b014ba227 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,7 +68,7 @@ just lint # actionlint + argo lint + registry allowlist | Task | Where to go | |---|---| | Authoring Argo workflow templates (YAML) | [`docs/skills/argo-workflows/SKILL.md`](docs/skills/argo-workflows/SKILL.md) | -| KubeVirt VM provisioning / boot failures | [`docs/skills/kubevirt-vms/SKILL.md`](docs/skills/kubevirt-vms/SKILL.md) | +| VM provisioning, Dakota VM boot test, or boot failures | [`docs/skills/kubevirt-vms/SKILL.md`](docs/skills/kubevirt-vms/SKILL.md) | | ArgoCD sync, GitOps rules, bootstrap vs managed | [`docs/skills/gitops-argocd/SKILL.md`](docs/skills/gitops-argocd/SKILL.md) | | GitHub Actions workflows | [`docs/skills/ci-tooling/SKILL.md`](docs/skills/ci-tooling/SKILL.md) | | bluefin-server bootc image | [`docs/skills/bluefin-server/SKILL.md`](docs/skills/bluefin-server/SKILL.md) | diff --git a/Justfile b/Justfile index cc2ae96078..bca9acf44c 100644 --- a/Justfile +++ b/Justfile @@ -115,6 +115,14 @@ run-dakota-container-qa image-tag="testing" variant="dakota": -p variant={{ variant }} \ -n {{ argo_ns }} --watch +# Run Dakota VM boot test in an ephemeral privileged pod. +# Usage: just run-dakota-vm-boot +# Usage: just run-dakota-vm-boot image="192.168.1.102:30500/dakota:pr-1577-gvisor" +run-dakota-vm-boot image="192.168.1.102:30500/dakota:testing": + argo submit --from workflowtemplate/dakota-vm-boot-test \ + -p image={{ image }} \ + -n {{ argo_ns }} --watch + # Promote one immutable Zot candidate to :testing after its lane passes QA. # Usage: just run-zot-promotion dakota-testing dakota candidate- sha256: run-zot-promotion lane repository candidate_tag expected_digest: diff --git a/argo/workflow-templates/dakota-container-qa-pipeline.yaml b/argo/workflow-templates/dakota-container-qa-pipeline.yaml index 6530131e3c..546dd3996e 100644 --- a/argo/workflow-templates/dakota-container-qa-pipeline.yaml +++ b/argo/workflow-templates/dakota-container-qa-pipeline.yaml @@ -11,9 +11,10 @@ metadata: composefs-oci backend currently cannot satisfy (no UKI/bootupd). notes: | Use this pipeline to verify dakota:testing and dakota-nvidia:testing - while the VM-based dakota-qa-pipeline remains blocked on UKI/bootupd - support (projectbluefin/lab#272). GUI behave suites cannot run inside - a pod because qecore-headless requires a full systemd/GDM session. + for fast container-level QA. For full VM boot and hardware virtualization + evidence, use dakota-vm-boot-test. GUI behave suites cannot run inside + a standard container pod because qecore-headless requires a full systemd/GDM + session. labels: app.kubernetes.io/component: qa-pipeline app.kubernetes.io/part-of: dakota-test-suite diff --git a/argo/workflow-templates/dakota-vm-boot-test.yaml b/argo/workflow-templates/dakota-vm-boot-test.yaml new file mode 100644 index 0000000000..7709f8ee91 --- /dev/null +++ b/argo/workflow-templates/dakota-vm-boot-test.yaml @@ -0,0 +1,164 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: dakota-vm-boot-test + namespace: argo + annotations: + description: | + Plain-root direct kernel boot verification for Dakota images in a privileged pod. + + Runs inside an ephemeral privileged pod whose container image is the Dakota image under + test (shipping qemu-system-x86_64, qemu-img, podman, ssh, ssh-keygen, skopeo, and e2fsprogs). + Unpacks the image layers directly into an ext4 root filesystem on raw disk via `mkfs.ext4 -d` + (no loop devices required), injects root SSH authorized keys, boots the guest kernel under + QEMU/KVM with host CPU passthrough (`-machine q35,accel=kvm -cpu host`), and executes + in-VM runtime evidence commands over SSH. + labels: + app.kubernetes.io/component: qa-pipeline + app.kubernetes.io/part-of: bluefin-test-suite +spec: + serviceAccountName: argo + entrypoint: boot-test + + arguments: + parameters: + - name: image + value: "192.168.1.102:30500/dakota:testing" + - name: commands + value: | + set -x + grep -E '^(PRETTY_NAME|IMAGE_VERSION|VERSION_ID)=' /etc/os-release + uname -r + systemctl is-system-running --wait || true + systemctl --failed --no-legend + sudo bootc status --booted 2>/dev/null | grep -iE 'image|digest' | head -4 + + templates: + - name: boot-test + activeDeadlineSeconds: 1800 + synchronization: + semaphores: + - configMapKeyRef: + name: workflow-semaphores + key: dakota-vm-boot + inputs: + parameters: + - name: image + default: "{{workflow.parameters.image}}" + - name: commands + default: "{{workflow.parameters.commands}}" + outputs: + parameters: + - name: evidence + valueFrom: + path: /work/evidence.log + - name: serial-tail + valueFrom: + path: /work/serial-tail.log + script: + image: "{{inputs.parameters.image}}" + command: [bash] + env: + - name: IMG + value: "{{inputs.parameters.image}}" + - name: IN_VM_COMMANDS + value: "{{inputs.parameters.commands}}" + securityContext: + privileged: true + runAsUser: 0 + resources: + requests: + cpu: "6" + memory: 12Gi + ephemeral-storage: 40Gi + limits: + cpu: "8" + memory: 14Gi + ephemeral-storage: 80Gi + volumeMounts: + - name: dev + mountPath: /dev + - name: work + mountPath: /work + source: | + set -euo pipefail + IMG="${IMG:?}" + W=/work; cd "$W" + + cleanup() { + local exit_code=$? + echo "Cleaning up VM resources (exit code ${exit_code})..." + if [ -f /work/qemu.pid ]; then + kill "$(cat /work/qemu.pid 2>/dev/null)" 2>/dev/null || true + fi + if [ -f /work/serial.log ]; then + tail -n 200 /work/serial.log > /work/serial-tail.log 2>/dev/null || true + else + echo "No serial log generated" > /work/serial-tail.log + fi + if [ ! -f /work/evidence.log ]; then + echo "No evidence log generated (exit code ${exit_code})" > /work/evidence.log + fi + } + trap cleanup EXIT + + log() { echo; echo "######## $*"; } + + log "unpack ${IMG} rootfs into an ext4 disk (no bootc install)" + skopeo copy -q --src-tls-verify=false --dest-decompress "docker://${IMG}" oci:/work/oci:img + mkdir -p /work/root + MANI=$(python3 -c 'import json;i=json.load(open("/work/oci/index.json"));print(i["manifests"][0]["digest"].split(":")[1])' 2>/dev/null || jq -r '.manifests[0].digest' /work/oci/index.json | cut -d: -f2) + for L in $(jq -r '.layers[].digest' "/work/oci/blobs/sha256/$MANI" | cut -d: -f2); do + tar -xpf "/work/oci/blobs/sha256/$L" -C /work/root --xattrs --xattrs-include='*' --numeric-owner + done + rm -rf /work/oci + R=/work/root + KVER=$(ls "$R/usr/lib/modules/" | head -1) + cp "$R/usr/lib/modules/$KVER/vmlinuz" vmlinuz + cp "$R/usr/lib/modules/$KVER/initramfs.img" initramfs.img + + ssh-keygen -q -t ed25519 -f key -N "" + mkdir -p "$R/etc/systemd/system/multi-user.target.wants" "$R/var/roothome/.ssh" + ln -sf /usr/lib/systemd/system/sshd.service "$R/etc/systemd/system/multi-user.target.wants/sshd.service" + ssh-keygen -A -f "$R" >/dev/null 2>&1 || true + cp key.pub "$R/var/roothome/.ssh/authorized_keys" + chmod 700 "$R/var/roothome/.ssh" + chmod 600 "$R/var/roothome/.ssh/authorized_keys" + + truncate -s 40G disk.raw + mkfs.ext4 -q -L root -d "$R" disk.raw + rm -rf "$R" + + OPTS="root=/dev/vda rw systemd.firstboot=no" + log "boot kernel ${KVER} under QEMU/KVM" + qemu-system-x86_64 -machine q35,accel=kvm -cpu host -m 8192 -smp 6 \ + -kernel vmlinuz -initrd initramfs.img -append "${OPTS} console=ttyS0,115200" \ + -drive if=none,id=disk,file=disk.raw,format=raw,cache=unsafe \ + -device virtio-blk-pci,drive=disk \ + -netdev user,id=net0,hostfwd=tcp:127.0.0.1:2222-:22 -device virtio-net-pci,netdev=net0 \ + -display none -serial file:serial.log -daemonize -pidfile qemu.pid + + SSH="ssh -i key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=3 -p 2222 root@127.0.0.1" + for i in $(seq 1 120); do + $SSH true 2>/dev/null && break + sleep 5 + done + + if ! $SSH true 2>/dev/null; then + echo "ERROR: SSH never came up after ~$((i*5))s" >&2 + cat serial.log >&2 || true + exit 1 + fi + echo "SSH up after ~$((i*5))s" + + log "VM evidence" + $SSH 'bash -s' <<< "${IN_VM_COMMANDS}" | tee evidence.log + tail -n 200 serial.log > serial-tail.log + + volumes: + - name: dev + hostPath: + path: /dev + - name: work + emptyDir: + sizeLimit: 80Gi diff --git a/docs/ops/RUNBOOK.md b/docs/ops/RUNBOOK.md index f3f042ed51..71159325bb 100644 --- a/docs/ops/RUNBOOK.md +++ b/docs/ops/RUNBOOK.md @@ -21,6 +21,7 @@ Three steady-state execution paths exist: |---|---|---| | BuildStream build lanes | Dakota and Bluefin Server images | Buildbarn CAS/AC shards, Zot registry; see [buildstream.md](../skills/cluster-tooling/buildstream.md) | | Container-only Dakota path | Image and PR QA | No persistent VM or host-disk state | +| Dakota VM boot test lane | Full kernel boot, ostree deployment, systemd, and runtime evidence | Ephemeral privileged pod running QEMU/KVM; no host-disk state | | Explicit VM-backed lanes | bluefin-server boot tests | Ephemeral KubeVirt resources | ### Container-only QA contract @@ -35,6 +36,16 @@ The Dakota digest poller runs at minute `:08` of each ten-minute interval for freshness tracking, but its QA trigger is disabled (`run-qa: "false"`). Daily suite coverage comes from `nightly-dakota` at 03:00 UTC. +### Dakota VM boot test lane + +`dakota-vm-boot-test` boots a published Dakota image as a hardware-accelerated +QEMU/KVM virtual machine inside an ephemeral privileged pod. The pod unpacks the +target image directly into a raw ext4 disk via `mkfs.ext4 -d` (without loop devices +or bootc-install constraints), injects root SSH credentials and systemd configurations, +boots QEMU with host CPU passthrough, and executes in-VM evidence commands over SSH. +The run is admitted through the `dakota-vm-boot: "1"` semaphore and leaves no residual +loop devices or persistent state. + ## Cluster topology | Host | Role | IP | Notes | diff --git a/docs/reference/workflow-reference.md b/docs/reference/workflow-reference.md index c6c0a0d1ad..1e086fe478 100644 --- a/docs/reference/workflow-reference.md +++ b/docs/reference/workflow-reference.md @@ -11,6 +11,7 @@ template add or rename. ## Table of Contents - [Pipelines](#pipelines) - [dakota-qa-pipeline](#dakota-qa-pipeline) + - [dakota-vm-boot-test](#dakota-vm-boot-test) - [dakota-build-pipeline](#dakota-build-pipeline) - [zot-candidate-lifecycle](#zot-candidate-lifecycle) - [bluefin-server-build-pipeline](#bluefin-server-build-pipeline) @@ -40,6 +41,24 @@ template add or rename. - **Just recipe:** `run-dakota-qa`. - **PR review:** use the [Dakota PR review skill](../skills/dakota-pr-review/SKILL.md). Build the exact PR SHA first, then run smoke and required E2E suites against the resulting image. If lab validation identifies a scoped PR defect, repair the PR branch, rebuild from its new SHA, rerun E2E, and merge directly only after a fresh pass. +### dakota-vm-boot-test +- **Purpose:** Direct kernel boot verification for Dakota images in a privileged pod. + Unpacks image layers directly into an ext4 raw disk via `mkfs.ext4 -d` (no loop devices + or bootc install), injects root SSH authorized keys, boots the guest kernel under QEMU/KVM + with host CPU passthrough (`-machine q35,accel=kvm -cpu host`), and executes in-VM runtime + evidence commands over SSH. +- **When to use:** Validating image contents, kernel boot, systemd initialization, + gVisor (`runsc`), `krun`/`libkrun` microVMs, and userspace filesystem tooling (`fuse2fs`, + `gocryptfs`) against published or PR candidate Dakota images. +- **Parameters:** `image` (default `192.168.1.102:30500/dakota:testing`), + `commands` (default: `/etc/os-release`, `uname -r`, `systemctl is-system-running`, + `systemctl --failed`, `bootc status`). +- **Admission & Resources:** Admitted through the `dakota-vm-boot` semaphore in + `manifests/workflow-semaphores.yaml`. Requests 6 CPUs, 12Gi memory, 40Gi ephemeral-storage + (limits 8 CPUs, 14Gi memory, 80Gi ephemeral-storage), hostPath `/dev`, and emptyDir `/work` (80Gi). +- **Outputs:** Captured evidence log (`evidence`), serial log tail (`serial-tail`), and pod logs. +- **Just recipe:** `run-dakota-vm-boot [image="..."]`. + ### dakota-build-pipeline - **Purpose:** The actual BuildStream compile step for Dakota — builds `oci/bluefin.bst` and pushes the image to the local Zot registry under the tag diff --git a/docs/skills/argo-workflows/patterns.md b/docs/skills/argo-workflows/patterns.md index 076f517044..f5cff5e693 100644 --- a/docs/skills/argo-workflows/patterns.md +++ b/docs/skills/argo-workflows/patterns.md @@ -103,24 +103,26 @@ stream and `:stable` is promoted separately from `main`. Keeping the cluster lan prevents accidental overwrites of the stable/production stream and makes the artifact identity obvious to downstream lab jobs. -### Dakota verification: containerized QA when the VM path is blocked +### Dakota verification: containerized QA and VM boot test pod -Dakota images are built from a composefs-oci backend that declares `bootloader = "systemd"` but -does not ship a UKI. The lab's standard VM QA path (containerDisk → `bootc install to-disk` -→ KubeVirt VM) therefore fails with `bootupd is required for ostree-based installs` because bootc -1.16.2 bails for systemd-boot ostree installs when no UKI is present. Until Dakota ships a UKI -(or bootc gains a composefs-oci install path), VM-boot verification is blocked. +Dakota images are built from a composefs-oci backend that declares `bootloader = "systemd"` and +ships no pre-baked UKI. While KubeVirt containerDisk builds require UKI/bootupd integration, +two testing lanes exist for Dakota: -WorkflowTemplate: `dakota-container-qa-pipeline` +1. **Containerized smoke checks (`dakota-container-qa-pipeline`)**: + Runs image-level checks directly inside a pod built from the target OCI image without provisioning a VM. + Verifies Dakota identity (`/etc/os-release`), presence of key binaries (`podman`, `flatpak`, + `gnome-shell`, `bootc`), bootc install config, and valid `bootc status` JSON. -- Runs image-level smoke checks directly inside a pod built from the target OCI image. -- Verifies Dakota identity (`/etc/os-release`), presence of key binaries (`podman`, `flatpak`, - `gnome-shell`, `bootc`), bootc install config, and valid `bootc status` JSON. -- Requires no `bootc install` and no containerDisk. -- GUI behave suites (`smoke`/`developer` via `qecore-headless`) cannot run inside a pod because - `qecore-headless` requires a full systemd/GDM session. +2. **Dakota VM boot test pod (`dakota-vm-boot-test`)**: + Runs an ephemeral privileged pod using the Dakota image under test. Inside the pod, image layers + are unpacked directly into a raw ext4 disk via `mkfs.ext4 -d` (no loop devices required), extracting + kernel/initramfs, injecting root SSH credentials, and booting QEMU/KVM with host CPU passthrough + (`-machine q35,accel=kvm -cpu host`). Once SSH is ready, it runs in-VM evidence commands (e.g. gVisor + `runsc`, `krun`/`libkrun` microVMs, `fuse2fs`, `gocryptfs`). Admitted via the `dakota-vm-boot: "1"` + semaphore in `manifests/workflow-semaphores.yaml`. -Default invocation for a fresh `dakota:testing` build: +Default invocation for containerized QA: ```bash argo submit --from workflowtemplate/dakota-container-qa-pipeline \ @@ -130,6 +132,12 @@ argo submit --from workflowtemplate/dakota-container-qa-pipeline \ -n argo --watch ``` +Default invocation for VM boot testing: + +```bash +just run-dakota-vm-boot image=":30500/dakota:testing" +``` + NVIDIA image builds are disabled for this clean distributed-build path. `dakota-qa-pipeline` has since migrated to the container-only path — it invokes diff --git a/docs/skills/dakota-pr-review/SKILL.md b/docs/skills/dakota-pr-review/SKILL.md index 6185fd8903..366985b077 100644 --- a/docs/skills/dakota-pr-review/SKILL.md +++ b/docs/skills/dakota-pr-review/SKILL.md @@ -33,7 +33,7 @@ Dakota PR review is a lab-backed admission process. GitHub Actions status is adv 1. Confirm the PR number, target branch, head SHA, and that it is still open and mergeable. 2. Do not treat `pr/needs-review`, `automerge`, or `chore/deps` labels as approval. Verify maintainer approval when the repository policy requires it. 3. Build the exact PR head SHA with `dakota-build-pipeline` in distributed (`re`) mode. Use the live WorkflowTemplate parameters; do not guess them. -4. Test the resulting image with `dakota-container-qa-pipeline` (image smoke checks) and, when the image supports it, `dakota-qa-pipeline`/`run-container-tests` for the full containerized BDD/GUI suites. +4. Test the resulting image with `dakota-container-qa-pipeline` (image smoke checks) and, when the image supports it, `dakota-qa-pipeline`/`run-container-tests` for the full containerized BDD/GUI suites. When runtime, virtualization, or kernel evidence is required (e.g. gVisor `runsc`, `krun`/`libkrun` microVMs, filesystem drivers), run `just run-dakota-vm-boot image=""` (`dakota-vm-boot-test`). 5. Let the `bst-build` semaphore (two lanes) admit builds. Two PR builds may run at once; submit no more than that and let the next PR wait for a free lane. 6. Capture Argo workflow names, exact SHAs, build mode, test result, and failure logs. A pass must be tied to the same commit that will be merged. 7. If build and E2E pass, recheck the PR head and mergeability, then merge directly. @@ -52,7 +52,7 @@ Dakota PR review is a lab-backed admission process. GitHub Actions status is adv 1. List open Dakota PRs and group them by base branch; process oldest/queue-ready first. 2. For each PR, inspect the current head SHA and changed paths, then dispatch the distributed build from the exact branch/ref. 3. Wait for `dakota-build-pipeline` to complete. A parent workflow marked successful with failed child nodes is not a clean pass; inspect the node summary and logs. -4. Run `dakota-container-qa-pipeline` against the built local registry image. Run the full `dakota-qa-pipeline` suites when the PR needs GUI/BDD coverage. +4. Run `dakota-container-qa-pipeline` against the built local registry image. Run the full `dakota-qa-pipeline` suites when the PR needs GUI/BDD coverage. For kernel, runtime, or virtualization validation, run `just run-dakota-vm-boot image="192.168.1.102:30500/dakota:"` (`dakota-vm-boot-test`). 5. Record pass/fail evidence and stop on infrastructure failures such as BuildBarn storage/DNS or BuildGrid worker loss; recover the lab before retrying. 6. On a clean pass, re-read the PR head SHA, confirm it remains mergeable, and merge directly. 7. After merge, verify the merge commit and watch the next build/publish workflow; do not report success merely because the merge API accepted the operation. diff --git a/docs/skills/kubevirt-vms/SKILL.md b/docs/skills/kubevirt-vms/SKILL.md index 6232b15eca..1607425857 100644 --- a/docs/skills/kubevirt-vms/SKILL.md +++ b/docs/skills/kubevirt-vms/SKILL.md @@ -16,6 +16,7 @@ metadata: ## When to Use - Editing `bluefin-server-boot-test.yaml` or other boot-test VM templates +- Running or debugging the Dakota VM boot test pod (`dakota-vm-boot-test.yaml`) - Debugging VM boot timeouts - Enabling a new KubeVirt feature gate - Understanding why a VM is stuck `Terminating` @@ -29,6 +30,43 @@ metadata: - [VM lifecycle](vm-lifecycle.md) — disks, scheduling, teardown. +## Dakota VM boot test pod (plain-root lane) + +The Dakota image under test cannot boot through KubeVirt containerDisk or standard `bootc install to-disk` due to upstream bootc/composefs splitstream limitations. Instead, `dakota-vm-boot-test.yaml` runs direct QEMU/KVM kernel boot inside an ephemeral privileged Kubernetes pod whose container image is the Dakota image under test. + +The Dakota container ships all required tooling: `qemu-system-x86_64`, `qemu-img`, `skopeo`, `ssh`, `ssh-keygen`, and `e2fsprogs` (`mkfs.ext4`). + +### When to Use +- Validating Dakota image contents and runtime services (kernel, systemd, networking, gVisor `runsc`, `krun`/`libkrun` microVMs, `fuse2fs`, `gocryptfs`). +- Testing PR images built by `just run-bst-build ` and pushed to the local Zot registry (`192.168.1.102:30500/dakota:`). +- Submitting on-demand via `just run-dakota-vm-boot image="192.168.1.102:30500/dakota:"`. + +### Architecture & Mechanics +1. **Unpack without loop devices**: + - `skopeo copy --src-tls-verify=false --dest-decompress docker://$IMG oci:/work/oci:img` + - Unpack every layer from the OCI manifest into `/work/root` using `tar -xpf ... --xattrs --xattrs-include='*' --numeric-owner`. + - Populate an ext4 disk image directly from `/work/root` via `truncate -s 40G disk.raw; mkfs.ext4 -q -L root -d /work/root disk.raw`. This populates the filesystem cleanly without kernel loop devices or partition tables. +2. **Credential and service injection**: + - Enable `sshd.service` via symlink in `/etc/systemd/system/multi-user.target.wants/`. + - Generate host keys (`ssh-keygen -A -f root`) and root keypair (`ssh-keygen -t ed25519 -f key -N ""`). + - Inject root public key into `/var/roothome/.ssh/authorized_keys` (mode 0600). +3. **Direct kernel boot under QEMU/KVM**: + - Copy `vmlinuz` and `initramfs.img` from `/usr/lib/modules//`. + - Boot QEMU with `-machine q35,accel=kvm -cpu host -m 8192 -smp 6`, `-kernel vmlinuz -initrd initramfs.img -append "root=/dev/vda rw systemd.firstboot=no console=ttyS0,115200"`. + - Forward host port 2222 to guest port 22 (`-netdev user,id=net0,hostfwd=tcp:127.0.0.1:2222-:22 -device virtio-net-pci,netdev=net0`). +4. **Execution & Evidence**: + - Poll SSH readiness on `127.0.0.1:2222` as `root` (typically ready in ~20 seconds). + - Stream evidence commands into the VM over SSH and capture outputs. + - Expected degradation: `systemctl is-system-running` reports `degraded` solely because `ublue-boot-timeout.service` fails due to the absence of a systemd-boot ESP on a plain-root disk. This is expected and normal for this test lane. + +### Failure Modes and Invariants (Timeless) +- **Plain-root runtime scope**: This lane validates image *contents and runtime behavior*, not the bootloader or ostree deployment update mechanics. +- **`bootc install to-disk` ostree backend failure**: Standard ostree installation fails with `bootupd is required for ostree-based installs` because bootc requires bootupd for systemd-boot ostree setups. Furthermore, no BLS entries or boot symlinks are created; even with synthetic symlinks, systemd detects `Running in initrd` post-switch-root and loops until service start limits force emergency mode. Not viable for Dakota. +- **`bootc install to-disk --composefs-backend` failure**: Dakota's native composefs installation path (bootc 1.16.13) fails with `Pulling image into composefs repository: Unexpected EOF in splitstream` across docker, containers-storage, and decompressed OCI layouts on both ext4 and btrfs (upstream: `bootc-dev/bootc#1703` and `composefs/composefs-rs#210`). The plain-root lane is the supported path until upstream resolves splitstream composefs import. +- **`bcvk ephemeral` dependency gap**: `bcvk ephemeral` requires host `virtiofsd`. The Dakota image does not ship `virtiofsd` and upstream publishes no standalone binary releases, preventing in-pod bcvk use without external package layering. +- **Nested KVM acceleration**: Lab nodes (AMD) run with `kvm_amd nested=1`. QEMU requires `-cpu host` to expose virtualization flags to the guest, enabling nested microVM runtimes like `libkrun`/`krun` and gVisor `runsc` inside the VM. +- **Script delivery**: Feeding long shell scripts via `kubectl run -i` stdin races against container startup. Workflows must inline scripts into Argo templates or mount them from ConfigMaps. + ## Console-driven VM control (imperative path) The KubeStellar Console ServiceAccount can start/stop/restart VMs diff --git a/manifests/workflow-semaphores.yaml b/manifests/workflow-semaphores.yaml index 0c1b19ee93..f905aee814 100644 --- a/manifests/workflow-semaphores.yaml +++ b/manifests/workflow-semaphores.yaml @@ -53,6 +53,11 @@ # tests/unit/test_semaphore_topology.py. # VMs are still scheduled natively and must not use this # semaphore (see docs/skills/argo-workflows/patterns.md). +# dakota-vm-boot — Serializes Dakota VM boot test pods on the cluster. +# Each pod runs QEMU/KVM with 6-8 CPUs and 12-14Gi RAM +# and performs plain-root disk unpack. One slot keeps +# host resources bounded without contending for memory +# or KVM acceleration. apiVersion: v1 kind: ConfigMap metadata: @@ -64,3 +69,4 @@ data: bst-build: "2" bst-cache-warm: "1" ghost-container-qa: "6" + dakota-vm-boot: "1" diff --git a/tests/unit/test_dakota_vm_boot_test.py b/tests/unit/test_dakota_vm_boot_test.py new file mode 100644 index 0000000000..41be5de011 --- /dev/null +++ b/tests/unit/test_dakota_vm_boot_test.py @@ -0,0 +1,71 @@ +from pathlib import Path +import yaml + +ROOT = Path(__file__).resolve().parents[2] +TEMPLATE_PATH = ROOT / "argo/workflow-templates/dakota-vm-boot-test.yaml" +SEMAPHORES_PATH = ROOT / "manifests/workflow-semaphores.yaml" + + +def test_dakota_vm_boot_test_template_structure(): + doc = yaml.safe_load(TEMPLATE_PATH.read_text(encoding="utf-8")) + assert doc["kind"] == "WorkflowTemplate" + assert doc["metadata"]["name"] == "dakota-vm-boot-test" + assert doc["metadata"]["namespace"] == "argo" + + spec = doc["spec"] + assert spec["entrypoint"] == "boot-test" + + params = {p["name"]: p.get("value") for p in spec.get("arguments", {}).get("parameters", [])} + assert "image" in params + assert params["image"] == "192.168.1.102:30500/dakota:testing" + assert "commands" in params + + templates = {t["name"]: t for t in spec.get("templates", [])} + assert "boot-test" in templates + + boot_test = templates["boot-test"] + assert boot_test.get("activeDeadlineSeconds") == 1800 + + # Semaphore check on the leaf template + sync = boot_test.get("synchronization", {}) + semaphores = sync.get("semaphores", []) + assert len(semaphores) == 1 + ref = semaphores[0].get("configMapKeyRef", {}) + assert ref.get("name") == "workflow-semaphores" + assert ref.get("key") == "dakota-vm-boot" + + # Semaphore exists in manifests/workflow-semaphores.yaml + sem_doc = yaml.safe_load(SEMAPHORES_PATH.read_text(encoding="utf-8")) + assert "dakota-vm-boot" in sem_doc["data"] + assert int(sem_doc["data"]["dakota-vm-boot"]) >= 1 + + # SecurityContext and resources + script = boot_test.get("script", {}) + sec_ctx = script.get("securityContext", {}) + assert sec_ctx.get("privileged") is True + assert sec_ctx.get("runAsUser") == 0 + + res = script.get("resources", {}) + req = res.get("requests", {}) + lim = res.get("limits", {}) + assert "cpu" in req and "cpu" in lim + assert "memory" in req and "memory" in lim + assert "ephemeral-storage" in req and "ephemeral-storage" in lim + + # Volumes and mounts + mounts = {m["name"]: m["mountPath"] for m in script.get("volumeMounts", [])} + assert mounts.get("dev") == "/dev" + assert mounts.get("work") == "/work" + + vols = {v["name"]: v for v in boot_test.get("volumes", [])} + assert "hostPath" in vols["dev"] + assert "emptyDir" in vols["work"] + + # Script assertions + source = script.get("source", "") + assert "skopeo copy" in source + assert "mkfs.ext4 -q -L root -d" in source + assert "qemu-system-x86_64" in source + assert "evidence.log" in source + assert "serial-tail.log" in source + assert "losetup" not in source