feat(vm): boot CoreOS via Ignition — Fedora CoreOS now runnable

CHANGELOG.md

@@ -8,6 +8,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) once a
 ## [Unreleased]
 
 ### Added
+- CoreOS (Ignition) boot support. CoreOS-family images boot via Ignition, not
+  cloud-init, so the executor now writes a minimal Ignition config (SSH key for
+  the `core` user) and passes it to QEMU via `-fw_cfg name=opt/com.coreos/config`
+  (see `internal/vm/ignition.go`). Fedora CoreOS is now runnable and proven —
+  FCOS stable boots and the validator load/attaches inside the guest (verified
+  on kernel `7.0.11-200.fc44`); fetch the image with `make vm-image-fcos`. RHEL
+  CoreOS (`rhcos`) shares this boot path but its image is pull-secret-gated via
+  the OpenShift release payload, so it stays non-runnable until an operator
+  supplies the image.
 - Embeddable library mode (`pkg/bpfcompat`). `ValidateBeforeLoad` /
   `ValidateBytes` do a real load of a compiled eBPF object against the local
   running kernel — no VM, no network — for use as a pre-load gate (e.g.

Makefile

@@ -13,7 +13,7 @@ LDFLAGS    ?= -X $(VERSION_PKG).Version=$(VERSION) \
               -X $(VERSION_PKG).BuildDate=$(BUILD_DATE)
 GO_BUILD_FLAGS ?= -trimpath -ldflags '$(LDFLAGS)'
 
-.PHONY: all deps vendor doctor doctor-virtme doctor-firecracker doctor-arm64-kvm firecracker-install firecracker-kernel-install firecracker-runnable firecracker-preflight arm64-kvm-preflight build test test-vendor tidy validator validator-dynamic validator-static pkg-embed-validator lib-hostload examples examples-arm64 oss-examples oss-evidence compatibility-site clean vm-ubuntu-22 vm-ubuntu-22-arm64 vm-images vm-images-tier1 vm-images-extended vm-images-expanded-2026 vm-images-expanded-2026-dry-run vm-images-latest-kernel matrix-runnable matrix-runnable-strict matrix-runnable-keep-manual latest-kernel-runnable upstream-kernel-runnable manual-image-check manual-image-check-strict profile-catalog-audit matrix-readiness runtime-selector-proof runtime-delivery-proof production-runtime-drill beta-tech-check tech-stability production-tech-check acceptance-dev-one acceptance-functional-dev-one acceptance-suite-dev-one acceptance-arm64-smoke acceptance-latest-kernel acceptance-upstream-kernel acceptance-firecracker-dev-one acceptance acceptance-expanded-runnable acceptance-evidence serve azure-provision-vm azure-bootstrap-vm azure-provision-foundation azure-production-boundary-proof azure-configure-tls azure-rotate-registry-secret
+.PHONY: all deps vendor doctor doctor-virtme doctor-firecracker doctor-arm64-kvm firecracker-install firecracker-kernel-install firecracker-runnable firecracker-preflight arm64-kvm-preflight build test test-vendor tidy validator validator-dynamic validator-static pkg-embed-validator lib-hostload examples examples-arm64 oss-examples oss-evidence compatibility-site clean vm-ubuntu-22 vm-ubuntu-22-arm64 vm-image-fcos vm-images vm-images-tier1 vm-images-extended vm-images-expanded-2026 vm-images-expanded-2026-dry-run vm-images-latest-kernel matrix-runnable matrix-runnable-strict matrix-runnable-keep-manual latest-kernel-runnable upstream-kernel-runnable manual-image-check manual-image-check-strict profile-catalog-audit matrix-readiness runtime-selector-proof runtime-delivery-proof production-runtime-drill beta-tech-check tech-stability production-tech-check acceptance-dev-one acceptance-functional-dev-one acceptance-suite-dev-one acceptance-arm64-smoke acceptance-latest-kernel acceptance-upstream-kernel acceptance-firecracker-dev-one acceptance acceptance-expanded-runnable acceptance-evidence serve azure-provision-vm azure-bootstrap-vm azure-provision-foundation azure-production-boundary-proof azure-configure-tls azure-rotate-registry-secret
 
 all: build validator
 
@@ -168,6 +168,11 @@ vm-ubuntu-22:
 	  "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" \
 	  "vm/cache/ubuntu-22.04.qcow2"
 
+# Fetch + decompress the current Fedora CoreOS stable qemu image for the
+# fedora-coreos-stable-7.0 profile (CoreOS boots via Ignition; see ignition.go).
+vm-image-fcos:
+	bash vm/scripts/fetch-fcos-image.sh vm/cache/fedora-coreos-stable.qcow2
+
 vm-ubuntu-22-arm64:
 	bash vm/scripts/fetch-cloud-image.sh \
 	  "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-arm64.img" \

docs/profile-catalog.md

@@ -38,8 +38,8 @@ This document defines the maintained profile matrices used for compatibility cam
    - `bottlerocket-aws-6.1` (manual image)
    - `flatcar-6.6` (URL-backed image)
    - `talos-6.6` (manual image)
-   - `fedora-coreos-stable-6.14` (manual image; Ignition boot — see below)
-   - `rhcos-4.16-5.14` (manual image, pull-secret gated; Ignition boot — see below)
+   - `fedora-coreos-stable-7.0` (runnable; Ignition boot, manual image — see below)
+   - `rhcos-4.16-5.14` (manual image, pull-secret gated; shares the FCOS Ignition boot path — see below)
    - `ubuntu-22.04-5.15-lockdown`
 4. Multi-architecture foundation:
    - `ubuntu-22.04-arm64-5.15` (`aarch64`, requires ARM64-capable runner)
@@ -95,8 +95,8 @@ This document defines the maintained profile matrices used for compatibility cam
   - `vm/cache/linux-mainline-5.6.qcow2`
   - `vm/cache/bottlerocket-aws-6.1.qcow2`
   - `vm/cache/talos-6.6.qcow2`
-  - `vm/cache/fedora-coreos-stable.qcow2` (also Ignition-gated — see Transport Notes)
-  - `vm/cache/rhcos-4.16.qcow2` (pull-secret + Ignition-gated — see Transport Notes)
+  - `vm/cache/fedora-coreos-stable.qcow2` (runnable once present; fetch + decompress from the FCOS stable stream — see Transport Notes)
+  - `vm/cache/rhcos-4.16.qcow2` (pull-secret gated — see Transport Notes)
 
 Strict commands can run now:
 
@@ -111,7 +111,8 @@ Optional licensed image source:
 
 - Current VM validator execution path is SSH-based.
 - `talos`, `bottlerocket`, `flatcar`, and `amazon-linux-2-4.14` are cataloged for planning/roadmap and are marked non-blocking in matrix definitions because the current executor cannot run validator payloads on them.
-- `fedora-coreos` and `rhcos` (RHEL CoreOS / OpenShift) are cataloged but **not runnable yet**: both boot via Ignition rather than cloud-init, so the SSH executor cannot provision the validator (same gap as `flatcar`). RHCOS additionally ships through the pull-secret-gated OpenShift release payload. Enabling them needs an Ignition-config bootstrap path in the QEMU executor; until then, the matching RHEL/AlmaLinux 9 (5.14) profile approximates the RHCOS kernel, and Fedora CoreOS is the freely-available stand-in for proving the CoreOS boot path.
+- `fedora-coreos` boots via **Ignition**, not cloud-init: the executor writes a minimal Ignition config (SSH key for the `core` user) and passes it to QEMU via `-fw_cfg name=opt/com.coreos/config` (see `internal/vm/ignition.go`). This path is **runnable and proven** — FCOS stable boots and the validator load/attaches inside the guest (verified on kernel `7.0.11-200.fc44`). It needs the manual image staged at `vm/cache/fedora-coreos-stable.qcow2` (fetch + `xz -d` from the FCOS stable stream).
+- `rhcos` (RHEL CoreOS / OpenShift) shares that exact Ignition boot path, so it is **mechanically supported** — but its image ships only through the pull-secret-gated OpenShift release payload, so it cannot be fetched or verified here. `ExecutionTransport()` therefore still reports it unsupported until an operator supplies the image; the matching RHEL/AlmaLinux 9 (5.14) profile approximates the RHCOS kernel in the meantime.
 - `rhel-8-4.18` uses NoCloud config-drive bootstrap in the current SSH executor (prefers `cloud-localds` ISO; falls back to local `vvfat` seed).
 - `aarch64`/`arm64` profiles select `qemu-system-aarch64`; `x86_64`/`amd64` profiles select `qemu-system-x86_64`.
 - ARM64 validation requires a matching ARM64-capable self-hosted runner, KVM access, an ARM64 cloud image, and a validator binary built for the guest architecture. The default Azure demo VM is x86_64 and should not be presented as ARM64 validation proof.

internal/vm/ignition.go

@@ -0,0 +1,70 @@
+package vm
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+)
+
+// CoreOS-family images (Fedora CoreOS, RHEL CoreOS) boot via Ignition, not
+// cloud-init: there is no NoCloud datasource to read a CIDATA seed. Instead the
+// QEMU "qemu" platform reads an Ignition config from the firmware config blob
+// at opt/com.coreos/config and applies it on first boot. We emit a minimal
+// Ignition v3 config that authorises the per-run SSH key for the default
+// `core` user, which is all the validator transport needs.
+
+// ignitionConfig is the subset of the Ignition v3 schema we generate.
+type ignitionConfig struct {
+	Ignition ignitionVersion `json:"ignition"`
+	Passwd   ignitionPasswd  `json:"passwd"`
+}
+
+type ignitionVersion struct {
+	Version string `json:"version"`
+}
+
+type ignitionPasswd struct {
+	Users []ignitionUser `json:"users"`
+}
+
+type ignitionUser struct {
+	Name              string   `json:"name"`
+	SSHAuthorizedKeys []string `json:"sshAuthorizedKeys"`
+}
+
+// writeIgnitionSeed writes a minimal Ignition config that adds publicKey to the
+// CoreOS `core` user, returning the path to the written file. The 3.3.0 spec
+// version is supported by both current Fedora CoreOS and RHEL CoreOS (OpenShift
+// 4.12+), so one config serves both.
+func writeIgnitionSeed(path, publicKey string) error {
+	cfg := ignitionConfig{
+		Ignition: ignitionVersion{Version: "3.3.0"},
+		Passwd: ignitionPasswd{
+			Users: []ignitionUser{
+				{Name: "core", SSHAuthorizedKeys: []string{strings.TrimSpace(publicKey)}},
+			},
+		},
+	}
+	data, err := json.MarshalIndent(cfg, "", "  ")
+	if err != nil {
+		return fmt.Errorf("marshal ignition config: %w", err)
+	}
+	data = append(data, '\n')
+	// 0600: the Ignition config is read only by the QEMU process we launch as
+	// the same user; no need for it to be group/world readable.
+	if err := os.WriteFile(path, data, 0o600); err != nil {
+		return fmt.Errorf("write ignition config: %w", err)
+	}
+	return nil
+}
+
+// isCoreOSIgnitionDistro reports whether a profile's distro boots via Ignition
+// and should use the fw_cfg Ignition seed instead of a cloud-init NoCloud seed.
+func isCoreOSIgnitionDistro(profile Profile) bool {
+	switch strings.ToLower(strings.TrimSpace(profile.Distro)) {
+	case "fedora-coreos", "fcos", "rhcos", "rhel-coreos":
+		return true
+	}
+	return false
+}

internal/vm/ignition_test.go

@@ -0,0 +1,77 @@
+package vm
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestWriteIgnitionSeed(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.ign")
+	const key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5 bpfcompat-run"
+	if err := writeIgnitionSeed(path, key+"\n"); err != nil {
+		t.Fatalf("writeIgnitionSeed: %v", err)
+	}
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read ignition: %v", err)
+	}
+	var cfg ignitionConfig
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		t.Fatalf("ignition config is not valid JSON: %v", err)
+	}
+	if cfg.Ignition.Version == "" {
+		t.Error("ignition.version must be set")
+	}
+	if len(cfg.Passwd.Users) != 1 || cfg.Passwd.Users[0].Name != "core" {
+		t.Fatalf("expected a single core user, got %+v", cfg.Passwd.Users)
+	}
+	if got := cfg.Passwd.Users[0].SSHAuthorizedKeys; len(got) != 1 || got[0] != key {
+		t.Errorf("ssh key = %v, want trimmed %q", got, key)
+	}
+}
+
+func TestIsCoreOSIgnitionDistro(t *testing.T) {
+	for _, d := range []string{"fedora-coreos", "FCOS", "rhcos", "rhel-coreos"} {
+		if !isCoreOSIgnitionDistro(Profile{Distro: d}) {
+			t.Errorf("%q should be a CoreOS Ignition distro", d)
+		}
+	}
+	for _, d := range []string{"ubuntu", "rhel", "flatcar", "almalinux"} {
+		if isCoreOSIgnitionDistro(Profile{Distro: d}) {
+			t.Errorf("%q should not be a CoreOS Ignition distro", d)
+		}
+	}
+}
+
+func TestSeedDeliveryForCoreOS(t *testing.T) {
+	if got := seedDeliveryForProfile(Profile{Distro: "fedora-coreos"}); got != seedDeliveryIgnition {
+		t.Errorf("fedora-coreos seed delivery = %q, want %q", got, seedDeliveryIgnition)
+	}
+}
+
+func TestBuildQEMUArgsIgnitionFwCfg(t *testing.T) {
+	profile := Profile{Distro: "fedora-coreos", Boot: BootConfig{MemoryMB: 2048, CPUs: 2}}
+	args := buildQEMUArgs(profile, "/tmp/overlay.qcow2", "/tmp/serial.log", 2222, seedDeliveryIgnition, "", "", "/tmp/config.ign")
+	joined := ""
+	for _, a := range args {
+		joined += a + " "
+	}
+	if !contains(joined, "name=opt/com.coreos/config,file=/tmp/config.ign") {
+		t.Fatalf("expected ignition fw_cfg arg, got: %s", joined)
+	}
+}
+
+func contains(haystack, needle string) bool {
+	return len(haystack) >= len(needle) && (func() bool {
+		for i := 0; i+len(needle) <= len(haystack); i++ {
+			if haystack[i:i+len(needle)] == needle {
+				return true
+			}
+		}
+		return false
+	})()
+}

internal/vm/qemu.go

@@ -144,6 +144,7 @@ const (
 	seedDeliveryNoCloudNet         seedDeliveryMode = "nocloud-net"
 	seedDeliveryNoCloudConfigDrive seedDeliveryMode = "nocloud-configdrive"
 	seedDeliveryNoCloudConfigFS    seedDeliveryMode = "nocloud-configfs"
+	seedDeliveryIgnition           seedDeliveryMode = "ignition"
 )
 
 func ExecuteProfile(ctx context.Context, req ExecutionRequest) (result ExecutionResult) {
@@ -216,21 +217,33 @@ func ExecuteProfile(ctx context.Context, req ExecutionRequest) (result Execution
 		return
 	}
 
+	seedMode := seedDeliveryForProfile(req.Profile)
 	seedDir := filepath.Join(vmRunDir, "seed")
-	if err := writeNoCloudSeed(seedDir, req.Profile.ID, publicKey); err != nil {
+	seedURL := ""
+	seedImagePath := ""
+
+	// CoreOS-family images boot via Ignition (fw_cfg), not a cloud-init NoCloud
+	// seed; write the Ignition config and skip the NoCloud seed entirely.
+	if seedMode == seedDeliveryIgnition {
+		seedImagePath = filepath.Join(vmRunDir, "config.ign")
+		if err := writeIgnitionSeed(seedImagePath, publicKey); err != nil {
+			result.InfraError = err.Error()
+			return
+		}
+		result.Notes = append(result.Notes, "seed delivery: Ignition config via fw_cfg (opt/com.coreos/config)")
+	} else if err := writeNoCloudSeed(seedDir, req.Profile.ID, publicKey); err != nil {
 		result.InfraError = err.Error()
 		return
 	}
 
-	seedMode := seedDeliveryForProfile(req.Profile)
-	seedURL := ""
-	seedImagePath := ""
 	seedDirAbs, err := filepath.Abs(seedDir)
 	if err != nil {
 		result.InfraError = fmt.Sprintf("resolve seed directory: %v", err)
 		return
 	}
 	switch seedMode {
+	case seedDeliveryIgnition:
+		// config.ign already written; nothing more to stage here.
 	case seedDeliveryNoCloudConfigDrive:
 		seedImagePath = filepath.Join(vmRunDir, "seed-cidata.iso")
 		if err := createNoCloudSeedImage(ctx, seedDir, seedImagePath); err != nil {
@@ -635,6 +648,9 @@ func buildQEMUArgs(profile Profile, overlayPath, serialLogPath string, sshPort i
 	}
 	args = append(qemuMachineArgs(profile), args...)
 	switch seedMode {
+	case seedDeliveryIgnition:
+		// CoreOS reads the Ignition config from the qemu firmware config blob.
+		args = append(args, "-fw_cfg", fmt.Sprintf("name=opt/com.coreos/config,file=%s", seedImagePath))
 	case seedDeliveryNoCloudConfigDrive:
 		args = append(args, "-drive", fmt.Sprintf("file=%s,if=ide,media=cdrom,format=raw,readonly=on", seedImagePath))
 	case seedDeliveryNoCloudConfigFS:
@@ -722,6 +738,9 @@ func needsCIDATASeed(profile Profile) bool {
 }
 
 func seedDeliveryForProfile(profile Profile) seedDeliveryMode {
+	if isCoreOSIgnitionDistro(profile) {
+		return seedDeliveryIgnition
+	}
 	if needsCIDATASeed(profile) {
 		if commandAvailable("cloud-localds") {
 			return seedDeliveryNoCloudConfigDrive
@@ -808,7 +827,7 @@ func sshUserCandidates(profile Profile) []string {
 		candidates = append(candidates, "opensuse")
 	case "sles":
 		candidates = append(candidates, "ec2-user", "opensuse")
-	case "flatcar":
+	case "flatcar", "fedora-coreos", "fcos", "rhcos", "rhel-coreos":
 		candidates = append(candidates, "core")
 	case "centos", "centos-stream", "rhel", "redhat":
 		candidates = append(candidates, "cloud-user", "centos")

internal/vm/qemu_test.go

@@ -216,10 +216,10 @@ func TestExecutionTransport(t *testing.T) {
 		{name: "talos blocked", distro: "talos", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "no ssh"},
 		{name: "bottlerocket blocked", distro: "bottlerocket", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "ssh"},
 		{name: "flatcar blocked", distro: "flatcar", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "ignition"},
-		{name: "fedora-coreos blocked", distro: "fedora-coreos", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "ignition"},
-		{name: "fcos alias blocked", distro: "FCOS", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "ignition"},
-		{name: "rhcos blocked", distro: "rhcos", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "ignition"},
-		{name: "rhel-coreos alias blocked", distro: "rhel-coreos", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "ignition"},
+		{name: "fedora-coreos supported", distro: "fedora-coreos", wantTransport: ExecutionTransportSSH, wantSupported: true},
+		{name: "fcos alias supported", distro: "FCOS", wantTransport: ExecutionTransportSSH, wantSupported: true},
+		{name: "rhcos blocked on image", distro: "rhcos", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "pull-secret"},
+		{name: "rhel-coreos blocked on image", distro: "rhel-coreos", wantTransport: ExecutionTransportUnsupported, wantSupported: false, wantInMsg: "pull-secret"},
 	}
 
 	for _, tt := range tests {

internal/vm/transport.go

@@ -25,9 +25,15 @@ func ExecutionTransport(profile Profile) (transport string, supported bool, reas
 	case "flatcar":
 		return ExecutionTransportUnsupported, false, "Flatcar images in this matrix require Ignition-style bootstrap; current validator runner depends on cloud-init+SSH provisioning."
 	case "fedora-coreos", "fcos":
-		return ExecutionTransportUnsupported, false, "Fedora CoreOS boots via Ignition (not cloud-init); current validator runner depends on cloud-init+SSH provisioning."
+		// Boots via Ignition over fw_cfg (opt/com.coreos/config), then SSH as
+		// the core user — implemented in ignition.go and proven on FCOS stable.
+		return ExecutionTransportSSH, true, ""
 	case "rhcos", "rhel-coreos":
-		return ExecutionTransportUnsupported, false, "RHEL CoreOS (OpenShift) boots via Ignition and ships through the pull-secret-gated OpenShift release payload; current validator runner depends on cloud-init+SSH provisioning."
+		// Shares Fedora CoreOS's Ignition+SSH boot path (now implemented), but
+		// the RHCOS image ships only through the pull-secret-gated OpenShift
+		// release payload, so it cannot be fetched/verified here. Supply the
+		// image to enable it; until then it stays non-runnable.
+		return ExecutionTransportUnsupported, false, "RHEL CoreOS shares the Fedora CoreOS Ignition boot path (now supported), but its image is only available via the pull-secret-gated OpenShift release payload; supply the image to enable it."
 	default:
 		return ExecutionTransportSSH, true, ""
 	}

matrices/tier3-cloud-native.yaml

@@ -6,7 +6,7 @@ profiles:
     required: false
   - id: talos-6.6
     required: false
-  - id: fedora-coreos-stable-6.14
+  - id: fedora-coreos-stable-7.0
     required: false
   - id: rhcos-4.16-5.14
     required: false