diff --git a/staging/mission-control/.gitattributes b/staging/mission-control/.gitattributes new file mode 100644 index 00000000..e686fe4d --- /dev/null +++ b/staging/mission-control/.gitattributes @@ -0,0 +1,4 @@ +*.sh text eol=lf +*.env text eol=lf +*.tsv text eol=lf +*.json text eol=lf diff --git a/staging/mission-control/README.md b/staging/mission-control/README.md new file mode 100644 index 00000000..cf0ff389 --- /dev/null +++ b/staging/mission-control/README.md @@ -0,0 +1,153 @@ +# mission-control — execute the ISO permutation matrix + +`ISO-PERMUTATION-MATRIX.md` says of itself: **"No builds were run."** Fourteen +of its sixteen rows are resolution predictions or code reading; only two were +ever installed. This directory is the part that was missing — it builds the +ISOs, stages the VMs, drives the installs, and verifies the result against an +oracle that names the PR behind every failure. + +It reuses the mechanics of `../vm-lab` (VMX discipline, thin disks, +stash-never-delete teardown, the serial-log liveness instrument) and drops +everything specific to the SPAGAT-Librarian appliance, which is out of scope. + +--- + +## The one thing that makes this tractable + +POI's `isoInstaller` does not only read `ks=` from the kernel command line. It +reads **`guestinfo.kickstart.data`** (base64) and `guestinfo.kickstart.url` +through `vmtoolsd`, and `/usr/bin/vmtoolsd` is present in the installer initrd +(`open-vm-tools` is in `packages_installer_initrd.json`). + +So a per-permutation kickstart is **one line in the VMX**. No ISO remaster, no +HTTP server, no typing at a boot menu. That splits the matrix cleanly: + +| Layer | Axes | Cost | +|---|---|---| +| **Build time** | ISO type × installer version | 4 ISOs, cached and reused | +| **Install time** | STIG × filesystem × kickstart-vs-UI | free | + +34 permutations, 4 ISOs. + +## Why both kickstart and UI + +Not thoroughness — they exercise different code, and each has a failure mode +the other cannot reach. + +- The **STIG menu is UI-only.** `stigenable.py` is reached solely from the + curses configurator, so a kickstart can never "answer yes"; it has to list + `KS_STIG_PACKAGES` by hand. That is what `variant=stigpkgs` does. +- The **`security:` key is kickstart-only** on POI 2.8. Rows `s01`/`s02` cover + it, and `s02` (`security: {fips: …}`) is reachable *exclusively* from a + kickstart on either installer version. +- The **same failure looks different** on the two paths. In UI mode a missing + package reduces to `InstallerError("Installer failed")` on screen with the + real cause only in `/var/log/installer`; in kickstart mode the tdnf error + surfaces directly. An oracle calibrated on one path misreads the other. + +--- + +## Layout + +``` +mission-control/ +├── config/ +│ ├── mission-control.env every value overridable: ${VAR:=default} +│ ├── permutations.tsv the matrix as data, with doc vs expected verdict +│ └── photon-matrix.vmx.template 10 placeholders, incl. per-permutation UUID +├── lib/ +│ ├── common.sh config locator, structured results, identity +│ └── oracle.sh the assertions - each names the PR it proves +└── bin/ + ├── mc-preflight.sh can this host do the job? + ├── mc-build-iso.sh build-axis tuple -> cached ISO + ├── mc-gen-kickstart.sh permutation -> kickstart JSON + ├── mc-create-vm.sh thin disk + VMX + guestinfo injection + ├── mc-install.sh autonomous or operator-driven install + ├── mc-verify.sh run the oracle, harvest the logs + ├── mc-teardown.sh stash the whole chain, keep the evidence + └── mc-run.sh drive it all, report at the end +``` + +## Use + +```bash +export MC_DIR=$PWD # or run from this directory +./bin/mc-preflight.sh # measured host readiness +./bin/mc-run.sh --all --plan # what would run, builds nothing +./bin/mc-run.sh --only k01,k03 # two autonomous permutations +./bin/mc-run.sh --only p03 # interactive: prompts, then waits +./bin/mc-run.sh --report # re-print from stored results +``` + +Runs are **sequential**: every ISO build shares `$PHOTON_TREE/stage`, and C: +has ~138 GB free, so VMs are torn down after verification rather than kept. + +## How a PR regression shows up + +Every assertion carries the PR it proves, so the report names the culprit: + +``` +ID ISO POI STIG FS MODE DOC RESULT PRs implicated +p03 minimal 2.8 yes ext4 ui fails FAIL(2) POI#11 +``` + +`DOC` is the verdict the matrix recorded *before* the PRs. A row whose result +reproduces `DOC`'s `fails` is a regression, and the PRs column says which one. +The oracle can currently implicate **PR#9, PR#21, PR#22, PR#24, POI#9, POI#10, +POI#11**. + +Results land in `$MC_RESULTS_DIR//`: `checks.jsonl` (one JSON object per +assertion), `kickstart.json`, and `logs/` with dmesg, `journalctl -b`, +`journalctl -p err`, failed units, `rpm -qa`, `/proc/cmdline`, mounts, +`/var/log/{installer,ansible-stig,messages}`, the mkinitrd log and the POI +manifest. + +The matrix supplies a *dependency-resolution* oracle only — `Error(1011)`, +media RPM presence, NEVRs. It gives no dmesg/journalctl/`/var/log` criteria at +all, so that layer is new here. + +--- + +## Things that will bite + +**`Error(1032)` is not a success signal.** It is the `--assumeno` dry-run +abort. A real install never emits it. Only `Error(1011)` means "a package the +installer asked for is not on the media". + +**Never match a single package name.** `list(set(packages))` in `installer.py` +randomises which of the six missing names tdnf reports first. The matrix +reproduced `rsyslog` where the user saw `libselinux-utils`; both were right. +Match on `Error(1011)` and set membership. + +**`grep -a` does not work here.** In a non-interactive shell `/usr/bin/grep` is +toybox, which has no `-a` and returns *zero matches* on a NUL-bearing serial +log rather than erroring. `mc_grep_count` strips NULs first. Interactively +`grep` is `ugrep`, which behaves differently again — so anything that works at +your prompt may still be wrong inside a script. + +**`sed \U` and `grep -P` are GNU extensions** and are absent here for the same +reason. `mc_win_path` uses `tr`. + +**UEFI ignores `bios.bootOrder`.** The NVRAM decides. Both install and teardown +stash `.nvram`, or the firmware's removable-media fallback finds the old ESP +and boots the *previous* image — which looks exactly like an install that +silently did nothing. + +**Purge stale installer RPMs before every build.** tdnf takes the highest +release it can see, so a months-old `photon-os-installer` left in +`stage/RPMS/` silently wins and ships on the ISO. `mc-build-iso.sh` deletes +them and records the NEVR that actually shipped. A run that exercises a stale +installer is worse than no run: it reports a verdict for code nobody ships. + +**Identity is positional, not hashed.** A `cksum`-based index collided on this +very matrix (`k04`/`k16` and `k09`/`s02` shared a MAC, UUID and IP) and could +reach `.240`, inside VMnet8's DHCP range. The ordinal in `permutations.tsv` is +unique by construction and keeps addresses at `.41–.74`. + +**Do not blanket-stop VMs.** This host runs other VMs, including live CI +runners. Every operation targets `mc-` by name. + +**Keep VMs off OneDrive.** The hand-made test VM lived under +`OneDrive/Dokumente/Virtual Machines` and disappeared mid-session. Mission +control uses `C:\photon-mc\vm`. diff --git a/staging/mission-control/bin/mc-build-iso.sh b/staging/mission-control/bin/mc-build-iso.sh new file mode 100755 index 00000000..4e9f34cb --- /dev/null +++ b/staging/mission-control/bin/mc-build-iso.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# mc-build-iso.sh - resolve one build-axis tuple to an ISO, building if needed. +# +# The build-time axes are ISO type and installer version, and nothing else. +# Everything the matrix varies at install time (STIG, filesystem, kickstart vs +# UI) is injected per VM, so 34 permutations need only 4 ISOs. +# +# usage: mc-build-iso.sh --iso-type minimal|full --poi 2.8|latest [--canister prebuilt|build|acvp|kat] [--force] +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" + +ISO_TYPE=minimal POI=2.8 CANISTER=prebuilt FORCE=0 +while [ $# -gt 0 ]; do + case "$1" in + --iso-type) ISO_TYPE="$2"; shift 2 ;; + --poi) POI="$2"; shift 2 ;; + --canister) CANISTER="$2"; shift 2 ;; + --force) FORCE=1; shift ;; + *) mc_die "unknown arg: $1" 64 ;; + esac +done + +case "$ISO_TYPE" in minimal) IMG=minimal-iso ;; full) IMG=iso ;; *) mc_die "bad --iso-type" 64 ;; esac +KEY="${ISO_TYPE}-poi${POI}-${CANISTER}" +DEST="$MC_ISO_CACHE/$KEY" +mkdir -p "$DEST" "$MC_BUILD_LOG_DIR" + +if [ "$FORCE" -eq 0 ] && [ -f "$DEST/photon.iso" ]; then + mc_log "cache hit: $KEY -> $DEST/photon.iso" + printf '%s\n' "$DEST/photon.iso"; exit 0 +fi + +# --- the stale-RPM landmine ---------------------------------------------- +# tdnf picks the highest release it can see, so a months-old +# photon-os-installer left in stage/RPMS silently wins and lands on the ISO. +# A test run that exercises a stale installer is worse than no test run: it +# reports a verdict for code nobody is shipping. +STAGE_RPMS="$PHOTON_TREE/stage/RPMS" +if [ -d "$STAGE_RPMS" ]; then + n=$(find "$STAGE_RPMS" -name 'photon-os-installer-*.rpm' 2>/dev/null | wc -l) + if [ "$n" -gt 0 ]; then + mc_log "purging $n cached photon-os-installer RPM(s) so the build cannot pick a stale one" + find "$STAGE_RPMS" -name 'photon-os-installer-*.rpm' -delete + fi +fi + +# --- installer version, without merging anything ------------------------- +# The point of this harness is to test PRs BEFORE they merge, so requiring a +# merge to reach the poi=latest rows would invert that. Instead each variant +# gets its own patch: poi=2.8 uses the downstream set as-is, poi=latest uses +# the same set with dcasota/photon#26 (installer v2.9) substituted for the +# 2.8-only installer PR. Both are generated from the PR branches and both are +# verified to apply to a pristine 5.0 before use. +# +# runPh5_normal.sh resolves its patch relative to its OWN directory, so the +# variant is selected by staging a script directory rather than by editing the +# build script. SCRIPT_DIR is used for nothing else (runPh5_normal.sh:75,163). +VARIANT_PATCH="$MC_VARIANT_PATCH_DIR/poi-${POI}.patch" +[ -f "$VARIANT_PATCH" ] || mc_die "no variant patch at $VARIANT_PATCH - run mc-make-variant-patches.sh" 3 + +STAGE_DIR="$MC_WORK/scriptdir/$KEY" +mkdir -p "$STAGE_DIR/photonos-patches" +cp "$PHOTON_SCRIPTS/runPh5_normal.sh" "$STAGE_DIR/runPh5_normal.sh" +cp "$VARIANT_PATCH" "$STAGE_DIR/photonos-patches/downstream-fixes.patch" +mc_log "staged build dir $STAGE_DIR with poi-${POI}.patch ($(grep -c '^+++ ' "$VARIANT_PATCH") files)" + +SPEC="$PHOTON_TREE/SPECS/photon-os-installer/photon-os-installer.spec" +have=$(awk '/^Version:/{print $2; exit}' "$SPEC" 2>/dev/null); [ -n "$have" ] || have='?' +mc_log "installer version in the pristine tree: $have (the variant patch sets the one under test)" + +BUILD_LOG="$MC_BUILD_LOG_DIR/${KEY}-$(date -u +%Y%m%dT%H%M%SZ).log" +mc_log "building $IMG (canister=$CANISTER) -> $BUILD_LOG" +mc_log "this takes hours; the run script polls rather than blocking" + +sh "$STAGE_DIR/runPh5_normal.sh" /root common 5.0 "$DEST" "$IMG" "$CANISTER" \ + > "$BUILD_LOG" 2>&1 +rc=$? +[ $rc -eq 0 ] || mc_die "build failed (rc=$rc), see $BUILD_LOG" "$rc" + +iso=$(find "$DEST" -maxdepth 1 -name '*.iso' -newer "$BUILD_LOG" 2>/dev/null | head -1) +[ -n "$iso" ] || iso=$(find "$DEST" -maxdepth 1 -name '*.iso' | head -1) +[ -n "$iso" ] || mc_die "build reported success but produced no ISO in $DEST" 4 +[ "$iso" = "$DEST/photon.iso" ] || ln -sf "$(basename "$iso")" "$DEST/photon.iso" + +# --- assert what actually shipped ---------------------------------------- +poi_on_media=$(xorriso -osirrox on -indev "$iso" -find /RPMS -name 'photon-os-installer-*.rpm' 2>/dev/null \ + | sed 's|.*/||' | tr -d "'" | head -1) +mc_log "installer on the produced media: ${poi_on_media:-ABSENT}" +printf '%s\n' "$poi_on_media" > "$DEST/poi-nevr.txt" +sha256sum "$iso" | awk '{print $1}' > "$DEST/photon.iso.sha256" +mc_log "cached: $DEST/photon.iso" +printf '%s\n' "$DEST/photon.iso" diff --git a/staging/mission-control/bin/mc-create-vm.sh b/staging/mission-control/bin/mc-create-vm.sh new file mode 100755 index 00000000..f27e7d7b --- /dev/null +++ b/staging/mission-control/bin/mc-create-vm.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# mc-create-vm.sh - VM directory, thin boot disk, and VMX for one permutation. +# +# vm-lab splits this into a .ps1 because vmware-vdiskmanager wants Windows +# paths. That split costs a whole second language with its own CRLF and +# ASCII-only-for-PowerShell-5.1 constraints, and cannot be tested from here. +# The .exe runs fine from WSL, so this stays in bash and converts the two +# paths it needs by hand. +# +# usage: mc-create-vm.sh --id --iso [--kickstart ] [--recreate] +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" + +PERM="" ISO="" KS="" RECREATE=0 +while [ $# -gt 0 ]; do + case "$1" in + --id) PERM="$2"; shift 2 ;; + --iso) ISO="$2"; shift 2 ;; + --kickstart) KS="$2"; shift 2 ;; + --recreate) RECREATE=1; shift ;; + *) mc_die "unknown arg: $1" 64 ;; + esac +done +[ -n "$PERM" ] || mc_die "--id is required" 64 +[ -n "$ISO" ] && [ -f "$ISO" ] || mc_die "--iso must name an existing file" 3 + +VM="mc-$PERM" +DIR_WSL="$MC_VM_ROOT_WSL/$VM" +DIR_WIN="$(mc_win_path "$DIR_WSL")" +IDX=$(mc_perm_index "$PERM") +MAC=$(mc_mac_for "$IDX") +UUID=$(mc_uuid_for "$IDX") +SERIAL_WSL="$DIR_WSL/${SERIAL_LOG_PREFIX}-${VM}.log" + +if [ -d "$DIR_WSL" ] && [ "$RECREATE" -eq 1 ]; then + mc_log "recreate: stashing $DIR_WSL" + mv "$DIR_WSL" "${DIR_WSL}.stashed-$(date -u +%Y%m%dT%H%M%SZ)" +fi +mkdir -p "$DIR_WSL" + +# --- boot disk ------------------------------------------------------------ +# -t 0 is monolithicSparse: one file, thin. A fresh 32 GB disk is a few MB and +# grows only as the guest writes. The hand-made test VM on this host is +# monolithicFlat and commits its full size up front; 34 of those would not fit +# in the free space on C:. +if [ ! -f "$DIR_WSL/$VM.vmdk" ]; then + "$VDISKMANAGER" -c -s "$BOOT_DISK_SIZE" -a "$BOOT_DISK_ADAPTER" -t "$BOOT_DISK_TYPE" \ + "$DIR_WIN\\$VM.vmdk" >/dev/null 2>&1 || mc_die "vmware-vdiskmanager failed for $VM" 5 + mc_log "created thin disk: $(du -h "$DIR_WSL/$VM.vmdk" 2>/dev/null | cut -f1) of $BOOT_DISK_SIZE" +else + mc_log "disk already present, keeping it" +fi + +# --- kickstart injection -------------------------------------------------- +# POI's isoInstaller reads guestinfo.kickstart.data via vmtoolsd, and +# /usr/bin/vmtoolsd is present in the installer initrd. So an autonomous +# permutation needs no ISO remaster and no typing at the boot menu. +# Omitting the line entirely is what selects the interactive path: with no +# kickstart the installer falls through to the curses configurator, which is +# the only place the STIG menu exists. +if [ -n "$KS" ] && [ -f "$KS" ]; then + GUESTINFO="guestinfo.kickstart.data = \"$(base64 -w0 < "$KS")\"" + mc_log "kickstart injected via guestinfo ($(wc -c < "$KS") bytes)" +else + GUESTINFO="# no kickstart: interactive install, operator drives the curses configurator" + mc_log "no kickstart - interactive permutation" +fi + +TPL="$_here/../config/photon-matrix.vmx.template" +python3 - "$TPL" "$DIR_WSL/$VM.vmx" "$VM" "$GUEST_VCPUS" "$GUEST_MEM_MB" "$MAC" "$UUID" \ + "$(mc_win_path "$ISO")" "$(mc_win_path "$SERIAL_WSL")" "$GUESTINFO" <<'PY' +import sys +tpl, out, vm, vcpu, mem, mac, uuid, iso, serial, guestinfo = sys.argv[1:11] +s = open(tpl).read() +for k, v in (("VM_NAME", vm), ("GUEST_VCPUS", vcpu), ("GUEST_MEM_MB", mem), + ("GUEST_MAC", mac), ("UUID_BIOS", uuid), ("ISO_PATH_WIN", iso), + ("SERIAL_LOG_WIN", serial), ("GUESTINFO_KICKSTART", guestinfo), + ("NIC_DEV", "vmxnet3"), ("SECUREBOOT", "FALSE")): + s = s.replace("@@%s@@" % k, v) +import re +left = re.findall(r"@@[A-Z_]+@@", s) +if left: + sys.exit("FAIL: unsubstituted placeholders survived: %s" % sorted(set(left))) +open(out, "w").write(s) +PY +[ $? -eq 0 ] || mc_die "VMX generation failed" 6 + +mc_log "vm=$VM ip=$(mc_ip_for "$IDX") mac=$MAC" +mc_log "vmx=$DIR_WSL/$VM.vmx" +printf '%s\n' "$DIR_WSL/$VM.vmx" diff --git a/staging/mission-control/bin/mc-gen-kickstart.sh b/staging/mission-control/bin/mc-gen-kickstart.sh new file mode 100755 index 00000000..377a4345 --- /dev/null +++ b/staging/mission-control/bin/mc-gen-kickstart.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# mc-gen-kickstart.sh - emit the kickstart JSON for one permutation. +# +# Written to stdout. mc-install.sh base64s it into guestinfo.kickstart.data, +# which POI's isoInstaller reads via vmtoolsd. That is why no permutation +# needs its own ISO: the install-time axes live here, not on the media. +# +# usage: mc-gen-kickstart.sh --fs ext4|btrfs --stig yes|no --variant none|selinux|fips|stigpkgs +# --id [--pubkey ] [--ip ] +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" + +FS=ext4 STIG=no VARIANT=none PERM=perm PUBKEY="" IPADDR="" +while [ $# -gt 0 ]; do + case "$1" in + --fs) FS="$2"; shift 2 ;; + --stig) STIG="$2"; shift 2 ;; + --variant) VARIANT="$2"; shift 2 ;; + --id) PERM="$2"; shift 2 ;; + --pubkey) PUBKEY="$2"; shift 2 ;; + --ip) IPADDR="$2"; shift 2 ;; + *) echo "unknown arg: $1" >&2; exit 64 ;; + esac +done + +KEY="" +[ -n "$PUBKEY" ] && [ -f "$PUBKEY" ] && KEY=$(cat "$PUBKEY") + +# The eight names stigenable.py requests when the STIG menu is answered yes. +# A kickstart cannot answer that menu - it is only reachable from the curses +# configurator - so a kickstart that wants STIG must list them itself. That is +# what variant=stigpkgs reproduces, and it is a genuinely different code path +# from the UI row, not a duplicate of it. +KS_STIG_PACKAGES='"audit","rsyslog","openssl-fips-provider","selinux-policy","aide"' + +python3 - "$FS" "$STIG" "$VARIANT" "$PERM" "$KEY" "$IPADDR" "$MC_GUEST_PASSWORD" <<'PY' +import json, sys +fs, stig, variant, perm, key, ipaddr, password = sys.argv[1:8] + +partitions = [ + {"mountpoint": "/boot/efi", "size": 512, "filesystem": "vfat"}, + {"mountpoint": "/boot", "size": 1024, "filesystem": "ext4"}, + {"mountpoint": "/", "size": 0, "filesystem": fs}, +] + +ks = { + # The hostname carries the permutation id so a guest self-identifies in + # every log line it ever emits. + "hostname": "mc-" + perm, + "password": {"crypted": False, "text": password}, + "disk": "/dev/sda", + "partitions": partitions, + "packagelist_file": "packages_minimal.json", + "linux_flavor": "linux-esx", + "bootmode": "efi", + "postinstall": [ + "#!/bin/sh", + "echo mc-%s > /etc/mission-control-permutation" % perm, + "systemctl enable sshd.service", + # Root ssh is how verification gets in. This is a disposable lab VM on + # a host-only NAT segment, torn down after the run. + "sed -i 's/^#*PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config", + ], +} +if key: + ks["public_key"] = key + +# variant=stigpkgs is the kickstart expression of "STIG = yes". +if variant == "stigpkgs" or stig == "yes": + ks["additional_packages"] = ["audit", "rsyslog", "openssl-fips-provider", + "selinux-policy", "aide"] + ks["ansible"] = [{ + "playbook": "/usr/share/ansible/stig-hardening/playbook.yml", + "logfile": "ansible-stig.log", + "verbosity": 2, + "extra-vars": "@/usr/share/ansible/stig-hardening/vars-chroot.yml", + # PHTN-50-000245 edits tmp.mount, which is package-owned and not + # %config. Editing it here shows as permanent rpm -V drift and is + # reverted by the next systemd upgrade, so the build side owns it. + "skip-tags": ["PHTN-50-000245"], + }] + +# The kickstart-only failure class. On POI 2.8 the security key is only +# present if the author writes it; POI master synthesises selinux for +# everyone. fips is never appended on the UI path on either version, so +# variant=fips is reachable exclusively from a kickstart. +if variant == "selinux": + ks["security"] = {"selinux": "permissive"} +elif variant == "fips": + ks["security"] = {"fips": 1} + +if ipaddr: + ks["network"] = {"type": "static", "ip_addr": ipaddr.split("/")[0], + "netmask": "255.255.255.0", "gateway": "", "nameserver": ""} +else: + ks["network"] = {"type": "dhcp"} + +print(json.dumps(ks, indent=4)) +PY diff --git a/staging/mission-control/bin/mc-install.sh b/staging/mission-control/bin/mc-install.sh new file mode 100755 index 00000000..2dbebca7 --- /dev/null +++ b/staging/mission-control/bin/mc-install.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# mc-install.sh - run one install, autonomously or with an operator. +# +# This is the part vm-lab delegates to spagat-vm-orchestrator, which is a +# cargo artifact of a repo we do not have. Everything it did is reachable with +# vmrun plus VMX edits; the only genuinely non-trivial piece is deciding when +# an install has finished, which is done here by watching the boot source +# change in the serial log. +# +# usage: mc-install.sh --id --mode auto|interactive [--timeout ] +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" + +PERM="" MODE=auto TIMEOUT="" NOWAIT=0 +while [ $# -gt 0 ]; do + case "$1" in + --id) PERM="$2"; shift 2 ;; + --mode) MODE="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + --no-wait) NOWAIT=1; shift ;; + *) mc_die "unknown arg: $1" 64 ;; + esac +done +[ -n "$PERM" ] || mc_die "--id is required" 64 +TIMEOUT="${TIMEOUT:-$MC_INSTALL_TIMEOUT_SEC}" + +VM="mc-$PERM" +DIR="$MC_VM_ROOT_WSL/$VM" +VMX="$DIR/$VM.vmx" +SER="$DIR/${SERIAL_LOG_PREFIX}-${VM}.log" +[ -f "$VMX" ] || mc_die "no VMX at $VMX - run mc-create-vm.sh first" 3 + +vm_is_up() { "$VMRUN" -T ws list 2>/dev/null | tr -d '\r' | grep -qi "$VM\.vmx"; } + +# UEFI ignores bios.bootOrder, so the only way to stop the firmware booting the +# PREVIOUS image out of the old ESP is to remove the NVRAM. vm-lab learned this +# the hard way; deleting the disk alone does not help because UEFI re-detects. +if [ -f "$DIR/$VM.nvram" ]; then + mv "$DIR/$VM.nvram" "$DIR/$VM.nvram.stashed-$(date -u +%Y%m%dT%H%M%SZ)" + mc_log "stashed NVRAM so UEFI cannot fall back to a previous image" +fi + +: > "$SER" 2>/dev/null || true +START_SIZE=$(stat -c%s "$SER" 2>/dev/null || echo 0) + +if [ "$MODE" = interactive ]; then + "$VMRUN" -T ws start "$(mc_win_path "$VMX")" gui >/dev/null 2>&1 \ + || mc_die "could not start $VM" 5 + cat </dev/null 2>&1 || mc_die "could not start $VM" 5 + mc_log "$VM started headless; kickstart supplied via guestinfo" +fi + +# With --no-wait the VM is left running for a human to drive; the caller +# decides when to verify. Used for the UI rows, where blocking here would hide +# the operator instructions behind a poll loop. +if [ "$NOWAIT" -eq 1 ]; then + mc_log "$VM is up and waiting for the operator" + exit 0 +fi + +# --- completion detection ------------------------------------------------- +# root=/dev/ram0 is the installer live environment; root=PARTUUID= is the +# installed system. The transition is the only unambiguous "the install +# finished and the machine came back on its own" signal. +mc_log "waiting up to ${TIMEOUT}s for the guest to boot off disk" +deadline=$(( $(date +%s) + TIMEOUT )) +last_size=0 stalled=0 result=timeout +while [ "$(date +%s)" -lt "$deadline" ]; do + sleep 15 + [ -f "$SER" ] || continue + size=$(stat -c%s "$SER" 2>/dev/null || echo 0) + if [ "$size" -eq "$last_size" ]; then stalled=$((stalled+1)); else stalled=0; fi + last_size=$size + if [ "$(mc_grep_count 'root=PARTUUID=' "$SER")" -gt 0 ]; then result=installed; break; fi + if [ "$(mc_grep_count 'Error(1011)' "$SER")" -gt 0 ]; then result=error1011; break; fi + # A long quiet stretch is not proof of a stall - vm-lab is explicit that + # no growth is not by itself a hang - so this only reports, never aborts. + [ $((stalled % 20)) -eq 19 ] && mc_log "serial log quiet for ~5min (size=${size}); still waiting" +done + +case "$result" in + installed) mc_log "install completed: guest is booting from disk" ;; + error1011) mc_log "install FAILED with Error(1011) - a package the installer requested is not on the media" ;; + timeout) mc_log "timed out after ${TIMEOUT}s with no boot-from-disk transition" ;; +esac + +# Detach the CDROM so a later boot cannot re-enter the installer. +sed -i 's|^sata0:1.startConnected = "TRUE"|sata0:1.startConnected = "FALSE"|' "$VMX" 2>/dev/null || true + +printf '%s\n' "$result" +[ "$result" = installed ] || exit 7 diff --git a/staging/mission-control/bin/mc-make-variant-patches.sh b/staging/mission-control/bin/mc-make-variant-patches.sh new file mode 100755 index 00000000..55e700ab --- /dev/null +++ b/staging/mission-control/bin/mc-make-variant-patches.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# mc-make-variant-patches.sh - build one patch per installer variant, from the +# PR branches, WITHOUT merging any of them. +# +# The whole point of the harness is to test PRs before they land, so a variant +# that required a merge first would be untestable by definition. Each variant +# is assembled by cherry-picking the PR branches onto a pristine 5.0 in a +# throwaway clone and diffing the result. +# +# poi-2.8.patch 5.0 + #9 #19 #21 #22 #23 #24 (installer stays 2.8) +# poi-latest.patch 5.0 + #9 #21 #22 #23 #24 #26 (#26 bumps it to v2.9) +# +# #19 and #26 are alternatives - #19 adds patches to 2.8, #26 moves to 2.9 +# where three of them are already upstream - so exactly one appears in each. +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" + +: "${MC_PHOTON_REMOTE:=https://github.com/dcasota/photon.git}" +CLONE="$MC_WORK/photon-variants" +mkdir -p "$MC_WORK" "$MC_VARIANT_PATCH_DIR" + +if [ ! -d "$CLONE/.git" ]; then + mc_log "cloning $MC_PHOTON_REMOTE (blobless) -> $CLONE" + git clone --quiet --filter=blob:none --no-checkout "$MC_PHOTON_REMOTE" "$CLONE" || mc_die "clone failed" 5 +fi +cd "$CLONE" || mc_die "cannot enter $CLONE" 5 +git fetch -q origin 5.0 fix-selinux-relabel fix/photon-os-installer-2.8-5-interactive-osrelease \ + fix/aide-libgcrypt-versioned-requires fix/systemd-groups-and-stig-variant \ + fix/stig-harden-reachable fix/kernel-shared-canister-config fix/poi-2.9-bump || mc_die "fetch failed" 5 + +build_variant() { + local name="$1"; shift + git checkout -q -B "variant-$name" origin/5.0 || return 1 + local br + for br in "$@"; do + git cherry-pick -x "$(git rev-parse "origin/$br")" >/dev/null 2>&1 || { + mc_log " CONFLICT applying $br to variant $name" + git cherry-pick --abort 2>/dev/null; return 1 + } + done + local out="$MC_VARIANT_PATCH_DIR/poi-${name}.patch" + git diff origin/5.0 "variant-$name" -- SPECS/ > "$out" + mc_log " poi-$name: $(grep -c '^+++ ' "$out") files, $(wc -l < "$out") lines" + + # Prove it applies to a pristine tree before anything relies on it. + local t; t=$(mktemp -d) + git archive origin/5.0 | tar x -C "$t" + if ( cd "$t" && patch -p1 --dry-run --forward < "$out" >/dev/null 2>&1 ); then + mc_log " poi-$name: applies to pristine 5.0" + else + mc_log " poi-$name: DOES NOT APPLY to pristine 5.0"; rm -rf "$t"; return 1 + fi + rm -rf "$t" +} + +rc=0 +mc_log "variant poi-2.8 (installer stays at 2.8, #19)" +build_variant 2.8 fix/photon-os-installer-2.8-5-interactive-osrelease \ + fix/aide-libgcrypt-versioned-requires fix-selinux-relabel \ + fix/systemd-groups-and-stig-variant fix/stig-harden-reachable \ + fix/kernel-shared-canister-config || rc=1 + +mc_log "variant poi-latest (installer v2.9 via #26, instead of #19)" +build_variant latest fix/poi-2.9-bump \ + fix/aide-libgcrypt-versioned-requires fix-selinux-relabel \ + fix/systemd-groups-and-stig-variant fix/stig-harden-reachable \ + fix/kernel-shared-canister-config || rc=1 + +exit $rc diff --git a/staging/mission-control/bin/mc-operator-card.sh b/staging/mission-control/bin/mc-operator-card.sh new file mode 100755 index 00000000..2d395a40 --- /dev/null +++ b/staging/mission-control/bin/mc-operator-card.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# mc-operator-card.sh - what a human must enter for one interactive permutation. +# +# Generated from permutations.tsv, never hand-written, so the instructions +# cannot drift from the matrix they are supposed to exercise. +# +# usage: mc-operator-card.sh --id +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" +TSV="$_here/../config/permutations.tsv" + +PERM="" +while [ $# -gt 0 ]; do + case "$1" in --id) PERM="$2"; shift 2 ;; *) mc_die "unknown arg: $1" 64 ;; esac +done +[ -n "$PERM" ] || mc_die "--id is required" 64 + +read -r _ ISO POI STIG FS MODE VARIANT DOC EXPECT < $VM + Matrix says : $DOC Expected with the PRs: $EXPECT + + ENTER IN THE INSTALLER + 1. License accept + 2. Disk /dev/sda -> choose CUSTOM partitioning + (auto-partition always makes ext4, so a + filesystem row can only be reached by hand) + /boot/efi 512 MB vfat + /boot 1024 MB ext4 + / rest $FS <-- the axis under test + 3. Hostname mc-$PERM + 4. Root password $MC_GUEST_PASSWORD + 5. "Apply STIG hardening" $(if [ "$STIG" = yes ]; then echo "YES <-- the axis under test"; else echo "NO"; fi) + (this menu is the reason $PERM cannot be automated: it exists only in + the curses configurator, so no kickstart can answer it) + 6. Let it install and reboot on its own. + + Tell me when the install has finished and I will verify it. +TXT diff --git a/staging/mission-control/bin/mc-preflight.sh b/staging/mission-control/bin/mc-preflight.sh new file mode 100755 index 00000000..bee41dff --- /dev/null +++ b/staging/mission-control/bin/mc-preflight.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# mc-preflight.sh - can this host run the matrix? +# Prints measured values, never a bare OK/FAIL: "tool missing" and "tool +# present but not executable by this user" need different fixes and are +# indistinguishable in a boolean. +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" + +fail=0 +say() { printf ' %-30s %s\n' "$1" "$2"; } +bad() { printf ' %-30s FAIL: %s\n' "$1" "$2"; fail=1; } + +echo "== identity ==" +say "user" "$(id -un) (uid $(id -u))" + +echo "== vmware tooling ==" +for t in "$VMRUN" "$VDISKMANAGER"; do + if [ ! -e "$t" ]; then bad "$(basename "$t")" "not found at $t" + elif [ ! -x "$t" ]; then bad "$(basename "$t")" "present but not executable by $(id -un)" + else say "$(basename "$t")" "executable"; fi +done +if [ -x "$VMRUN" ]; then + say "vmrun list" "$("$VMRUN" -T ws list 2>&1 | head -1)" + # Never blanket-stop VMs: other VMs on this host may be live CI runners. + other=$("$VMRUN" -T ws list 2>/dev/null | tr -d "\r" | grep -c "\.vmx$") || other=0 + say "VMs already running" "$other (mission control only ever touches its own)" +fi + +echo "== disk ==" +for d in "$MC_VM_ROOT_WSL" "$MC_ISO_CACHE" "$MC_RESULTS_DIR"; do + mkdir -p "$d" 2>/dev/null || true + if [ -d "$d" ]; then + avail=$(df -BG --output=avail "$d" 2>/dev/null | tail -1 | tr -dc '0-9') + say "$d" "${avail:-?} GiB free" + # A thin 32 GB disk starts ~6 MB, but a completed Photon install is + # ~12 GB. Sequential runs with teardown keep this bounded. + [ -n "${avail:-}" ] && [ "$avail" -lt 25 ] && bad "$d" "under 25 GiB free" + else bad "$d" "could not create"; fi +done + +echo "== iso build tree ==" +[ -d "$PHOTON_TREE" ] && say "photon tree" "$PHOTON_TREE ($(git -C "$PHOTON_TREE" rev-parse --short HEAD 2>/dev/null || echo 'not a repo'))" \ + || bad "photon tree" "$PHOTON_TREE missing" +[ -f "$DOWNSTREAM_PATCH" ] && say "downstream patch" "$(grep -c '^+++ ' "$DOWNSTREAM_PATCH") files" \ + || bad "downstream patch" "$DOWNSTREAM_PATCH missing" +if [ -f "$DOWNSTREAM_PATCH" ] && [ -d "$PHOTON_TREE" ]; then + if git -C "$PHOTON_TREE" apply --check "$DOWNSTREAM_PATCH" 2>/dev/null; then + say "patch applies" "yes" + else + bad "patch applies" "no - rebase it or the build guard will refuse" + fi +fi + +# runPh5_normal.sh resolves its patch relative to its own directory, so the +# patch preflight validates must be the patch the build will actually use. +# Two copies of each exist on this host and they had diverged. +RESOLVED="$PHOTON_SCRIPTS/photonos-patches/downstream-fixes.patch" +if [ -f "$RESOLVED" ]; then + if [ "$(realpath "$RESOLVED" 2>/dev/null)" = "$(realpath "$DOWNSTREAM_PATCH" 2>/dev/null)" ]; then + say "build resolves patch" "same file preflight checked" + else + bad "build resolves patch" "$RESOLVED ($(grep -c '^+++ ' "$RESOLVED") files) != $DOWNSTREAM_PATCH ($(grep -c '^+++ ' "$DOWNSTREAM_PATCH") files)" + fi +else + bad "build resolves patch" "$PHOTON_SCRIPTS/runPh5_normal.sh would find no patch at $RESOLVED" +fi + +echo "== guest tooling ==" +for t in xorriso python3 ssh sshpass; do + command -v "$t" >/dev/null 2>&1 && say "$t" "$(command -v "$t")" || bad "$t" "not installed" +done + +echo "== ssh key ==" +if [ -f "$SSH_KEY_DIR/$SSH_KEY_NAME" ]; then say "keypair" "$SSH_KEY_DIR/$SSH_KEY_NAME" +else say "keypair" "absent - mc-run.sh will create it"; fi + +echo +[ "$fail" -eq 0 ] && echo "preflight: PASS" || echo "preflight: FAIL" +exit $fail diff --git a/staging/mission-control/bin/mc-run.sh b/staging/mission-control/bin/mc-run.sh new file mode 100755 index 00000000..bc60c92d --- /dev/null +++ b/staging/mission-control/bin/mc-run.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# mc-run.sh - mission control. Drive permutations end to end. +# +# usage: +# mc-run.sh --plan show what would run, build nothing +# mc-run.sh --only p01,p04 [--keep] run named permutations +# mc-run.sh --all run every permutation in the matrix +# mc-run.sh --report re-print the summary from stored results +# +# Runs are SEQUENTIAL by design: every ISO build shares $PHOTON_TREE/stage, and +# C: does not have room for many installed VMs at once. +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" +TSV="$_here/../config/permutations.tsv" + +ONLY="" ALL=0 PLAN=0 REPORT=0 KEEP=0 +while [ $# -gt 0 ]; do + case "$1" in + --only) ONLY="$2"; shift 2 ;; + --all) ALL=1; shift ;; + --plan) PLAN=1; shift ;; + --report) REPORT=1; shift ;; + --keep) KEEP=1; shift ;; + *) mc_die "unknown arg: $1" 64 ;; + esac +done + +rows() { grep -vE '^#|^$' "$TSV"; } + +select_rows() { + if [ -n "$ONLY" ]; then + printf '%s' "$ONLY" | tr ',' '\n' | while read -r id; do + [ -n "$id" ] && rows | awk -v p="$id" '$1==p' + done + elif [ "$ALL" -eq 1 ]; then rows + fi +} + +# ---- report -------------------------------------------------------------- +mc_report() { + printf '\n%-6s %-8s %-7s %-5s %-6s %-5s %-9s %-9s %s\n' \ + ID ISO POI STIG FS MODE DOC RESULT "PRs implicated" + printf '%s\n' "--------------------------------------------------------------------------------" + local id f pass fail verdict prs + rows | while read -r id iso poi stig fs mode variant doc expect; do + f="$MC_RESULTS_DIR/$id/checks.jsonl" + if [ ! -f "$f" ]; then verdict="-"; prs="" + else + fail=$(grep -c '"status":"fail"' "$f") || fail=0 + if [ "$fail" -eq 0 ]; then verdict="PASS"; prs="" + else + verdict="FAIL($fail)" + prs=$(grep '"status":"fail"' "$f" | sed -n 's/.*"pr":"\([^"]*\)".*/\1/p' \ + | grep -v '^-$' | sort -u | tr '\n' ' ') + fi + fi + printf '%-6s %-8s %-7s %-5s %-6s %-5s %-9s %-9s %s\n' \ + "$id" "$iso" "$poi" "$stig" "$fs" "$mode" "$doc" "$verdict" "$prs" + done + echo + echo "DOC is what ISO-PERMUTATION-MATRIX.md recorded before the PRs." + echo "A row whose RESULT reproduces DOC's 'fails' is a PR regression;" + echo "the PRs column names which PR the failing assertions belong to." +} + +[ "$REPORT" -eq 1 ] && { mc_report; exit 0; } + +SEL=$(select_rows) +[ -n "$SEL" ] || mc_die "nothing selected - pass --only , --all, or --plan" 64 + +if [ "$PLAN" -eq 1 ]; then + echo "ISOs required (build-time axes only):" + printf '%s\n' "$SEL" | awk '{print " "$2"/"$3}' | sort -u + echo + echo "Permutations:" + printf '%s\n' "$SEL" | while read -r id iso poi stig fs mode variant doc expect; do + printf ' %-5s %-8s poi=%-7s stig=%-4s fs=%-6s mode=%-5s variant=%-9s doc=%s\n' \ + "$id" "$iso" "$poi" "$stig" "$fs" "$mode" "$variant" "$doc" + done + exit 0 +fi + +[ -f "$SSH_KEY_DIR/$SSH_KEY_NAME" ] || { + mkdir -p "$SSH_KEY_DIR"; chmod 700 "$SSH_KEY_DIR" + ssh-keygen -t ed25519 -N '' -C "photon-mc@$(hostname)" -f "$SSH_KEY_DIR/$SSH_KEY_NAME" >/dev/null \ + || mc_die "ssh-keygen failed" 5 + mc_log "created lab keypair $SSH_KEY_DIR/$SSH_KEY_NAME" +} + +total=0 failed=0 +printf '%s\n' "$SEL" | while read -r id iso poi stig fs mode variant doc expect; do + total=$((total+1)) + echo + echo "################ $id ################" + + ISO_PATH=$("$_here/mc-build-iso.sh" --iso-type "$iso" --poi "$poi" 2>&1 | tail -1) + if [ ! -f "$ISO_PATH" ]; then + mc_log "$id: no ISO for $iso/$poi - $ISO_PATH" + mc_result_init "$id" + mc_check iso.available "-" fail "built" "missing" "$ISO_PATH" + continue + fi + + KS="" + if [ "$mode" = ks ]; then + KS="$MC_RESULTS_DIR/$id/kickstart.json" + mkdir -p "$(dirname "$KS")" + "$_here/mc-gen-kickstart.sh" --fs "$fs" --stig "$stig" --variant "$variant" \ + --id "$id" --pubkey "$SSH_KEY_DIR/$SSH_KEY_NAME.pub" > "$KS" + fi + + "$_here/mc-create-vm.sh" --id "$id" --iso "$ISO_PATH" ${KS:+--kickstart "$KS"} --recreate >/dev/null \ + || { mc_log "$id: VM creation failed"; continue; } + + if [ "$mode" = ks ]; then + "$_here/mc-install.sh" --id "$id" --mode auto || true + else + "$_here/mc-install.sh" --id "$id" --mode interactive || true + fi + + "$_here/mc-verify.sh" --id "$id" || failed=$((failed+1)) + + [ "$KEEP" -eq 1 ] || "$_here/mc-teardown.sh" --id "$id" --purge >/dev/null +done + +mc_report diff --git a/staging/mission-control/bin/mc-teardown.sh b/staging/mission-control/bin/mc-teardown.sh new file mode 100755 index 00000000..9f78861a --- /dev/null +++ b/staging/mission-control/bin/mc-teardown.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# mc-teardown.sh - return one permutation's VM to a fresh-disk state. +# +# Nothing is deleted; files are renamed .stashed-. Recovery is a rename +# back. The whole chain goes, not just the disk: if any piece survives, UEFI's +# removable-media fallback finds the old ESP and boots the PREVIOUS image - +# and bios.bootOrder is ignored on EFI VMs, so that is the only control. +# +# The serial log and the results directory are always preserved: they are the +# evidence the run produced. +# +# usage: mc-teardown.sh --id [--purge] (--purge deletes old stashes) +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$(mc_find_config "$_here")" + +PERM="" PURGE=0 +while [ $# -gt 0 ]; do + case "$1" in + --id) PERM="$2"; shift 2 ;; + --purge) PURGE=1; shift ;; + *) mc_die "unknown arg: $1" 64 ;; + esac +done +[ -n "$PERM" ] || mc_die "--id is required" 64 + +VM="mc-$PERM"; DIR="$MC_VM_ROOT_WSL/$VM" +[ -d "$DIR" ] || { mc_log "$DIR does not exist, nothing to tear down"; exit 0; } + +# Only ever stop our own VM. Other VMs on this host may be live CI runners. +if "$VMRUN" -T ws list 2>/dev/null | tr -d '\r' | grep -qi "$VM\.vmx"; then + mc_log "stopping $VM" + "$VMRUN" -T ws stop "$(mc_win_path "$DIR/$VM.vmx")" hard >/dev/null 2>&1 || true + sleep 3 +fi + +TS=$(date -u +%Y%m%dT%H%M%SZ) +n=0 +# Globbed, not enumerated: a fixed list of two snapshot deltas silently leaves +# an orphan on a VM that reached -000003.vmdk. +for f in "$DIR"/*.vmdk "$DIR"/*.vmsn "$DIR"/*.vmsd "$DIR"/*.nvram "$DIR"/*.vmss; do + [ -e "$f" ] || continue + case "$f" in *.stashed-*) continue ;; esac + mv "$f" "${f}.stashed-$TS" && n=$((n+1)) +done +rm -rf "$DIR"/*.lck 2>/dev/null || true +mc_log "stashed $n file(s) with suffix .stashed-$TS" + +kept=$(ls "$DIR"/${SERIAL_LOG_PREFIX}-*.log 2>/dev/null | wc -l) +mc_log "preserved $kept serial log(s) - they are this run's evidence" + +if [ "$PURGE" -eq 1 ]; then + old=$(find "$DIR" -name '*.stashed-*' 2>/dev/null | wc -l) + find "$DIR" -name '*.stashed-*' -delete 2>/dev/null || true + mc_log "purged $old stashed file(s) to reclaim space" +fi diff --git a/staging/mission-control/bin/mc-verify.sh b/staging/mission-control/bin/mc-verify.sh new file mode 100755 index 00000000..01a08bf3 --- /dev/null +++ b/staging/mission-control/bin/mc-verify.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# mc-verify.sh - run the oracle against one installed permutation and harvest logs. +# usage: mc-verify.sh --id [--ip ] +set -u +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "$_here/../lib/common.sh" +. "$_here/../lib/oracle.sh" +. "$(mc_find_config "$_here")" + +PERM="" IP="" +while [ $# -gt 0 ]; do + case "$1" in + --id) PERM="$2"; shift 2 ;; + --ip) IP="$2"; shift 2 ;; + *) mc_die "unknown arg: $1" 64 ;; + esac +done +[ -n "$PERM" ] || mc_die "--id is required" 64 + +TSV="$_here/../config/permutations.tsv" +read -r _ ISO_TYPE POI STIG FS MODE VARIANT DOC EXPECT </dev/null || true +mc_oracle_install "$SER" + +# --- guest --------------------------------------------------------------- +# Discover the address rather than assuming it: an interactive install may +# have taken a DHCP lease the kickstart never pinned. +if [ -z "$IP" ]; then + IP=$("$VMRUN" -T ws getGuestIPAddress "$DIR/$VM.vmx" -wait 2>/dev/null | tr -d '\r' | tail -1) + case "$IP" in *.*.*.*) ;; *) IP="" ;; esac +fi + +if [ -n "$IP" ]; then + mc_check guest.ip "-" info "" "$IP" "" + export SSHPASS="$MC_GUEST_PASSWORD" + guest_run() { + sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o BatchMode=no -o LogLevel=ERROR \ + "${SSH_USER}@${IP}" "$@" 2>/dev/null + } + if guest_run true; then + mc_oracle_guest guest_run "$STIG" "$FS" + mc_oracle_harvest guest_run "$HARVEST" + else + mc_check guest.ssh "-" fail "reachable" "unreachable" "no ssh to $IP" + fi +else + mc_check guest.ip "-" fail "discovered" "none" "vmrun getGuestIPAddress returned nothing" +fi + +mc_result_summary diff --git a/staging/mission-control/config/mission-control.env b/staging/mission-control/config/mission-control.env new file mode 100644 index 00000000..dfa851ac --- /dev/null +++ b/staging/mission-control/config/mission-control.env @@ -0,0 +1,77 @@ +# mission-control.env - single source of paths, sizes, network and identity. +# +# EVERY assignment is ${VAR:-default} on purpose. vm-lab/config/vm-lab.env +# assigns unconditionally, so exporting a variable before calling its scripts +# is silently overwritten - the documented per-run overrides do not work there. +# A permutation driver has to be able to set these per run, so this file must +# never regress to plain assignment. + +# ---- host tooling ------------------------------------------------------- +: "${VMWARE_DIR_WIN:=C:\\Program Files\\VMware\\VMware Workstation}" +: "${VMRUN:=/mnt/c/Program Files/VMware/VMware Workstation/vmrun.exe}" +: "${VDISKMANAGER:=/mnt/c/Program Files/VMware/VMware Workstation/vmware-vdiskmanager.exe}" + +# ---- where VMs live ----------------------------------------------------- +# Deliberately NOT under OneDrive: the existing hand-made test VM sits in +# OneDrive/Dokumente/Virtual Machines, where the sync client can touch a +# running VM's files. C: is also ~97% full, so mission control tears each VM +# down after verification instead of keeping 16 alive. +: "${MC_VM_ROOT_WIN:=C:\\photon-mc\\vm}" +: "${MC_VM_ROOT_WSL:=/mnt/c/photon-mc/vm}" + +# ---- artefacts ---------------------------------------------------------- +: "${MC_ISO_CACHE:=/root/photon-mc/iso-cache}" # one ISO per build-axis tuple +: "${MC_RESULTS_DIR:=/root/photon-mc/results}" # per-run JSON + harvested logs +: "${MC_BUILD_LOG_DIR:=/root/photon-mc/build-logs}" +: "${MC_WORK:=/root/photon-mc/work}" +: "${MC_VARIANT_PATCH_DIR:=/root/photon-mc/variant-patches}" + +# ---- photon build tree -------------------------------------------------- +: "${PHOTON_TREE:=/root/5.0}" +# NOTE: runPh5_normal.sh resolves downstream-fixes.patch RELATIVE TO ITSELF +# ($SCRIPT_DIR/photonos-patches/...). There are two copies of both the script +# and the patch on this host, and the one in the scripts repo carried a stale +# 8-file patch that no longer applies, while /root carries the live 27-file +# one. Pointing at the wrong pair fails the build guard with +# "does not apply", which reads like a rebase problem rather than a path +# problem. mc-preflight.sh now asserts the two agree. +: "${PHOTON_SCRIPTS:=/root}" +: "${DOWNSTREAM_PATCH:=/root/photonos-patches/downstream-fixes.patch}" + +# ---- guest defaults ----------------------------------------------------- +: "${GUEST_VCPUS:=2}" +: "${GUEST_MEM_MB:=4096}" +: "${BOOT_DISK_SIZE:=32GB}" +: "${BOOT_DISK_ADAPTER:=lsilogic}" +# -t 0 = monolithicSparse: single file, thin. The hand-made test VM is +# monolithicFlat and commits its full size up front - 16 of those will not fit +# in the 138 GB free on C:. +: "${BOOT_DISK_TYPE:=0}" + +# ---- network ------------------------------------------------------------ +# VMnet8 (NAT) is 192.168.225.0/24, DHCP .128-.254, gateway .1 - read from +# C:\ProgramData\VMware\vmnetdhcp.conf. Mission control assigns each +# permutation a static address BELOW the DHCP floor so it can never collide +# with a lease, and pins MAC+UUID so the address is reproducible. +: "${MC_NET_PREFIX:=192.168.225}" +: "${MC_NET_GATEWAY:=${MC_NET_PREFIX}.2}" +: "${MC_NET_DNS:=${MC_NET_PREFIX}.2}" +: "${MC_NET_CIDR:=24}" +: "${MC_IP_BASE:=40}" # permutation N -> .(MC_IP_BASE + N), i.e. .41 upward +: "${MC_MAC_PREFIX:=00:0c:29}" # VMware OUI + +# ---- ssh ---------------------------------------------------------------- +: "${SSH_KEY_DIR:=${HOME}/.ssh}" +: "${SSH_KEY_NAME:=photon-mc-ed25519}" +# Stock Photon has no 'operator' user; vm-lab's default is SPAGAT-specific. +: "${SSH_USER:=root}" +# The lab root password baked into generated kickstarts. Overridable so the +# oracle can also be pointed at a VM that mission control did not install. +: "${MC_GUEST_PASSWORD:=MissionControl123!}" + +# ---- timing ------------------------------------------------------------- +: "${SERIAL_LOG_PREFIX:=serial0}" +: "${MC_INSTALL_TIMEOUT_SEC:=2400}" +: "${MC_BOOT_TIMEOUT_SEC:=600}" +: "${MC_SSH_TIMEOUT_SEC:=300}" +: "${MC_SAMPLE_SEC:=25}" diff --git a/staging/mission-control/config/permutations.tsv b/staging/mission-control/config/permutations.tsv new file mode 100644 index 00000000..eb006e9a --- /dev/null +++ b/staging/mission-control/config/permutations.tsv @@ -0,0 +1,61 @@ +# Photon OS ISO permutation matrix - the executable form of ISO-PERMUTATION-MATRIX.md +# +# Columns +# id stable identifier; also the VM name and the last IP octet offset +# iso_type minimal | full BUILD-TIME axis -> selects/creates an ISO +# poi 2.8 | latest BUILD-TIME axis -> selects/creates an ISO +# stig no | yes INSTALL-TIME +# fs ext4 | btrfs INSTALL-TIME +# mode ui | ks INSTALL-TIME (the fifth dimension) +# ks_variant - ks only: none|selinux|fips|stigpkgs +# doc the verdict ISO-PERMUTATION-MATRIX.md records (pre-PR) +# expect what mission control expects WITH all PRs applied +# +# doc vs expect is the whole point: where they differ, the PRs are doing work, +# and a run that matches doc instead of expect is a PR regression. +# +# On mode=ks + stig=yes: the STIG menu is UI-only (stigenable.py is reached +# only from the curses configurator), so a kickstart cannot "answer yes". The +# ks rows express STIG by listing KS_STIG_PACKAGES manually - which is what +# ks_variant=stigpkgs does. This is a genuinely different code path, not a +# duplicate of the UI row. +# +#id iso_type poi stig fs mode ks_variant doc expect +p01 minimal 2.8 no ext4 ui - works pass +p02 minimal 2.8 no btrfs ui - untested pass +p03 minimal 2.8 yes ext4 ui - fails pass +p04 minimal 2.8 yes btrfs ui - fails pass +p05 minimal latest no ext4 ui - fails pass +p06 minimal latest no btrfs ui - fails pass +p07 minimal latest yes ext4 ui - fails pass +p08 minimal latest yes btrfs ui - fails pass +p09 full 2.8 no ext4 ui - works pass +p10 full 2.8 no btrfs ui - untested pass +p11 full 2.8 yes ext4 ui - works pass +p12 full 2.8 yes btrfs ui - untested pass +p13 full latest no ext4 ui - untested pass +p14 full latest no btrfs ui - untested pass +p15 full latest yes ext4 ui - untested pass +p16 full latest yes btrfs ui - untested pass +k01 minimal 2.8 no ext4 ks none works pass +k02 minimal 2.8 no btrfs ks none untested pass +k03 minimal 2.8 yes ext4 ks stigpkgs fails pass +k04 minimal 2.8 yes btrfs ks stigpkgs fails pass +k05 minimal latest no ext4 ks none fails pass +k06 minimal latest no btrfs ks none fails pass +k07 minimal latest yes ext4 ks stigpkgs fails pass +k08 minimal latest yes btrfs ks stigpkgs fails pass +k09 full 2.8 no ext4 ks none untested pass +k10 full 2.8 no btrfs ks none untested pass +k11 full 2.8 yes ext4 ks stigpkgs untested pass +k12 full 2.8 yes btrfs ks stigpkgs untested pass +k13 full latest no ext4 ks none untested pass +k14 full latest no btrfs ks none untested pass +k15 full latest yes ext4 ks stigpkgs untested pass +k16 full latest yes btrfs ks stigpkgs untested pass +# Kickstart-only failure class (section 11.3). These have NO UI counterpart: +# the security: key is synthesised for everyone on POI latest, but on 2.8 it +# is only present if the kickstart author writes it. fips is never appended on +# the UI path on either version, so s02 is reachable exclusively via kickstart. +s01 minimal 2.8 no ext4 ks selinux fails pass +s02 minimal 2.8 no ext4 ks fips fails pass diff --git a/staging/mission-control/config/photon-matrix.vmx.template b/staging/mission-control/config/photon-matrix.vmx.template new file mode 100644 index 00000000..4f20f4fc --- /dev/null +++ b/staging/mission-control/config/photon-matrix.vmx.template @@ -0,0 +1,92 @@ +#!/usr/bin/vmware +.encoding = "UTF-8" +config.version = "8" +virtualHW.version = "21" +displayName = "@@VM_NAME@@" + +# Photon guest type - matches the hand-verified test VM on this host. +guestOS = "vmware-photon-64" + +numvcpus = "@@GUEST_VCPUS@@" +memSize = "@@GUEST_MEM_MB@@" + +# --- boot disk ------------------------------------------------------------ +# lsilogic to match what the Photon initrd expects. +scsi0.present = "TRUE" +scsi0.virtualDev = "lsilogic" +scsi0.pciSlotNumber = "16" +scsi0:0.present = "TRUE" +scsi0:0.deviceType = "scsi-hardDisk" +scsi0:0.fileName = "@@VM_NAME@@.vmdk" + +# --- install media -------------------------------------------------------- +# The CDROM MUST be on SATA, not IDE. On ide1:0 the installer boots but +# userspace cannot find /dev/sr0, and the resulting "volume label not found" +# failure reads exactly like a corrupt ISO. The verified-working test VM on +# this host also carries the ISO on sata0:1. +sata0.present = "TRUE" +sata0.pciSlotNumber = "35" +sata0:1.present = "TRUE" +sata0:1.deviceType = "cdrom-image" +sata0:1.fileName = "@@ISO_PATH_WIN@@" +sata0:1.startConnected = "TRUE" + +# --- firmware ------------------------------------------------------------- +# EFI, matching the shipped ISO's grub2/EFI boot path. +# NOTE: bios.bootOrder is IGNORED on EFI VMs. Boot source is controlled by +# NVRAM, which is why teardown must move the .nvram aside - otherwise UEFI's +# removable-media fallback finds the previous ESP and boots the OLD image. +firmware = "efi" +uefi.secureBoot.enabled = "@@SECUREBOOT@@" + +# --- network -------------------------------------------------------------- +# Static MAC in VMware's manual OUI (00:50:56:00:00:00 - 00:50:56:3F:FF:FF). +# vm-lab pins addressType="generated" plus the BIOS UUID, because VMware +# DERIVES a generated MAC from the UUID - which couples two values that then +# must be changed in lockstep. addressType="static" decouples them, so each +# permutation gets its own MAC without any UUID arithmetic. +ethernet0.present = "TRUE" +ethernet0.connectionType = "nat" +ethernet0.virtualDev = "@@NIC_DEV@@" +ethernet0.addressType = "static" +ethernet0.address = "@@GUEST_MAC@@" +ethernet0.pciSlotNumber = "160" + +# Still pinned, and still distinct per permutation: a shared UUID would make +# VMware treat every clone as the same machine. +uuid.action = "keep" +uuid.bios = "@@UUID_BIOS@@" +uuid.location = "@@UUID_BIOS@@" + +# --- serial console ------------------------------------------------------- +# The only unambiguous liveness instrument during an unattended install: +# whether this file grows. Everything else (vmrun list, CPU%) gives false +# readings. vmrun list in particular OMITS GUI-started VMs. +serial0.present = "TRUE" +serial0.fileType = "file" +serial0.fileName = "@@SERIAL_LOG_WIN@@" +serial0.startConnected = "TRUE" +serial0.yieldOnMsrRead = "TRUE" + +# --- unattended-install hygiene ------------------------------------------ +# Without autoAnswer a modal dialog blocks power-on forever, with no output. +msg.autoAnswer = "TRUE" +# The guest clock must not be host-slaved; time-sync behaviour is under test. +tools.syncTime = "FALSE" + +# --- kickstart injection -------------------------------------------------- +# POI's isoInstaller reads guestinfo.kickstart.data (base64) via vmtoolsd, +# verified present in the installer initrd as /usr/bin/vmtoolsd with +# open-vm-tools in packages_installer_initrd.json. This is why an autonomous +# permutation needs NO ISO remaster and no boot-menu typing. +# For mode=ui this line is removed entirely, so the installer finds no +# kickstart and falls through to the curses configurator. +@@GUESTINFO_KICKSTART@@ + +# --- trimmed hardware ----------------------------------------------------- +usb.present = "FALSE" +ehci.present = "FALSE" +sound.present = "FALSE" +floppy0.present = "FALSE" +vmci0.present = "TRUE" +svga.autodetect = "TRUE" diff --git a/staging/mission-control/lib/common.sh b/staging/mission-control/lib/common.sh new file mode 100644 index 00000000..fc3f9d35 --- /dev/null +++ b/staging/mission-control/lib/common.sh @@ -0,0 +1,137 @@ +# common.sh - shared helpers. Source, do not execute. +# +# Conventions carried over from vm-lab because they were each learned the hard +# way there: +# * always 'grep -a' a serial log - it contains NUL bytes and plain grep +# silently prints nothing +# * 'n=$(grep -c ...) || n=0', never '$(grep -c ... || echo 0)' +# * every check that can be vacuous carries a negative control +# * print the measured value, not a bare OK/FAIL - "tool missing" and "tool +# present but unreadable by this user" need different fixes and look +# identical in a boolean +# What is new here: every check emits a machine-readable record and a non-zero +# exit propagates. vm-lab's 40-check-staging.sh never exits non-zero, which is +# fine for an inspection tool and useless for a matrix. + +set -u + +mc_find_config() { + local here="${1:?}" c + for c in "${MC_DIR:-}/config/mission-control.env" \ + "${here}/../config/mission-control.env" \ + "${here}/config/mission-control.env" \ + "${PWD}/config/mission-control.env" \ + "${PWD}/../config/mission-control.env"; do + [ -n "$c" ] && [ -f "$c" ] && { printf '%s\n' "$c"; return 0; } + done + echo "FAIL: mission-control.env not found. Export MC_DIR=/path/to/mission-control." >&2 + exit 78 +} + +# ---- structured results ------------------------------------------------- +# Every assertion lands in $MC_RESULT_FILE as one JSON object per line. +# 'pr' names the pull request the assertion proves, so a failure reads as +# "PR #22 regressed" rather than "something broke". +mc_result_init() { + MC_PERM_ID="${1:?perm id}" + MC_RESULT_FILE="${MC_RESULTS_DIR}/${MC_PERM_ID}/checks.jsonl" + mkdir -p "$(dirname "$MC_RESULT_FILE")" + : > "$MC_RESULT_FILE" + MC_FAILED=0 +} + +mc_json_escape() { printf '%s' "${1-}" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read())[1:-1])'; } + +# mc_check +mc_check() { + local id="$1" pr="$2" st="$3" exp="$4" act="$5" det="${6-}" + printf '{"perm":"%s","check":"%s","pr":"%s","status":"%s","expected":"%s","actual":"%s","detail":"%s"}\n' \ + "$MC_PERM_ID" "$id" "$pr" "$st" \ + "$(mc_json_escape "$exp")" "$(mc_json_escape "$act")" "$(mc_json_escape "$det")" \ + >> "$MC_RESULT_FILE" + case "$st" in + pass) printf ' PASS %-34s %-10s %s\n' "$id" "$pr" "$act" ;; + fail) printf ' FAIL %-34s %-10s expected=%s actual=%s\n' "$id" "$pr" "$exp" "$act"; MC_FAILED=$((MC_FAILED+1)) ;; + skip) printf ' skip %-34s %-10s %s\n' "$id" "$pr" "$det" ;; + *) printf ' info %-34s %-10s %s\n' "$id" "$pr" "$act" ;; + esac +} + +# mc_expect [detail] +mc_expect() { + local id="$1" pr="$2" exp="$3" act="$4" det="${5-}" + if [ "$exp" = "$act" ]; then mc_check "$id" "$pr" pass "$exp" "$act" "$det" + else mc_check "$id" "$pr" fail "$exp" "$act" "$det"; fi +} + +mc_result_summary() { + local total pass fail + total=$(wc -l < "$MC_RESULT_FILE") + pass=$(grep -c '"status":"pass"' "$MC_RESULT_FILE") || pass=0 + fail=$(grep -c '"status":"fail"' "$MC_RESULT_FILE") || fail=0 + printf '\n %s: %s checks, %s pass, %s fail\n' "$MC_PERM_ID" "$total" "$pass" "$fail" + if [ "$fail" -gt 0 ]; then + printf ' PRs implicated:\n' + grep '"status":"fail"' "$MC_RESULT_FILE" \ + | sed -n 's/.*"pr":"\([^"]*\)".*/ \1/p' | sort -u + fi + return "$([ "$fail" -eq 0 ] && echo 0 || echo 1)" +} + +# ---- serial log helpers ------------------------------------------------- +# Strip NULs and SGR sequences. vm-lab had this right on one line and wrong on +# another (s/...*g//g instead of *m//g); keeping it in one function so it +# cannot drift again. +mc_clean_log() { tr -d '\000' | sed -E 's/\x1b\[[0-9;]*m//g'; } + +# Do NOT use "grep -a" here. On this host /usr/bin/grep is toybox in a +# non-interactive shell (interactively it is ugrep), and toybox grep has no +# -a: it returns 0 matches on a NUL-bearing file instead of erroring. That is +# the same silent-zero trap vm-lab documents, reached by a different route. +# Stripping NULs first is portable across all three greps. +mc_grep_count() { + local n + n=$(tr -d '\000' < "$2" 2>/dev/null | grep -c "$1" 2>/dev/null) || n=0 + printf '%s' "$n" +} + +# ---- identity ----------------------------------------------------------- +# Deterministic per-permutation MAC/UUID/IP. VMware's manual-assignment OUI is +# 00:50:56:00:00:00-00:50:56:3F:FF:FF; staying inside it means the address is +# ours and is never derived from the UUID. +# Index = the permutation's ordinal in permutations.tsv, NOT a hash of its id. +# A cksum-based index collided on this very matrix (k04/k16 and k09/s02 shared +# an index, and therefore a MAC, a UUID and an IP), and could reach .240 - +# inside VMnet8's DHCP range of .128-.254. An ordinal is unique by +# construction and stays bounded, so the addresses can never collide with a +# lease or with each other. +mc_perm_index() { + local id="$1" tsv="${MC_PERM_FILE:-${MC_DIR:-}/config/permutations.tsv}" n + [ -f "$tsv" ] || tsv="$(dirname "${BASH_SOURCE[0]}")/../config/permutations.tsv" + n=$(grep -vE '^#|^$' "$tsv" | awk -v want="$id" '$1==want{print NR; exit}') + [ -n "$n" ] || mc_die "permutation '$id' is not in $tsv" 65 + # .41 upward; the matrix would have to exceed 80 rows to reach the DHCP floor. + [ "$n" -le 80 ] || mc_die "permutation ordinal $n would push the IP into the DHCP range" 65 + printf '%s' "$n" +} + +mc_mac_for() { printf '00:50:56:3a:%02x:%02x' $(( ${1} / 256 )) $(( ${1} % 256 )); } +mc_uuid_for() { printf '56 4d 6d 63 00 00 00 00-00 00 00 00 00 00 %02x %02x' $(( ${1} / 256 )) $(( ${1} % 256 )); } +mc_ip_for() { printf '%s.%d' "$MC_NET_PREFIX" $(( MC_IP_BASE + ${1} )); } + +# /mnt/c/foo/bar -> C:\foo\bar. tr, not sed's \U: /usr/bin/sed here is +# toybox in a non-interactive shell and emits a literal "U" for that GNU +# extension. vmrun.exe and vmware-vdiskmanager.exe both need Windows form. +mc_win_path() { + local p="$1" drive rest + case "$p" in + /mnt/?/*) + drive=$(printf '%s' "$p" | cut -c6 | tr 'a-z' 'A-Z') + rest=$(printf '%s' "$p" | cut -c7- | tr '/' '\\') + printf '%s:%s' "$drive" "$rest" ;; + *) printf '%s' "$p" | tr '/' '\\' ;; + esac +} + +mc_log() { printf '[mc] %s\n' "$*"; } +mc_die() { printf '[mc] FAIL: %s\n' "$*" >&2; exit "${2:-1}"; } diff --git a/staging/mission-control/lib/oracle.sh b/staging/mission-control/lib/oracle.sh new file mode 100644 index 00000000..261c731c --- /dev/null +++ b/staging/mission-control/lib/oracle.sh @@ -0,0 +1,164 @@ +# oracle.sh - the pass/fail assertions. Source after common.sh. +# +# Every assertion names the PR it proves. That is the point: a run does not +# report "something broke", it reports "PR #22 regressed", because the whole +# purpose of this harness is to make PR issues identifiable. +# +# The permutation matrix supplies a *dependency-resolution* oracle only +# (Error(1011) vs Error(1032), media RPM presence). It supplies nothing for +# dmesg / journalctl / /var/log. Sections C and D below are new work. + +# ---- A. media, before any VM exists ------------------------------------- +# Six packages that the matrix records as ABSENT from minimal media. Their +# presence is what POI#11 (the doc's FIX-1b) delivers, and their absence is +# the root cause of matrix rows 3,4,7,8 - and, via selinux-policy, 5,6. +MC_STIG_MEDIA_PKGS="rsyslog openssl-fips-provider selinux-policy libselinux-utils aide" + +mc_oracle_media() { + local iso="$1" iso_type="$2" + local list; list=$(xorriso -osirrox on -indev "$iso" -find /RPMS -name '*.rpm' 2>/dev/null | sed 's|.*/||' | tr -d "'") + local n; n=$(printf '%s\n' "$list" | grep -c '\.rpm$') || n=0 + mc_check media.rpm_count "-" info "" "$n" "RPMs on media" + + # Negative control: a name that must never resolve. Without it a broken + # extraction would make every presence check vacuously pass. + local ctl; ctl=$(printf '%s\n' "$list" | grep -cE '^zzz-not-a-real-package-[0-9]') || ctl=0 + mc_expect media.negative_control "-" "0" "$ctl" "control must find nothing" + + local missing="" p c + for p in $MC_STIG_MEDIA_PKGS; do + c=$(printf '%s\n' "$list" | grep -cE "^${p}-[0-9]") || c=0 + [ "$c" -eq 0 ] && missing="$missing $p" + done + # ntp is a capability satisfied by ntpsec; no package is literally named ntp. + c=$(printf '%s\n' "$list" | grep -cE '^ntpsec-[0-9]') || c=0 + [ "$c" -eq 0 ] && missing="$missing ntpsec" + mc_expect media.stig_packages "POI#11" "" "${missing# }" "STIG set must be on the media for minimal-iso" + + # Stale-RPM shadowing: tdnf picks the highest release, so a months-old + # photon-os-installer left in stage/RPMS silently wins and ends up on the + # ISO. Record what actually shipped. + local poi; poi=$(printf '%s\n' "$list" | grep -oE '^photon-os-installer-[0-9][^ ]*\.rpm' | head -1) + mc_check media.poi_rpm "-" info "" "${poi:-ABSENT}" "installer actually on the media" +} + +# ---- B. install phase, from the serial log ------------------------------ +mc_oracle_install() { + local serial="$1" + [ -f "$serial" ] || { mc_check install.serial_log "-" fail "present" "missing" "$serial"; return 1; } + + # Error(1011) is a genuine resolution failure. Error(1032) is only ever a + # --assumeno dry-run artifact and must NOT be treated as a real-install + # signal. Never match a specific package name: list(set(packages)) makes + # which of the six tdnf reports first non-deterministic. + local e1011; e1011=$(mc_grep_count 'Error(1011)' "$serial") + mc_expect install.no_error_1011 "POI#11" "0" "$e1011" "No matching packages" + + local efail; efail=$(mc_grep_count 'Failed to install some packages' "$serial") + mc_expect install.packages_installed "POI#11" "0" "$efail" "" + + # The i18n error proves the locale.conf ordering fix did NOT apply. + local i18n; i18n=$(mc_grep_count 'i18n_vars not set' "$serial") + mc_expect install.no_i18n_error "POI#10" "0" "$i18n" "dracut 20i18n needs /etc/locale.conf at initrd build time" + + # The single most valuable completion signal: the boot source moves from + # the installer live env to the installed disk. + local ram parts + ram=$(mc_grep_count 'root=/dev/ram0' "$serial") + parts=$(mc_grep_count 'root=PARTUUID=' "$serial") + mc_check install.boot_ram0 "-" info "" "$ram" "installer live-env boots" + mc_expect install.booted_from_disk "-" "yes" "$([ "$parts" -gt 0 ] && echo yes || echo no)" \ + "root=PARTUUID= means the install completed and the VM rebooted off disk" + + local ansfail; ansfail=$(mc_grep_count 'AssertionError' "$serial") + mc_expect install.ansible_no_assert "PR#9" "0" "$ansfail" "installer.py asserts on playbook returncode" +} + +# ---- C. post-boot, over ssh --------------------------------------------- +# $1 is a function name that runs a command in the guest and prints stdout. +mc_oracle_guest() { + local run="$1" stig="$2" fs="$3" canister="${4:-prebuilt}" + local v + + v=$($run 'findmnt -no FSTYPE /' 2>/dev/null | tr -d ' ') + mc_expect guest.root_fstype "-" "$fs" "${v:-unknown}" "the filesystem axis actually took effect" + + v=$($run 'getenforce' 2>/dev/null | tr -d ' ') + if [ "$stig" = yes ]; then + mc_expect guest.selinux "PR#9" "Enforcing" "${v:-unknown}" "" + else + mc_check guest.selinux "PR#9" info "" "${v:-unknown}" "" + fi + + # PR#22: both group regressions are visible in the journal of every boot. + v=$($run "journalctl -b --no-pager 2>/dev/null | grep -c \"Unknown group 'render'\"" 2>/dev/null | tr -d ' ') + mc_expect guest.no_render_group "PR#22" "0" "${v:-?}" "dangling accel rule in 50-udev-default.rules" + v=$($run "journalctl -b --no-pager 2>/dev/null | grep -c \"resolve group 'systemd-journal'\"" 2>/dev/null | tr -d ' ') + mc_expect guest.no_journal_group "PR#22" "0" "${v:-?}" "initrd sysusers snippet emptied by systemd patch 0004" + + # PR#22 again: /tmp hardening is delivered at build time because + # tmp.mount is package-owned and not %config; the installer deliberately + # skips the equivalent ansible control PHTN-50-000245. + if [ "$stig" = yes ]; then + v=$($run 'findmnt -no OPTIONS /tmp' 2>/dev/null | grep -c noexec) || v=0 + mc_check guest.tmp_noexec "PR#22" info "" "$v" "1 once STIG_HARDEN builds are enabled" + fi + + # POI#9: exactly five STIG packages requested, not eight. + v=$($run "zcat /var/log/poi/manifest.json.gz 2>/dev/null | python3 -c \"import json,sys;print(len(json.load(sys.stdin)['install_config'].get('additional_packages',[])))\"" 2>/dev/null | tr -d ' ') + if [ "$stig" = yes ]; then + mc_expect guest.stig_pkg_count "POI#9" "5" "${v:-?}" "libselinux-utils, ntp, libgcrypt dropped as redundant" + fi + + # The matrix's own cheap assertion: stig-hardening runs from the initrd + # and must never land on the target. + v=$($run 'rpm -q stig-hardening >/dev/null 2>&1 && echo installed || echo absent' 2>/dev/null | tr -d ' ') + mc_expect guest.stig_not_on_target "-" "absent" "${v:-?}" "stig-hardening is not in KS_STIG_PACKAGES" + + # PR#21: versioned libgcrypt only at subrelease >= 91. + v=$($run 'rpm -q --requires aide 2>/dev/null | grep -c "libgcrypt >= 1.10.4"' 2>/dev/null | tr -d ' ') + mc_check guest.aide_libgcrypt "PR#21" info "" "${v:-0}" "1 only when built at subrelease >= 91" + + # POI#9 counterpart: time sync works without ntp being installed. + v=$($run 'timedatectl show -p NTPSynchronized --value' 2>/dev/null | tr -d ' ') + mc_check guest.time_synced "POI#9" info "" "${v:-?}" "systemd-timesyncd, not ntp" + + # Canister/FIPS, when the ISO was built with one. + v=$($run 'cat /proc/sys/crypto/fips_enabled 2>/dev/null' 2>/dev/null | tr -d ' ') + mc_check guest.fips_enabled "PR#24" info "" "${v:-0}" "" + v=$($run 'dmesg 2>/dev/null | grep -c "canister verification passed"' 2>/dev/null | tr -d ' ') + mc_check guest.fips_canister "PR#24" info "" "${v:-0}" "" + + v=$($run 'systemctl --failed --no-legend --no-pager 2>/dev/null | wc -l' 2>/dev/null | tr -d ' ') + mc_expect guest.failed_units "PR#9" "0" "${v:-?}" "first boot may race the SELinux relabel; second boot must be clean" + + v=$($run 'journalctl -b --no-pager 2>/dev/null | grep -ci "avc: *denied"' 2>/dev/null | tr -d ' ') + mc_check guest.avc_denials "PR#9" info "" "${v:-?}" "non-zero on first boot is the documented relabel race" +} + +# ---- D. log harvest ------------------------------------------------------ +# The matrix defines no dmesg/journalctl//var/log criteria at all, so this +# collects the evidence rather than asserting on it - except for the two +# counts, which are cheap regression detectors. +mc_oracle_harvest() { + local run="$1" dest="$2" + mkdir -p "$dest" + $run 'dmesg' > "$dest/dmesg.txt" 2>/dev/null + $run 'journalctl -b --no-pager' > "$dest/journal-boot.txt" 2>/dev/null + $run 'journalctl -p err -b --no-pager' > "$dest/journal-err.txt" 2>/dev/null + $run 'systemctl --failed --no-pager' > "$dest/failed-units.txt" 2>/dev/null + $run 'rpm -qa | sort' > "$dest/rpm-qa.txt" 2>/dev/null + $run 'cat /proc/cmdline' > "$dest/cmdline.txt" 2>/dev/null + $run 'findmnt -A' > "$dest/mounts.txt" 2>/dev/null + for f in installer.log ansible-stig.log messages; do + $run "cat /var/log/$f 2>/dev/null" > "$dest/varlog-$f" 2>/dev/null + done + $run 'cat /var/log/mkinitrd-*.log 2>/dev/null' > "$dest/varlog-mkinitrd.txt" 2>/dev/null + $run 'zcat /var/log/poi/manifest.json.gz 2>/dev/null' > "$dest/poi-manifest.json" 2>/dev/null + + local d j + d=$(grep -cE '\] (BUG|WARNING|Oops|Call Trace)' "$dest/dmesg.txt" 2>/dev/null) || d=0 + mc_expect logs.dmesg_no_bug "-" "0" "$d" "kernel BUG/WARNING/Oops in dmesg" + j=$(wc -l < "$dest/journal-err.txt" 2>/dev/null) || j=0 + mc_check logs.journal_err_lines "-" info "" "$j" "harvested to journal-err.txt" +}