From 8ab23e248f844c6691a6634d61f7c15210c86b91 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sun, 20 Sep 2026 18:34:18 +0000 Subject: [PATCH 1/4] test(installer): behavioural contract for the QEMU boot-proof readiness gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test-installer-artifact (the recipe show-me-the-future delegates to) polls /healthz and / after booting the installed target in QEMU, and only breaks out of the loop when the healthz body reports status "ok" AND the root probe answers HTTP 200. Nothing exercised that decision logic before this change: a prior attempt at issue #130 (PR #179) sliced the recipe body out of the Justfile as text and asserted substrings, so every assertion stayed green whether the underlying checks were correct, weakened, or deleted. This suite instead runs the real recipe in a private sandbox copy of the Justfile (the tests/unit/flash-installer_test.bats pattern), with qemu-system-x86_64, curl and zstd replaced by logging stubs on PATH so 'just test-installer-artifact' actually executes. It proves, by running the recipe rather than describing it: - readiness requires both conditions together, not either alone - a failing/malformed healthz body short-circuits before the root probe is ever called - a QEMU process that dies before answering is detected and reported, independent of curl/jq state - the serial log tail is dumped on both failure paths (timeout and death) One substitution is applied to the sandboxed Justfile, asserted the same way flash-installer_test.bats asserts its own: the OVMF_CODE lookup only searches hardcoded absolute host paths, none of which exist on an unprivileged CI runner, so the recipe would abort before ever reaching the readiness loop. An env-gated candidate is added to the front of the existing first_existing() call; unset, it defaults to a nonexistent path, so production behaviour is unchanged. Verified non-vacuous: dropping the root-code check (collapsing the two conditions into one, as a plausible regression) fails 2 of the 10 cases; restoring it is green again. os-base: flatcar (#130's acceptance line) does not exist on main yet — no such build option exists anywhere in the tree, and the whole Flatcar-base cutover cluster (#132/#133/#136/#140/#163/#191) is parked 2-discussing pending the hard-rule-1 ADR decision on #133. This does not close #130; it pins the readiness contract that show-me-the-future/test-installer-artifact already enforce today, independent of which payload eventually boots under it, so the gate stays guarded while that decision is pending. Assisted-by: Claude via Claude Code Co-authored-by: Claude Signed-off-by: Danathar --- tests/unit/test-installer-artifact_test.bats | 339 +++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100755 tests/unit/test-installer-artifact_test.bats diff --git a/tests/unit/test-installer-artifact_test.bats b/tests/unit/test-installer-artifact_test.bats new file mode 100755 index 00000000..d85f2360 --- /dev/null +++ b/tests/unit/test-installer-artifact_test.bats @@ -0,0 +1,339 @@ +#!/usr/bin/env bats +# +# Behavioural contract for the readiness-decision logic inside the +# `test-installer-artifact` recipe (the recipe `show-me-the-future` delegates +# to after `build-installer`/`export-installer`). This is the boot-proof gate +# named in issue #130. +# +# A prior attempt at this issue (#179) was closed because its test sliced +# recipe bodies out of the Justfile as text and asserted substrings — every +# assertion passed whether the underlying health check was correct, weakened, +# or deleted, because it matched echo/log wording rather than the checks +# themselves. This suite instead runs the real recipe: a private sandbox copy +# of the Justfile, with `qemu-system-x86_64`, `curl` and `zstd` replaced by +# logging stubs on PATH so `just test-installer-artifact` executes for real +# and the decision logic (readiness requires BOTH a `{"status":"ok"}` +# `/healthz` body AND an HTTP 200 on `/`, exactly `flash-installer_test.bats`' +# behavioural style) is exercised rather than paraphrased. +# +# One substitution is applied to the sandboxed Justfile, and asserted exactly +# like `flash-installer_test.bats` asserts its own: the recipe's OVMF +# firmware lookup only searches hardcoded absolute host paths +# (/usr/share/OVMF/..., linuxbrew Cellar paths, ...), none of which exist on +# an unprivileged CI runner, so the recipe would abort before ever reaching +# the readiness loop. `first_existing` is given one extra, env-controlled +# candidate at the front of its list; left unset it defaults to a path that +# does not exist, so production behaviour is unchanged. QEMU itself never +# reads the firmware content the stub points at, so the file only needs to +# exist. +# +# `os-base: flatcar` (issue #130's acceptance line) does not exist on `main` +# — there is no such build option anywhere in the tree — so this suite proves +# the readiness contract that `show-me-the-future`/`test-installer-artifact` +# already enforce today, independent of which payload eventually boots under +# it. It does not, and cannot yet, close #130. + +setup() { + if ! command -v just >/dev/null 2>&1; then + skip "just is not installed" + fi + if ! command -v jq >/dev/null 2>&1; then + skip "jq is not installed" + fi + + REPO_ROOT="$(cd "${BATS_TEST_DIRNAME}/../.." && pwd)" + JUSTFILE="${REPO_ROOT}/Justfile" + SANDBOX="${BATS_TEST_TMPDIR}/sandbox" + STUB_DIR="${BATS_TEST_TMPDIR}/bin" + LOG="${BATS_TEST_TMPDIR}/calls.log" + + mkdir -p "$STUB_DIR" "$SANDBOX/dist" "$SANDBOX/cache" "$SANDBOX/elements" + : > "$LOG" + + # The recipe reads no repository state other than dist/, but the + # Justfile's top-level backtick assignments inspect + # elements/freedesktop-sdk.bst regardless of which recipe runs. Copy it + # so loading the sandbox Justfile behaves exactly like the real one. + cp "${REPO_ROOT}/elements/freedesktop-sdk.bst" "${SANDBOX}/elements/" + + # Seed the exported artefacts the recipe copies out of dist/. + : > "${SANDBOX}/dist/bluefin-server-installer-1.0.raw.zst" + : > "${SANDBOX}/dist/bluefin-server-pxe-vmlinuz-1.0" + : > "${SANDBOX}/dist/bluefin-server-pxe-initrd-1.0.cpio.gz" + + # A stand-in OVMF firmware file. QEMU is stubbed and never reads it; it + # only has to exist so the recipe's firmware lookup succeeds. + OVMF_STUB="${SANDBOX}/OVMF_CODE_stub.fd" + truncate -s 1024 "$OVMF_STUB" + + apply_ovmf_override_substitution + + make_zstd_stub + make_qemu_stub + make_curl_stub +} + +# The OVMF_CODE lookup (`Justfile:254`, duplicated verbatim at the same +# recipe boundary in `install-vm`) only searches hardcoded absolute paths +# that do not exist on an unprivileged CI runner. Insert one extra, +# env-gated candidate at the front of the `first_existing` call so the +# sandbox can point it at a stub file; asserted below so a future edit to +# that line is caught instead of silently un-sandboxing the suite. +apply_ovmf_override_substitution() { + python3 - "$JUSTFILE" "${SANDBOX}/Justfile" <<'PY' +import sys + +src, dst = sys.argv[1], sys.argv[2] +text = open(src, encoding="utf-8").read() + +old = ' OVMF_CODE=$(first_existing \\\n' +new = ' OVMF_CODE=$(first_existing "${OVMF_CODE_TEST_OVERRIDE:-/nonexistent-ovmf-code}" \\\n' +count = text.count(old) +assert count == 2, f"expected exactly 2 occurrences of the OVMF_CODE lookup, found {count}" + +open(dst, "w", encoding="utf-8").write(text.replace(old, new)) +PY +} + +@test "sandbox substitution: the OVMF_CODE lookup is patched exactly where expected" { + run grep -cF 'OVMF_CODE=$(first_existing "${OVMF_CODE_TEST_OVERRIDE:-' "${SANDBOX}/Justfile" + [ "$status" -eq 0 ] + [ "$output" -eq 2 ] +} + +# make_zstd_stub +# +# Records the invocation and materialises the `-o ` target so the +# recipe's subsequent `truncate`/`cp`/qemu-stub steps find the file they +# expect, without actually decompressing anything. +make_zstd_stub() { + cat > "${STUB_DIR}/zstd" <> "${LOG}" +args=("\$@") +for ((i = 0; i < \${#args[@]}; i++)); do + if [ "\${args[\$i]}" = "-o" ]; then + : > "\${args[\$((i + 1))]}" + fi +done +exit 0 +EOF + chmod +x "${STUB_DIR}/zstd" +} + +# make_qemu_stub +# +# The recipe invokes qemu-system-x86_64 twice: once in the foreground to run +# the installer to completion (`-serial mon:stdio`), once backgrounded to +# boot the installed target (`-serial file:`). The stub tells the two +# apart by the `-serial` value. The foreground call exits immediately +# (installer "completed"). The backgrounded call either dies immediately +# (QEMU_TARGET_DIES=1, simulating a boot crash) or stays alive, handling +# SIGTERM cleanly, until the recipe's own cleanup trap kills it — exactly +# what `kill -0 "$TARGET_QEMU_PID"` in the polling loop needs to observe. +make_qemu_stub() { + cat > "${STUB_DIR}/qemu-system-x86_64" <> "${LOG}" + +serial_arg="" +prev="" +for a in "\$@"; do + if [ "\$prev" = "-serial" ]; then + serial_arg="\$a" + fi + prev="\$a" +done + +case "\$serial_arg" in + file:*) + serial_file="\${serial_arg#file:}" + : > "\$serial_file" + if [ -n "\${QEMU_SERIAL_LOG_CONTENT:-}" ]; then + printf '%s\n' "\${QEMU_SERIAL_LOG_CONTENT}" >> "\$serial_file" + fi + if [ "\${QEMU_TARGET_DIES:-0}" = "1" ]; then + exit 1 + fi + trap 'exit 0' TERM INT + while true; do sleep 1; done + ;; + *) + exit "\${QEMU_INSTALL_EXIT:-0}" + ;; +esac +EOF + chmod +x "${STUB_DIR}/qemu-system-x86_64" +} + +# make_curl_stub +# +# Answers the two probes the readiness loop makes, driven entirely by env +# vars set per test: CURL_HEALTHZ_EXIT/CURL_HEALTHZ_BODY for `/healthz`, +# CURL_ROOT_EXIT/CURL_ROOT_CODE for `/`. Real `jq` (not stubbed) parses +# whatever body this stub prints, so the suite exercises the actual +# `jq -e '.status == "ok"'` check rather than a paraphrase of it. +make_curl_stub() { + cat > "${STUB_DIR}/curl" <> "${LOG}" + +url="" +for a in "\$@"; do url="\$a"; done + +case "\$url" in + */healthz) + [ "\${CURL_HEALTHZ_EXIT:-0}" = "0" ] || exit "\${CURL_HEALTHZ_EXIT}" + printf '%s' "\${CURL_HEALTHZ_BODY:-}" + exit 0 + ;; + *) + [ "\${CURL_ROOT_EXIT:-0}" = "0" ] || exit "\${CURL_ROOT_EXIT}" + printf '%s' "\${CURL_ROOT_CODE:-000}" + exit 0 + ;; +esac +EOF + chmod +x "${STUB_DIR}/curl" +} + +# run_test_installer_artifact [deadline-seconds] +# +# All CURL_*/QEMU_*/OVMF_CODE_TEST_OVERRIDE knobs are read from the calling +# test's exported environment; only the deadline is parameterised here since +# every test needs one. +run_test_installer_artifact() { + local deadline="${1:-5}" + run env PATH="${STUB_DIR}:${PATH}" \ + XDG_CACHE_HOME="${SANDBOX}/cache" \ + OVMF_CODE_TEST_OVERRIDE="${OVMF_STUB}" \ + SHOW_ME_THE_FUTURE_DEADLINE="${deadline}" \ + CURL_HEALTHZ_EXIT="${CURL_HEALTHZ_EXIT:-0}" \ + CURL_HEALTHZ_BODY="${CURL_HEALTHZ_BODY:-}" \ + CURL_ROOT_EXIT="${CURL_ROOT_EXIT:-0}" \ + CURL_ROOT_CODE="${CURL_ROOT_CODE:-}" \ + QEMU_TARGET_DIES="${QEMU_TARGET_DIES:-0}" \ + QEMU_SERIAL_LOG_CONTENT="${QEMU_SERIAL_LOG_CONTENT:-}" \ + just --justfile "${SANDBOX}/Justfile" \ + --working-directory "${SANDBOX}" \ + test-installer-artifact +} + +assert_log() { + if ! grep -qF -- "$1" "$LOG"; then + echo "expected call log to contain: $1" >&2 + cat "$LOG" >&2 + return 1 + fi +} + +refute_log() { + if grep -qF -- "$1" "$LOG"; then + echo "expected call log NOT to contain: $1" >&2 + cat "$LOG" >&2 + return 1 + fi +} + +# The root probe's URL has no path suffix; matching end-of-line distinguishes +# a real call to it from a call to /healthz (which also contains "8080/"). +assert_root_probe_attempted() { + if ! grep -qE 'curl .* http://127\.0\.0\.1:8080/$' "$LOG"; then + echo "expected the root probe (http://127.0.0.1:8080/) to have been called" >&2 + cat "$LOG" >&2 + return 1 + fi +} + +refute_root_probe_attempted() { + if grep -qE 'curl .* http://127\.0\.0\.1:8080/$' "$LOG"; then + echo "expected the root probe (http://127.0.0.1:8080/) NOT to have been called" >&2 + cat "$LOG" >&2 + return 1 + fi +} + +# --- guards: the checks this suite depends on are still the real ones ----- + +@test "guard: readiness still requires the healthz body to report status ok" { + run grep -cF 'jq -e '"'"'.status == "ok"'"'"'' "$JUSTFILE" + [ "$status" -eq 0 ] + [ "$output" -eq 1 ] +} + +@test "guard: readiness still requires the root probe to answer exactly 200" { + run grep -cF '[ "$ROOT_CODE" = "200" ]' "$JUSTFILE" + [ "$status" -eq 0 ] + [ "$output" -eq 1 ] +} + +# --- the readiness decision ------------------------------------------------- + +@test "declares readiness only once healthz reports ok status and root answers 200" { + export CURL_HEALTHZ_BODY='{"status":"ok"}' + export CURL_ROOT_CODE="200" + run_test_installer_artifact 10 + [ "$status" -eq 0 ] + [[ "$output" == *"KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200"* ]] + assert_root_probe_attempted +} + +@test "does not declare readiness when healthz is ok but the root probe never returns 200" { + export CURL_HEALTHZ_BODY='{"status":"ok"}' + export CURL_ROOT_CODE="503" + run_test_installer_artifact 1 + [ "$status" -ne 0 ] + [[ "$output" == *"Timed out after 1s waiting for KubeStellar Console readiness"* ]] + [[ "$output" != *"KubeStellar Console is healthy"* ]] + # The root probe having been reached proves the failure is the "never + # 200" branch, not an accidental short-circuit before it. + assert_root_probe_attempted +} + +@test "does not declare readiness when root answers 200 but healthz never reports ok" { + export CURL_HEALTHZ_BODY='{"status":"degraded"}' + export CURL_ROOT_CODE="200" + run_test_installer_artifact 1 + [ "$status" -ne 0 ] + [[ "$output" == *"Timed out after 1s waiting for KubeStellar Console readiness"* ]] + [[ "$output" != *"KubeStellar Console is healthy"* ]] + # healthz status must gate the root probe: a failing/absent status must + # never let the loop even ask the root probe. + refute_root_probe_attempted +} + +@test "does not declare readiness when the healthz probe cannot connect at all" { + export CURL_HEALTHZ_EXIT="7" + export CURL_ROOT_CODE="200" + run_test_installer_artifact 1 + [ "$status" -ne 0 ] + [[ "$output" == *"Timed out after 1s waiting for KubeStellar Console readiness"* ]] + refute_root_probe_attempted +} + +@test "exits nonzero and reports the failure once the target QEMU process dies" { + export QEMU_TARGET_DIES="1" + run_test_installer_artifact 30 + [ "$status" -ne 0 ] + [[ "$output" == *"died unexpectedly"* ]] + [[ "$output" != *"KubeStellar Console is healthy"* ]] +} + +@test "dumps the serial log tail when the target QEMU process dies" { + export QEMU_TARGET_DIES="1" + export QEMU_SERIAL_LOG_CONTENT="kernel: this is the last thing the guest printed" + run_test_installer_artifact 30 + [ "$status" -ne 0 ] + [[ "$output" == *"Serial log tail"* ]] + [[ "$output" == *"this is the last thing the guest printed"* ]] +} + +@test "dumps the serial log tail when readiness times out instead of the target dying" { + export CURL_HEALTHZ_EXIT="7" + export QEMU_SERIAL_LOG_CONTENT="kernel: still booting, never became ready" + run_test_installer_artifact 1 + [ "$status" -ne 0 ] + [[ "$output" == *"Serial log tail"* ]] + [[ "$output" == *"still booting, never became ready"* ]] + [[ "$output" != *"died unexpectedly"* ]] +} From c495cb1582fb133eff47f598ab03a5b12d65d2ea Mon Sep 17 00:00:00 2001 From: Douglas Baggett Date: Tue, 22 Sep 2026 07:07:54 -0400 Subject: [PATCH 2/4] test(installer): rewrite the readiness contract around the in-guest probe #220 moved the readiness decision into the guest: a systemd unit injected over an SMBIOS credential polls the kiosk proxy and prints KIOSK_CONSOLE_READY on the console, and the host only greps the serial log for that marker. Four tests still drove a host-side curl loop that no longer exists and failed at the merged head. Drive the host decision through the serial log instead: the marker present, a progress line only, no marker at all, and a near-miss substring. Guard the two conditions and the credential delivery as text in the unit, and assert this recipe runs no host-side probe (scoped to its own body, since install-vm legitimately forwards its own ports). Four recipe mutations each fail between one and four tests. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YQCk7xJDvhtPoc4mWPzVCK Signed-off-by: Douglas Baggett --- tests/unit/test-installer-artifact_test.bats | 98 ++++++++++++-------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/tests/unit/test-installer-artifact_test.bats b/tests/unit/test-installer-artifact_test.bats index d85f2360..0f912402 100755 --- a/tests/unit/test-installer-artifact_test.bats +++ b/tests/unit/test-installer-artifact_test.bats @@ -237,17 +237,9 @@ refute_log() { # The root probe's URL has no path suffix; matching end-of-line distinguishes # a real call to it from a call to /healthz (which also contains "8080/"). -assert_root_probe_attempted() { - if ! grep -qE 'curl .* http://127\.0\.0\.1:8080/$' "$LOG"; then - echo "expected the root probe (http://127.0.0.1:8080/) to have been called" >&2 - cat "$LOG" >&2 - return 1 - fi -} - -refute_root_probe_attempted() { - if grep -qE 'curl .* http://127\.0\.0\.1:8080/$' "$LOG"; then - echo "expected the root probe (http://127.0.0.1:8080/) NOT to have been called" >&2 +refute_host_probe_attempted() { + if grep -qE 'curl ' "$LOG"; then + echo "expected the host NOT to have run curl at all (the probe lives in the guest)" >&2 cat "$LOG" >&2 return 1 fi @@ -255,60 +247,91 @@ refute_root_probe_attempted() { # --- guards: the checks this suite depends on are still the real ones ----- -@test "guard: readiness still requires the healthz body to report status ok" { - run grep -cF 'jq -e '"'"'.status == "ok"'"'"'' "$JUSTFILE" +@test "guard: the in-guest probe still requires the healthz body to report status ok" { + # The readiness decision moved into the guest with #220: a systemd unit + # injected over an SMBIOS credential polls the kiosk proxy from inside + # and prints KIOSK_CONSOLE_READY on the console. The two conditions are + # now text inside that unit's ExecStart, so they are guarded as text. + run grep -cF "curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)\" = ok ]" "$JUSTFILE" [ "$status" -eq 0 ] [ "$output" -eq 1 ] } -@test "guard: readiness still requires the root probe to answer exactly 200" { - run grep -cF '[ "$ROOT_CODE" = "200" ]' "$JUSTFILE" +@test "guard: the in-guest probe still requires the root probe to answer exactly 200" { + run grep -cF '"%%{http_code}" https://127.0.0.1:8080/)" = 200 ]' "$JUSTFILE" [ "$status" -eq 0 ] [ "$output" -eq 1 ] } -# --- the readiness decision ------------------------------------------------- +@test "guard: the unit is delivered as an SMBIOS credential and pulled in by the cmdline" { + run grep -cF "io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service=" "$JUSTFILE" + [ "$status" -eq 0 ] + [ "$output" -eq 1 ] + run grep -cF "systemd.wants=bluefin-kiosk-ready.service" "$JUSTFILE" + [ "$status" -eq 0 ] + [ "$output" -eq 1 ] +} -@test "declares readiness only once healthz reports ok status and root answers 200" { - export CURL_HEALTHZ_BODY='{"status":"ok"}' - export CURL_ROOT_CODE="200" +@test "guard: this recipe never probes the kiosk from the host" { + # The proxy is bound to the guest's loopback, which a QEMU hostfwd cannot + # reach, so a host-side probe would be a probe of nothing. The only host + # decision is the serial-log marker. Scoped to this recipe's body: + # install-vm is a different recipe and forwards its own ports. + recipe_body() { + awk '/^test-installer-artifact:/{p=1;next} p&&/^[a-z][a-z0-9-]*:/{exit} p' "$JUSTFILE" + } + # http, not https: the in-guest probe speaks https to the TLS-terminating + # proxy, so a plaintext probe here could only be a host-side one. + run bash -c 'recipe_body() { awk "/^test-installer-artifact:/{p=1;next} p&&/^[a-z][a-z0-9-]*:/{exit} p" "$1"; }; recipe_body "$1" | grep -cE "curl [^|]*http://127\.0\.0\.1:8080"' _ "$JUSTFILE" + [ "$output" -eq 0 ] + run bash -c 'recipe_body() { awk "/^test-installer-artifact:/{p=1;next} p&&/^[a-z][a-z0-9-]*:/{exit} p" "$1"; }; recipe_body "$1" | grep -cF "hostfwd=tcp:127.0.0.1:8080-:8080"' _ "$JUSTFILE" + [ "$output" -eq 0 ] +} + +# --- the readiness decision ------------------------------------------------- +# +# From the host's side the decision is: the marker the in-guest unit prints +# appears in the serial log. The QEMU stub writes whatever +# QEMU_SERIAL_LOG_CONTENT holds into the serial file, standing in for the +# guest console, so each case below seeds the log with what a guest in that +# state would have printed. + +@test "declares readiness once the in-guest probe prints the marker on the console" { + export QEMU_SERIAL_LOG_CONTENT="<4>kiosk-ready: polling https://127.0.0.1:8080 +<4>KIOSK_CONSOLE_READY" run_test_installer_artifact 10 [ "$status" -eq 0 ] [[ "$output" == *"KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200"* ]] - assert_root_probe_attempted + refute_host_probe_attempted } -@test "does not declare readiness when healthz is ok but the root probe never returns 200" { - export CURL_HEALTHZ_BODY='{"status":"ok"}' - export CURL_ROOT_CODE="503" +@test "does not declare readiness while the in-guest probe is still waiting" { + # The unit prints a progress line every 15 polls while either condition + # is unmet; that line must never be mistaken for the marker. + export QEMU_SERIAL_LOG_CONTENT="<4>kiosk-ready: polling https://127.0.0.1:8080 +<4>kiosk-ready: waiting k0s=active hz=200 root=503 pods=12 running=9" run_test_installer_artifact 1 [ "$status" -ne 0 ] [[ "$output" == *"Timed out after 1s waiting for KubeStellar Console readiness"* ]] [[ "$output" != *"KubeStellar Console is healthy"* ]] - # The root probe having been reached proves the failure is the "never - # 200" branch, not an accidental short-circuit before it. - assert_root_probe_attempted } -@test "does not declare readiness when root answers 200 but healthz never reports ok" { - export CURL_HEALTHZ_BODY='{"status":"degraded"}' - export CURL_ROOT_CODE="200" +@test "does not declare readiness when the guest never prints the marker at all" { + export QEMU_SERIAL_LOG_CONTENT="kernel: still booting, never became ready" run_test_installer_artifact 1 [ "$status" -ne 0 ] [[ "$output" == *"Timed out after 1s waiting for KubeStellar Console readiness"* ]] [[ "$output" != *"KubeStellar Console is healthy"* ]] - # healthz status must gate the root probe: a failing/absent status must - # never let the loop even ask the root probe. - refute_root_probe_attempted } -@test "does not declare readiness when the healthz probe cannot connect at all" { - export CURL_HEALTHZ_EXIT="7" - export CURL_ROOT_CODE="200" +@test "does not declare readiness on a substring that merely resembles the marker" { + # The marker is grepped as a fixed string, so a progress line quoting it + # in prose is the closest false positive. The unit never prints one, but + # the guard is that only the exact marker line counts. + export QEMU_SERIAL_LOG_CONTENT="<4>kiosk-ready: waiting for KIOSK_CONSOLE_READ" run_test_installer_artifact 1 [ "$status" -ne 0 ] - [[ "$output" == *"Timed out after 1s waiting for KubeStellar Console readiness"* ]] - refute_root_probe_attempted + [[ "$output" != *"KubeStellar Console is healthy"* ]] } @test "exits nonzero and reports the failure once the target QEMU process dies" { @@ -329,7 +352,6 @@ refute_root_probe_attempted() { } @test "dumps the serial log tail when readiness times out instead of the target dying" { - export CURL_HEALTHZ_EXIT="7" export QEMU_SERIAL_LOG_CONTENT="kernel: still booting, never became ready" run_test_installer_artifact 1 [ "$status" -ne 0 ] From 4d680767b8c91f125715ac2dbffa3d73d2dd87db Mon Sep 17 00:00:00 2001 From: Danathar Date: Tue, 22 Sep 2026 16:29:35 -0400 Subject: [PATCH 3/4] fix(test): scope the SMBIOS credential guard to test-installer-artifact install-vm (#227) reuses the same kiosk-ready credential and cmdline mechanism, so it now carries its own copy of both strings. A whole-file grep -c counts both recipes' copies and fails against current main (count 2, not 1). Scope the guard to this recipe's body, the same way the host-probe guard just below it already does. Co-Authored-By: Claude Sonnet 5 --- tests/unit/test-installer-artifact_test.bats | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/test-installer-artifact_test.bats b/tests/unit/test-installer-artifact_test.bats index 0f912402..caecc6be 100755 --- a/tests/unit/test-installer-artifact_test.bats +++ b/tests/unit/test-installer-artifact_test.bats @@ -264,10 +264,14 @@ refute_host_probe_attempted() { } @test "guard: the unit is delivered as an SMBIOS credential and pulled in by the cmdline" { - run grep -cF "io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service=" "$JUSTFILE" + # Scoped to this recipe's body, like the host-probe guard below: install-vm + # also carries this credential/cmdline pair (it reuses the same in-guest + # kiosk-ready mechanism), so a whole-file grep would count both recipes' + # copies instead of just this one's. + run bash -c 'recipe_body() { awk "/^test-installer-artifact:/{p=1;next} p&&/^[a-z][a-z0-9-]*:/{exit} p" "$1"; }; recipe_body "$1" | grep -cF "io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service="' _ "$JUSTFILE" [ "$status" -eq 0 ] [ "$output" -eq 1 ] - run grep -cF "systemd.wants=bluefin-kiosk-ready.service" "$JUSTFILE" + run bash -c 'recipe_body() { awk "/^test-installer-artifact:/{p=1;next} p&&/^[a-z][a-z0-9-]*:/{exit} p" "$1"; }; recipe_body "$1" | grep -cF "systemd.wants=bluefin-kiosk-ready.service"' _ "$JUSTFILE" [ "$status" -eq 0 ] [ "$output" -eq 1 ] } From 7b84134074b0d9d029a1fbba715fc99699fa5dae Mon Sep 17 00:00:00 2001 From: hive-scanner <223556219+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 01:11:35 -0400 Subject: [PATCH 4/4] docs(test): describe what the readiness suite actually proves The file header and curl-stub comments described an earlier revision of this suite, from before the readiness probe moved into the guest (#220): they claimed the two-condition check was exercised by routing requests through the curl stub and parsing the body with jq. Nothing does that at head, and the check is jq -r .status, not jq -e. Rewrite the header to state the split the suite actually implements: the host-side loop (marker, target death, deadline, log tail) is executed for real against the sandboxed recipe, while the in-guest two-condition check is pinned by fixed-string greps because it can only run inside the booted guest. Correct the near-miss test's comment, which claimed exact-line matching that the recipe's unanchored grep does not do, and the refute_host_probe_attempted comment, which described a URL-shaped match it no longer performs. Drop the CURL_HEALTHZ_EXIT/CURL_HEALTHZ_BODY/CURL_ROOT_EXIT/CURL_ROOT_CODE knobs: no test sets them, the host never invokes curl, and one test asserts exactly that. The stub keeps its call logging, which refute_host_probe_ attempted needs, and now fails loudly if the host ever does probe. No behavioural assertion changes; 12/12 still pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/unit/test-installer-artifact_test.bats | 82 ++++++++++---------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/tests/unit/test-installer-artifact_test.bats b/tests/unit/test-installer-artifact_test.bats index caecc6be..173a7925 100755 --- a/tests/unit/test-installer-artifact_test.bats +++ b/tests/unit/test-installer-artifact_test.bats @@ -9,12 +9,27 @@ # recipe bodies out of the Justfile as text and asserted substrings — every # assertion passed whether the underlying health check was correct, weakened, # or deleted, because it matched echo/log wording rather than the checks -# themselves. This suite instead runs the real recipe: a private sandbox copy -# of the Justfile, with `qemu-system-x86_64`, `curl` and `zstd` replaced by -# logging stubs on PATH so `just test-installer-artifact` executes for real -# and the decision logic (readiness requires BOTH a `{"status":"ok"}` -# `/healthz` body AND an HTTP 200 on `/`, exactly `flash-installer_test.bats`' -# behavioural style) is exercised rather than paraphrased. +# themselves. +# +# The readiness decision is split across two sides, and this suite covers +# them differently because only one of them is reachable from a test host: +# +# * Host side (executed for real): the recipe boots the target under QEMU +# and waits for the in-guest unit's `KIOSK_CONSOLE_READY` marker to +# appear in the serial log, failing on target death or deadline. The +# suite runs the real recipe — a private sandbox copy of the Justfile, +# with `qemu-system-x86_64`, `curl` and `zstd` replaced by logging stubs +# on PATH, the same pattern as `flash-installer_test.bats` — so +# `just test-installer-artifact` actually executes and the marker / +# PID-death / timeout / log-tail behaviour is exercised, not paraphrased. +# +# * Guest side (guarded as text): since #220 the two-condition check +# (`/healthz` reporting `status: ok` AND `/` answering HTTP 200) lives in +# a systemd unit injected over an SMBIOS credential and only ever runs +# inside the booted guest, against a proxy bound to the guest's +# loopback. Nothing on the host can execute it, so it is pinned by +# fixed-string greps of the unit's ExecStart (the `guard:` tests below). +# Those guards fail if either condition is weakened or dropped. # # One substitution is applied to the sandboxed Justfile, and asserted exactly # like `flash-installer_test.bats` asserts its own: the recipe's OVMF @@ -168,50 +183,32 @@ EOF # make_curl_stub # -# Answers the two probes the readiness loop makes, driven entirely by env -# vars set per test: CURL_HEALTHZ_EXIT/CURL_HEALTHZ_BODY for `/healthz`, -# CURL_ROOT_EXIT/CURL_ROOT_CODE for `/`. Real `jq` (not stubbed) parses -# whatever body this stub prints, so the suite exercises the actual -# `jq -e '.status == "ok"'` check rather than a paraphrase of it. +# The readiness probe runs inside the guest, so the host should never invoke +# curl at all. This stub exists only to record any invocation in the call +# log, so `refute_host_probe_attempted` can prove that. It shapes no +# response: if the host ever did probe, the stub fails the call loudly +# rather than pretending to be a reachable kiosk proxy. make_curl_stub() { cat > "${STUB_DIR}/curl" <> "${LOG}" - -url="" -for a in "\$@"; do url="\$a"; done - -case "\$url" in - */healthz) - [ "\${CURL_HEALTHZ_EXIT:-0}" = "0" ] || exit "\${CURL_HEALTHZ_EXIT}" - printf '%s' "\${CURL_HEALTHZ_BODY:-}" - exit 0 - ;; - *) - [ "\${CURL_ROOT_EXIT:-0}" = "0" ] || exit "\${CURL_ROOT_EXIT}" - printf '%s' "\${CURL_ROOT_CODE:-000}" - exit 0 - ;; -esac +echo "curl stub: the host is not expected to probe the kiosk" >&2 +exit 1 EOF chmod +x "${STUB_DIR}/curl" } # run_test_installer_artifact [deadline-seconds] # -# All CURL_*/QEMU_*/OVMF_CODE_TEST_OVERRIDE knobs are read from the calling -# test's exported environment; only the deadline is parameterised here since -# every test needs one. +# The QEMU_*/OVMF_CODE_TEST_OVERRIDE knobs are read from the calling test's +# exported environment; only the deadline is parameterised here since every +# test needs one. run_test_installer_artifact() { local deadline="${1:-5}" run env PATH="${STUB_DIR}:${PATH}" \ XDG_CACHE_HOME="${SANDBOX}/cache" \ OVMF_CODE_TEST_OVERRIDE="${OVMF_STUB}" \ SHOW_ME_THE_FUTURE_DEADLINE="${deadline}" \ - CURL_HEALTHZ_EXIT="${CURL_HEALTHZ_EXIT:-0}" \ - CURL_HEALTHZ_BODY="${CURL_HEALTHZ_BODY:-}" \ - CURL_ROOT_EXIT="${CURL_ROOT_EXIT:-0}" \ - CURL_ROOT_CODE="${CURL_ROOT_CODE:-}" \ QEMU_TARGET_DIES="${QEMU_TARGET_DIES:-0}" \ QEMU_SERIAL_LOG_CONTENT="${QEMU_SERIAL_LOG_CONTENT:-}" \ just --justfile "${SANDBOX}/Justfile" \ @@ -235,10 +232,10 @@ refute_log() { fi } -# The root probe's URL has no path suffix; matching end-of-line distinguishes -# a real call to it from a call to /healthz (which also contains "8080/"). +# The readiness probe lives in the guest, so no host-side curl invocation is +# legitimate: any entry in the call log at all is a failure. refute_host_probe_attempted() { - if grep -qE 'curl ' "$LOG"; then + if grep -qF 'curl ' "$LOG"; then echo "expected the host NOT to have run curl at all (the probe lives in the guest)" >&2 cat "$LOG" >&2 return 1 @@ -328,10 +325,13 @@ refute_host_probe_attempted() { [[ "$output" != *"KubeStellar Console is healthy"* ]] } -@test "does not declare readiness on a substring that merely resembles the marker" { - # The marker is grepped as a fixed string, so a progress line quoting it - # in prose is the closest false positive. The unit never prints one, but - # the guard is that only the exact marker line counts. +@test "does not declare readiness on a near-miss of the marker" { + # The host greps the serial log for the marker as an unanchored fixed + # string, so any line *containing* KIOSK_CONSOLE_READY declares + # readiness — that is the contract, not exact-line matching. What this + # case pins is the other direction: console text that merely approaches + # the marker without containing it (here, a truncated spelling) must not + # trip the gate. export QEMU_SERIAL_LOG_CONTENT="<4>kiosk-ready: waiting for KIOSK_CONSOLE_READ" run_test_installer_artifact 1 [ "$status" -ne 0 ]