diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fb679324..abcf32699 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,8 +4,8 @@ on: pull_request: push: branches: [testing] - schedule: - - cron: "20 22 * * *" + # Nightly scheduling lives in sync-main-to-testing so signatures and E2E + # consistently refer to refs/heads/testing, not the default main branch. workflow_dispatch: permissions: {} diff --git a/.github/workflows/post-testing-e2e.yml b/.github/workflows/post-testing-e2e.yml index 7bbfcc484..fff736b7c 100644 --- a/.github/workflows/post-testing-e2e.yml +++ b/.github/workflows/post-testing-e2e.yml @@ -5,35 +5,204 @@ on: workflows: ["Build Utah"] types: [completed] branches: [testing] + workflow_dispatch: + inputs: + build_run_id: + description: Successful Build Utah run on testing to validate + required: true + type: string -permissions: read-all +permissions: {} + +concurrency: + group: utah-iso-e2e + cancel-in-progress: false jobs: - promote-to-testing: + resolve: if: >- - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event == 'push' && - github.event.workflow_run.head_branch == 'testing' - runs-on: ubuntu-latest + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event != 'pull_request' && + github.event.workflow_run.head_repository.full_name == github.repository) + runs-on: ubuntu-24.04 permissions: - packages: write + contents: read actions: read + outputs: + matrix: ${{ steps.inputs.outputs.matrix }} + sha: ${{ steps.inputs.outputs.sha }} + run: ${{ steps.inputs.outputs.run }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + - id: inputs + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + BUILD_RUN: ${{ inputs.build_run_id || github.event.workflow_run.id }} + run: | + set -euo pipefail + [[ "$BUILD_RUN" =~ ^[0-9]+$ ]] + gh api "repos/$REPOSITORY/actions/runs/$BUILD_RUN" > run.json + gh run download "$BUILD_RUN" --repo "$REPOSITORY" \ + --pattern 'image-digest-testing-*' --dir digests + matrix=$(python3 scripts/resolve-e2e-inputs.py run.json digests "$REPOSITORY") + { + echo "matrix=$matrix" + echo "sha=$(jq -r .head_sha run.json)" + echo "run=$BUILD_RUN" + } >> "$GITHUB_OUTPUT" + + luks: + needs: resolve + runs-on: ubuntu-24.04 + timeout-minutes: 180 + permissions: + contents: read + packages: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.matrix) }} steps: - - name: Download all flavor digests + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + ref: ${{ needs.resolve.outputs.sha }} + persist-credentials: false + - name: Prepare ISO and QEMU host + run: | + set -euo pipefail + # These hosted-runner SDKs are unused by ISO composition. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt-get update + sudo apt-get install -y podman skopeo uidmap slirp4netns fuse-overlayfs \ + squashfs-tools xorriso dosfstools mtools qemu-system-x86 qemu-utils \ + ovmf sshpass ffmpeg tesseract-ocr + test -c /dev/kvm + sudo chmod a+rw /dev/kvm + test -r /dev/kvm && test -w /dev/kvm + mkdir -p output evidence + df -h . + - name: Pull exact published payload env: GH_TOKEN: ${{ github.token }} - run: gh run download "${{ github.event.workflow_run.id }}" --repo "${{ github.repository }}" --pattern 'image-digest-testing-*' --dir /tmp/digests - - name: Promote verified images to testing + IMAGE_REF: ${{ matrix.ref }} + run: | + set -euo pipefail + echo "$GH_TOKEN" | podman login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + podman pull "$IMAGE_REF" + podman logout ghcr.io + - name: Compose disposable debug ISO + env: + IMAGE_REF: ${{ matrix.ref }} + UTAH_ISO_WORKDIR: ${{ runner.temp }} + run: | + set -euo pipefail + bash iso/scripts/build-iso.sh "$IMAGE_REF" "$PWD/output/utah-live.iso" \ + "Utah CI Live" 1 "$IMAGE_REF" 2>&1 | tee evidence/iso-build.log + - name: Run existing LUKS install and desktop test + env: + IMAGE_REF: ${{ matrix.ref }} + UTAH_E2E_WORK: ${{ runner.temp }}/utah-luks-e2e + UTAH_E2E_REQUIRE_SCREENSHOTS: "1" + UTAH_E2E_REQUIRE_FASTFETCH: "1" + UTAH_E2E_RAM: "6144" + UTAH_E2E_CPUS: "2" + run: | + set -euo pipefail + bash iso/scripts/luks-e2e.sh "$PWD/output/utah-live.iso" "$IMAGE_REF" \ + 2>&1 | tee evidence/luks-test.log + - name: Record exact test provenance + env: + IMAGE_REF: ${{ matrix.ref }} + SOURCE_SHA: ${{ needs.resolve.outputs.sha }} + BUILD_RUN: ${{ needs.resolve.outputs.run }} + run: | + set -euo pipefail + sha256sum output/utah-live.iso > evidence/iso.sha256 + jq -n --arg image "$IMAGE_REF" --arg sha "$SOURCE_SHA" \ + --arg build "$BUILD_RUN" --arg run "$GITHUB_RUN_ID" \ + '{image:$image,source_sha:$sha,build_run:$build,e2e_run:$run}' \ + > docs/verification/provenance.json + cp evidence/iso.sha256 docs/verification/ + - name: Preserve diagnostics even on failure + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: iso-diagnostics-${{ matrix.image }} + retention-days: 14 + if-no-files-found: warn + path: | + evidence/ + ${{ runner.temp }}/utah-luks-e2e/*.log + ${{ runner.temp }}/utah-luks-e2e/fastfetch-ocr.txt + ${{ runner.temp }}/utah-luks-e2e/screenshots/ + - name: Preserve passing verification record + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: iso-verified-${{ matrix.image }} + if-no-files-found: error + retention-days: 30 + path: docs/verification/ + # Never upload debug ISOs or test disks: they contain test passwords. + + promote-to-testing: + needs: [resolve, luks] + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + steps: + - name: Promote only the tested digests if still current env: GH_TOKEN: ${{ github.token }} - REGISTRY: ghcr.io/${{ github.repository_owner }} + REPOSITORY: ${{ github.repository }} + SOURCE_SHA: ${{ needs.resolve.outputs.sha }} + MATRIX: ${{ needs.resolve.outputs.matrix }} run: | set -euo pipefail - echo "$GH_TOKEN" | skopeo login ghcr.io --username "${{ github.actor }}" --password-stdin - count=0 - while IFS='=' read -r name digest; do - [[ -n "$name" && "$digest" == sha256:* ]] || continue - skopeo copy --all "docker://${REGISTRY}/${name}@${digest}" "docker://${REGISTRY}/${name}:testing" - count=$((count + 1)) - done < <(find /tmp/digests -name '*.txt' -exec cat {} +) - test "$count" -eq 4 + current=$(gh api "repos/$REPOSITORY/commits/testing" --jq .sha) + if [[ "$current" != "$SOURCE_SHA" ]]; then + echo "Superseded build: retaining evidence without moving tags." + exit 0 + fi + sudo apt-get update && sudo apt-get install -y skopeo + echo "$GH_TOKEN" | skopeo login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + while IFS= read -r ref; do + skopeo copy --all --preserve-digests "docker://$ref" "docker://${ref%@*}:testing" + done < <(jq -r '.include[].ref' <<< "$MATRIX") + + documentation: + needs: [resolve, luks] + runs-on: ubuntu-24.04 + permissions: + contents: write + pull-requests: write + actions: read + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + ref: main + - name: Download passing main-desktop screenshots + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: iso-verified-utah + path: docs/verification + - name: Refresh README evidence link + run: python3 scripts/update-e2e-readme.py + - name: Propose updated verification record + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7 + with: + branch: automation/iso-verification + title: 'docs: refresh verified ISO screenshots' + commit-message: 'docs: refresh verified ISO screenshots' + body: | + Screenshots from the existing LUKS ISO test, after every configured flavor passed. + Source commit: ${{ needs.resolve.outputs.sha }} + Build: https://github.com/${{ github.repository }}/actions/runs/${{ needs.resolve.outputs.run }} + ISO E2E: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + Exact image reference and ISO checksum are in docs/verification/. + add-paths: | + README.md + docs/verification/ diff --git a/.github/workflows/sync-main-to-testing.yml b/.github/workflows/sync-main-to-testing.yml index 0c99deff6..3d04fd4c4 100644 --- a/.github/workflows/sync-main-to-testing.yml +++ b/.github/workflows/sync-main-to-testing.yml @@ -3,12 +3,31 @@ name: Sync main to testing on: push: branches: [main] + paths-ignore: + - README.md + - docs/verification/** + schedule: + - cron: "20 22 * * *" + workflow_dispatch: permissions: contents: write + actions: write jobs: sync: uses: projectbluefin/actions/.github/workflows/reusable-sync-branches.yml@eb4c546389f19ad9e42f1af052623de73c620ebb # v1 permissions: contents: write + + build: + needs: sync + runs-on: ubuntu-24.04 + permissions: + actions: write + steps: + - name: Start the testing build explicitly after token-authenticated sync + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + run: gh workflow run build.yml --repo "$REPOSITORY" --ref testing diff --git a/Justfile b/Justfile index 74f35af47..d0c9d3220 100644 --- a/Justfile +++ b/Justfile @@ -66,7 +66,9 @@ check: python3 -m py_compile scripts/check-repo-availability.py python3 scripts/install-packages.py --check packages/bluefin.toml python3 scripts/verify-rpm-contract.py --check packages/bluefin.toml - python3 -m unittest discover -s tests -p 'test_package_resolution.py' + pip install --quiet pyyaml 2>/dev/null || true + python3 -m unittest discover -s tests -p 'test_*.py' + bash -n iso/scripts/luks-e2e.sh grep -qE 'reusable-build\.yml@(v1|[0-9a-f]{40} # v1)$' .github/workflows/build.yml test -f Containerfile.kernel bash -n scripts/install-ogc-kernel.sh @@ -80,7 +82,6 @@ check: <(grep -m1 '^ARG BASE_IMAGE=' Containerfile.kernel) python3 -m py_compile scripts/flavors.py python3 scripts/flavors.py list >/dev/null - pip install --quiet pyyaml 2>/dev/null || true python3 scripts/check_workflow_outputs.py pip install --quiet jsonschema 2>/dev/null || true bash scripts/check-skill-frontmatter.sh diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 7e0ea07ee..d069fbb44 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -25,11 +25,15 @@ Three workflows, all thin callers into `projectbluefin/actions@v1` reusables, each pinned to a SHA tagged `v1`: - `.github/workflows/build.yml` -- pull requests, pushes to `testing`, a - nightly cron, and manual dispatch. Top-level `permissions: {}`; each job + manual dispatch. Top-level `permissions: {}`; each job grants its own. Cancels in-progress runs per workflow and ref. - `.github/workflows/promote-testing-to-main.yml` -- pushes to `testing`, a nightly cron, and manual dispatch. -- `.github/workflows/sync-main-to-testing.yml` -- every push to `main`. +- `.github/workflows/sync-main-to-testing.yml` -- source pushes to `main`, + nightly cron, and manual dispatch; explicitly dispatches the testing build + after syncing. Token-authenticated branch pushes alone do not start CI. +- `.github/workflows/post-testing-e2e.yml` -- successful non-PR testing builds + and manual dispatch with a successful testing build run ID. CI delegates builds, vulnerability reporting, SBOMs, keyless signatures, provenance, caching, and rechunking to `projectbluefin/actions@v1` (originated @@ -118,9 +122,42 @@ and confusingly, so this reads the same list (comment, ## sync-main-to-testing -Every push to `main` calls `reusable-sync-branches.yml@v1` (same SHA pin) -with `contents: write` and nothing else -- there is no local logic to drift -(workflow, `.github/workflows/sync-main-to-testing.yml`). +Source pushes to `main` and the nightly schedule call +`reusable-sync-branches.yml@v1`, then explicitly dispatch `build.yml` on +`testing` with `actions: write`. README/verification-only pushes are excluded +to avoid evidence-update build loops. Nightly runs still sync those changes. + +## ISO LUKS gate and screenshots + +`post-testing-e2e.yml` downloads the originating build's digest artifacts. +`scripts/resolve-e2e-inputs.py` rejects PRs, foreign repositories, failed runs, +wrong branches/workflows, mutable references, conflicting digests, and missing +flavors. The expected set comes from `scripts/flavors.py images`. + +Each configured image is pulled by digest, composed into a disposable debug +ISO, and passed to the existing `iso/scripts/luks-e2e.sh`. Both guests have +restricted networking. CI requires KVM, PNG screenshots, and OCR evidence +that fastfetch ran in the graphical terminal. Test credentials are confined +to the disposable ISO/disk; neither is uploaded or released. + +Every matrix job preserves build/test logs, serial logs, and screenshots, +including on failure. Only passing jobs upload `docs/verification` with the +source commit, original build run, E2E run, image digest and ISO checksum. +The promotion job depends on the entire matrix; a superseded testing commit +cannot move tags. Registry tag copies are sequential, not an atomic multi-tag +transaction: a registry failure can interrupt promotion after a partial copy. + +A separate least-privilege job proposes the main desktop's screenshots in +`automation/iso-verification`, a documentation PR. It updates only the README +evidence block and `docs/verification/`, preserving the current README's other +content. The repository must allow Actions to create pull requests; a denied +write fails this job visibly, without deleting test artifacts. It does not +auto-merge the evidence PR or imply a fresh pass for a different commit. + +For a deliberate rerun, dispatch Post-Testing E2E with `build_run_id` from a +successful testing build containing the current harness. Do not pass a PR +build: PRs do not publish immutable images. This matrix validates emulated +UEFI desktop installation, not Secure Boot, TPM unlock, or physical GPUs. ## Verification diff --git a/docs/skills/local-testing.md b/docs/skills/local-testing.md index e2824fde2..153d658f5 100644 --- a/docs/skills/local-testing.md +++ b/docs/skills/local-testing.md @@ -141,9 +141,17 @@ accounts and requires local QEMU/KVM access, not a production installation. Passing runs refresh `docs/verification/README.md`, its screenshots, and the delimited verification block in the root README. These are historical local test records, not proof that the current commit passed CI. In particular, -the fastfetch capture currently waits after terminal autostart; it does not -independently assert that fastfetch rendered. Automated CI publication still -needs to be wired up and must retain the tested commit/image digest. +local fastfetch capture waits after terminal autostart by default. CI sets +`UTAH_E2E_REQUIRE_FASTFETCH=1` to require OCR of its completion marker and +kernel output, and `UTAH_E2E_REQUIRE_SCREENSHOTS=1` to reject missing PNGs. +CI retains the tested commit/image digest and proposes evidence updates in a +documentation PR only after all flavors pass. See [ci-workflows.md](ci-workflows.md). + +The harness blocks outbound guest networking while retaining loopback-only +SSH forwards, so installation cannot silently fall back to an online pull. +It enables sshd through a boot argument on the disposable installed disk, +never by rebuilding or changing the published image. `UTAH_E2E_RAM` and +`UTAH_E2E_CPUS` control VM resources (defaults 8192 MiB and four CPUs). When integrating this harness with newer image-build fixes, retain the currently verified package-image digest and available-package contract. diff --git a/iso/scripts/luks-e2e.sh b/iso/scripts/luks-e2e.sh index ad3c22e89..a2aa70196 100755 --- a/iso/scripts/luks-e2e.sh +++ b/iso/scripts/luks-e2e.sh @@ -25,6 +25,8 @@ ISO="${1:?live ISO path is required}" PAYLOAD_IMAGE="${2:?installed image reference is required}" PASSPHRASE="${3:-testpassphrase}" WORK="${UTAH_E2E_WORK:-/var/tmp/utah-luks-e2e}" +VM_RAM="${UTAH_E2E_RAM:-8192}" +VM_CPUS="${UTAH_E2E_CPUS:-4}" ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" # The account the installer creates, and the one phase 6 logs in as. @@ -103,6 +105,8 @@ PY shot() { local label="$1" sock="$2" local ppm="${SHOTS}/${label}.ppm" png="${SHOTS}/${label}.png" + # A failed capture must not reuse evidence from an earlier attempt. + rm -f "${ppm}" "${png}" monitor "${sock}" "screendump ${ppm}" || return 0 sleep 1 if command -v ffmpeg >/dev/null 2>&1 && [[ -s "${ppm}" ]]; then @@ -184,7 +188,7 @@ qemu-img create -f qcow2 "${INSTALL_DISK}" 64G >/dev/null cp -f "${OVMF_VARS_SRC}" "${VARS}" "${QEMU}" \ - -machine q35 -cpu host -m 8192 -smp 4 ${ACCEL} \ + -machine q35 -cpu host -m "${VM_RAM}" -smp "${VM_CPUS}" ${ACCEL} \ -drive "if=pflash,format=raw,readonly=on,file=${OVMF_CODE}" \ -drive "if=pflash,format=raw,file=${VARS}" \ -drive "if=none,id=iso,file=${ISO},media=cdrom,readonly=on,format=raw" \ @@ -192,7 +196,7 @@ cp -f "${OVMF_VARS_SRC}" "${VARS}" -device scsi-cd,drive=iso \ -drive "if=none,id=disk,file=${INSTALL_DISK},format=qcow2" \ -device virtio-blk-pci,drive=disk \ - -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22" \ + -netdev "user,id=net0,restrict=on,hostfwd=tcp:127.0.0.1:${SSH_PORT}-:22" \ -device virtio-net-pci,netdev=net0 \ -monitor "unix:${MONITOR_LIVE},server,nowait" \ -serial "file:${SERIAL_LIVE}" \ @@ -305,6 +309,10 @@ ssh_live 'sudo bash -euc " sed -i \"s|^options .*|& console=tty0 console=ttyS0|\" \"\$entry\" grep -q \"forward_to_console\" \"\$entry\" || \ sed -i \"s|^options .*|& systemd.journald.forward_to_console=yes|\" \"\$entry\" + # Enable diagnostics on this disposable installed disk only. + # The published payload keeps sshd disabled. + grep -q \"systemd.wants=sshd.service\" \"\$entry\" || \ + sed -i \"s|^options .*|& systemd.wants=sshd.service|\" \"\$entry\" # Unlocking this LUKS2 volume costs ~30s of argon2 before userspace # starts, and on a loaded host udev can still be settling when # systemd gives up on /boot at its 45s default -- the installed @@ -380,12 +388,12 @@ echo "=== Phase 4/6: boot the installed disk ===" # the passphrase at a firmware shell. Real hardware keeps its NVRAM; so do we. cp -f "${VARS}" "${WORK}/ovmf-vars-installed.fd" "${QEMU}" \ - -machine q35 -cpu host -m 8192 -smp 4 ${ACCEL} \ + -machine q35 -cpu host -m "${VM_RAM}" -smp "${VM_CPUS}" ${ACCEL} \ -drive "if=pflash,format=raw,readonly=on,file=${OVMF_CODE}" \ -drive "if=pflash,format=raw,file=${WORK}/ovmf-vars-installed.fd" \ -drive "if=none,id=disk,file=${INSTALL_DISK},format=qcow2" \ -device virtio-blk-pci,drive=disk \ - -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT_INSTALLED}-:22" \ + -netdev "user,id=net0,restrict=on,hostfwd=tcp:127.0.0.1:${SSH_PORT_INSTALLED}-:22" \ -device virtio-net-pci,netdev=net0 \ -monitor "unix:${MONITOR_INSTALLED},server,nowait" \ -serial "file:${SERIAL_INSTALLED}" \ @@ -460,7 +468,7 @@ shot installed-greeter "${MONITOR_INSTALLED}" # runs inside the session with a bus and a display of its own. echo "Arranging for a terminal to open in the session..." ssh_target " - grep -q 'utah-e2e fastfetch' ~/.bashrc 2>/dev/null || printf '%s\n' '[[ \$- == *i* ]] && fastfetch # utah-e2e fastfetch' >> ~/.bashrc + grep -q 'utah-e2e fastfetch' ~/.bashrc 2>/dev/null || printf '%s\n' '[[ \$- == *i* ]] && { fastfetch --logo none && echo UTAH-E2E-FASTFETCH; } # utah-e2e fastfetch' >> ~/.bashrc mkdir -p ~/.config/autostart cat > ~/.config/autostart/${TERMINAL_APP}.desktop </dev/null || echo " could not write the autostart entry" >&2 +" || fail "could not write the terminal autostart entry" # Type the password at the greeter. GDM offers the single account already # selected, so Enter opens the password field and the password submits it. @@ -553,8 +561,31 @@ shot installed-desktop "${MONITOR_INSTALLED}" # shot a human actually reads: it names the OS, the kernel and the desktop # from inside the installed system, so one image carries what half a dozen # assertions above prove separately. -sleep 20 -shot installed-fastfetch "${MONITOR_INSTALLED}" +if [[ "${UTAH_E2E_REQUIRE_FASTFETCH:-0}" == 1 ]]; then + command -v tesseract >/dev/null || fail "tesseract is required for CI screenshot verification" + fastfetch_seen=0 + for _ in $(seq 1 24); do + shot installed-fastfetch "${MONITOR_INSTALLED}" + if [[ -s "${SHOTS}/installed-fastfetch.png" ]]; then + tesseract "${SHOTS}/installed-fastfetch.png" "${WORK}/fastfetch-ocr" 2>/dev/null + if grep -qi 'UTAH.E2E.FASTFETCH' "${WORK}/fastfetch-ocr.txt" \ + && grep -qi 'Kernel' "${WORK}/fastfetch-ocr.txt"; then + fastfetch_seen=1 + break + fi + fi + sleep 5 + done + (( fastfetch_seen )) || fail "fastfetch output was not visible in the desktop screenshot" +else + sleep 20 + shot installed-fastfetch "${MONITOR_INSTALLED}" +fi +if [[ "${UTAH_E2E_REQUIRE_SCREENSHOTS:-0}" == 1 ]]; then + for label in live-desktop installed-greeter installed-desktop installed-fastfetch; do + [[ -s "${SHOTS}/${label}.png" ]] || fail "missing required screenshot: ${label}" + done +fi echo echo "PASS: Utah installed to an encrypted disk, unlocked, and ${TEST_USER} logged" diff --git a/scripts/resolve-e2e-inputs.py b/scripts/resolve-e2e-inputs.py new file mode 100644 index 000000000..b55cf7153 --- /dev/null +++ b/scripts/resolve-e2e-inputs.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Fail closed when selecting immutable images from one trusted build run.""" +import argparse +import json +from pathlib import Path +import re +import subprocess + + +def resolve(run, expected, artifact_dir, repository): + if (run.get("conclusion") != "success" or run.get("head_branch") != "testing" + or run.get("event") not in {"push", "workflow_dispatch", "schedule"} + or run.get("repository", {}).get("full_name") != repository + or run.get("head_repository", {}).get("full_name") != repository + or run.get("path") != ".github/workflows/build.yml" + or not re.fullmatch(r"[0-9a-f]{40}", run.get("head_sha", ""))): + raise ValueError("not a successful trusted testing image build") + images = {} + for path in Path(artifact_dir).rglob("*.txt"): + for line in path.read_text().splitlines(): + if not line: + continue + if "|" in line: + name, arch, digest = line.split("|") + if arch != "amd64": + raise ValueError("unexpected architecture") + else: + name, digest = line.split("=") + if name not in expected or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest): + raise ValueError("unexpected image name or digest") + if name in images and images[name] != digest: + raise ValueError("conflicting image digests") + images[name] = digest + if set(images) != set(expected): + raise ValueError("build artifacts do not cover the configured flavor set") + owner = repository.split("/")[0].lower() + return {"include": [{"image": name, "digest": images[name], + "ref": f"ghcr.io/{owner}/{name}@{images[name]}"} + for name in expected]} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("run") + parser.add_argument("artifacts") + parser.add_argument("repository") + args = parser.parse_args() + expected = [item["image"] for item in json.loads(subprocess.check_output( + ["python3", "scripts/flavors.py", "images"], text=True))] + print(json.dumps(resolve(json.loads(Path(args.run).read_text()), expected, + args.artifacts, args.repository))) diff --git a/scripts/update-e2e-readme.py b/scripts/update-e2e-readme.py new file mode 100644 index 000000000..a0b743b74 --- /dev/null +++ b/scripts/update-e2e-readme.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Embed a passing CI record without copying an old source README over main.""" +import json +from pathlib import Path +import re + + +def update(text, provenance): + sha, run = provenance["source_sha"], provenance["e2e_run"] + if not re.fullmatch(r"[0-9a-f]{40}", sha) or not re.fullmatch(r"[0-9]+", run): + raise ValueError("invalid provenance") + begin, end = "", "" + block = (f"{begin}\n" + "[![Verified ISO desktop](docs/verification/screenshots/installed-fastfetch.png)]" + "(docs/verification/README.md)\n\n" + f"*LUKS ISO test passed for commit `{sha[:12]}`. " + f"[CI run](https://github.com/projectbluefin/utah/actions/runs/{run}); " + "[screenshots and provenance](docs/verification/README.md).*\n" + f"{end}") + pattern = re.compile(re.escape(begin) + r".*?" + re.escape(end), re.S) + if pattern.search(text): + return pattern.sub(lambda _: block, text, count=1) + title, separator, rest = text.partition("\n") + return title + separator + "\n" + block + "\n" + rest + + +if __name__ == "__main__": + provenance = json.loads(Path("docs/verification/provenance.json").read_text()) + if not Path("docs/verification/screenshots/installed-fastfetch.png").is_file(): + raise SystemExit("missing verified screenshot") + path = Path("README.md") + path.write_text(update(path.read_text(), provenance)) diff --git a/tests/test_iso_ci.py b/tests/test_iso_ci.py new file mode 100644 index 000000000..069403f70 --- /dev/null +++ b/tests/test_iso_ci.py @@ -0,0 +1,94 @@ +"""CI must test the complete exact-digest set before publication.""" +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +def load(name): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class InputsTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) / "digests.txt" + self.digest = "sha256:" + "a" * 64 + self.path.write_text(f"utah={self.digest}\nutah|amd64|{self.digest}\n") + self.run = dict(conclusion="success", head_branch="testing", event="workflow_dispatch", + repository={"full_name": "projectbluefin/utah"}, + head_repository={"full_name": "projectbluefin/utah"}, + path=".github/workflows/build.yml", head_sha="b" * 40) + + def resolve(self, expected=None): + return load("resolve-e2e-inputs").resolve( + self.run, expected or ["utah"], self.tmp.name, "projectbluefin/utah") + + def test_dual_artifact_formats_agree(self): + self.assertEqual(self.resolve()["include"][0]["ref"], + f"ghcr.io/projectbluefin/utah@{self.digest}") + + def test_rejects_untrusted_or_failed_builds(self): + for field, value in [("event", "pull_request"), ("conclusion", "failure"), + ("head_branch", "main"), ("head_sha", "bad"), + ("path", "other.yml"), + ("head_repository", {"full_name": "attacker/utah"})]: + with self.subTest(field=field): + old = self.run[field] + self.run[field] = value + with self.assertRaises(ValueError): + self.resolve() + self.run[field] = old + + def test_missing_flavor_is_fatal(self): + with self.assertRaises(ValueError): + self.resolve(["utah", "second-image"]) + + def test_rejects_conflicts_tags_unknown_names_and_architectures(self): + for line in ["utah=latest", "intruder=" + self.digest, + "utah|arm64|" + self.digest, "utah=sha256:" + "c" * 64]: + with self.subTest(line=line): + self.path.write_text(f"utah={self.digest}\n{line}\n") + with self.assertRaises(ValueError): + self.resolve() + + +class EvidenceTests(unittest.TestCase): + def test_readme_update_is_idempotent_and_preserves_other_text(self): + update = load("update-e2e-readme").update + proof = {"source_sha": "a" * 40, "e2e_run": "123"} + text = "# Utah\n\nKeep this paragraph.\n" + result = update(text, proof) + self.assertEqual(update(result, proof), result) + self.assertIn("Keep this paragraph.", result) + self.assertIn("actions/runs/123", result) + + def test_publication_needs_all_luks_jobs_and_debug_images_are_not_uploaded(self): + import yaml + jobs = yaml.safe_load((ROOT / ".github/workflows/post-testing-e2e.yml").read_text())["jobs"] + for name in ["promote-to-testing", "documentation"]: + self.assertIn("luks", jobs[name]["needs"]) + self.assertNotIn("if", jobs[name]) + steps = jobs["luks"]["steps"] + self.assertFalse(jobs["luks"]["strategy"]["fail-fast"]) + test = next(step for step in steps if "Run existing LUKS" in step.get("name", "")) + self.assertEqual(test["env"]["UTAH_E2E_REQUIRE_FASTFETCH"], "1") + uploads = [s for s in steps if "upload-artifact@" in s.get("uses", "")] + for step in uploads: + self.assertNotIn("output/", step["with"]["path"]) + self.assertNotIn("qcow2", step["with"]["path"]) + self.assertTrue(any(s.get("if") == "always()" for s in uploads)) + + def test_harness_restricts_both_guests_and_requires_real_screenshots(self): + script = (ROOT / "iso/scripts/luks-e2e.sh").read_text() + self.assertEqual(script.count("restrict=on,hostfwd=tcp:127.0.0.1:"), 2) + self.assertIn("systemd.wants=sshd.service", script) + self.assertIn("fastfetch output was not visible", script) + self.assertIn("missing required screenshot", script)