From 72a0792d1a3e5498fa6a4e07d5877ab57fb94026 Mon Sep 17 00:00:00 2001 From: lex00 <121451605+lex00@users.noreply.github.com> Date: Thu, 6 Aug 2026 22:09:02 -0600 Subject: [PATCH] uat: their release gate runs on every commit, held to the recorded matrix (#61) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream's gate is the 63-case Robot UAT, run by hand against a billed cluster before each release. It now runs here per commit, free: the stack up on k3d with the image built from the commit, their suite unmodified from a pinned checkout, and assert-matrix.py holding the result to uat/expected-failures.txt in both directions — a new failure is a regression, an expected failure that starts passing fails too until the record moves with it. The full local run that seeded the expectations doubles as the re-measure the pass matrix lacked: 50 of 63 on v0.4.0, identical to the v0.1.0 recording, each failure still accounted for. A smoke job runs first: the landing page's own commands against the candidate — docker run, the list-microvm-images call, /_m80/health at 29/29 — because a docs prompt nobody executes is a docs prompt that rots. Both jobs proven locally before this commit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WVmBmk6XERjKxHa4HyezvT --- .github/workflows/uat.yml | 116 ++++++++++++++++++++++++++++++++++++++ docs/kubemicrovm.md | 2 +- uat/assert-matrix.py | 68 ++++++++++++++++++++++ uat/expected-failures.txt | 22 ++++++++ 4 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/uat.yml create mode 100644 uat/assert-matrix.py create mode 100644 uat/expected-failures.txt diff --git a/.github/workflows/uat.yml b/.github/workflows/uat.yml new file mode 100644 index 0000000..6f49875 --- /dev/null +++ b/.github/workflows/uat.yml @@ -0,0 +1,116 @@ +name: uat + +# KubeMicroVM's own release gate, run per commit (#61). +# +# Their gate is the 63-case Robot Framework UAT, which upstream runs by hand +# against a live, billed cluster before each release candidate. m80 exists to +# make most of that suite free, so this workflow runs it the way the docs say +# a person does: the stack up on k3d with the image built from this commit, +# their suite unmodified from a pinned checkout, and the pass matrix gated in +# BOTH directions — a new failure is a regression, and an expected failure +# that starts passing means the recorded matrix must move with reality +# (uat/expected-failures.txt and docs/kubemicrovm.md), not drift from it. +# +# The smoke job runs first and cheap: the landing page's own commands against +# the same candidate image, because a docs prompt nobody executes is a docs +# prompt that rots (the kubemicrovm-ops local-e2e's founding rule, applied +# here). + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: uat-${{ github.ref }} + cancel-in-progress: true + +env: + # The KubeMicroVM checkout the suite runs from. Bumping it is a deliberate + # commit: their suite moving is exactly the event the matrix gate exists to + # notice, so it must not move silently underneath a green run. + KUBEMICROVM_REF: b3886991604593ebe21fd9b690e8686f01291207 + +jobs: + # The landing page's commands, literally. `docker run` the candidate, the + # CLI call the "Give this to your agent" block opens with, and the health + # line the Status section claims. The CLI runs from the UAT runner image + # rather than the host so its version is pinned by a Dockerfile, not by + # whatever the runner shipped this week — same command, deterministic CLI. + smoke: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: build the candidate + run: docker build -q -t m80:candidate . + + - name: the landing page, executed + run: | + docker network create m80-smoke + docker run -d --rm --name m80 --network m80-smoke m80:candidate + docker build -q -t m80-uat-runner uat/ + docker run --rm --network m80-smoke --entrypoint aws \ + -e AWS_ACCESS_KEY_ID=test -e AWS_SECRET_ACCESS_KEY=test \ + m80-uat-runner --endpoint-url http://m80:4290 --region us-east-2 \ + lambda-microvms list-microvm-images | grep -q '"items"' + docker run --rm --network m80-smoke --entrypoint sh m80-uat-runner \ + -c "curl -s http://m80:4290/_m80/health" > health.json + python3 - <<'EOF' + import json + c = json.load(open("health.json"))["coverage"] + assert c["implemented"] == 29 and not c["notImplementedYet"], c + print(f"health: {c['implemented']}/29, nothing pending") + EOF + + uat: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - uses: actions/checkout@v4 + with: + repository: codriverlabs/KubeMicroVM + ref: ${{ env.KUBEMICROVM_REF }} + path: kubemicrovm + + - uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: install k3d + run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + + - name: build the candidate + run: docker build -q -t m80:candidate . + + # up.sh imports a locally built image instead of pulling, which is the + # whole point here: the operator's own gate runs against this commit's + # m80, not against the last release. + - name: the stack, on the candidate + run: M80_IMAGE=m80:candidate ./uat/up.sh + + - name: their suite, unmodified + run: KUBEMICROVM="${GITHUB_WORKSPACE}/kubemicrovm" ./uat/run.sh || true + + # The suite exiting non-zero is expected — thirteen failures are the + # recorded matrix. The gate is the exact set, both directions. + - name: the matrix holds + run: python3 uat/assert-matrix.py uat-results/output.xml uat/expected-failures.txt + + - name: what the operator saw + if: always() + run: kubectl -n kube-microvm logs deploy/kube-microvm-operator --tail=100 || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: uat-results + path: uat-results/ + retention-days: 14 diff --git a/docs/kubemicrovm.md b/docs/kubemicrovm.md index 399ecc9..599b19c 100644 --- a/docs/kubemicrovm.md +++ b/docs/kubemicrovm.md @@ -140,7 +140,7 @@ Each of these is a difference between this harness and the EKS run the suite was ## Pass matrix -Run 2026-08-01 against m80 v0.1.0 and operator 1.0.11, excluding the performance suite. Not re-measured against v0.2.0 — the only behaviour that changed there is the per-VM endpoint, and the cases that touch it never reach m80 (see the first failure group) — nor against v0.3–v0.4 (current: v0.4.0). A standing re-measure is exactly what [#61](https://github.com/INTENTIUS/m80/issues/61) proposes: their UAT in CI, so this number stops being a snapshot. +First recorded 2026-08-01 against m80 v0.1.0 and operator 1.0.11, excluding the performance suite; re-measured 2026-08-06 against v0.4.0 with the identical result. Since #61 landed this is no longer a snapshot: CI runs the whole suite against every commit's build (`.github/workflows/uat.yml`) and holds the run to exactly this matrix in both directions — a new failure fails the build, and a listed failure that starts passing fails it too, until this page and `uat/expected-failures.txt` move with it. **50 of 63 cases pass.** diff --git a/uat/assert-matrix.py b/uat/assert-matrix.py new file mode 100644 index 0000000..3ed0e0a --- /dev/null +++ b/uat/assert-matrix.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Gate a UAT run against the recorded pass matrix, in both directions. + + python3 uat/assert-matrix.py uat-results/output.xml uat/expected-failures.txt + +A failure not in the expected list is a regression and fails the gate. An +expected failure that now PASSES also fails the gate — good news, but the +recorded matrix (docs/kubemicrovm.md and the expectations file) must move +with reality, and a gate that lets the record rot in the flattering +direction is how "50 of 63" stops being true without anyone deciding it. + +The expectations file is one full Robot test name per line, `#` comments and +blank lines ignored. Skipped tests count as neither passed nor failed. +""" +from __future__ import annotations + +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +def main() -> int: + if len(sys.argv) != 3: + print(__doc__) + return 2 + output = ET.parse(sys.argv[1]).getroot() + expected = { + line.strip() + for line in Path(sys.argv[2]).read_text().splitlines() + if line.strip() and not line.strip().startswith("#") + } + + passed, failed = set(), set() + for test in output.iter("test"): + status = test.find("status") + verdict = status.get("status", "FAIL") if status is not None else "FAIL" + name = test.get("name", "(unnamed)") + (passed if verdict == "PASS" else failed if verdict == "FAIL" else set()).add(name) + + regressions = sorted(failed - expected) + surprises = sorted(expected & passed) + missing = sorted(expected - failed - passed) + + print(f"{len(passed)} passed, {len(failed)} failed ({len(expected)} expected)") + ok = True + if regressions: + ok = False + print("\nNEW failures — regressions:") + for name in regressions: + print(f" ✗ {name}") + if surprises: + ok = False + print("\nExpected failures that now PASS — update uat/expected-failures.txt") + print("and the pass matrix in docs/kubemicrovm.md, then re-run:") + for name in surprises: + print(f" ✓ {name}") + if missing: + ok = False + print("\nExpected failures that did not run at all — the suite moved underneath us:") + for name in missing: + print(f" ? {name}") + if ok: + print("matrix holds: every failure expected, every expectation still failing") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/uat/expected-failures.txt b/uat/expected-failures.txt new file mode 100644 index 0000000..254d4eb --- /dev/null +++ b/uat/expected-failures.txt @@ -0,0 +1,22 @@ +# The recorded pass matrix's failure set — one Robot test name per line. +# +# uat/assert-matrix.py holds a run to exactly this set, both directions: a +# failure not listed here is a regression, and a listed failure that starts +# passing means this file and docs/kubemicrovm.md's matrix must move with +# reality. Why each of these fails is accounted for, line by line, in +# docs/kubemicrovm.md's "Why the thirteen fail". +# +# Recorded 2026-08-06 against m80 v0.4.0 (source build) and operator 1.0.11. +AUTO-02 Auto-Resume On Traffic +INJ-08 Token Works To Call MicroVM +MEM-07 Run VM From 4096 MiB Image +NET-01 Internet Egress Connects To Public Internet +NET-02 Default Egress Has Internet Access +NET-04 VPC Egress VM Connects +Pod Identity Association Exists +QS-07 Curl Endpoint Returns OK +QS-08 Teardown Delete VM +RBAC-05 Authorized SA Gets Token Via Operator +RBAC-06 Authorized SA Rejected For Different VM +RS-06 Delete ReplicaSet Terminates All VMs +Verify No Resources Remaining