From 5a731a6e644bdf1bc4b382f45441bad5f5246199 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Fri, 14 Aug 2026 11:03:44 -0700 Subject: [PATCH 1/2] ci: add an IPv6-only kind e2e job No CI job exercises IPv6-only, so nothing catches a change that breaks it, and the IPv6 work is a set of changes that do little apart. This builds a single-stack IPv6 kind cluster, installs the full system and runs the networking suite, where every test that can tell one address family from another lives. It asserts the cluster really is single-stack, and that cluster DNS is the shape an IPv6-only install needs, before installing anything -- so a green run cannot be vacuous. It carries no copy of what it exercises: dispatch it with a list of pull requests and it merges them onto main for the length of the run. It stays out of the e2e-test gate, so it never blocks a pull request. --- .github/workflows/e2e-ipv6.yaml | 381 ++++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 .github/workflows/e2e-ipv6.yaml diff --git a/.github/workflows/e2e-ipv6.yaml b/.github/workflows/e2e-ipv6.yaml new file mode 100644 index 0000000000..6e93e99189 --- /dev/null +++ b/.github/workflows/e2e-ipv6.yaml @@ -0,0 +1,381 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: e2e-ipv6 +# Separate from pr-workflow.yaml so this can be gated independently -- and so a +# 40-minute IPv6 run never delays that workflow's merge-gating jobs. +# +# Only the networking suite runs. Every test that can tell one address family +# from another lives in it, and the demo suite is a large share of the wall +# clock for coverage that is not family-specific. The demos still deploy, +# because networking builds its actors from their templates. +# +# Two ways in. A pull request that edits this file runs it: the paths filter is +# evaluated before the job, so no unrelated request pays for a v6 cluster, and +# the lane can demonstrate itself without a label someone has to create first. +# Dispatch it with a list of pull requests and it merges them onto main in that +# order and runs the result: the IPv6 work is a set of changes that do nothing +# apart, so a per-request run cannot show the feature working, only that it did +# no harm. The stack exists for the length of the run and is never pushed, +# which is what lets this branch carry the workflow file alone instead of a +# copy of every change it wants to exercise. +# +# Stacking is dispatch-only, so the pull_request path runs main plus this file +# and nothing else. That exercises the lane, not the IPv6 work, and it stays +# red until the changes this gates are on main. +# +# Either way this job stays out of the `e2e-test` gate, so it never blocks a PR. +on: + pull_request: + paths: ['.github/workflows/e2e-ipv6.yaml'] + workflow_dispatch: + inputs: + prs: + description: 'Pull requests to merge onto main, in order, e.g. "958 938 1083". Blank runs main alone.' + required: false + type: string +permissions: + contents: read +jobs: + e2e-test-ipv6: + runs-on: ubuntu-latest + # Nothing else in this workflow sets a timeout, so jobs inherit GitHub's + # 6-hour default. A broken IPv6 cluster does not crash, it misses + # 10-minute ActorTemplate deadlines, so an uncapped job burns hours. + timeout-minutes: 40 + env: + # Non-default name so these steps can be replayed locally without + # touching an existing cluster. install-ate-kind.sh does not derive + # KUBECTL_CONTEXT from the cluster name the way run-e2e-kind.sh does, + # so both have to be set here. + KIND_CLUSTER_NAME: ate-ipv6 + KUBECTL_CONTEXT: kind-ate-ipv6 + # 8.8.8.8 reached through the well-known NAT64 prefix. CoreDNS is a + # v6-only pod on a runner with no IPv6 egress of its own, so this is the + # only shape of upstream resolver it can reach. Read back below, not + # passed in: the script derives it from the prefix, and a job that + # supplied it would not notice if that stopped working. + EXPECT_DNS_UPSTREAM: 64:ff9b::0808:0808 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + # The merges below need refs this clone would not otherwise have. + fetch-depth: 0 + - name: Stack the pull requests under test + # Merge, not cherry-pick: a request that has been squash-merged since it + # was opened re-applies as a revert of what is already on main, and only + # a merge notices there is nothing to do. Order is the order given, so a + # conflict names the request that has to rebase rather than the stack. + # + # This builds and runs the requests' code. The token is read-only and no + # secrets reach it; a run happens only when someone asks for one. + if: github.event_name == 'workflow_dispatch' && inputs.prs != '' + env: + GH_TOKEN: ${{ github.token }} + PRS: ${{ inputs.prs }} + # The numbers in `prs` are always upstream pull requests, so they are + # resolved against upstream by name rather than against whichever repo + # the run happens to be in. Reading them needs no token. + UPSTREAM: https://github.com/agent-substrate/substrate + run: | + git config user.name 'e2e-ipv6' + git config user.email 'e2e-ipv6@invalid' + { + echo "### Stack under test" + echo "" + echo "Merged onto \`main\` at \`$(git rev-parse --short HEAD)\`, in order:" + echo "" + echo "| pull request | head | title |" + echo "|---|---|---|" + } >> "$GITHUB_STEP_SUMMARY" + for pr in ${PRS//,/ }; do + case "${pr}" in + ''|*[!0-9]*) echo "::error::'${pr}' is not a pull request number"; exit 1 ;; + esac + git fetch -q "${UPSTREAM}" "refs/pull/${pr}/head:pr-${pr}" \ + || { echo "::error::no such pull request: #${pr}"; exit 1; } + if ! git merge --no-edit -q "pr-${pr}"; then + echo "::error::#${pr} does not merge onto the stack; it needs a rebase" + git merge --abort || true + exit 1 + fi + title=$(gh pr view "${pr}" --repo "${UPSTREAM}" --json title -q .title 2>/dev/null || echo '?') + echo "| #${pr} | \`$(git rev-parse --short "pr-${pr}")\` | ${title} |" >> "$GITHUB_STEP_SUMMARY" + echo "merged #${pr}: ${title}" + done + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: 'go.mod' + - name: Free disk space + # kind node image + control-plane images + snapshots are tight on the + # ~14GB runner disk even without the micro-VM assets. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Enable IPv6 in the Docker daemon + # ubuntu-latest ships dockerd with IPv6 off, so kind would create its + # network v4-only and create-kind-cluster.sh would reject the cluster. + # Merge the two keys into whatever daemon.json the runner image ships + # rather than replacing the file. + run: | + sudo mkdir -p /etc/docker + [ -s /etc/docker/daemon.json ] || echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null + sudo cat /etc/docker/daemon.json \ + | jq '. + {"ipv6": true, "ip6tables": true}' \ + | sudo tee /etc/docker/daemon.json.new >/dev/null + sudo mv /etc/docker/daemon.json.new /etc/docker/daemon.json + sudo systemctl restart docker + # Assert, don't print: if the merge or the restart silently did not take, + # kind builds a v4-only network and the failure surfaces much later as a + # cluster that is not the one this job exists to test. + docker network inspect bridge --format '{{.EnableIPv6}}' | grep -qx true || { + echo "::error::dockerd did not come back with IPv6 enabled" + sudo cat /etc/docker/daemon.json; exit 1; + } + echo "bridge EnableIPv6=true" + - name: Enable forwarding for the NAT64 agent + # ubuntu-latest has no IPv6 egress whatsoever -- measured, not assumed: + # every curl -6 fails in ~2ms. A v6-only cluster still has to reach real + # v4 destinations (atelet fetches the gVisor tarball from GCS, + # TestActorEgress fetches example.com), so the cluster translates for + # itself. What upstream's own e2e sets, and ordered after the dockerd + # restart, which rebuilds the chains this affects. + run: | + sudo sysctl -w net.ipv4.ip_forward=1 + sudo sysctl -w net.ipv6.conf.all.forwarding=1 + - name: Create cluster + # NAT64 is opt-in in the script and this is the case it exists for: the + # names this job needs have real AAAA records pointing at addresses the + # runner cannot reach, so the prefix deploys the translator and routes + # cluster DNS through it. The script gates on a pod fetching through the + # prefix, so a broken translator fails here rather than as a rollout + # timeout ten minutes later. Everything the Corefile ends up containing + # is the script's doing -- the steps below only read it back. + env: + IP_FAMILY: ipv6 + IPV6_DNS64_PREFIX: '64:ff9b::/96' + run: | + # Named check rather than letting the script report a missing file: + # this is the first step that touches the IPv6-only kind support, so on + # a tree without it this is where the run stops, and it should say why. + [ -f hack/third_party/nat64/install.yaml ] || { + echo "::error::hack/third_party/nat64/install.yaml is missing -- the IPv6-only kind support this job gates is not in this tree" + exit 1 + } + hack/create-kind-cluster.sh + - name: Wait for the node to be ready + # podCIDRs is assigned by the controller-manager after the node registers, + # so the assertion below can read an empty field on a cluster that is + # perfectly fine and report "cannot confirm IP family". Poll for the field + # this job actually reads, then wait on Ready for everything after it. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + for _ in $(seq 60); do + [ -n "$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}' 2>/dev/null)" ] && break + sleep 2 + done + k wait --for=condition=Ready nodes --all --timeout=180s + - name: Assert the cluster is single-stack IPv6 + # This job is worthless if the cluster is not actually v6-only, and a + # green run leaves no evidence either way -- the diagnostics dump below + # only runs on failure. A kind default change or an IP_FAMILY regression + # would otherwise turn this into a second IPv4 run that reports success. + # Checked here rather than later so it fails as itself. + # + # PreferDualStack Services resolving to a single clusterIP is the + # positive signal: on a dual-stack cluster they would get two. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + pod_cidrs=$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}') + svc_ips=$(k -n default get svc kubernetes -o jsonpath='{.spec.clusterIPs[*]}') + node_ips=$(k get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}') + for pair in "podCIDRs=${pod_cidrs}" "kubernetes.clusterIPs=${svc_ips}" "node.InternalIP=${node_ips}"; do + case "${pair#*=}" in + *.*) echo "::error::not single-stack IPv6 -- ${pair}"; exit 1 ;; + "") echo "::error::empty, cannot confirm IP family -- ${pair}"; exit 1 ;; + esac + echo " ${pair}" + done + echo "single-stack IPv6 confirmed" + - name: Assert create-kind-cluster.sh built the Corefile this job needs + # The script owns the whole Corefile, so this reads it back instead of + # rewriting it. That is the point: a green run is then evidence for what + # the script produced, where a job that patched the Corefile itself would + # paper over the absence of the very change it is here to gate. Each of + # these is something the install would otherwise fail on ten minutes + # later and far less legibly. + run: | + Corefile=$(kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}') + echo "${Corefile}" + check() { + echo "${Corefile}" | grep -q "$1" || { echo "::error::$2"; exit 1; } + } + if echo "${Corefile}" | grep -q 'forward .* /etc/resolv.conf'; then + echo "::error::CoreDNS still forwards to /etc/resolv.conf -- the IPv6-only DNS fix is not in this tree" + exit 1 + fi + check "forward .* ${EXPECT_DNS_UPSTREAM}" \ + "CoreDNS does not forward to ${EXPECT_DNS_UPSTREAM}" + check '^kind-registry:53' \ + "no kind-registry server block -- atelet cannot pull from its own netns" + check '^cluster.local:53' \ + "the cluster zones have no server block of their own -- dns64 would take out service discovery" + check 'translate_all' \ + "no dns64 translate_all -- external names resolve to addresses this runner cannot reach" + echo "Corefile is the IPv6-only shape this job needs" + - name: Verify cluster DNS answers both internal and external names + # The install is the next step and it takes ten minutes to fail. A DNS + # regression is the failure this Corefile is most likely to cause, so + # assert all three cases here where the message is unambiguous. + # Reads the log rather than attaching. `kubectl run --attach` has to + # connect before the container writes, and on a cluster still settling it + # loses that race far more often than it wins it -- a probe that fails on + # a healthy cluster is worse than no probe. Create, wait, read the log, + # and retry the whole thing. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + probe() { + k delete pod dnscheck --ignore-not-found --wait --timeout=60s >/dev/null 2>&1 + k run dnscheck --restart=Never --image=busybox:1.36 --command -- \ + sh -c ' + nslookup kubernetes.default.svc.cluster.local >/dev/null 2>&1 \ + || { echo "FAIL: an in-cluster Service does not resolve"; exit 1; } + nslookup storage.googleapis.com 2>/dev/null | grep -q "64:ff9b" \ + || { echo "FAIL: external names are not synthesized through NAT64"; exit 1; } + # Absolute name deliberately. The registry has its own server + # block and no search domain leads to it, so the bare name is + # NXDOMAIN under ndots:5 -- busybox gives up there, while glibc + # and Go go on to try the name as given, which is what atelet + # does when it pulls from its own netns. Querying what those + # resolvers end up querying is what makes this worth failing on. + nslookup kind-registry. >/dev/null 2>&1 \ + || { echo "FAIL: kind-registry does not resolve; atelet cannot pull"; exit 1; } + echo "DNS-OK" + ' >/dev/null 2>&1 || return 1 + k wait --for=jsonpath='{.status.phase}'=Succeeded pod/dnscheck --timeout=120s \ + >/dev/null 2>&1 + k logs dnscheck > /tmp/dnscheck.log 2>&1 || return 1 + grep -q DNS-OK /tmp/dnscheck.log + } + for attempt in $(seq 6); do + if probe; then cat /tmp/dnscheck.log; exit 0; fi + echo "dnscheck attempt ${attempt} failed" + cat /tmp/dnscheck.log 2>/dev/null || true + sleep 10 + done + echo "::error::cluster DNS did not come up" + exit 1 + - name: Install Agent Substrate + run: hack/install-ate-kind.sh --deploy-ate-system + - name: Assert the control plane is up + # install-ate.sh runs under pipefail, so a failed apply does propagate. + # What it would not catch is a Deployment that rolls out and then + # crash-loops. Re-check everything deploy_ate_system waits on -- + # atenet-egress included, since a non-dual-stack Envoy listener fails + # there first, by way of a readiness probe the kubelet cannot reach. + run: | + for r in deployment/ate-api-server deployment/ate-controller \ + deployment/atenet-router deployment/atenet-egress \ + deployment/dns statefulset/postgres daemonset/atelet; do + kubectl --context="$KUBECTL_CONTEXT" -n ate-system rollout status "$r" --timeout=120s + done + kubectl --context="$KUBECTL_CONTEXT" -n podcertificate-controller-system \ + rollout status deployment/podcertificate-controller --timeout=120s + if kubectl --context="$KUBECTL_CONTEXT" -n ate-system get pods \ + -o jsonpath='{.items[*].status.containerStatuses[*].state.waiting.reason}' \ + | grep -q CrashLoopBackOff; then + echo "::error::a pod in ate-system is in CrashLoopBackOff" + exit 1 + fi + - name: Deploy substrate gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter-substrate + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Assert the demo fixtures exist + # A failed demo deploy exits 0: install-ate.sh dispatches demos through + # `if "${demo}_cmdline" "$1"`, which suspends errexit, and _cmdline ends + # in an unconditional `return 0`. Without this the suites fail later with + # "ActorTemplate not found", pointing at the tests instead of the install. + # + # Two lookups, because the two fixtures live in different layers: the + # egress template is a CRD, while the substrate counter's is a store + # object the CRD API never sees. + run: | + kubectl --context="$KUBECTL_CONTEXT" -n ate-demo-egress get actortemplate egress \ + || { echo "::error::ate-demo-egress/egress was not created -- the demo deploy failed silently"; exit 1; } + go run ./cmd/kubectl-ate --context="$KUBECTL_CONTEXT" \ + get actor-template counter -a ate-demo-counter-substrate \ + || { echo "::error::ate-demo-counter-substrate/counter was not created -- the demo deploy failed silently"; exit 1; } + - name: Run E2E tests (networking) + id: e2e-networking + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ + | tee /tmp/e2e-networking.log + - name: Guard against a vacuously green run + # A test that gates on a dual-stack Service skips itself on v6-only and + # exits 0, so a suite that only skipped would otherwise read as a pass. + # The bar is one real PASS, not zero skips: a test with nothing to compare + # on a single-family cluster is right to skip. Skips are printed so a + # growing list gets noticed. + # Skipped when the suite never ran: with no log to count, this step would + # otherwise report a reassuring zero on a job that failed earlier. + if: always() && steps.e2e-networking.outcome != 'skipped' + run: | + for f in /tmp/e2e-networking.log; do + [ -s "$f" ] || { echo "::error::${f} is missing or empty"; exit 1; } + passed=$(grep -c -- '--- PASS' "$f" || true) + echo "${f}: ${passed} passed, $(grep -c -- '--- SKIP' "$f" || true) skipped" + grep -h -- '--- SKIP' "$f" || true + if [ "${passed}" -eq 0 ]; then + echo "::error::${f} has no passing tests -- a suite that only skips proves nothing" + exit 1 + fi + done + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context="$KUBECTL_CONTEXT" get actortemplate,workerpool,pods -A -o wide || true + dump() { + echo "=== logs: $1/$2 ===" + kubectl --context="$KUBECTL_CONTEXT" logs -n "$1" "$2" --all-containers --tail=300 2>/dev/null || true + } + for p in $(kubectl --context="$KUBECTL_CONTEXT" get pods -n ate-system -o name 2>/dev/null); do + dump ate-system "$p" + done + # Every worker pod in any namespace: the demo pools plus the e2e suites' + # randomly-named per-test namespaces, which the suites keep on failure. + kubectl --context="$KUBECTL_CONTEXT" get pods -A -l ate.dev/worker-pool \ + -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ + | while read -r ns name; do dump "$ns" "$name"; done + # IPv6-specific: the Corefile create-kind-cluster.sh built, and what each + # Service actually got assigned, are the two things that differ from the + # IPv4 job. + kubectl --context="$KUBECTL_CONTEXT" -n kube-system logs -l k8s-app=kube-dns --tail=100 || true + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' || true + kubectl --context="$KUBECTL_CONTEXT" get svc -A \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,POLICY:.spec.ipFamilyPolicy,IPS:.spec.clusterIPs || true + # The agent logs the interface and eBPF setup it did, which is the only + # view of an egress failure that is not a bare timeout. Its pod is the + # one thing here that can be absent on a run that failed early, so say + # so rather than letting kubectl's stderr through: those diagnostics + # most need to not end on something that reads like a second failure. + echo "=== nat64 agent ===" + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get pods -l app=nat64 -o wide \ + 2>/dev/null || echo "(no agent; NAT64 never started)" + kubectl --context="$KUBECTL_CONTEXT" -n kube-system logs -l app=nat64 --tail=100 \ + --prefix 2>/dev/null || true From d311465ae1e19f028f60fe578377a4d9250fb9d5 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Mon, 31 Aug 2026 16:09:12 -0700 Subject: [PATCH 2/2] ci: run the IPv6-only e2e job on both sandbox classes The IPv6-only lane only ever ran gVisor, so nothing in CI booted a micro-VM guest on a cluster that has IPv6 -- which is the one configuration where the two classes diverge. gVisor's runsc adopts the interior netns wholesale and picks the actor's addresses up for free, while a micro-VM's eth0 is a tap cross-connected to that netns at L2 and the guest has to be told over the kata-agent channel. The job now enables KVM, asserts the node really got /dev/kvm, deploys the micro-VM counter and egress demos, and replays the networking suite under E2E_SANDBOX_CLASS=microvm. Asset staging is the step a v6-only cluster breaks, because the rustfs endpoint is built from a ClusterIP that here is a bare IPv6 literal, so the job fails early and by name when that fix is not in the tree. The vacuous-green guard covers the second log but deliberately does not gain the dual-stack job's per-test assertions: on a single-stack cluster TestActorIngressPerFamily skips itself by design. Part of #246. --- .github/workflows/e2e-ipv6.yaml | 147 +++++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 12 deletions(-) diff --git a/.github/workflows/e2e-ipv6.yaml b/.github/workflows/e2e-ipv6.yaml index 6e93e99189..ab99627f86 100644 --- a/.github/workflows/e2e-ipv6.yaml +++ b/.github/workflows/e2e-ipv6.yaml @@ -13,14 +13,22 @@ # limitations under the License. name: e2e-ipv6 -# Separate from pr-workflow.yaml so this can be gated independently -- and so a -# 40-minute IPv6 run never delays that workflow's merge-gating jobs. +# Separate from pr-workflow.yaml so this can be gated independently -- and so an +# hour-long IPv6 run never delays that workflow's merge-gating jobs. # # Only the networking suite runs. Every test that can tell one address family # from another lives in it, and the demo suite is a large share of the wall # clock for coverage that is not family-specific. The demos still deploy, # because networking builds its actors from their templates. # +# Both sandbox classes run, gVisor then micro-VM, off the one cluster and the +# one E2E_SANDBOX_CLASS knob. The class is where address-family support +# diverges: gVisor's runsc adopts the interior netns wholesale and picks the +# actor's IPv6 up for free, while a micro-VM's eth0 is a tap cross-connected to +# that netns at L2, so the guest has to be told over the kata-agent channel. A +# lane that ran gVisor alone could not tell a working micro-VM guest from one +# that never got an address. +# # Two ways in. A pull request that edits this file runs it: the paths filter is # evaluated before the job, so no unrelated request pays for a v6 cluster, and # the lane can demonstrate itself without a label someone has to create first. @@ -53,7 +61,10 @@ jobs: # Nothing else in this workflow sets a timeout, so jobs inherit GitHub's # 6-hour default. A broken IPv6 cluster does not crash, it misses # 10-minute ActorTemplate deadlines, so an uncapped job burns hours. - timeout-minutes: 40 + # Raised from 40 for the second sandbox class: a micro-VM golden snapshot is + # a cloud-hypervisor cold boot plus a checkpoint on nested KVM, and the + # networking suite then runs twice. + timeout-minutes: 75 env: # Non-default name so these steps can be replayed locally without # touching an existing cluster. install-ate-kind.sh does not derive @@ -121,10 +132,30 @@ jobs: go-version-file: 'go.mod' - name: Free disk space # kind node image + control-plane images + snapshots are tight on the - # ~14GB runner disk even without the micro-VM assets. + # ~14GB runner disk, and the micro-VM assets -- a 1GB kata-static tarball + # unpacked to a 256MB rootfs, plus a kernel -- are on top of that. run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL df -h / + - name: Assert the micro-VM IPv6 staging fix is in this tree + # Named check for the same reason as the nat64 one below. On a v6-only + # cluster the rustfs ClusterIP is a bare IPv6 literal, which is not a URL, + # so asset staging fails outright without this -- ten minutes in, as an + # aws-cli error about an endpoint, naming nothing that would lead here. + run: | + grep -q 'RUSTFS_IP="\[' hack/microvm-assets/stage-to-rustfs.sh || { + echo "::error::hack/microvm-assets/stage-to-rustfs.sh does not bracket an IPv6 rustfs ClusterIP -- stack #1122 to run the micro-VM lane" + exit 1 + } + - name: Cache micro-VM assets + # restore, not the composite cache action: pr-workflow.yaml's e2e-test + # writes this key on every push to main, and a second saver would race it + # for no gain -- the assets are identical and assemble.sh fully pins them. + # A miss just means run-microvm-demo-kind.sh assembles them itself. + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} - name: Enable IPv6 in the Docker daemon # ubuntu-latest ships dockerd with IPv6 off, so kind would create its # network v4-only and create-kind-cluster.sh would reject the cluster. @@ -156,6 +187,18 @@ jobs: run: | sudo sysctl -w net.ipv4.ip_forward=1 sudo sysctl -w net.ipv6.conf.all.forwarding=1 + - name: Enable KVM + # Grant the runner access to /dev/kvm so create-kind-cluster.sh makes it + # usable inside the node. No node label follows from this: a micro-VM + # WorkerPool reaches the node through the ate.dev/kvm extended resource + # atelet's device plugin advertises, which is why the check that this + # worked has to wait until after the install. After the dockerd restart + # above, which is what the script's probe container runs under. + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm - name: Create cluster # NAT64 is opt-in in the script and this is the case it exists for: the # names this job needs have real AAAA records pointing at addresses the @@ -188,6 +231,18 @@ jobs: sleep 2 done k wait --for=condition=Ready nodes --all --timeout=180s + - name: Assert the node has /dev/kvm + # create-kind-cluster.sh probes for KVM and degrades silently to + # gVisor-only when the probe fails, so without this a "micro-VM lane" + # could pass by never having been one. Only the device is checkable here; + # what the scheduler actually places on is asserted after the install, + # because atelet is what advertises it. + run: | + for node in $(hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do + docker exec "${node}" test -c /dev/kvm \ + || { echo "::error::${node} has no /dev/kvm -- the micro-VM lane would be gVisor in disguise"; exit 1; } + echo " ${node}: /dev/kvm present" + done - name: Assert the cluster is single-stack IPv6 # This job is worthless if the cluster is not actually v6-only, and a # green run leaves no evidence either way -- the diagnostics dump below @@ -301,10 +356,43 @@ jobs: echo "::error::a pod in ate-system is in CrashLoopBackOff" exit 1 fi + - name: Assert the node advertises ate.dev/kvm + # A micro-VM WorkerPool requests this extended resource, so an unschedulable + # worker pod is how its absence would otherwise surface -- as a demo deploy + # that hangs until its golden-snapshot deadline. atelet's device plugin + # registers a moment after the DaemonSet reports rolled out, hence the poll. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + for node in $(hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do + kvm="" + for _ in $(seq 30); do + n=$(k get node "${node}" -o jsonpath='{.status.allocatable.ate\.dev/kvm}') + case "${n}" in ''|0) sleep 2 ;; *) kvm="${n}"; break ;; esac + done + [ -n "${kvm}" ] || { + echo "::error::${node} does not advertise ate.dev/kvm -- atelet's device plugin never registered it, so no micro-VM worker can be scheduled" + k -n ate-system logs -l app=atelet --tail=50 --prefix || true + exit 1 + } + echo " ${node}: allocatable ate.dev/kvm=${kvm}" + done + - name: Deploy substrate micro-VM counter demo + # Stages the (cached) assets into the cluster's rustfs and applies the + # counter-substrate-microvm demo onto the control plane installed above. + # First of the demos: it is the step a v6-only cluster breaks, and its + # golden snapshot is the long pole, so failing here fails early. The + # substrate demo handler waits for that snapshot itself. + run: | + hack/run-microvm-demo-kind.sh --substrate + df -h / - name: Deploy substrate gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter-substrate - - name: Deploy egress demo - run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Deploy egress demos + # One per class: networking builds its egress Actor from the fixture for + # the class under test, so both have to exist before either lane runs. + run: | + hack/install-ate-kind.sh --deploy-demo-egress + hack/install-ate-kind.sh --deploy-demo-egress-microvm - name: Assert the demo fixtures exist # A failed demo deploy exits 0: install-ate.sh dispatches demos through # `if "${demo}_cmdline" "$1"`, which suspends errexit, and _cmdline ends @@ -315,17 +403,42 @@ jobs: # egress template is a CRD, while the substrate counter's is a store # object the CRD API never sees. run: | - kubectl --context="$KUBECTL_CONTEXT" -n ate-demo-egress get actortemplate egress \ - || { echo "::error::ate-demo-egress/egress was not created -- the demo deploy failed silently"; exit 1; } - go run ./cmd/kubectl-ate --context="$KUBECTL_CONTEXT" \ - get actor-template counter -a ate-demo-counter-substrate \ - || { echo "::error::ate-demo-counter-substrate/counter was not created -- the demo deploy failed silently"; exit 1; } + for ns_tmpl in ate-demo-egress/egress ate-demo-egress-microvm/egress-microvm; do + ns=${ns_tmpl%/*}; tmpl=${ns_tmpl#*/} + kubectl --context="$KUBECTL_CONTEXT" -n "${ns}" get actortemplate "${tmpl}" \ + || { echo "::error::${ns_tmpl} was not created -- the demo deploy failed silently"; exit 1; } + done + for as_tmpl in ate-demo-counter-substrate/counter \ + ate-demo-counter-substrate-microvm/counter-microvm; do + as=${as_tmpl%/*}; tmpl=${as_tmpl#*/} + go run ./cmd/kubectl-ate --context="$KUBECTL_CONTEXT" \ + get actor-template "${tmpl}" -a "${as}" \ + || { echo "::error::${as_tmpl} was not created -- the demo deploy failed silently"; exit 1; } + done - name: Run E2E tests (networking) id: e2e-networking run: | set -o pipefail hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ | tee /tmp/e2e-networking.log + - name: Run E2E tests (networking, micro-VM) + # The same suite again, with every fixture repointed at its micro-VM + # variant by the single E2E_SANDBOX_CLASS knob (internal/e2e/sandbox.go). + # Sequential with the gVisor lane above, not concurrent: a suite releases + # its namespace -- and with it its worker pods -- as each test passes, so + # the two runs do not contend for the one kind node. + # + # always(), so a gVisor failure does not hide the micro-VM result. That is + # the whole point of the lane: the two classes reach an actor's addresses + # by different routes and can fail independently. + id: e2e-networking-microvm + if: always() && steps.e2e-networking.outcome != 'skipped' + env: + E2E_SANDBOX_CLASS: microvm + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ + | tee /tmp/e2e-networking-microvm.log - name: Guard against a vacuously green run # A test that gates on a dual-stack Service skips itself on v6-only and # exits 0, so a suite that only skipped would otherwise read as a pass. @@ -335,8 +448,16 @@ jobs: # Skipped when the suite never ran: with no log to count, this step would # otherwise report a reassuring zero on a job that failed earlier. if: always() && steps.e2e-networking.outcome != 'skipped' + env: + MICROVM_RAN: ${{ steps.e2e-networking-microvm.outcome != 'skipped' }} run: | - for f in /tmp/e2e-networking.log; do + logs=/tmp/e2e-networking.log + if [ "${MICROVM_RAN}" = "true" ]; then + logs="${logs} /tmp/e2e-networking-microvm.log" + else + echo "::warning::the micro-VM lane did not run; only gVisor was checked" + fi + for f in ${logs}; do [ -s "$f" ] || { echo "::error::${f} is missing or empty"; exit 1; } passed=$(grep -c -- '--- PASS' "$f" || true) echo "${f}: ${passed} passed, $(grep -c -- '--- SKIP' "$f" || true) skipped" @@ -359,6 +480,8 @@ jobs: done # Every worker pod in any namespace: the demo pools plus the e2e suites' # randomly-named per-test namespaces, which the suites keep on failure. + # The failing actor runs in one of these, so this is where its ateom logs + # -- and, for a micro-VM worker, the guest console tail -- live. kubectl --context="$KUBECTL_CONTEXT" get pods -A -l ate.dev/worker-pool \ -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ | while read -r ns name; do dump "$ns" "$name"; done