From 474137ec97acea3b2d76ad702742ccd958cfc939 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Fri, 21 Aug 2026 09:50:05 -0700 Subject: [PATCH] e2e: check an actor's address families against its worker pod's Nothing outside a sandbox can see which address families the actor inside it ended up with, and the two sandbox classes reach that state by different routes -- gVisor's runsc adopts the interior netns wholesale, while a micro-VM guest has to be told over the kata-agent channel. The counter demo now serves /netinfo, reporting its own interfaces and the addresses on them, and TestActorAddressFamilies compares that against the worker pod's own IPs. The assertion is an equivalence rather than "expect IPv6", so it means something on every cluster: on a dual-stack one it is the positive check, and on the IPv4-only clusters CI runs today it asserts that an actor was not given an address its pod cannot route. Link-local does not count as IPv6 -- every link has an fe80::, so counting it would make the check vacuous. Part of #246 --- demos/counter/counter.go | 90 +++++++++++++ demos/counter/counter_test.go | 121 ++++++++++++++++++ .../suites/networking/addressfamily_test.go | 118 +++++++++++++++++ 3 files changed, 329 insertions(+) create mode 100644 demos/counter/counter_test.go create mode 100644 internal/e2e/suites/networking/addressfamily_test.go diff --git a/demos/counter/counter.go b/demos/counter/counter.go index a3927a6dc8..873ee5e80b 100644 --- a/demos/counter/counter.go +++ b/demos/counter/counter.go @@ -21,11 +21,13 @@ import ( "crypto/rand" "crypto/sha256" "encoding/base64" + "encoding/json" "fmt" "io" "log/slog" "net" "net/http" + "net/netip" "os" "os/signal" "path/filepath" @@ -149,6 +151,23 @@ func main() { w.Write([]byte(response)) }) + // /netinfo reports the address families the sandbox itself has, which is + // the only place they can be observed: an actor's interior network is + // configured differently by each sandbox class, and nothing outside the + // sandbox can see what the actor ended up with. + defaultMux.HandleFunc("/netinfo", func(w http.ResponseWriter, r *http.Request) { + info, err := currentNetInfo() + if err != nil { + slog.ErrorContext(r.Context(), "Error collecting interface addresses", slog.Any("err", err)) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(info); err != nil { + slog.ErrorContext(r.Context(), "Error writing netinfo response", slog.Any("err", err)) + } + }) + go func() { slog.InfoContext(ctx, "Starting counter server on port 80") if err := http.ListenAndServe(":80", defaultMux); err != nil { @@ -247,6 +266,77 @@ func hashRandomFile() string { return base64.RawStdEncoding.EncodeToString(hash[:]) } +// netInfo is the /netinfo response: the address families the sandbox actually +// has, plus the raw per-interface CIDRs they were derived from, so a failing +// assertion reports what the actor saw rather than just that it disagreed. +type netInfo struct { + Interfaces []ifaceInfo `json:"interfaces"` + IPv4 []string `json:"ipv4"` + IPv6 []string `json:"ipv6"` +} + +type ifaceInfo struct { + Name string `json:"name"` + CIDRs []string `json:"cidrs"` +} + +func currentNetInfo() (netInfo, error) { + ifaces, err := net.Interfaces() + if err != nil { + return netInfo{}, fmt.Errorf("listing interfaces: %w", err) + } + info := netInfo{Interfaces: make([]ifaceInfo, 0, len(ifaces))} + for _, iface := range ifaces { + addrs, err := iface.Addrs() + if err != nil { + // A link can disappear between the two calls; report the rest. + slog.Warn("Error getting interface addresses", slog.String("iface", iface.Name), slog.Any("err", err)) + continue + } + entry := ifaceInfo{Name: iface.Name, CIDRs: make([]string, 0, len(addrs))} + for _, addr := range addrs { + entry.CIDRs = append(entry.CIDRs, addr.String()) + } + info.Interfaces = append(info.Interfaces, entry) + } + info.IPv4, info.IPv6 = assignedAddrs(info.Interfaces) + return info, nil +} + +// assignedAddrs splits the CIDRs on ifaces into the IPv4 and IPv6 addresses +// somebody assigned to this sandbox, which is not the same question as which +// are globally routable: the actor veth's IPv4 is 169.254.17.2, link-local by +// RFC 3927, and its IPv6 is a ULA. Both are kept. +// +// IPv6 link-local is the one exclusion, and the asymmetry with IPv4 link-local +// is the point: the kernel puts an fe80:: on every link that has IPv6 compiled +// in, so counting it would make "does this actor have IPv6" vacuously true, +// whereas 169.254.17.2 was deliberately assigned and is the actor's only IPv4. +func assignedAddrs(ifaces []ifaceInfo) (v4, v6 []string) { + v4, v6 = []string{}, []string{} + for _, iface := range ifaces { + for _, cidr := range iface.CIDRs { + prefix, err := netip.ParsePrefix(cidr) + if err != nil { + continue + } + addr := prefix.Addr().Unmap() + if addr.IsLoopback() || addr.IsUnspecified() || addr.IsMulticast() { + continue + } + if addr.Is4() { + v4 = append(v4, addr.String()) + continue + } + if addr.IsLinkLocalUnicast() { + continue + } + v6 = append(v6, addr.String()) + } + } + return v4, v6 +} + func getCurrentIP() string { addrs, err := net.InterfaceAddrs() if err != nil { diff --git a/demos/counter/counter_test.go b/demos/counter/counter_test.go new file mode 100644 index 0000000000..de08aa6147 --- /dev/null +++ b/demos/counter/counter_test.go @@ -0,0 +1,121 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "slices" + "testing" +) + +func TestAssignedAddrs(t *testing.T) { + for _, tc := range []struct { + name string + ifaces []ifaceInfo + wantV4 []string + wantV6 []string + }{{ + name: "no interfaces", + ifaces: nil, + wantV4: []string{}, + wantV6: []string{}, + }, { + // The sandbox as it looks today: the actor veth plus loopback. + name: "actor veth IPv4 only", + ifaces: []ifaceInfo{ + {Name: "lo", CIDRs: []string{"127.0.0.1/8", "::1/128"}}, + {Name: "eth0", CIDRs: []string{"169.254.17.2/30", "fe80::a8:1eff:fe00:2/64"}}, + }, + wantV4: []string{"169.254.17.2"}, + wantV6: []string{}, + }, { + // The whole point of the endpoint: fe80:: alone must not read as IPv6, + // or the dual-stack assertion passes on an IPv4-only actor. + name: "link-local IPv6 alone is not IPv6", + ifaces: []ifaceInfo{ + {Name: "eth0", CIDRs: []string{"fe80::a8:1eff:fe00:2/64"}}, + }, + wantV4: []string{}, + wantV6: []string{}, + }, { + name: "actor veth dual stack", + ifaces: []ifaceInfo{ + {Name: "lo", CIDRs: []string{"127.0.0.1/8", "::1/128"}}, + {Name: "eth0", CIDRs: []string{"169.254.17.2/30", "fd00:169:254::2/126", "fe80::a8:1eff:fe00:2/64"}}, + }, + wantV4: []string{"169.254.17.2"}, + wantV6: []string{"fd00:169:254::2"}, + }, { + // A ULA is the only kind of IPv6 the actor veth ever carries. + name: "ULA counts as IPv6", + ifaces: []ifaceInfo{ + {Name: "eth0", CIDRs: []string{"fd00:10:244::22/64"}}, + }, + wantV4: []string{}, + wantV6: []string{"fd00:10:244::22"}, + }, { + name: "loopback alone is neither family", + ifaces: []ifaceInfo{ + {Name: "lo", CIDRs: []string{"127.0.0.1/8", "::1/128"}}, + }, + wantV4: []string{}, + wantV6: []string{}, + }, { + // net.Addr.String() never spells an IPv4 address this way, but a + // v4-mapped form must classify as IPv4 rather than inflate the v6 list. + name: "v4-mapped IPv6 is IPv4", + ifaces: []ifaceInfo{ + {Name: "eth0", CIDRs: []string{"::ffff:169.254.17.2/126"}}, + }, + wantV4: []string{"169.254.17.2"}, + wantV6: []string{}, + }, { + name: "unparseable entries are skipped", + ifaces: []ifaceInfo{ + {Name: "eth0", CIDRs: []string{"not-an-address", "169.254.17.2/30"}}, + }, + wantV4: []string{"169.254.17.2"}, + wantV6: []string{}, + }} { + t.Run(tc.name, func(t *testing.T) { + gotV4, gotV6 := assignedAddrs(tc.ifaces) + if !slices.Equal(gotV4, tc.wantV4) { + t.Errorf("assignedAddrs() v4 = %v, want %v", gotV4, tc.wantV4) + } + if !slices.Equal(gotV6, tc.wantV6) { + t.Errorf("assignedAddrs() v6 = %v, want %v", gotV6, tc.wantV6) + } + }) + } +} + +// TestCurrentNetInfo runs the collector against the machine's real interfaces. +// It cannot assert which families are present, only that every interface is +// reported and that the classification is a subset of what was collected. +func TestCurrentNetInfo(t *testing.T) { + info, err := currentNetInfo() + if err != nil { + t.Fatalf("currentNetInfo() error = %v", err) + } + if len(info.Interfaces) == 0 { + t.Fatal("currentNetInfo() reported no interfaces") + } + wantV4, wantV6 := assignedAddrs(info.Interfaces) + if !slices.Equal(info.IPv4, wantV4) { + t.Errorf("IPv4 = %v, want %v", info.IPv4, wantV4) + } + if !slices.Equal(info.IPv6, wantV6) { + t.Errorf("IPv6 = %v, want %v", info.IPv6, wantV6) + } +} diff --git a/internal/e2e/suites/networking/addressfamily_test.go b/internal/e2e/suites/networking/addressfamily_test.go new file mode 100644 index 0000000000..62b0633089 --- /dev/null +++ b/internal/e2e/suites/networking/addressfamily_test.go @@ -0,0 +1,118 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package networking + +import ( + "context" + "encoding/json" + "net/http" + "net/netip" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// netInfo mirrors the counter demo's /netinfo response. +type netInfo struct { + Interfaces []struct { + Name string `json:"name"` + CIDRs []string `json:"cidrs"` + } `json:"interfaces"` + IPv4 []string `json:"ipv4"` + IPv6 []string `json:"ipv6"` +} + +// TestActorAddressFamilies asserts that an actor gets exactly the address +// families its worker pod has. +// +// It is written as an equivalence rather than "expect IPv6" so it means +// something on every cluster. On a dual-stack one it is the positive check that +// the actor was actually given IPv6; on the IPv4-only clusters CI runs today it +// asserts the opposite -- that the actor was *not* given an address its pod +// cannot route -- which guards the gate in ateomnet.SetupActorNetwork and the +// dangerous direction of the micro-VM restore reconcile, where a dual-stack +// golden lands on an IPv4-only pod. +// +// Both sandbox classes run this via E2E_SANDBOX_CLASS, and they reach the +// answer by completely different routes: gVisor's runsc adopts the interior +// netns wholesale, while the micro-VM guest has to be told over the kata-agent +// channel. Only the actor can report what it ended up with, which is what +// /netinfo is for. +func TestActorAddressFamilies(t *testing.T) { + ctx := context.Background() + actorName, actor := createAndResumeSubstrateActor(t, ctx, "family", e2e.SubstrateCounterFixture()) + + assignment := actor.GetStatus().GetWorkerAssignment() + if assignment.GetWorkerNamespace() == "" || assignment.GetWorkerPod() == "" { + t.Skipf("resumed Actor has no worker pod assignment, so there is nothing to compare against: %+v", actor) + } + podV4, podV6 := workerPodFamilies(t, ctx, assignment) + + router := mustRouterClient(t, ctx) + defer router.Close() + actorRef := resources.ActorRef{Atespace: networkingAtespace, Name: actorName} + body := waitForRouteReady(t, "Actor /netinfo", func() (*http.Response, error) { + return router.Get(ctx, actorRef, "/netinfo") + }) + + var info netInfo + if err := json.Unmarshal([]byte(body), &info); err != nil { + t.Fatalf("parsing /netinfo response %q: %v", body, err) + } + for _, iface := range info.Interfaces { + t.Logf("actor interface %s: %v", iface.Name, iface.CIDRs) + } + + // An actor with no IPv4 has failed at something far more basic than address + // families, and would otherwise let the IPv6 legs pass vacuously. + if len(info.IPv4) == 0 { + t.Errorf("actor has no IPv4 address; interfaces: %+v", info.Interfaces) + } + if got, want := len(info.IPv6) > 0, len(podV6) > 0; got != want { + t.Errorf("actor has IPv6 = %v, but its worker pod %s/%s has IPv6 = %v\n\tpod IPv4: %v\n\tpod IPv6: %v\n\tactor IPv4: %v\n\tactor IPv6: %v\n\tactor interfaces: %+v", + got, assignment.GetWorkerNamespace(), assignment.GetWorkerPod(), want, + podV4, podV6, info.IPv4, info.IPv6, info.Interfaces) + } +} + +// workerPodFamilies splits the assigned worker pod's addresses by family. Pod +// IPs are already the global addresses the CNI handed out, so unlike the +// actor's own interfaces there is no link-local to filter -- but they are +// classified by parsing rather than by counting Status.PodIPs, because a +// single-stack pod on a dual-stack cluster still has exactly one entry. +func workerPodFamilies(t *testing.T, ctx context.Context, assignment *ateapipb.WorkerAssignment) (v4, v6 []string) { + t.Helper() + pod, err := e2e.GetClients().K8s.CoreV1().Pods(assignment.GetWorkerNamespace()). + Get(ctx, assignment.GetWorkerPod(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("getting worker pod %s/%s: %v", assignment.GetWorkerNamespace(), assignment.GetWorkerPod(), err) + } + for _, podIP := range pod.Status.PodIPs { + addr, err := netip.ParseAddr(podIP.IP) + if err != nil { + t.Fatalf("worker pod %s/%s has unparseable IP %q: %v", pod.Namespace, pod.Name, podIP.IP, err) + } + if addr.Unmap().Is4() { + v4 = append(v4, addr.String()) + } else { + v6 = append(v6, addr.String()) + } + } + t.Logf("worker pod %s/%s has IPv4 %v, IPv6 %v", pod.Namespace, pod.Name, v4, v6) + return v4, v6 +}