From ded6132f73f586f160c33fc7c1208ebd72bfae2a Mon Sep 17 00:00:00 2001 From: Peter Sprygada Date: Tue, 27 Oct 2026 11:24:23 -0400 Subject: [PATCH] fix: Retry every interrupted netlink listing Most link, address, route, neighbor and rule listings in the node binaries made a single netlink attempt. The kernel interrupts a listing when another process changes the state it walks, so any link churn on a node failed garbage collection's VRF lookups, datapath repair passes and the host gateway's tap neighbor sweep, which silently resolved nothing that pass. Under steady link churn about half of garbage collection's VRF lookups failed. nldump now also lists neighbors and rules, counted under new neighbor and rule kinds on the existing galactic_netlink_dump_* counters. A Lister from nldump.On retries over a handle the caller already holds, for callers whose handle lives in another network namespace, such as the edge agent's host attachment code and the CNI plugins' container-namespace checks. Every remaining single-attempt listing in garbage collection, datapath repair, the host gateway, host configuration, the CNI installer and plugins, the edge agent, the eBPF attach code, the egress route map, SRv6 egress and plain routes, VIP binding and loopback address detection now goes through nldump, so each discards an interrupted result and retries within the existing bound. On Linux 7.1 only link and address listings were seen interrupted under churn; route, neighbor and rule listings are retried the same way if a kernel reports one, as netlink documents they can. The new nldumptest package churns links and addresses in a test namespace and repeats a pass until the kernel has interrupted its listings. Root tests use it to require that garbage collection's VRF resolution, a full datapath repair pass and the tap neighbor sweep each complete 200 passes under churn, with at least one link listing interrupted; all three fail on the previous revision within the first few passes. Resolves #1133 --- docs/observability/netlink.md | 31 +-- internal/cni/netns.go | 12 +- internal/cni/ops_check.go | 3 +- internal/cniroute/ops_check.go | 3 +- internal/edge/hostattach/hostattach.go | 6 +- internal/edge/hostattach/inventory.go | 6 +- internal/edge/hostattach/readback.go | 14 +- internal/gc/churn_root_test.go | 178 ++++++++++++++++++ internal/gc/datapath_repair.go | 5 +- internal/gc/gc.go | 3 +- internal/hostconf/hostconf.go | 4 +- internal/hostgw/churn_root_test.go | 73 +++++++ internal/hostgw/hostgw.go | 3 +- internal/hostgw/tapneigh.go | 8 +- internal/installer/installer.go | 2 +- internal/installer/locatorroute.go | 3 +- internal/installer/sidecarreturn.go | 3 +- .../plumbing/ebpf/attach/egress_reattach.go | 5 +- internal/plumbing/ebpf/attach/interfaces.go | 5 +- internal/plumbing/ebpf/edgeattach/attach.go | 3 +- .../ebpf/egressroutemap/egressroute.go | 3 +- internal/plumbing/ebpf/natattach/attach.go | 3 +- internal/plumbing/ebpf/xdpattach/set.go | 3 +- internal/plumbing/loaddr/loaddr.go | 4 +- .../plumbing/nldump/churn_internal_test.go | 115 ++++++++--- internal/plumbing/nldump/nldump.go | 113 ++++++++--- .../plumbing/nldump/nldump_internal_test.go | 2 +- internal/plumbing/nldump/nldump_test.go | 127 +++++++++++++ .../plumbing/nldump/nldumptest/nldumptest.go | 177 +++++++++++++++++ internal/plumbing/srv6/egress.go | 3 +- internal/plumbing/srv6/plainroute.go | 10 +- internal/plumbing/vip/vip.go | 4 +- 32 files changed, 829 insertions(+), 105 deletions(-) create mode 100644 internal/gc/churn_root_test.go create mode 100644 internal/hostgw/churn_root_test.go create mode 100644 internal/plumbing/nldump/nldumptest/nldumptest.go diff --git a/docs/observability/netlink.md b/docs/observability/netlink.md index 0fc1ba05..20de103f 100644 --- a/docs/observability/netlink.md +++ b/docs/observability/netlink.md @@ -2,16 +2,22 @@ See [scraping and component ports](README.md#where-the-metrics-come-from). -The kernel interrupts a listing of links, addresses or routes when another -process changes them while it runs. The listing is then discarded and -retried: up to 5 rounds of 10 back-to-back attempts, with a pause of 20, 40, -60 and 80 ms between rounds. These counters show how often that happens on a -node, so sustained churn is visible before it makes a component fail. - -| Metric | Type | Labels | Meaning | -| ----------------------------------------- | ------- | ----------------------------------- | ---------------------------------------------------------------------------------------------- | -| `galactic_netlink_dump_interrupted_total` | counter | `kind` (`link`, `address`, `route`) | Listing attempts the kernel interrupted. Each one is discarded and retried | -| `galactic_netlink_dump_failures_total` | counter | `kind` (`link`, `address`, `route`) | Listings still interrupted on their last attempt. The caller got an error and acted on nothing | +The kernel interrupts a listing of links, addresses, routes, neighbors or +rules when another process changes them while it runs. Every such listing in +the node binaries discards an interrupted result and retries: up to 5 rounds +of 10 back-to-back attempts, with a pause of 20, 40, 60 and 80 ms between +rounds. These counters show how often that happens on a node, so sustained +churn is visible before it makes a component fail. + +On Linux 7.1, listings of thousands of routes, neighbors or rules were never +interrupted under churn, while link and address listings were, so the +`route`, `neighbor` and `rule` series can stay at 0. A non-zero value there +is not a fault. + +| Metric | Type | Labels | Meaning | +| ----------------------------------------- | ------- | ------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| `galactic_netlink_dump_interrupted_total` | counter | `kind` (`link`, `address`, `route`, `neighbor`, `rule`) | Listing attempts the kernel interrupted. Each one is discarded and retried | +| `galactic_netlink_dump_failures_total` | counter | `kind` (`link`, `address`, `route`, `neighbor`, `rule`) | Listings still interrupted on their last attempt. The caller got an error and acted on nothing | Each process counts only its own listings, on its existing metrics port: @@ -29,9 +35,8 @@ Until one exists, the queries below show nothing for them, which is not the same as no churn. The CNI plugins run once per attach and expose no metrics, so their -listings are not counted; an exhausted retry there fails the attach or -detach with the interruption error. Listings that do not go through the -shared retry are not counted either. +retries are not counted; an exhausted retry there fails the attach or +detach with the interruption error. Interruptions are normal while pods start and stop. A failure is not: the component returned an error for that pass and retries it later. diff --git a/internal/cni/netns.go b/internal/cni/netns.go index a5a005d3..e4528350 100644 --- a/internal/cni/netns.go +++ b/internal/cni/netns.go @@ -11,6 +11,8 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" + + "go.datum.net/galactic/internal/plumbing/nldump" ) // configureInterfaceInNetns applies addresses and default routes to the guest @@ -101,7 +103,7 @@ func addrFamily(ip net.IP) int { func addAddrIfMissing(handle *netlink.Handle, link netlink.Link, ifName string, ipNet *net.IPNet) error { want := netlink.Addr{IPNet: ipNet} - existing, err := handle.AddrList(link, addrFamily(ipNet.IP)) + existing, err := nldump.On(handle).AddrList(link, addrFamily(ipNet.IP)) if err != nil { return fmt.Errorf("list addresses on %q: %w", ifName, err) } @@ -122,7 +124,7 @@ func addAddrIfMissing(handle *netlink.Handle, link netlink.Link, ifName string, // gateway is a real misconfiguration rather than something a retried ADD should // paper over, so it returns an error. func addDefaultRouteIfMissing(handle *netlink.Handle, link netlink.Link, ifName string, gateway net.IP) error { - existing, err := handle.RouteList(link, addrFamily(gateway)) + existing, err := nldump.On(handle).RouteList(link, addrFamily(gateway)) if err != nil { return fmt.Errorf("list routes on %q: %w", ifName, err) } @@ -136,7 +138,7 @@ func addDefaultRouteIfMissing(handle *netlink.Handle, link netlink.Link, ifName return fmt.Errorf("default route on %q already points via %s, refusing to add conflicting route via %s", ifName, r.Gw, gateway) } - routes, err := handle.RouteList(nil, addrFamily(gateway)) + routes, err := nldump.On(handle).RouteList(nil, addrFamily(gateway)) if err != nil { return fmt.Errorf("list namespace routes: %w", err) } @@ -208,7 +210,7 @@ func flushGuestNetnsConfig(netnsPath, ifName string) error { } for _, family := range []int{netlink.FAMILY_V6, netlink.FAMILY_V4} { - routes, err := handle.RouteList(link, family) + routes, err := nldump.On(handle).RouteList(link, family) if err != nil { return fmt.Errorf("list routes on %q: %w", ifName, err) } @@ -221,7 +223,7 @@ func flushGuestNetnsConfig(netnsPath, ifName string) error { } } - addrs, err := handle.AddrList(link, family) + addrs, err := nldump.On(handle).AddrList(link, family) if err != nil { return fmt.Errorf("list addresses on %q: %w", ifName, err) } diff --git a/internal/cni/ops_check.go b/internal/cni/ops_check.go index dd63dfd5..67df3acc 100644 --- a/internal/cni/ops_check.go +++ b/internal/cni/ops_check.go @@ -19,6 +19,7 @@ import ( "go.datum.net/galactic/internal/cnimaster" "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/nldump" ) // cmdCheck validates that the container's network state matches what ADD @@ -215,7 +216,7 @@ func validateIPOnInterface(ip net.IP, name, netns string) error { return fmt.Errorf("find link: %w", err) } - addrs, err := handle.AddrList(link, netlink.FAMILY_V6) + addrs, err := nldump.On(handle).AddrList(link, netlink.FAMILY_V6) if err != nil { return fmt.Errorf("list addresses: %w", err) } diff --git a/internal/cniroute/ops_check.go b/internal/cniroute/ops_check.go index 435d7779..0149630c 100644 --- a/internal/cniroute/ops_check.go +++ b/internal/cniroute/ops_check.go @@ -13,6 +13,7 @@ import ( "github.com/vishvananda/netlink" "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/nldump" "go.datum.net/galactic/internal/plumbing/vrf" ) @@ -63,7 +64,7 @@ func checkTerminationRoutes(vpc, vpcAttachment string, terminations []Terminatio } defer handle.Close() //nolint:errcheck // netlink cleanup on teardown - routes, err := handle.RouteListFiltered( + routes, err := nldump.On(handle).RouteListFiltered( netlink.FAMILY_V6, &netlink.Route{Table: int(tableID)}, netlink.RT_FILTER_TABLE, diff --git a/internal/edge/hostattach/hostattach.go b/internal/edge/hostattach/hostattach.go index 17b76564..d3fe244f 100644 --- a/internal/edge/hostattach/hostattach.go +++ b/internal/edge/hostattach/hostattach.go @@ -43,6 +43,8 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/plumbing/nldump" ) // MinMTU is the smallest MTU an attachment link or packet device may have: @@ -459,7 +461,7 @@ func ensureMTU(h *netlink.Handle, link netlink.Link, mtu int) error { // route for it from the VRF table, after which the address no longer accepts // traffic. func ensureAddr(h *netlink.Handle, link netlink.Link, addr *netlink.Addr) error { - addrs, err := h.AddrList(link, unix.AF_INET6) + addrs, err := nldump.On(h).AddrList(link, unix.AF_INET6) if err != nil { return err } @@ -535,7 +537,7 @@ func configureHostLink( // attachment or to a workload, and choosing between them is not the // provisioner's decision. func checkRouteOwner(h *netlink.Handle, r *netlink.Route) error { - existing, err := h.RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: r.Table, Dst: r.Dst}, + existing, err := nldump.On(h).RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: r.Table, Dst: r.Dst}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_DST) if err != nil { return fmt.Errorf("hostattach: list routes for %s in table %d: %w", r.Dst, r.Table, err) diff --git a/internal/edge/hostattach/inventory.go b/internal/edge/hostattach/inventory.go index 446abf71..c5f6ae35 100644 --- a/internal/edge/hostattach/inventory.go +++ b/internal/edge/hostattach/inventory.go @@ -11,6 +11,8 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/plumbing/nldump" ) const aliasPrefix = "galactic-edge:hostattach:" @@ -69,7 +71,7 @@ func ListHost() ([]ObservedAttachment, error) { } func listOwned(h *netlink.Handle) ([]ObservedAttachment, error) { - links, err := h.LinkList() + links, err := nldump.On(h).LinkList() if err != nil { return nil, fmt.Errorf("hostattach: list owned links: %w", err) } @@ -95,7 +97,7 @@ func hostPrefixes(a Attachment) []netip.Prefix { // no longer appear in desired state. Kernel-derived local/link-local routes // and routes using other links are never candidates for deletion. func extraRoutes(h *netlink.Handle, index int, table uint32, want []netip.Prefix) ([]netlink.Route, error) { - routes, err := h.RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: int(table), LinkIndex: index}, + routes, err := nldump.On(h).RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: int(table), LinkIndex: index}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) if err != nil { return nil, fmt.Errorf("hostattach: list attachment routes: %w", err) diff --git a/internal/edge/hostattach/readback.go b/internal/edge/hostattach/readback.go index 3816864b..441ba061 100644 --- a/internal/edge/hostattach/readback.go +++ b/internal/edge/hostattach/readback.go @@ -15,6 +15,8 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/plumbing/nldump" ) // ErrDrift is wrapped by Readback when the kernel does not hold the state the @@ -138,7 +140,7 @@ func checkLink(h *netlink.Handle, name, master string, add func(string, ...any)) } func checkAddr(h *netlink.Handle, link netlink.Link, want netip.Addr, add func(string, ...any)) { - addrs, err := h.AddrList(link, unix.AF_INET6) + addrs, err := nldump.On(h).AddrList(link, unix.AF_INET6) if err != nil { add("list addresses on %q: %v", link.Attrs().Name, err) return @@ -155,7 +157,7 @@ func checkAddr(h *netlink.Handle, link netlink.Link, want netip.Addr, add func(s } func checkNeigh(h *netlink.Handle, link netlink.Link, ip netip.Addr, mac net.HardwareAddr, add func(string, ...any)) { - neighs, err := h.NeighList(link.Attrs().Index, unix.AF_INET6) + neighs, err := nldump.On(h).NeighList(link.Attrs().Index, unix.AF_INET6) if err != nil { add("list neighbors on %q: %v", link.Attrs().Name, err) return @@ -177,7 +179,8 @@ func checkNeigh(h *netlink.Handle, link netlink.Link, ip netip.Addr, mac net.Har func checkRoute( h *netlink.Handle, table uint32, dst netip.Prefix, ifindex int, gw netip.Addr, add func(string, ...any), ) { - routes, err := h.RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: int(table)}, netlink.RT_FILTER_TABLE) + routes, err := nldump.On(h).RouteListFiltered( + unix.AF_INET6, &netlink.Route{Table: int(table)}, netlink.RT_FILTER_TABLE) if err != nil { add("list routes in table %d: %v", table, err) return @@ -198,7 +201,8 @@ func checkRoute( } func checkUnreachableDefault(h *netlink.Handle, table uint32, add func(string, ...any)) { - routes, err := h.RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: int(table)}, netlink.RT_FILTER_TABLE) + routes, err := nldump.On(h).RouteListFiltered( + unix.AF_INET6, &netlink.Route{Table: int(table)}, netlink.RT_FILTER_TABLE) if err != nil { add("list routes in table %d: %v", table, err) return @@ -214,7 +218,7 @@ func checkUnreachableDefault(h *netlink.Handle, table uint32, add func(string, . // checkLocalRoute requires the kernel's local route for addr in table, without // which the pod VRF does not deliver traffic addressed to the attachment. func checkLocalRoute(h *netlink.Handle, table uint32, addr netip.Addr, add func(string, ...any)) { - routes, err := h.RouteListFiltered(unix.AF_INET6, + routes, err := nldump.On(h).RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: int(table), Type: unix.RTN_LOCAL}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_TYPE) if err != nil { add("list local routes in table %d: %v", table, err) diff --git a/internal/gc/churn_root_test.go b/internal/gc/churn_root_test.go new file mode 100644 index 00000000..ca5e64e9 --- /dev/null +++ b/internal/gc/churn_root_test.go @@ -0,0 +1,178 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package gc + +import ( + "context" + "fmt" + "net" + "os" + "testing" + + "github.com/containernetworking/plugins/pkg/ns" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" + "go.datum.net/galactic/internal/cni/tap" + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/crdnames" + "go.datum.net/galactic/internal/plumbing/ebpf/attach" + "go.datum.net/galactic/internal/plumbing/ebpf/attachreg" + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/nldump/nldumptest" + "go.datum.net/galactic/internal/plumbing/vrf" +) + +// churnPasses is how many passes each churn test runs at least, enough for a +// single-attempt listing to meet an interruption. +const churnPasses = 200 + +// TestResolveVRFKernelState_UnderLinkChurn requires the VRF table and egress +// kind lookup the eBPF vrf_table sweep repairs from to succeed while links +// are added and removed on the node. +func TestResolveVRFKernelState_UnderLinkChurn(t *testing.T) { + requireRoot(t) + const ( + nodeName = "churn-node" + vpc = "jU" + ) + nsObj, err := ns.TempNetNS() + if err != nil { + t.Fatalf("create test netns: %v", err) + } + t.Cleanup(func() { _ = nsObj.Close() }) + + var wantTable uint32 + err = nsObj.Do(func(ns.NetNS) error { + if err := vrf.Add(vpc); err != nil { + return fmt.Errorf("vrf.Add: %w", err) + } + if _, err := tap.Add(vpc, "c1a", "owner", 1500); err != nil { + return fmt.Errorf("tap.Add: %w", err) + } + wantTable, err = vrf.TableID(vpc) + return err + }) + if err != nil { + t.Fatal(err) + } + nldumptest.Churn(t, nsObj) + + inst := crdnames.BGPVRFInstanceName(vpc, nodeName) + passes := nldumptest.UntilInterrupted(t, nsObj, []string{nldumptest.KindLink}, churnPasses, func() error { + table, kind, ok, err := resolveVRFKernelState(nodeName, inst) + if err != nil { + return err + } + if !ok || table != wantTable || kind != usidmap.EgressKindTap { + return fmt.Errorf("resolved table %d kind %d ok %v, want table %d kind %d", + table, kind, ok, wantTable, usidmap.EgressKindTap) + } + return nil + }) + t.Logf("%d passes", passes) +} + +// TestRepairAttachmentDatapath_UnderChurn requires a datapath repair pass to +// complete, examining every attachment, while links and addresses are added +// and removed on the node. +func TestRepairAttachmentDatapath_UnderChurn(t *testing.T) { + requireRoot(t) + const ( + nodeName = "churn-node" + uplinkName = "chup0" + shardSID = "2001:db8:ff01:9:e001::" + shardLocator = "2001:db8:ff01:9::/64" + ) + node := attachreg.Node{Locator: testLocator, NodeID: 5} + egress := attachreg.EgressConfig{ShardSIDs: shardSID, NAT64Prefix: "64:ff9b::/96"} + attachments := []rootAttachment{ + {vpc: "jU", vpcAttachment: "c1a", ifaceType: attachreg.InterfaceTypeVeth, vrfID: 0x21, ipam: &cniipam.IPAMResult{ + IPv6Subnet: mustCIDR(t, "fd20:30:ff01::/96"), + IPv6Gateway: net.ParseIP("fd20:30::1"), + IPv4Address: net.ParseIP("10.1.0.5"), + IPv4Gateway: net.ParseIP("172.21.1.1"), + }}, + {vpc: "kV", vpcAttachment: "c1b", ifaceType: attachreg.InterfaceTypeTap, vrfID: 0x22, ipam: &cniipam.IPAMResult{ + IPv6Subnet: mustCIDR(t, "fd20:31:ff02::/96"), + IPv6Gateway: net.ParseIP("fd20:31::1"), + }}, + } + + objects := make([]client.Object, 0, 1+2*len(attachments)) + objects = append(objects, &bgpv1alpha1.BGPRouter{ + ObjectMeta: metav1.ObjectMeta{Name: repairRouter, Namespace: repairNamespace}, + Spec: bgpv1alpha1.BGPRouterSpec{ + TargetRef: bgpv1alpha1.TargetRef{Kind: targetKindNode, Name: nodeName}, LocalASN: 65000, + SRv6Locator: node.Locator, NodeID: node.NodeID, + }, + }) + for _, a := range attachments { + objects = append(objects, &bgpv1alpha1.BGPVRFInstance{ + ObjectMeta: metav1.ObjectMeta{Name: crdnames.BGPVRFInstanceName(a.vpc, nodeName), Namespace: repairNamespace}, + Spec: bgpv1alpha1.BGPVRFInstanceSpec{ + RouterTarget: bgpv1alpha1.RouterTarget{RouterRef: &bgpv1alpha1.RouterRef{Name: repairRouter}}, + VRFID: a.vrfID, + ImportRouteTargets: []bgpv1alpha1.RouteTarget{{Value: testRouteTarget}}, + ExportRouteTargets: []bgpv1alpha1.RouteTarget{{Value: testRouteTarget}}, + }, + }, a.advertisement(nodeName)) + } + k8s := fake.NewClientBuilder().WithScheme(gcTestScheme(t)).WithObjects(objects...).Build() + + pinDir := fmt.Sprintf("/sys/fs/bpf/galactic-churn-test-%d", os.Getpid()) + t.Cleanup(func() { _ = os.RemoveAll(pinDir) }) + objs, err := attach.Load(pinDir) + if err != nil { + t.Fatalf("attach.Load: %v", err) + } + t.Cleanup(func() { _ = objs.Close() }) + + t.Setenv(config.EnvCNIEBPFInterfaces, uplinkName) + attach.InvalidateUplinkIndexes() + t.Cleanup(attach.InvalidateUplinkIndexes) + + nsObj, err := ns.TempNetNS() + if err != nil { + t.Fatalf("create test netns: %v", err) + } + t.Cleanup(func() { _ = nsObj.Close() }) + + err = nsObj.Do(func(ns.NetNS) error { + if err := setUpUplink(uplinkName, shardLocator); err != nil { + return err + } + for _, a := range attachments { + if err := createAttachment(a); err != nil { + return err + } + if err := registerAttachment(a, node, pinDir, egress); err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("register attachments: %v", err) + } + nldumptest.Churn(t, nsObj) + + passes := nldumptest.UntilInterrupted(t, nsObj, []string{nldumptest.KindLink}, churnPasses, func() error { + result, err := RepairAttachmentDatapath(context.Background(), k8s, DatapathRepairConfig{ + Namespace: repairNamespace, NodeName: nodeName, PinDir: pinDir, Egress: egress, + }) + if err != nil { + return err + } + if result.Attachments != len(attachments) { + return fmt.Errorf("repair examined %d attachments, want %d", result.Attachments, len(attachments)) + } + return nil + }) + t.Logf("%d passes", passes) +} diff --git a/internal/gc/datapath_repair.go b/internal/gc/datapath_repair.go index c3161055..c9401632 100644 --- a/internal/gc/datapath_repair.go +++ b/internal/gc/datapath_repair.go @@ -24,6 +24,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/attachreg" "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" + "go.datum.net/galactic/internal/plumbing/nldump" "go.datum.net/galactic/internal/plumbing/srv6" ) @@ -37,9 +38,9 @@ var hostLinkNameRegex = regexp.MustCompile(`^G([A-Za-z0-9]{9})([A-Za-z0-9]{3})H$ var ( openDatapathMapsFn = attachreg.OpenPinnedMaps listRoutesFn = func(tableID uint32, family int) ([]netlink.Route, error) { - return netlink.RouteListFiltered(family, &netlink.Route{Table: int(tableID)}, netlink.RT_FILTER_TABLE) + return nldump.RouteListFiltered(family, &netlink.Route{Table: int(tableID)}, netlink.RT_FILTER_TABLE) } - listAddrsFn = netlink.AddrList + listAddrsFn = nldump.AddrList linkByIndexFn = netlink.LinkByIndex resolvePublicUplinkFn attachreg.UplinkResolver = srv6.ResolvePublicUplink ) diff --git a/internal/gc/gc.go b/internal/gc/gc.go index 45de2eae..bd7c0db4 100644 --- a/internal/gc/gc.go +++ b/internal/gc/gc.go @@ -28,6 +28,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/nptv6map" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/nldump" "go.datum.net/galactic/internal/plumbing/nptv6" "go.datum.net/galactic/internal/plumbing/vrf" ) @@ -850,7 +851,7 @@ func edgeLinkVRFKeys(pinDir string) (map[usidmap.VRFKey]struct{}, error) { // needing no real VRF interface or CAP_NET_ADMIN. var ( listVRFLinksFn = vrf.ListVRFLinks - listAllLinksFn = netlink.LinkList + listAllLinksFn = nldump.LinkList ) // resolveVRFKernelState recovers the Linux VRF routing table ID and egress diff --git a/internal/hostconf/hostconf.go b/internal/hostconf/hostconf.go index 16323618..306719c2 100644 --- a/internal/hostconf/hostconf.go +++ b/internal/hostconf/hostconf.go @@ -34,6 +34,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/galactic/internal/plumbing/nldump" ) // PluginType is the "type" value the installer writes into the static @@ -267,7 +269,7 @@ func DetectNodeNameFromAPI() (string, error) { return "", fmt.Errorf("list nodes: %w", err) } - addrs, err := netlink.AddrList(nil, netlink.FAMILY_ALL) + addrs, err := nldump.AddrList(nil, netlink.FAMILY_ALL) if err != nil { return "", fmt.Errorf("list local addresses: %w", err) } diff --git a/internal/hostgw/churn_root_test.go b/internal/hostgw/churn_root_test.go new file mode 100644 index 00000000..11ebe41b --- /dev/null +++ b/internal/hostgw/churn_root_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package hostgw + +import ( + "fmt" + "net" + "testing" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" + + "go.datum.net/galactic/internal/plumbing/nldump/nldumptest" +) + +// churnPasses is how many passes the churn test runs at least, enough for a +// single-attempt link listing to meet an interruption. +const churnPasses = 200 + +// TestEnsureTapGuestNeighbors_UnderLinkChurn requires the tap guest neighbor +// sweep to find and resolve a VPC tap's guest on every pass while links are +// added and removed on the node. The guest's neighbor entry is already +// present, so a pass that finds the tap resolves it without soliciting. +func TestEnsureTapGuestNeighbors_UnderLinkChurn(t *testing.T) { + requireRoot(t) + const table = 1000 + nsObj, err := ns.TempNetNS() + if err != nil { + t.Fatalf("create test netns: %v", err) + } + t.Cleanup(func() { _ = nsObj.Close() }) + + err = nsObj.Do(func(ns.NetNS) error { + vrfLink := &netlink.Vrf{LinkAttrs: netlink.LinkAttrs{Name: "churnvrf"}, Table: table} + if err := netlink.LinkAdd(vrfLink); err != nil { + return fmt.Errorf("add VRF: %w", err) + } + tap := &netlink.Tuntap{LinkAttrs: netlink.LinkAttrs{Name: "churntap"}, Mode: netlink.TUNTAP_MODE_TAP} + if err := netlink.LinkAdd(tap); err != nil { + return fmt.Errorf("add tap: %w", err) + } + if err := netlink.LinkSetMaster(tap, vrfLink); err != nil { + return fmt.Errorf("enslave tap: %w", err) + } + for _, l := range []netlink.Link{vrfLink, tap} { + if err := netlink.LinkSetUp(l); err != nil { + return fmt.Errorf("set %s up: %w", l.Attrs().Name, err) + } + } + guest := mustParseCIDR(t, "fd20:40:ff01::/96") + if err := netlink.RouteAdd(&netlink.Route{LinkIndex: tap.Attrs().Index, Dst: guest, Table: table}); err != nil { + return fmt.Errorf("add guest route: %w", err) + } + return netlink.NeighAdd(&netlink.Neigh{ + LinkIndex: tap.Attrs().Index, Family: netlink.FAMILY_V6, State: netlink.NUD_PERMANENT, + IP: guest.IP, HardwareAddr: net.HardwareAddr{0x02, 0, 0, 0, 0, 1}, + }) + }) + if err != nil { + t.Fatal(err) + } + nldumptest.Churn(t, nsObj) + + passes := nldumptest.UntilInterrupted(t, nsObj, []string{nldumptest.KindLink}, churnPasses, func() error { + if resolved, pending := EnsureTapGuestNeighbors(); resolved != 1 || pending != 0 { + return fmt.Errorf("EnsureTapGuestNeighbors() = (%d, %d), want (1, 0)", resolved, pending) + } + return nil + }) + t.Logf("%d passes", passes) +} diff --git a/internal/hostgw/hostgw.go b/internal/hostgw/hostgw.go index 4ffc3975..b27d71fb 100644 --- a/internal/hostgw/hostgw.go +++ b/internal/hostgw/hostgw.go @@ -25,6 +25,7 @@ import ( "go.datum.net/galactic/internal/cniipam" "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/nldump" "go.datum.net/galactic/internal/plumbing/vrf" ) @@ -230,7 +231,7 @@ func installPodSubnetRoute(hostLink netlink.Link, subnet *net.IPNet, family, tab desiredRoute.Flags = int(netlink.FLAG_ONLINK) } - existingRoutes, err := netlink.RouteListFiltered( + existingRoutes, err := nldump.RouteListFiltered( family, &netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE, diff --git a/internal/hostgw/tapneigh.go b/internal/hostgw/tapneigh.go index 034eb6ee..672f0930 100644 --- a/internal/hostgw/tapneigh.go +++ b/internal/hostgw/tapneigh.go @@ -14,6 +14,8 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/plumbing/nldump" ) const ( @@ -57,7 +59,7 @@ const ( // Errors are per attachment and never abort the sweep: one guest that is down // must not stop the others. func EnsureTapGuestNeighbors() (resolved, pending int) { - links, err := netlink.LinkList() + links, err := nldump.LinkList() if err != nil { slog.Warn("hostgw: could not list links to resolve tap guest neighbors", "err", err) return 0, 0 @@ -124,7 +126,7 @@ func EnsureTapGuestNeighbors() (resolved, pending int) { func guestAddrsForTap(tableID, tapIndex int) []net.IP { var out []net.IP for _, family := range []int{netlink.FAMILY_V6, netlink.FAMILY_V4} { - routes, err := netlink.RouteListFiltered(family, + routes, err := nldump.RouteListFiltered(family, &netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE) if err != nil { continue @@ -182,7 +184,7 @@ func lookupNeighborMAC(linkIndex int, guest net.IP) (net.HardwareAddr, error) { if guest.To4() != nil { family = netlink.FAMILY_V4 } - neighs, err := netlink.NeighList(linkIndex, family) + neighs, err := nldump.NeighList(linkIndex, family) if err != nil { return nil, fmt.Errorf("list neighbors on link %d: %w", linkIndex, err) } diff --git a/internal/installer/installer.go b/internal/installer/installer.go index aa2722c1..000b1d59 100644 --- a/internal/installer/installer.go +++ b/internal/installer/installer.go @@ -199,7 +199,7 @@ var newK8sClientFn = func() (client.Client, error) { // addrListFn can be overridden in tests to mock netlink interface addresses. var addrListFn = func(family int) ([]netlink.Addr, error) { - return netlink.AddrList(nil, family) + return nldump.AddrList(nil, family) } // ebpfStartFn loads, pins, and attaches the eBPF uSID datapath, then keeps its diff --git a/internal/installer/locatorroute.go b/internal/installer/locatorroute.go index 2d3abbcb..33cc1b41 100644 --- a/internal/installer/locatorroute.go +++ b/internal/installer/locatorroute.go @@ -18,6 +18,7 @@ import ( bgpv1alpha1 "go.datum.net/galactic/api/network/v1alpha1" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" + "go.datum.net/galactic/internal/plumbing/nldump" ) // ensureLocatorLocalRoute installs a local route for this node's uSID locator @@ -96,7 +97,7 @@ func ensureLocatorLocalRoute(ctx context.Context, k8s client.Client, namespace, // pruneStaleLocatorLocalRoutes removes every locator local route on lo except // keep. See staleLocatorLocalRoute for what it will and will not touch. func pruneStaleLocatorLocalRoutes(loIndex int, keep netip.Prefix) error { - routes, err := netlink.RouteListFiltered(unix.AF_INET6, + routes, err := nldump.RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: unix.RT_TABLE_LOCAL, LinkIndex: loIndex}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF) if err != nil { diff --git a/internal/installer/sidecarreturn.go b/internal/installer/sidecarreturn.go index e1d821f0..cf93939d 100644 --- a/internal/installer/sidecarreturn.go +++ b/internal/installer/sidecarreturn.go @@ -26,6 +26,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/metrics" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/nldump" ) // The ingress sidecar's return path, host side. @@ -355,7 +356,7 @@ func ensureSidecarReturnNeighbor(addr netip.Addr, hostIfindex int, mac net.Hardw // routes, and a range check that drifted from the constants would delete // someone else's. func pruneSidecarReturnRoutes(live map[uint32]netip.Addr) error { - routes, err := netlink.RouteListFiltered(netlink.FAMILY_V6, + routes, err := nldump.RouteListFiltered(netlink.FAMILY_V6, &netlink.Route{Table: unix.RT_TABLE_UNSPEC}, netlink.RT_FILTER_TABLE) if err != nil { return fmt.Errorf("list routes to prune the sidecar return path: %w", err) diff --git a/internal/plumbing/ebpf/attach/egress_reattach.go b/internal/plumbing/ebpf/attach/egress_reattach.go index ad8d0408..a39840c9 100644 --- a/internal/plumbing/ebpf/attach/egress_reattach.go +++ b/internal/plumbing/ebpf/attach/egress_reattach.go @@ -13,6 +13,7 @@ import ( "golang.org/x/sys/unix" "go.datum.net/galactic/internal/plumbing/ebpf/attachmentidentity" + "go.datum.net/galactic/internal/plumbing/nldump" ) // ReattachEgress moves every usid_egress attachment on this node's veths and @@ -156,9 +157,9 @@ func linkGone(err error) bool { return errors.As(err, ¬Found) || errors.Is(err, unix.ENODEV) || errors.Is(err, unix.ENOENT) } -// listLinksFn indirects netlink.LinkList so a test can confine ReattachEgress +// listLinksFn indirects nldump.LinkList so a test can confine ReattachEgress // to its own namespace's links. -var listLinksFn = netlink.LinkList +var listLinksFn = nldump.LinkList // ReattachDeliver attaches tp_deliver to the tc egress of every interface // carrying the managed egress chain, replacing any filter there in place, and diff --git a/internal/plumbing/ebpf/attach/interfaces.go b/internal/plumbing/ebpf/attach/interfaces.go index 089d7f30..c4835f7c 100644 --- a/internal/plumbing/ebpf/attach/interfaces.go +++ b/internal/plumbing/ebpf/attach/interfaces.go @@ -17,6 +17,7 @@ import ( "go.datum.net/galactic/internal/config" "go.datum.net/galactic/internal/plumbing/bond" + "go.datum.net/galactic/internal/plumbing/nldump" ) // routeListFn and linkByIndexFn are package vars so tests can substitute a fake @@ -29,7 +30,7 @@ import ( // VRF-scoped underlay, and would otherwise be undetectable. var ( routeListFn = func() ([]netlink.Route, error) { - return netlink.RouteListFiltered(netlink.FAMILY_V6, + return nldump.RouteListFiltered(netlink.FAMILY_V6, &netlink.Route{Table: unix.RT_TABLE_UNSPEC}, netlink.RT_FILTER_TABLE) } linkByIndexFn = func(index int) (netlink.Link, error) { @@ -38,7 +39,7 @@ var ( // linkListFn enumerates every link on the host, for expandBondSlaves. // linkByNameFn is declared in health.go and reused here. linkListFn = func() ([]netlink.Link, error) { - return netlink.LinkList() + return nldump.LinkList() } ) diff --git a/internal/plumbing/ebpf/edgeattach/attach.go b/internal/plumbing/ebpf/edgeattach/attach.go index e0eb4801..8afd05e9 100644 --- a/internal/plumbing/ebpf/edgeattach/attach.go +++ b/internal/plumbing/ebpf/edgeattach/attach.go @@ -22,6 +22,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/mappin" "go.datum.net/galactic/internal/plumbing/ebpf/xdpattach" "go.datum.net/galactic/internal/plumbing/ebpf/xdpmode" + "go.datum.net/galactic/internal/plumbing/nldump" ) // linkByNameFn and linkListFn are override points, as elsewhere in this @@ -29,7 +30,7 @@ import ( // touching the host network stack. var ( linkByNameFn = netlink.LinkByName - linkListFn = netlink.LinkList + linkListFn = nldump.LinkList ) // PinDir is the default bpffs directory every edge program map is pinned under, diff --git a/internal/plumbing/ebpf/egressroutemap/egressroute.go b/internal/plumbing/ebpf/egressroutemap/egressroute.go index d7de9e52..6e5f33b9 100644 --- a/internal/plumbing/ebpf/egressroutemap/egressroute.go +++ b/internal/plumbing/ebpf/egressroutemap/egressroute.go @@ -22,6 +22,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/prog" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/nldump" ) // neighborResolveTimeout and neighborResolvePollInterval bound how long @@ -270,7 +271,7 @@ func solicitNeighbor(linkIndex int, nextHop net.IP) { // soliciting. It is resolveNeighbor's passive fast path and its poll step once // a solicit is in flight. func lookupNeighbor(linkIndex int, nextHop net.IP) (net.HardwareAddr, error) { - neighs, err := netlink.NeighList(linkIndex, netlink.FAMILY_V6) + neighs, err := nldump.NeighList(linkIndex, netlink.FAMILY_V6) if err != nil { return nil, fmt.Errorf("list neighbors on link %d: %w", linkIndex, err) } diff --git a/internal/plumbing/ebpf/natattach/attach.go b/internal/plumbing/ebpf/natattach/attach.go index 9a1830b7..289a93cf 100644 --- a/internal/plumbing/ebpf/natattach/attach.go +++ b/internal/plumbing/ebpf/natattach/attach.go @@ -20,6 +20,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/natprog" "go.datum.net/galactic/internal/plumbing/ebpf/xdpattach" "go.datum.net/galactic/internal/plumbing/ebpf/xdpmode" + "go.datum.net/galactic/internal/plumbing/nldump" ) // linkByNameFn and linkListFn are override points, as elsewhere in this @@ -27,7 +28,7 @@ import ( // touching the host network stack. var ( linkByNameFn = netlink.LinkByName - linkListFn = netlink.LinkList + linkListFn = nldump.LinkList ) // PinDir is the default bpffs directory every NAT66 map is pinned under, diff --git a/internal/plumbing/ebpf/xdpattach/set.go b/internal/plumbing/ebpf/xdpattach/set.go index 45a8bd95..7e3a65c5 100644 --- a/internal/plumbing/ebpf/xdpattach/set.go +++ b/internal/plumbing/ebpf/xdpattach/set.go @@ -16,11 +16,12 @@ import ( "github.com/vishvananda/netlink" "go.datum.net/galactic/internal/plumbing/ebpf/xdpmode" + "go.datum.net/galactic/internal/plumbing/nldump" ) // linkListFn is an override point, matching linkByNameFn, so the bond sibling // lookup can be faked in tests. -var linkListFn = netlink.LinkList +var linkListFn = nldump.LinkList // attachFn is an override point so Set's tests can drive attachment without a // real program or a real NIC. diff --git a/internal/plumbing/loaddr/loaddr.go b/internal/plumbing/loaddr/loaddr.go index 8d5eb929..ede22465 100644 --- a/internal/plumbing/loaddr/loaddr.go +++ b/internal/plumbing/loaddr/loaddr.go @@ -13,6 +13,8 @@ import ( "fmt" "github.com/vishvananda/netlink" + + "go.datum.net/galactic/internal/plumbing/nldump" ) // Detect returns the first global-unicast IPv6 address on the loopback, @@ -24,7 +26,7 @@ func Detect() (string, error) { return "", fmt.Errorf("find lo interface: %w", err) } - addrs, err := netlink.AddrList(link, netlink.FAMILY_V6) + addrs, err := nldump.AddrList(link, netlink.FAMILY_V6) if err != nil { return "", fmt.Errorf("list lo addresses: %w", err) } diff --git a/internal/plumbing/nldump/churn_internal_test.go b/internal/plumbing/nldump/churn_internal_test.go index eb4bf044..f1b8292e 100644 --- a/internal/plumbing/nldump/churn_internal_test.go +++ b/internal/plumbing/nldump/churn_internal_test.go @@ -6,6 +6,7 @@ package nldump import ( "fmt" + "net" "os" "sync" "testing" @@ -14,6 +15,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" ) func requireRoot(t *testing.T) { @@ -23,27 +25,80 @@ func requireRoot(t *testing.T) { } } -// TestLinkList_CountsRealInterruptions lists links on a real kernel while -// other goroutines add and remove links in the same namespace. Every listing -// succeeds, and the interrupted counter records the interruptions the -// kernel reported along the way. -func TestLinkList_CountsRealInterruptions(t *testing.T) { +// churnCase is one listing kind exercised under churn: setup adds the stable +// entries the listing walks, change adds and removes one transient entry, and +// list makes one retrying listing. +type churnCase struct { + kind string + setup func() error + change func(churner, i int) + list func() error +} + +// TestListings_CountRealInterruptions lists each kind on a real kernel while +// other goroutines add and remove entries of that kind in the same namespace. +// Every listing succeeds, and the interrupted counter records the +// interruptions the kernel reported along the way. +func TestListings_CountRealInterruptions(t *testing.T) { requireRoot(t) + var link netlink.Link + cases := []churnCase{ + { + kind: kindLink, + setup: func() error { + for i := range 300 { + if err := netlink.LinkAdd(dummy(fmt.Sprintf("stab%d", i))); err != nil { + return fmt.Errorf("add stable link %d: %w", i, err) + } + } + return nil + }, + change: func(c, i int) { + d := dummy(fmt.Sprintf("churn%d_%d", c, i%4)) + if err := netlink.LinkAdd(d); err == nil { + _ = netlink.LinkDel(d) + } + }, + list: func() error { _, err := LinkList(); return err }, + }, + { + kind: kindAddress, + setup: func() error { + d := dummy("addr0") + if err := netlink.LinkAdd(d); err != nil { + return fmt.Errorf("add link: %w", err) + } + link = d + for i := range 300 { + if err := netlink.AddrAdd(link, addr(0, i)); err != nil { + return fmt.Errorf("add stable address %d: %w", i, err) + } + } + return nil + }, + change: func(c, i int) { + a := addr(c+1, i%4) + if err := netlink.AddrAdd(link, a); err == nil { + _ = netlink.AddrDel(link, a) + } + }, + list: func() error { _, err := AddrList(nil, netlink.FAMILY_V6); return err }, + }, + } + for _, tc := range cases { + t.Run(tc.kind, func(t *testing.T) { runChurn(t, tc) }) + } +} + +func runChurn(t *testing.T, tc churnCase) { + t.Helper() nsObj, err := ns.TempNetNS() if err != nil { t.Fatalf("create test netns: %v", err) } t.Cleanup(func() { _ = nsObj.Close() }) - err = nsObj.Do(func(_ ns.NetNS) error { - for i := range 300 { - if err := netlink.LinkAdd(&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: fmt.Sprintf("stab%d", i)}}); err != nil { - return fmt.Errorf("add stable link %d: %w", i, err) - } - } - return nil - }) - if err != nil { + if err := nsObj.Do(func(_ ns.NetNS) error { return tc.setup() }); err != nil { t.Fatal(err) } @@ -60,10 +115,7 @@ func TestLinkList_CountsRealInterruptions(t *testing.T) { return nil default: } - dummy := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: fmt.Sprintf("churn%d_%d", c, i%4)}} - if err := netlink.LinkAdd(dummy); err == nil { - _ = netlink.LinkDel(dummy) - } + tc.change(c, i) // Bursts of changes, not a saturated loop that could // outlast any bounded retry. time.Sleep(2 * time.Millisecond) @@ -76,18 +128,18 @@ func TestLinkList_CountsRealInterruptions(t *testing.T) { wg.Wait() }) - before := testutil.ToFloat64(interrupted.WithLabelValues(kindLink)) - beforeFailures := testutil.ToFloat64(failures.WithLabelValues(kindLink)) + before := testutil.ToFloat64(interrupted.WithLabelValues(tc.kind)) + beforeFailures := testutil.ToFloat64(failures.WithLabelValues(tc.kind)) err = nsObj.Do(func(_ ns.NetNS) error { // Churn interrupts a listing within a few iterations; the deadline // only bounds a kernel that never reports one. deadline := time.Now().Add(10 * time.Second) - for i := 0; testutil.ToFloat64(interrupted.WithLabelValues(kindLink)) == before; i++ { + for i := 0; testutil.ToFloat64(interrupted.WithLabelValues(tc.kind)) == before; i++ { if time.Now().After(deadline) { - return fmt.Errorf("no interruption counted after %d listings under link churn", i) + return fmt.Errorf("no interruption counted after %d listings under %s churn", i, tc.kind) } - if _, err := LinkList(); err != nil { - return fmt.Errorf("LinkList failed on iteration %d while links were changing: %w", i, err) + if err := tc.list(); err != nil { + return fmt.Errorf("%s listing failed on iteration %d while entries were changing: %w", tc.kind, i, err) } } return nil @@ -95,7 +147,20 @@ func TestLinkList_CountsRealInterruptions(t *testing.T) { if err != nil { t.Fatal(err) } - if d := testutil.ToFloat64(failures.WithLabelValues(kindLink)) - beforeFailures; d != 0 { + if d := testutil.ToFloat64(failures.WithLabelValues(tc.kind)) - beforeFailures; d != 0 { t.Errorf("failure counter rose by %v, want 0", d) } } + +func dummy(name string) *netlink.Dummy { + return &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: name}} +} + +// addr returns an IPv6 address unique per (group, i), with DAD off so it is +// usable at once. +func addr(group, i int) *netlink.Addr { + return &netlink.Addr{ + IPNet: &net.IPNet{IP: net.ParseIP(fmt.Sprintf("fd00:%x::%x", group, i+1)), Mask: net.CIDRMask(128, 128)}, + Flags: unix.IFA_F_NODAD, + } +} diff --git a/internal/plumbing/nldump/nldump.go b/internal/plumbing/nldump/nldump.go index 6ea3d527..8f2529f2 100644 --- a/internal/plumbing/nldump/nldump.go +++ b/internal/plumbing/nldump/nldump.go @@ -2,22 +2,25 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later -// Package nldump lists links, addresses and routes over netlink without -// failing when another process changes them mid-dump. +// Package nldump lists links, addresses, routes, neighbors and rules over +// netlink without failing when another process changes them mid-dump. // // The kernel flags a dump interrupted when the state it walks changes while // the dump runs, as it does whenever another process adds or removes a link, -// address or route. The package-level netlink list functions make a single -// attempt and return netlink.ErrDumpInterrupted, so any churn on the node -// fails the caller. These functions retry an interrupted dump a bounded -// number of times and never return a partial result: an entry missing from an -// interrupted dump would look absent. Every interrupted attempt, and every -// dump still interrupted on its last attempt, is counted in the metrics -// MustRegister exposes, so churn is visible before it fails a caller. +// address, route, neighbor or rule. The netlink list functions and Handle +// methods make a single attempt and return netlink.ErrDumpInterrupted, so any +// churn on the node fails the caller. These functions retry an interrupted +// dump a bounded number of times and never return a partial result: an entry +// missing from an interrupted dump would look absent. Every interrupted +// attempt, and every dump still interrupted on its last attempt, is counted +// in the metrics MustRegister exposes, so churn is visible before it fails a +// caller. // -// Each call opens its own netlink socket in the calling thread's network -// namespace, with the same send and receive timeout, as the package-level -// functions do. +// Each package-level function opens its own netlink socket in the calling +// thread's network namespace, with the same send and receive timeout, as the +// package-level netlink functions do. On returns a Lister that lists over a +// handle the caller already holds instead, such as one opened in another +// network namespace. package nldump import ( @@ -45,9 +48,11 @@ const labelKind = "kind" // Listing kinds, the values of labelKind. const ( - kindLink = "link" - kindAddress = "address" - kindRoute = "route" + kindLink = "link" + kindAddress = "address" + kindRoute = "route" + kindNeighbor = "neighbor" + kindRule = "rule" ) var ( @@ -67,7 +72,7 @@ var ( ) func init() { - for _, kind := range []string{kindLink, kindAddress, kindRoute} { + for _, kind := range []string{kindLink, kindAddress, kindRoute, kindNeighbor, kindRule} { interrupted.WithLabelValues(kind) failures.WithLabelValues(kind) } @@ -85,26 +90,86 @@ var sleep = time.Sleep // LinkList returns every link in the calling thread's network namespace. func LinkList() ([]netlink.Link, error) { - return dump(kindLink, func(h *netlink.Handle) ([]netlink.Link, error) { return h.LinkList() }) + return dump(Lister.LinkList) } // AddrList returns link's addresses of family, or every link's when link is // nil. Errors other than an interrupted dump are returned as netlink returns // them. func AddrList(link netlink.Link, family int) ([]netlink.Addr, error) { - return dump(kindAddress, func(h *netlink.Handle) ([]netlink.Addr, error) { return h.AddrList(link, family) }) + return dump(func(l Lister) ([]netlink.Addr, error) { return l.AddrList(link, family) }) +} + +// RouteList returns the main-table routes of family through link, or through +// any link when link is nil, as netlink.RouteList does. +func RouteList(link netlink.Link, family int) ([]netlink.Route, error) { + return dump(func(l Lister) ([]netlink.Route, error) { return l.RouteList(link, family) }) } // RouteListFiltered returns the routes of family that match filter on the // fields filterMask selects, as netlink.RouteListFiltered does. func RouteListFiltered(family int, filter *netlink.Route, filterMask uint64) ([]netlink.Route, error) { - return dump(kindRoute, func(h *netlink.Handle) ([]netlink.Route, error) { - return h.RouteListFiltered(family, filter, filterMask) - }) + return dump(func(l Lister) ([]netlink.Route, error) { return l.RouteListFiltered(family, filter, filterMask) }) +} + +// NeighList returns the neighbor entries of family on the link with index +// linkIndex, or on every link when linkIndex is 0, as netlink.NeighList does. +func NeighList(linkIndex, family int) ([]netlink.Neigh, error) { + return dump(func(l Lister) ([]netlink.Neigh, error) { return l.NeighList(linkIndex, family) }) +} + +// RuleList returns the policy routing rules of family, as netlink.RuleList +// does. +func RuleList(family int) ([]netlink.Rule, error) { + return dump(func(l Lister) ([]netlink.Rule, error) { return l.RuleList(family) }) +} + +// Lister runs the retrying listings over a netlink handle its caller owns, in +// whatever network namespace that handle was opened. It neither closes the +// handle nor changes its socket timeout. +type Lister struct { + h *netlink.Handle +} + +// On returns a Lister over h. +func On(h *netlink.Handle) Lister { + return Lister{h: h} +} + +// LinkList returns every link, as LinkList does. +func (l Lister) LinkList() ([]netlink.Link, error) { + return retry(kindLink, l.h.LinkList) +} + +// AddrList returns link's addresses of family, as AddrList does. +func (l Lister) AddrList(link netlink.Link, family int) ([]netlink.Addr, error) { + return retry(kindAddress, func() ([]netlink.Addr, error) { return l.h.AddrList(link, family) }) +} + +// RouteList returns the routes of family through link, as RouteList does. +func (l Lister) RouteList(link netlink.Link, family int) ([]netlink.Route, error) { + return retry(kindRoute, func() ([]netlink.Route, error) { return l.h.RouteList(link, family) }) +} + +// RouteListFiltered returns the routes matching filter, as RouteListFiltered +// does. +func (l Lister) RouteListFiltered(family int, filter *netlink.Route, filterMask uint64) ([]netlink.Route, error) { + return retry(kindRoute, func() ([]netlink.Route, error) { return l.h.RouteListFiltered(family, filter, filterMask) }) +} + +// NeighList returns the neighbor entries on linkIndex, as NeighList does. +func (l Lister) NeighList(linkIndex, family int) ([]netlink.Neigh, error) { + return retry(kindNeighbor, func() ([]netlink.Neigh, error) { return l.h.NeighList(linkIndex, family) }) +} + +// RuleList returns the policy routing rules of family, as RuleList does. +func (l Lister) RuleList(family int) ([]netlink.Rule, error) { + return retry(kindRule, func() ([]netlink.Rule, error) { return l.h.RuleList(family) }) } -// dump runs list over an rtnetlink handle, through retry. -func dump[T any](kind string, list func(*netlink.Handle) (T, error)) (T, error) { +// dump runs list over a Lister on a fresh rtnetlink handle in the calling +// thread's network namespace. +func dump[T any](list func(Lister) (T, error)) (T, error) { var zero T // The handle makes one attempt per call, so retry sees, and counts, every // interruption. @@ -119,7 +184,7 @@ func dump[T any](kind string, list func(*netlink.Handle) (T, error)) (T, error) if err := handle.SetSocketTimeout(netlink.GetSocketTimeout()); err != nil { return zero, err } - return retry(kind, func() (T, error) { return list(handle) }) + return list(On(handle)) } // retry calls list until it returns anything but netlink.ErrDumpInterrupted, diff --git a/internal/plumbing/nldump/nldump_internal_test.go b/internal/plumbing/nldump/nldump_internal_test.go index 77f8c8db..a692e0f8 100644 --- a/internal/plumbing/nldump/nldump_internal_test.go +++ b/internal/plumbing/nldump/nldump_internal_test.go @@ -149,7 +149,7 @@ func TestMustRegister_ExposesEveryKind(t *testing.T) { } } } - want := []string{kindAddress, kindLink, kindRoute} + want := []string{kindAddress, kindLink, kindNeighbor, kindRoute, kindRule} for _, name := range []string{"galactic_netlink_dump_interrupted_total", "galactic_netlink_dump_failures_total"} { slices.Sort(got[name]) if !slices.Equal(got[name], want) { diff --git a/internal/plumbing/nldump/nldump_test.go b/internal/plumbing/nldump/nldump_test.go index 64f1d139..45f492d5 100644 --- a/internal/plumbing/nldump/nldump_test.go +++ b/internal/plumbing/nldump/nldump_test.go @@ -14,10 +14,13 @@ import ( "testing" "github.com/containernetworking/plugins/pkg/ns" + "github.com/prometheus/client_golang/prometheus" "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" "golang.org/x/sys/unix" "go.datum.net/galactic/internal/plumbing/nldump" + "go.datum.net/galactic/internal/plumbing/nldump/nldumptest" ) // TestDumps_MatchPackageLevelFunctions pins that the retrying dumps run in the @@ -97,6 +100,130 @@ func TestDumps_MatchPackageLevelFunctions(t *testing.T) { } } +// TestLister_ListsTheHandlesNamespace pins that a Lister lists the network +// namespace its handle was opened in, not the calling thread's, and returns +// what the handle's own single-attempt methods return for every kind. +func TestLister_ListsTheHandlesNamespace(t *testing.T) { + requireRoot(t) + nsObj, err := ns.TempNetNS() + if err != nil { + t.Fatalf("create test netns: %v", err) + } + t.Cleanup(func() { _ = nsObj.Close() }) + // nsObj owns this descriptor. Its Path names the namespace of the thread + // that created it, wherever that thread is now, so it cannot be opened + // instead. + h, err := netlink.NewHandleAt(netns.NsHandle(int(nsObj.Fd()))) + if err != nil { + t.Fatalf("open handle in test netns: %v", err) + } + t.Cleanup(func() { _ = h.Close() }) + + link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "nldumplister"}} + if err := h.LinkAdd(link); err != nil { + t.Fatalf("add link: %v", err) + } + if err := h.LinkSetUp(link); err != nil { + t.Fatalf("set link up: %v", err) + } + ip := net.ParseIP("fd00:2::1") + addr := &netlink.Addr{IPNet: &net.IPNet{IP: ip, Mask: net.CIDRMask(64, 128)}, Flags: unix.IFA_F_NODAD} + if err := h.AddrAdd(link, addr); err != nil { + t.Fatalf("add address: %v", err) + } + neighIP := net.ParseIP("fd00:2::2") + if err := h.NeighAdd(&netlink.Neigh{ + LinkIndex: link.Attrs().Index, Family: netlink.FAMILY_V6, State: netlink.NUD_PERMANENT, + IP: neighIP, HardwareAddr: net.HardwareAddr{0x02, 0, 0, 0, 0, 2}, + }); err != nil { + t.Fatalf("add neighbor: %v", err) + } + rule := netlink.NewRule() + rule.Family, rule.Priority, rule.Table = netlink.FAMILY_V6, 1234, 1234 + if err := h.RuleAdd(rule); err != nil { + t.Fatalf("add rule: %v", err) + } + + l := nldump.On(h) + links, err := l.LinkList() + if err != nil { + t.Fatalf("LinkList: %v", err) + } + if !strings.Contains(names(links), link.Name) { + t.Errorf("LinkList = %s, want it to hold %s from the handle's namespace", names(links), link.Name) + } + host, err := nldump.LinkList() + if err != nil { + t.Fatalf("package LinkList: %v", err) + } + if strings.Contains(names(host), link.Name) { + t.Errorf("package LinkList listed %s from another namespace", link.Name) + } + + addrs, err := l.AddrList(link, netlink.FAMILY_V6) + if err != nil || !slices.ContainsFunc(addrs, func(a netlink.Addr) bool { return a.IP.Equal(ip) }) { + t.Errorf("AddrList = %v, %v; want it to hold %s", addrs, err, ip) + } + routes, err := l.RouteList(link, netlink.FAMILY_V6) + wantRoutes, wantErr := h.RouteList(link, netlink.FAMILY_V6) + if err != nil || wantErr != nil || len(routes) == 0 || len(routes) != len(wantRoutes) { + t.Errorf("RouteList returned %d routes (%v), handle returned %d (%v); want equal and at least one", + len(routes), err, len(wantRoutes), wantErr) + } + neighs, err := l.NeighList(link.Attrs().Index, netlink.FAMILY_V6) + if err != nil || !slices.ContainsFunc(neighs, func(n netlink.Neigh) bool { return n.IP.Equal(neighIP) }) { + t.Errorf("NeighList = %v, %v; want it to hold %s", neighs, err, neighIP) + } + rules, err := l.RuleList(netlink.FAMILY_V6) + if err != nil || !slices.ContainsFunc(rules, func(r netlink.Rule) bool { return r.Priority == rule.Priority }) { + t.Errorf("RuleList = %v, %v; want it to hold priority %d", rules, err, rule.Priority) + } + + // The package-level forms list the calling thread's namespace. + err = nsObj.Do(func(ns.NetNS) error { + if got, err := nldump.NeighList(link.Attrs().Index, netlink.FAMILY_V6); err != nil || len(got) != len(neighs) { + return fmt.Errorf("package NeighList = %v, %w; want %d entries", got, err, len(neighs)) + } + if got, err := nldump.RuleList(netlink.FAMILY_V6); err != nil || len(got) != len(rules) { + return fmt.Errorf("package RuleList = %v, %w; want %d rules", got, err, len(rules)) + } + if got, err := nldump.RouteList(link, netlink.FAMILY_V6); err != nil || len(got) != len(routes) { + return fmt.Errorf("package RouteList = %v, %w; want %d routes", got, err, len(routes)) + } + return nil + }) + if err != nil { + t.Error(err) + } +} + +// TestNldumptest_KindsMatchMetricLabels pins nldumptest's kind names to the +// label values the counters carry, so the helper never waits on a series +// that does not exist. +func TestNldumptest_KindsMatchMetricLabels(t *testing.T) { + reg := prometheus.NewRegistry() + nldump.MustRegister(reg) + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + kinds := map[string]bool{} + for _, f := range families { + for _, m := range f.GetMetric() { + for _, l := range m.GetLabel() { + if l.GetName() == "kind" { + kinds[l.GetValue()] = true + } + } + } + } + for _, k := range []string{nldumptest.KindLink, nldumptest.KindAddress} { + if !kinds[k] { + t.Errorf("no counter carries kind %q; have %v", k, kinds) + } + } +} + func requireRoot(t *testing.T) { t.Helper() if os.Geteuid() != 0 { diff --git a/internal/plumbing/nldump/nldumptest/nldumptest.go b/internal/plumbing/nldump/nldumptest/nldumptest.go new file mode 100644 index 00000000..85623174 --- /dev/null +++ b/internal/plumbing/nldump/nldumptest/nldumptest.go @@ -0,0 +1,177 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package nldumptest drives netlink listings through kernel interruptions in +// tests: it churns links and addresses in a network namespace and repeats a +// pass until the kernel has interrupted the listings it makes. +package nldumptest + +import ( + "fmt" + "net" + "sync" + "testing" + "time" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/prometheus/client_golang/prometheus" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/plumbing/nldump" +) + +// Listing kinds, the kind label values of nldump's counters. +const ( + KindLink = "link" + KindAddress = "address" +) + +// stableLinks is how many links, each with an address, Churn adds before it +// starts. The kernel only flags a dump interrupted when it spans several +// reads, which a handful of links or addresses never does. +const stableLinks = 300 + +// Churn adds stableLinks dummy links with an address each to netns, then adds and removes links +// carrying an address there until the test ends, so link and address +// listings in netns are interrupted. Link names start with "nlchurn", which +// no Galactic interface name matches. +func Churn(t testing.TB, netns ns.NetNS) { + t.Helper() + err := netns.Do(func(ns.NetNS) error { + for i := range stableLinks { + d := dummy(fmt.Sprintf("nlchurns%d", i)) + if err := netlink.LinkAdd(d); err != nil { + return fmt.Errorf("add stable link %d: %w", i, err) + } + if err := netlink.AddrAdd(d, churnAddr(0xff, i)); err != nil { + return fmt.Errorf("add stable address %d: %w", i, err) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + stop := make(chan struct{}) + var wg sync.WaitGroup + for c := range 2 { + wg.Add(1) + go func() { + defer wg.Done() + _ = netns.Do(func(ns.NetNS) error { + for i := 0; ; i++ { + select { + case <-stop: + return nil + default: + } + d := dummy(fmt.Sprintf("nlchurn%d_%d", c, i%4)) + if err := netlink.LinkAdd(d); err == nil { + _ = netlink.AddrAdd(d, churnAddr(c, i%4)) + _ = netlink.LinkDel(d) + } + // Bursts of changes, not a saturated loop that could + // outlast any bounded retry. + time.Sleep(2 * time.Millisecond) + } + }) + }() + } + t.Cleanup(func() { + close(stop) + wg.Wait() + }) +} + +// UntilInterrupted runs pass in netns at least minPasses times and until +// nldump has counted an interrupted listing of every kind in kinds since the +// call. It fails the test if any pass returns an error, if nldump counts a +// failed listing, if no interruption of some kind arrives within 10 seconds, +// or if the passes take longer than 2 minutes in all. A pass makes several +// listings and only some are interrupted, so minPasses keeps every listing it +// makes under churn long enough to meet an interruption. It returns how many +// passes ran. +func UntilInterrupted(t testing.TB, netns ns.NetNS, kinds []string, minPasses int, pass func() error) int { + t.Helper() + interruptedBefore := counts(t, "galactic_netlink_dump_interrupted_total") + failuresBefore := counts(t, "galactic_netlink_dump_failures_total") + pending := func() []string { + now := counts(t, "galactic_netlink_dump_interrupted_total") + var out []string + for _, k := range kinds { + if now[k] == interruptedBefore[k] { + out = append(out, k) + } + } + return out + } + + passes := 0 + err := netns.Do(func(ns.NetNS) error { + start := time.Now() + for left := pending(); len(left) > 0 || passes < minPasses; left = pending() { + if len(left) > 0 && time.Since(start) > 10*time.Second { + return fmt.Errorf("no %v listing interrupted after %d passes", left, passes) + } + if time.Since(start) > 2*time.Minute { + return fmt.Errorf("only %d of %d passes ran in 2 minutes", passes, minPasses) + } + if err := pass(); err != nil { + return fmt.Errorf("pass %d failed while links and addresses were changing: %w", passes, err) + } + passes++ + } + return nil + }) + if err != nil { + t.Fatal(err) + } + failuresAfter := counts(t, "galactic_netlink_dump_failures_total") + for kind, n := range failuresAfter { + if d := n - failuresBefore[kind]; d != 0 { + t.Errorf("%s listing failures rose by %v, want 0", kind, d) + } + } + return passes +} + +// counts returns metric's value per kind label. +func counts(t testing.TB, metric string) map[string]float64 { + t.Helper() + reg := prometheus.NewRegistry() + nldump.MustRegister(reg) + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather netlink dump metrics: %v", err) + } + out := map[string]float64{} + for _, f := range families { + if f.GetName() != metric { + continue + } + for _, m := range f.GetMetric() { + for _, l := range m.GetLabel() { + if l.GetName() == "kind" { + out[l.GetValue()] = m.GetCounter().GetValue() + } + } + } + } + return out +} + +// churnAddr returns an address unique per (group, i), with DAD off so adding +// it changes nothing else. +func churnAddr(group, i int) *netlink.Addr { + return &netlink.Addr{ + IPNet: &net.IPNet{IP: net.ParseIP(fmt.Sprintf("fd00:c%x::%x", group, i+1)), Mask: net.CIDRMask(128, 128)}, + Flags: unix.IFA_F_NODAD, + } +} + +func dummy(name string) *netlink.Dummy { + return &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: name}} +} diff --git a/internal/plumbing/srv6/egress.go b/internal/plumbing/srv6/egress.go index 0c9bfbb0..1d5b5f75 100644 --- a/internal/plumbing/srv6/egress.go +++ b/internal/plumbing/srv6/egress.go @@ -14,6 +14,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/attach" "go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap" + "go.datum.net/galactic/internal/plumbing/nldump" ) // ErrNoShardResolvable is wrapped by EgressPrefixRouteAdd and @@ -210,7 +211,7 @@ func resolveUplinkNeighbor(name string) (linkIndex int, dmac, smac net.HardwareA smac = link.Attrs().HardwareAddr linkIndex = link.Attrs().Index - neighs, err := netlink.NeighList(linkIndex, netlink.FAMILY_V6) + neighs, err := nldump.NeighList(linkIndex, netlink.FAMILY_V6) if err != nil { return 0, nil, nil, fmt.Errorf("list neighbors on %s: %w", name, err) } diff --git a/internal/plumbing/srv6/plainroute.go b/internal/plumbing/srv6/plainroute.go index 54219268..3d88a73b 100644 --- a/internal/plumbing/srv6/plainroute.go +++ b/internal/plumbing/srv6/plainroute.go @@ -11,6 +11,8 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/plumbing/nldump" ) // Plain routes are the kernel routes installed for EVPN Type 5 paths carrying @@ -64,7 +66,7 @@ const mainTable = unix.RT_TABLE_MAIN // before its default route. Idempotent: a rule already present is left alone. func EnsurePlainRouteRules() error { for _, family := range []int{netlink.FAMILY_V4, netlink.FAMILY_V6} { - existing, err := netlink.RuleList(family) + existing, err := nldump.RuleList(family) if err != nil { return fmt.Errorf("list %s policy rules: %w", familyName(family), err) } @@ -201,7 +203,7 @@ func sameForwarding(existing, desired *netlink.Route) bool { // plainRoutesFor returns the routes in PlainRouteTable for exactly prefix. func plainRoutesFor(prefix *net.IPNet) ([]netlink.Route, error) { - routes, err := netlink.RouteListFiltered(prefixFamily(prefix), + routes, err := nldump.RouteListFiltered(prefixFamily(prefix), &netlink.Route{Table: int(PlainRouteTable), Dst: prefix}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_DST) if err != nil { @@ -229,7 +231,7 @@ func PlainRouteDel(prefix *net.IPNet) error { func PlainRouteList() ([]*net.IPNet, error) { var dsts []*net.IPNet for _, family := range []int{netlink.FAMILY_V4, netlink.FAMILY_V6} { - routes, err := netlink.RouteListFiltered(family, + routes, err := nldump.RouteListFiltered(family, &netlink.Route{Table: int(PlainRouteTable), Protocol: PlainRouteProtocol}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_PROTOCOL) if err != nil { @@ -253,7 +255,7 @@ func PlainRouteList() ([]*net.IPNet, error) { // matches; a route FRR or the kernel installed carries its own protocol and is // left alone. func RemoveLegacyPlainRoute(prefix *net.IPNet) (int, error) { - routes, err := netlink.RouteListFiltered(prefixFamily(prefix), + routes, err := nldump.RouteListFiltered(prefixFamily(prefix), &netlink.Route{Table: mainTable, Dst: prefix}, netlink.RT_FILTER_TABLE|netlink.RT_FILTER_DST) if err != nil { diff --git a/internal/plumbing/vip/vip.go b/internal/plumbing/vip/vip.go index 42f133ae..5077d4f2 100644 --- a/internal/plumbing/vip/vip.go +++ b/internal/plumbing/vip/vip.go @@ -22,6 +22,8 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/plumbing/nldump" ) // InterfaceName is the dummy link this package owns exclusively for VIP @@ -92,7 +94,7 @@ func Verify(addr net.IP) error { if addr.To4() != nil { family = unix.AF_INET } - addrs, err := netlink.AddrList(link, family) + addrs, err := nldump.AddrList(link, family) if err != nil { return fmt.Errorf("vip: list addresses on %s: %w", InterfaceName, err) }