fix(edge): Remove stray IPv6 routes and addresses on Edge attachments

docs/designs/masque/acceptance.md

@@ -10,7 +10,7 @@ Packet cases use sequence-numbered traffic in both directions, capture at sender
 
 [#985](https://github.com/datum-cloud/galactic/issues/985) packages setup, fault injection, collection and cleanup commands in a clean-checkout fixture. It asserts removal of test-created links, routes, maps, namespaces and API resources and preservation of unrelated resources. The ledger's retained/tombstoned allocations are expected state, identified explicitly rather than called a leak. Cleanup errors fail the run. Tests never deploy to production as part of this milestone.
 
-`task verify:masque-conformance`, run from `deploy/containerlab`, runs the [#974](https://github.com/datum-cloud/galactic/issues/974) packet-access checks on the lab's dfw cluster. It covers M16 and M07, NAT egress from the attachment node, two connectors sharing one attachment while one request ends, router garbage collection while connectors forward, a provider process killed and restarted in place, pod recreation, a partly set-up instance killed without shutdown, and asserted teardown. The provider is the test-only fixture in `tests/edgeconformance/masqueprovider`. It runs the [MASQUE provider adapter](../../edge/masque-provider.md) with `internal/edge/connectip`, and admits static test credentials in place of the [#977](https://github.com/datum-cloud/galactic/issues/977) authority. A lab result does not satisfy a row that requires the target platform.
+`task verify:masque-conformance`, run from `deploy/containerlab`, runs the [#974](https://github.com/datum-cloud/galactic/issues/974) packet-access checks on the lab's dfw cluster. It covers M16, M07, and the #989 findings A06, A14 and A16, plus NAT egress from the attachment node, two connectors sharing one attachment while one request ends, router garbage collection while connectors forward, a provider process killed and restarted in place, pod recreation, a partly set-up instance killed without shutdown, and asserted teardown. A14's namespace-only impostor is a real-kernel test of `internal/edge/hostattach`. The provider is the test-only fixture in `tests/edgeconformance/masqueprovider`. It runs the [MASQUE provider adapter](../../edge/masque-provider.md) with `internal/edge/connectip`, and admits static test credentials in place of the [#977](https://github.com/datum-cloud/galactic/issues/977) authority. A lab result does not satisfy a row that requires the target platform. [#985](https://github.com/datum-cloud/galactic/issues/985) owns running these checks on the target platform.
 
 ## Epic and implementation checks
 

docs/edge/provider-contract.md

@@ -180,6 +180,8 @@ stateDiagram-v2
 
 A failed step leaves the attachment in its last state with a condition naming the reason. Partial host state carries the attachment ID in the host link's alias, so the node finds and finishes it on restart (`internal/edge/hostattach/inventory.go:33`).
 
+The node provisions every attachment again at least once a minute (`agent.Resync`), which repairs host state that drifted under it. It removes every IPv6 unicast route through the host link, the pod link or the packet device in the table it owns, other than the ones it installs, matching destination, gateway and metric. The only kernel routes it leaves are the connected routes of the addresses it assigns itself, and it removes any other global address from the pod link and the packet device, so an address added in the pod brings no route of its own. A route or address that appears after that pass, or that the node fails to remove, fails `Readback`, and the attachment leaves `Programmed`. IPv4 routes, other tables and policy rules are outside this repair.
+
 ### Forwarding claim
 
 Each `EdgeRoute` is the claim for one prefix in one gVPC. An instance renews one lease, not one per route, so the claim has two levels:

internal/edge/hostattach/closed_gate_test.go

@@ -99,7 +99,7 @@ func TestClosedGateNeverReachesShard(t *testing.T) {
 			return err
 		}
 		defer func() { _ = h.Close() }()
-		return pruneRoutes(h, link, table, []netip.Prefix{own})
+		return pruneRoutes(h, link, table, []installed{{dst: own}}, connected())
 	})
 	echoes := f.tunA.echoes.Load()
 	for range 3 {

internal/edge/hostattach/hostattach.go

@@ -518,8 +518,8 @@ func configureHostLink(
 			LinkIndex: idx, Table: int(table), Dst: prefixNet(p), Gw: a.Address.AsSlice(),
 		})
 	}
-	if err := pruneRoutes(h, link.Attrs().Index, table, hostPrefixes(a)); err != nil {
-		return err
+	if err := pruneRoutes(h, link.Attrs().Index, table, hostRoutes(a), connected()); err != nil {
+		return fmt.Errorf("hostattach: host link %q: %w", a.HostLink, err)
 	}
 	for _, r := range routes {
 		if err := checkRouteOwner(h, r); err != nil {
@@ -578,6 +578,16 @@ func configurePodLink(h *netlink.Handle, link netlink.Link, a Attachment, hostMA
 	}); err != nil {
 		return fmt.Errorf("hostattach: neighbor %s on %q: %w", hostLinkLocal, a.PodLink, err)
 	}
+	// The pod link carries only the node's addresses and the default route
+	// toward the host. Any other address brings a connected route, and any
+	// other route sends pod VRF traffic past the uSID datapath's view of the
+	// attachment.
+	if err := pruneAddrs(h, link, []netip.Addr{a.Address}); err != nil {
+		return fmt.Errorf("hostattach: pod link %q: %w", a.PodLink, err)
+	}
+	if err := pruneRoutes(h, idx, a.PodTable, podRoutes, connected(a.Address)); err != nil {
+		return fmt.Errorf("hostattach: pod link %q: %w", a.PodLink, err)
+	}
 	def := &netlink.Route{
 		LinkIndex: idx, Table: int(a.PodTable),
 		Dst: &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}, Gw: hostLinkLocal.AsSlice(),
@@ -629,8 +639,11 @@ func ensureTUN(h *netlink.Handle, pod *Namespace, a Attachment) error {
 	if err := h.LinkSetUp(tun); err != nil {
 		return fmt.Errorf("hostattach: set TUN %q up: %w", a.TUN.Name, err)
 	}
-	if err := pruneRoutes(h, tun.Attrs().Index, a.PodTable, a.Prefixes); err != nil {
-		return err
+	if err := pruneAddrs(h, tun, nil); err != nil {
+		return fmt.Errorf("hostattach: TUN %q: %w", a.TUN.Name, err)
+	}
+	if err := pruneRoutes(h, tun.Attrs().Index, a.PodTable, tunRoutes(a), connected()); err != nil {
+		return fmt.Errorf("hostattach: TUN %q: %w", a.TUN.Name, err)
 	}
 	for _, p := range a.Prefixes {
 		r := &netlink.Route{LinkIndex: tun.Attrs().Index, Table: int(a.PodTable), Dst: prefixNet(p)}

internal/edge/hostattach/inventory.go

@@ -7,6 +7,7 @@ package hostattach
 import (
 	"fmt"
 	"net/netip"
+	"slices"
 	"strings"
 
 	"github.com/vishvananda/netlink"
@@ -87,42 +88,138 @@ func listOwned(h *netlink.Handle) ([]ObservedAttachment, error) {
 	return result, nil
 }
 
-func hostPrefixes(a Attachment) []netip.Prefix {
-	prefixes := make([]netip.Prefix, 0, len(a.Prefixes)+1)
-	prefixes = append(prefixes, netip.PrefixFrom(a.Address, 128))
-	return append(prefixes, a.Prefixes...)
+// podDefault is the destination of the only route an attachment's pod link
+// carries in the pod VRF.
+var podDefault = netip.PrefixFrom(netip.IPv6Unspecified(), 0)
+
+// installed is a route the node installs through an attachment's link: its
+// destination and, for a route with one, its gateway.
+type installed struct {
+	dst netip.Prefix
+	gw  netip.Addr
 }
 
-// extraRoutes lists static unicast destinations on a dedicated owned link that
-// no longer appear in desired state. Kernel-derived local/link-local routes
-// and routes using other links are never candidates for deletion.
-func extraRoutes(h *netlink.Handle, index int, table uint32, want []netip.Prefix) ([]netlink.Route, error) {
+// hostRoutes are the routes the node installs through the host link: the
+// attachment's address, and each prefix via that address.
+func hostRoutes(a Attachment) []installed {
+	routes := make([]installed, 0, len(a.Prefixes)+1)
+	routes = append(routes, installed{dst: netip.PrefixFrom(a.Address, 128)})
+	for _, p := range a.Prefixes {
+		routes = append(routes, installed{dst: p, gw: a.Address})
+	}
+	return routes
+}
+
+// podRoutes is the route the node installs through the pod link: the pod
+// VRF's default, via the host link's link-local address.
+var podRoutes = []installed{{dst: podDefault, gw: hostLinkLocal}}
+
+// tunRoutes are the routes the node installs through the packet device: one
+// per prefix, without a gateway.
+func tunRoutes(a Attachment) []installed {
+	routes := make([]installed, 0, len(a.Prefixes))
+	for _, p := range a.Prefixes {
+		routes = append(routes, installed{dst: p})
+	}
+	return routes
+}
+
+// matches reports whether r is the installed route: the same destination
+// and gateway, at the default metric the node's writes leave.
+func (want installed) matches(r netlink.Route) bool {
+	if !sameDst(r.Dst, want.dst) || (r.Priority != 0 && r.Priority != defaultRouteMetric) {
+		return false
+	}
+	gw, _ := netip.AddrFromSlice(r.Gw)
+	return gw.Unmap() == want.gw
+}
+
+// defaultRouteMetric is the metric the kernel gives an IPv6 route added
+// without one.
+const defaultRouteMetric = 1024
+
+// linkLocalPrefix is the kernel's connected route for any link-local
+// address, which every link carries.
+var linkLocalPrefix = netip.MustParsePrefix("fe80::/64")
+
+// connected lists the kernel's connected routes the node expects through
+// one of an attachment's links: the link-local prefix, and the prefixes of
+// the addresses the node itself assigns to the link.
+func connected(own ...netip.Addr) []netip.Prefix {
+	prefixes := make([]netip.Prefix, 0, len(own)+1)
+	prefixes = append(prefixes, linkLocalPrefix)
+	for _, a := range own {
+		prefixes = append(prefixes, netip.PrefixFrom(a, a.BitLen()))
+	}
+	return prefixes
+}
+
+// extraRoutes lists the unicast routes through a dedicated owned link that
+// the node does not install: any destination, gateway or metric other than
+// want's. A kernel route counts as the kernel's own only when it is one of
+// the connected routes the node expects (see connected), never because the
+// link happens to carry an address now, so neither a route written with
+// "proto kernel" nor the connected route of an address someone else added
+// is exempt. Local and multicast routes, and routes through other links, are
+// never candidates.
+func extraRoutes(h *netlink.Handle, index int, table uint32, want []installed, kernel []netip.Prefix,
+) ([]netlink.Route, error) {
 	routes, err := nldump.On(h).RouteListFiltered(unix.AF_INET6, &netlink.Route{Table: int(table), LinkIndex: index},
 		netlink.RT_FILTER_TABLE|netlink.RT_FILTER_OIF)
 	if err != nil {
 		return nil, fmt.Errorf("hostattach: list attachment routes: %w", err)
 	}
 	var extras []netlink.Route
 	for _, r := range routes {
-		if r.Type != unix.RTN_UNICAST || r.Protocol == unix.RTPROT_KERNEL {
+		if r.Type != unix.RTN_UNICAST {
 			continue
 		}
-		found := false
-		for _, p := range want {
-			if sameDst(r.Dst, p) {
-				found = true
-				break
-			}
+		if r.Protocol == unix.RTPROT_KERNEL && slices.ContainsFunc(kernel, func(p netip.Prefix) bool {
+			return sameDst(r.Dst, p)
+		}) {
+			continue
 		}
-		if !found {
+		if !slices.ContainsFunc(want, func(w installed) bool { return w.matches(r) }) {
 			extras = append(extras, r)
 		}
 	}
 	return extras, nil
 }
 
-func pruneRoutes(h *netlink.Handle, index int, table uint32, want []netip.Prefix) error {
-	extras, err := extraRoutes(h, index, table, want)
+// extraAddrs lists the IPv6 addresses on link other than link-local ones and
+// own, the addresses the node assigns to it.
+func extraAddrs(h *netlink.Handle, link netlink.Link, own []netip.Addr) ([]netlink.Addr, error) {
+	addrs, err := nldump.On(h).AddrList(link, unix.AF_INET6)
+	if err != nil {
+		return nil, fmt.Errorf("hostattach: list addresses on %q: %w", link.Attrs().Name, err)
+	}
+	var extras []netlink.Addr
+	for _, a := range addrs {
+		addr, ok := netip.AddrFromSlice(a.IP)
+		if !ok || addr.Unmap().IsLinkLocalUnicast() || slices.Contains(own, addr.Unmap()) {
+			continue
+		}
+		extras = append(extras, a)
+	}
+	return extras, nil
+}
+
+// pruneAddrs removes the addresses extraAddrs finds on link.
+func pruneAddrs(h *netlink.Handle, link netlink.Link, own []netip.Addr) error {
+	extras, err := extraAddrs(h, link, own)
+	if err != nil {
+		return err
+	}
+	for _, a := range extras {
+		if err := h.AddrDel(link, &a); err != nil {
+			return fmt.Errorf("hostattach: remove stray address %s from %q: %w", a.IPNet, link.Attrs().Name, err)
+		}
+	}
+	return nil
+}
+
+func pruneRoutes(h *netlink.Handle, index int, table uint32, want []installed, kernel []netip.Prefix) error {
+	extras, err := extraRoutes(h, index, table, want, kernel)
 	if err != nil {
 		return err
 	}

internal/edge/hostattach/readback.go

@@ -76,12 +76,14 @@ func Readback(host, pod *Namespace, a Attachment) error {
 	checkNeigh(podH, podLink, hostLinkLocal, hostLink.Attrs().HardwareAddr, add)
 
 	hostIdx := hostLink.Attrs().Index
-	checkExtraRoutes(hostH, hostIdx, hostTable, hostPrefixes(a), add)
+	checkExtraRoutes(hostH, hostIdx, hostTable, hostRoutes(a), connected(), add)
 	checkRoute(hostH, hostTable, netip.PrefixFrom(a.Address, 128), hostIdx, netip.Addr{}, add)
 	for _, p := range a.Prefixes {
 		checkRoute(hostH, hostTable, p, hostIdx, a.Address, add)
 	}
-	checkRoute(podH, a.PodTable, netip.PrefixFrom(netip.IPv6Unspecified(), 0), podLink.Attrs().Index, hostLinkLocal, add)
+	checkExtraAddrs(podH, podLink, []netip.Addr{a.Address}, add)
+	checkExtraRoutes(podH, podLink.Attrs().Index, a.PodTable, podRoutes, connected(a.Address), add)
+	checkRoute(podH, a.PodTable, podDefault, podLink.Attrs().Index, hostLinkLocal, add)
 	checkUnreachableDefault(podH, a.PodTable, add)
 	checkLocalRoute(podH, a.PodTable, a.Address, add)
 
@@ -91,7 +93,8 @@ func Readback(host, pod *Namespace, a Attachment) error {
 			if want := a.TUN.effectiveMTU(a.MTU); tun.Attrs().MTU != want {
 				add("TUN %q MTU %d, want %d", a.TUN.Name, tun.Attrs().MTU, want)
 			}
-			checkExtraRoutes(podH, tun.Attrs().Index, a.PodTable, a.Prefixes, add)
+			checkExtraAddrs(podH, tun, nil, add)
+			checkExtraRoutes(podH, tun.Attrs().Index, a.PodTable, tunRoutes(a), connected(), add)
 			if tun.Attrs().Alias != ownerAlias(a.ID) {
 				add("TUN ownership does not match attachment")
 			}
@@ -244,13 +247,26 @@ func sameDst(got *net.IPNet, want netip.Prefix) bool {
 	return netip.PrefixFrom(addr.Unmap(), ones) == want
 }
 
-func checkExtraRoutes(h *netlink.Handle, index int, table uint32, want []netip.Prefix, add func(string, ...any)) {
-	extras, err := extraRoutes(h, index, table, want)
+func checkExtraAddrs(h *netlink.Handle, link netlink.Link, own []netip.Addr, add func(string, ...any)) {
+	extras, err := extraAddrs(h, link, own)
+	if err != nil {
+		add("%v", err)
+		return
+	}
+	for _, a := range extras {
+		add("unexpected address %s on %q", a.IPNet, link.Attrs().Name)
+	}
+}
+
+func checkExtraRoutes(h *netlink.Handle, index int, table uint32, want []installed, kernel []netip.Prefix,
+	add func(string, ...any)) {
+	extras, err := extraRoutes(h, index, table, want, kernel)
 	if err != nil {
 		add("%v", err)
 		return
 	}
 	for _, r := range extras {
-		add("unexpected route %s in table %d through ifindex %d", r.Dst, table, index)
+		add("unexpected route %s via %s metric %d in table %d through ifindex %d", r.Dst, r.Gw, r.Priority, table,
+			index)
 	}
 }