fix(router): Remove VIP translation rows no binding owns

cmd/galactic-router/root.go

@@ -167,13 +167,20 @@ func runCmd(cfg *config.RouterConfig) error {
 	// matters only once a tap-kind binding is reconciled here, at which point
 	// the reconciler reports a clear error rather than silently doing nothing.
 	// Veth bindings work regardless.
-	var vipTranslationTable controller.VIPTranslationTable
+	//
+	// The same handle backs the GC ticker's vip_xlat_table sweep, which is
+	// skipped while the pin is missing.
+	var (
+		vipTranslationTable controller.VIPTranslationTable
+		vipXlatSweeper      *controller.VIPXlatSweeper
+	)
 	vipXlatTable, vipXlatCloser, vipXlatErr := vipxlatmap.OpenPinnedVipXlatTable(attach.PinDir)
 	if vipXlatErr != nil {
 		ctrl.Log.Error(vipXlatErr, "vip_xlat_table not available on this node; "+
 			"tap-kind ServiceVIPBinding objects will fail to bind until the eBPF uSID datapath is loaded")
 	} else {
 		vipTranslationTable = vipXlatTable
+		vipXlatSweeper = &controller.VIPXlatSweeper{Client: mgr.GetClient(), NodeName: nodeName, Table: vipXlatTable}
 		defer vipXlatCloser.Close() //nolint:errcheck // best-effort close of our own fd at shutdown
 	}
 
@@ -294,28 +301,7 @@ func runCmd(cfg *config.RouterConfig) error {
 		return fmt.Errorf("setup GC controller: %w", err)
 	}
 
-	// The GC ticker runs until the manager's context is cancelled. The
-	// first pass waits for informer caches to sync, so it does not see an
-	// empty advertisement list and delete live VRFs.
-	go func() {
-		ticker := time.NewTicker(cfg.GCInterval)
-		defer ticker.Stop()
-
-		if !mgr.GetCache().WaitForCacheSync(ctx) {
-			log.Printf("GC: cache sync failed, skipping initial pass")
-			return
-		}
-		gcRec.RunGC(ctx)
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticker.C:
-				gcRec.RunGC(ctx)
-			}
-		}
-	}()
+	go runGCTicker(ctx, mgr, cfg.GCInterval, gcRec, vipXlatSweeper)
 
 	if err := mgr.Start(ctx); err != nil {
 		return fmt.Errorf("manager exited: %w", err)
@@ -346,6 +332,39 @@ func runCmd(cfg *config.RouterConfig) error {
 	return nil
 }
 
+// runGCTicker runs the GC controller's passes every interval until ctx is
+// cancelled. The first pass waits for informer caches to sync, so it does not
+// see an empty advertisement list and delete live VRFs.
+//
+// vipXlatSweeper, nil when vip_xlat_table is not open, runs only on ticks,
+// never in that first pass, so the ServiceVIPBinding reconciler re-registers
+// every live binding before the first sweep.
+func runGCTicker(
+	ctx context.Context, mgr ctrl.Manager, interval time.Duration,
+	gcRec *controller.GCReconciler, vipXlatSweeper *controller.VIPXlatSweeper,
+) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	if !mgr.GetCache().WaitForCacheSync(ctx) {
+		log.Printf("GC: cache sync failed, skipping initial pass")
+		return
+	}
+	gcRec.RunGC(ctx)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			gcRec.RunGC(ctx)
+			if vipXlatSweeper != nil {
+				vipXlatSweeper.Sweep(ctx)
+			}
+		}
+	}
+}
+
 // setupVIPBindingControllers registers the two halves of this node's DSR
 // backend side: NetworkRuleBindingReconciler, which writes a ServiceVIPBinding
 // for every NetworkRule backend whose VPCAttachment is on this node, and

docs/agents/ARCHITECTURE-ROUTER.md

@@ -82,7 +82,8 @@ galactic/
 │   ├── metadata/            # Build-time version info (Version, GitCommit, etc.)
 │   ├── gc/                  # Orphaned BGPAdvertisement/BGPVRFInstance CRD, stale
 │   │                        #   kernel VRF, and eBPF vrf_table entry cleanup,
-│   │                        #   driven by the GC controller
+│   │                        #   driven by the GC controller (the vip_xlat_table
+│   │                        #   sweep is internal/controller/vipxlat_sweep.go)
 │   └── plumbing/            # Low-level kernel and network primitives (shared with
 │       ├── intf/            #   the CNI chain — see ARCHITECTURE-CNI.md's own copy
 │       ├── srv6/             #   of this tree for the CNI-side entries)
@@ -180,7 +181,8 @@ lives in `root.go`'s `runCmd`:
     NetworkRuleBinding, NodeDeparture, ServiceRoutePolicy, and GC (the GC
     controller also starts a ticker goroutine that waits for cache sync,
     then runs on `--gc-interval`, default 5m; NodeDeparture sweeps on the
-    same interval).
+    same interval, and when `vip_xlat_table` opened in step 7, each GC tick
+    after the first pass also sweeps it).
 11. `mgr.Start(ctx)` — blocks until the signal-handler context is cancelled.
 
 This is identical whether the pod is running in the plain role
@@ -280,7 +282,8 @@ co-located `galactic-gateway` container's own health port on the same
 - **A tenant VRF route that fails to install is retried, not dropped.** `processEVPNPath` writes a VRF-scoped path's route into `egress_route_table` through `internal/runtime/gobgp/vrfroutes.go`. `Register` resolves the SID's uplink and neighbor MAC at write time, which fails while an uplink is still coming up after a restart, and BGP never redelivers an unchanged best path, so before #676 such a route stayed missing until the path changed upstream. `vrfRoutes` keeps each failed route in a pending set and retries it whenever the main table changes (1s debounce) and every 30s, until it installs or its path is withdrawn; removing a VRF drops its pending routes. One mutex serializes every write to the map from this runtime, taken per route during a retry so an unresolvable neighbor (up to 2s) does not hold up the RIB watcher for the whole pass. `EgressRouteTable.Refresh` cannot cover this: it walks existing entries only. The pending count, and the plain-route reconciler's failing count, are exported as `galactic_router_route_install_failing{kind="vrf"|"plain"}`, and `GalacticRouterRoutesNotInstalled` fires when either stays above 0 for 10 minutes.
 - **`ServiceVIPBinding`s are generated, one per node per backend.** `NetworkRuleBindingReconciler` (`internal/controller/networkrule_binding_controller.go`) runs on every node and writes a binding for each IPv6 VIP of each accepted `NetworkRule` and each backend the rule's `backendSelector` places on this node, through the same `selectRuleBackends` the gateway uses, so the gateway never sends a backend's flows to a node with no binding planned for it. Each node writes and deletes only its own bindings (labelled `galactic.datum.net/binding-node` and `app.kubernetes.io/managed-by: galactic-router`), so there is one writer per object and no leader election. A binding is owned by its rule, records the rule's `vpcRef` (which names the VRF its rows go in) and takes `egressKind` from the attachment's interface mode. The node summarizes its bindings on the rule as `<node>/BackendsBound`; for an invalid rule only the nodes that served it (held bindings going in, or already report `InvalidRule`) write one, since the gateways already report the error and a condition from every node would be a status write per node per edit. Several backends of one VIP can share a node: the gateway writes each backend's slot into the uSID and `usid.c` keys the ingress row on it (#799).
 - **Surviving routers clean up after a departed node.** Only a node's own `NetworkRuleBindingReconciler` clears its `<node>/BackendsBound` conditions and deletes its generated bindings, and only its own `ServiceVIPBindingReconciler` removes the teardown finalizer from bindings targeting it. So once a node's `Node` object is deleted, `NodeDepartureReconciler` (`internal/controller/node_departure_controller.go`) on every other router removes that node's `BackendsBound` condition from every `NetworkRule`, deletes its generated bindings and removes the finalizer from every binding whose `targetRef.name` names it, hand-written ones included; the kernel rows went with the node. Without it the conditions stayed forever and a deleted rule's bindings stayed `Terminating`. Every Node create and delete maps to one sweep key, so the sweep runs on the initial Node list (startup), on every Node delete and every `--gc-interval`, and a router down during the delete still converges. A node missing from the cached Node list counts as departed only once an uncached `Get` returns NotFound. Every router sweeps, with no leader election: status writes and finalizer patches retry on conflict (the patch carries the resourceVersion) and NotFound counts as done.
-- **Deleting a `ServiceVIPBinding` does not need its VRF on the node.** A binding's two `vip_xlat_table` rows are keyed on its VRF's uSID Argument. `ServiceVIPBindingReconciler` reads it from the `BGPVRFInstance` its `vpcRef` names on this node, and that is gone once the VPC leaves the node. Before #750, deletion re-derived it, failed, held the finalizer forever and left both rows in the map. Teardown now finds the rows by value with `vipxlatmap.UnregisterBinding`: since #764 each key carries the VIP or backend address, and since #799 the ingress key also carries the backend's slot, so the binding's ingress row (its VIP, port and slot, rewriting to its backend) and egress row (its backend and port, rewriting to its VIP) are found exactly, under any Argument. Each row is removed only while it still holds the binding's values, so a row another binding has since written stays. When the VRF context still resolves, `UnregisterBindingAt` applies the same checks at that location too. Neither pass deletes a row without first reading it, and a process-wide lock keeps a bind from landing between the read and the delete. A value check cannot tell two bindings with identical values apart, so while such a twin is live on the node, deleting one removes nothing. Rows exist only for IPv6 addresses, so an IPv4 or invalid address skips both passes instead of blocking deletion. Only a failed kernel delete keeps the finalizer. Nothing sweeps `vip_xlat_table` periodically, so rows left behind by a finalizer cleared by hand stay until removed with `galactic-router vip xlat list` and `vip xlat remove --protocol --vip --port --backend --backend-port` (`--slot` defaults to the backend's own), run in that node's `galactic-router` pod (`kubectl exec <pod> -- /galactic-router vip xlat list`).
+- **Deleting a `ServiceVIPBinding` does not need its VRF on the node.** A binding's two `vip_xlat_table` rows are keyed on its VRF's uSID Argument. `ServiceVIPBindingReconciler` reads it from the `BGPVRFInstance` its `vpcRef` names on this node, and that is gone once the VPC leaves the node. Before #750, deletion re-derived it, failed, held the finalizer forever and left both rows in the map. Teardown now finds the rows by value with `vipxlatmap.UnregisterBinding`: since #764 each key carries the VIP or backend address, and since #799 the ingress key also carries the backend's slot, so the binding's ingress row (its VIP, port and slot, rewriting to its backend) and egress row (its backend and port, rewriting to its VIP) are found exactly, under any Argument. Each row is removed only while it still holds the binding's values, so a row another binding has since written stays. When the VRF context still resolves, `UnregisterBindingAt` applies the same checks at that location too. Neither pass deletes a row without first reading it, and a process-wide lock keeps a bind from landing between the read and the delete. A value check cannot tell two bindings with identical values apart, so while such a twin is live on the node, deleting one removes nothing. Rows exist only for IPv6 addresses, so an IPv4 or invalid address skips both passes instead of blocking deletion. Only a failed kernel delete keeps the finalizer. Rows left behind some other way, by a finalizer cleared by hand, a crash or a manual write, are removed by the periodic sweep below; to remove one sooner, use `galactic-router vip xlat list` and `vip xlat remove --protocol --vip --port --backend --backend-port` (`--slot` defaults to the backend's own), run in that node's `galactic-router` pod (`kubectl exec <pod> -- /galactic-router vip xlat list`).
+- **`vip_xlat_table` is swept on the GC ticker.** `controller.VIPXlatSweeper` (`internal/controller/vipxlat_sweep.go`) runs on every GC tick after the first, never in the initial pass at startup, so the `ServiceVIPBinding` reconciler has re-registered every live binding first. It takes the table's generation, lists every `ServiceVIPBinding` targeting this node (including ones being deleted, whose finalizer owns their rows), resolves each one's two rows the way the reconciler does, and calls `VipXlatTable.Reconcile`, which removes every row not in that set unless it was registered after the snapshot. The live set is by key, so a row is kept whatever value it holds. A binding whose VRF context does not resolve right now has no known Argument, so every row shaped like one of its rows (direction, slot, protocol, address and port, under any Block and Argument) is kept instead. A binding whose spec does not parse never wrote a row and protects nothing, and any error listing bindings or building the index skips the whole pass. This is what removes the slot-0 ingress rows a router built before #799 wrote: the current key carries the backend's slot, so nothing else ever matches them. `usid.c` has no slot-0 fallback, so `galactic-gateway` and `galactic-router` have to be upgraded together across #799; a node whose router has not yet rewritten its rows with slots drops the flows a new gateway sends it.
 
 ---
 

docs/gateway/configuration.md

@@ -584,6 +584,12 @@ and protocol, in the same VPC. The older binding keeps serving. The newer
 one is not programmed and reports `Bound=False` with reason `Conflict`,
 naming the binding that holds the row.
 
+`usid.c` matches only the slot the gateway sent, with no fallback to the
+slot-0 ingress rows a `galactic-router` built before #799 wrote, so upgrade
+`galactic-gateway` and `galactic-router` together. Each router rewrites its
+bindings' rows with slots when it starts, and its periodic `vip_xlat_table`
+sweep removes the old slot-0 rows.
+
 Each node summarizes its bindings on the rule as
 `<backend-node>/BackendsBound`: `True` with reason `Bound` once every one
 reports `Bound`, `False` with reason `BindingsNotBound` naming each that

internal/controller/servicevipbinding_controller.go

@@ -497,6 +497,20 @@ type vipBindingRows struct {
 func resolveVIPBindingRows(
 	idx *backendSIDIndex, nodeName string, binding *bgpv1alpha1.ServiceVIPBinding,
 ) (vipBindingRows, error) {
+	rows, err := parseVIPBindingRows(binding)
+	if err != nil {
+		return vipBindingRows{}, err
+	}
+	block, argument, err := resolveVIPBindingContextFromIndex(idx, nodeName, binding.Spec.VPCRef)
+	if err != nil {
+		return vipBindingRows{}, fmt.Errorf("resolve VRF context for VIP binding: %w", err)
+	}
+	return rows.at(block, argument), nil
+}
+
+// parseVIPBindingRows derives the two rows binding claims from its spec alone,
+// with their block and argument left zero for the caller to fill in with at.
+func parseVIPBindingRows(binding *bgpv1alpha1.ServiceVIPBinding) (vipBindingRows, error) {
 	vipIP, err := netip.ParseAddr(binding.Spec.VIPAddress)
 	if err != nil {
 		return vipBindingRows{}, fmt.Errorf("invalid vipAddress %q: %w", binding.Spec.VIPAddress, err)
@@ -513,26 +527,28 @@ func resolveVIPBindingRows(
 		return vipBindingRows{}, err
 	}
 
-	block, argument, err := resolveVIPBindingContextFromIndex(idx, nodeName, binding.Spec.VPCRef)
-	if err != nil {
-		return vipBindingRows{}, fmt.Errorf("resolve VRF context for VIP binding: %w", err)
-	}
-
 	vipPort := uint16(binding.Spec.Port)            //nolint:gosec // kubebuilder-validated 1-65535
 	backendPort := uint16(binding.Spec.BackendPort) //nolint:gosec // kubebuilder-validated 1-65535
 
 	slot := srv6.BackendSlot(backendIP, backendPort)
 
 	return vipBindingRows{
-		ingress:     vipRow{block: block, argument: argument, slot: slot, proto: proto, addr: vipIP, port: vipPort},
-		egress:      vipRow{egress: true, block: block, argument: argument, proto: proto, addr: backendIP, port: backendPort},
+		ingress:     vipRow{slot: slot, proto: proto, addr: vipIP, port: vipPort},
+		egress:      vipRow{egress: true, proto: proto, addr: backendIP, port: backendPort},
 		vipAddr:     net.IP(vipIP.AsSlice()),
 		backendAddr: net.IP(backendIP.AsSlice()),
 		vipPort:     vipPort,
 		backendPort: backendPort,
 	}, nil
 }
 
+// at returns rows placed in the VRF identified by (block, argument).
+func (rows vipBindingRows) at(block uint64, argument uint16) vipBindingRows {
+	rows.ingress.block, rows.ingress.argument = block, argument
+	rows.egress.block, rows.egress.argument = block, argument
+	return rows
+}
+
 // vipPeer is another live binding on this node and the rows it claims.
 type vipPeer struct {
 	binding *bgpv1alpha1.ServiceVIPBinding