feat: Pick load balancer backends by label and bind them automatically

cmd/galactic-gateway/main.go

@@ -30,6 +30,11 @@ const (
 	resourceNetworkRules      = "networkrules"
 	resourceBGPAdvertisements = "bgpadvertisements"
 	resourceBGPRouters        = "bgprouters"
+	resourceBGPVRFInstances   = "bgpvrfinstances"
+
+	cloudAPIGroup          = "cloud.datumapis.com"
+	cloudAPIVersion        = "v1alpha1"
+	resourceVPCAttachments = "vpcattachments"
 )
 
 func main() {
@@ -68,6 +73,8 @@ func checkWatchPermissions(mgr ctrl.Manager) {
 		{group: networkAPIGroup, version: networkAPIVersion, resource: resourceNetworkRules},
 		{group: networkAPIGroup, version: networkAPIVersion, resource: resourceBGPAdvertisements},
 		{group: networkAPIGroup, version: networkAPIVersion, resource: resourceBGPRouters},
+		{group: networkAPIGroup, version: networkAPIVersion, resource: resourceBGPVRFInstances},
+		{group: cloudAPIGroup, version: cloudAPIVersion, resource: resourceVPCAttachments},
 	}
 
 	for _, r := range resources {

cmd/galactic-gateway/root.go

@@ -23,6 +23,7 @@ import (
 	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
+	cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1"
 	"go.datum.net/galactic/internal/config"
 	"go.datum.net/galactic/internal/controller"
 	"go.datum.net/galactic/internal/gateway"
@@ -54,6 +55,7 @@ func runCmd(cfg *config.GatewayConfig) error {
 	scheme := runtime.NewScheme()
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(bgpv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(cloudv1alpha1.AddToScheme(scheme))
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,

cmd/galactic-router/main.go

@@ -26,6 +26,7 @@ const (
 	resourceBGPPolicies        = "bgppolicies"
 	resourceBGPVRFInstances    = "bgpvrfinstances"
 	resourceServiceVIPBindings = "servicevipbindings"
+	resourceNetworkRules       = "networkrules"
 	resourceServiceEndpoints   = "serviceendpoints"
 	resourceServiceRoutePolicy = "serviceroutepolicies"
 	resourceVPCAttachments     = "vpcattachments"
@@ -68,6 +69,7 @@ func checkWatchPermissions(mgr ctrl.Manager) {
 		{group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceBGPPolicies},
 		{group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceBGPVRFInstances},
 		{group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceServiceVIPBindings},
+		{group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceNetworkRules},
 		{group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceServiceEndpoints},
 		{group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceServiceRoutePolicy},
 		{group: "cloud.datumapis.com", version: "v1alpha1", resource: resourceVPCAttachments},

cmd/galactic-router/root.go

@@ -265,14 +265,10 @@ func runCmd(cfg *config.RouterConfig) error {
 		return fmt.Errorf("setup Node controller: %w", err)
 	}
 
-	// Register ServiceVIPBinding controller.
-	if err := (&controller.ServiceVIPBindingReconciler{
-		Client:              mgr.GetClient(),
-		Scheme:              mgr.GetScheme(),
-		NodeName:            nodeName,
-		VIPTranslationTable: vipTranslationTable,
-	}).SetupWithManager(mgr); err != nil {
-		return fmt.Errorf("setup ServiceVIPBinding controller: %w", err)
+	// Register the ServiceVIPBinding controller and the NetworkRule binding
+	// writer that generates its bindings.
+	if err := setupVIPBindingControllers(mgr, nodeName, vipTranslationTable); err != nil {
+		return err
 	}
 
 	// Register the node-local platform service route controller. It resolves
@@ -350,6 +346,30 @@ func runCmd(cfg *config.RouterConfig) error {
 	return nil
 }
 
+// setupVIPBindingControllers registers the two halves of this node's DSR
+// backend side: NetworkRuleBindingReconciler, which writes a ServiceVIPBinding
+// for every NetworkRule backend whose VPCAttachment is on this node, and
+// ServiceVIPBindingReconciler, which programs vip_xlat_table from those
+// bindings.
+func setupVIPBindingControllers(mgr ctrl.Manager, nodeName string, table controller.VIPTranslationTable) error {
+	if err := (&controller.ServiceVIPBindingReconciler{
+		Client:              mgr.GetClient(),
+		Scheme:              mgr.GetScheme(),
+		NodeName:            nodeName,
+		VIPTranslationTable: table,
+	}).SetupWithManager(mgr); err != nil {
+		return fmt.Errorf("setup ServiceVIPBinding controller: %w", err)
+	}
+	if err := (&controller.NetworkRuleBindingReconciler{
+		Client:   mgr.GetClient(),
+		Scheme:   mgr.GetScheme(),
+		NodeName: nodeName,
+	}).SetupWithManager(mgr); err != nil {
+		return fmt.Errorf("setup NetworkRule binding controller: %w", err)
+	}
+	return nil
+}
+
 func setupServiceRouteController(
 	mgr ctrl.Manager,
 	nodeName, bgpNamespace string, frontendEnabled bool,

cmd/galactic-router/vipxlat.go

@@ -18,13 +18,14 @@ import (
 
 	"go.datum.net/galactic/internal/plumbing/ebpf/attach"
 	"go.datum.net/galactic/internal/plumbing/ebpf/vipxlatmap"
+	"go.datum.net/galactic/internal/plumbing/srv6"
 )
 
 // vipXlatTable is the part of *vipxlatmap.VipXlatTable the xlat subcommands
 // use, so tests can run them against a fake.
 type vipXlatTable interface {
 	List() ([]vipxlatmap.Entry, error)
-	UnregisterBinding(proto uint8, vipAddr net.IP, vipPort uint16,
+	UnregisterBinding(slot uint16, proto uint8, vipAddr net.IP, vipPort uint16,
 		backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error)
 }
 
@@ -56,9 +57,9 @@ func newVIPXlatListCommand(pinDir *string) *cobra.Command {
 	return &cobra.Command{
 		Use:   "list",
 		Short: "Print every vip_xlat_table row",
-		Long: `list prints every vip_xlat_table row. An ingress row is keyed on the VIP
-and its port and rewrites to the backend. An egress row is keyed on the backend
-and its port and rewrites to the VIP.`,
+		Long: `list prints every vip_xlat_table row. An ingress row is keyed on the VIP,
+its port and the backend's slot, and rewrites to the backend. An egress row is
+keyed on the backend and its port, with slot 0, and rewrites to the VIP.`,
 		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			table, closer, err := openVipXlatTable(*pinDir)
@@ -83,14 +84,17 @@ func newVIPXlatRemoveCommand(pinDir *string) *cobra.Command {
 		vipPort     uint16
 		backendAddr string
 		backendPort uint16
+		slot        uint16
 	)
 	cmd := &cobra.Command{
 		Use:   "remove",
 		Short: "Remove the rows one ServiceVIPBinding wrote",
 		Long: `remove deletes the two vip_xlat_table rows a ServiceVIPBinding with these
 spec values wrote, under whatever VRF they were written, and prints them. A row
 is removed only while it still rewrites to these values. One another binding
-has since written its own values to is left alone.`,
+has since written its own values to is left alone. The ingress row's slot is
+derived from the backend address and port, as galactic-router derives it, unless
+--slot names another.`,
 		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			proto, err := parseXlatProtocol(protocol)
@@ -115,7 +119,10 @@ has since written its own values to is left alone.`,
 			}
 			defer closer.Close() //nolint:errcheck // our own fd; the pinned map outlives it
 
-			removed, err := table.UnregisterBinding(proto, vip, vipPort, backend, backendPort)
+			if slot == 0 {
+				slot = srv6.BackendSlot(netip.AddrFrom16([16]byte(backend.To16())), backendPort)
+			}
+			removed, err := table.UnregisterBinding(slot, proto, vip, vipPort, backend, backendPort)
 			if len(removed) > 0 {
 				if werr := writeVipXlatEntries(cmd.OutOrStdout(), removed); werr != nil {
 					err = errors.Join(err, werr)
@@ -131,6 +138,8 @@ has since written its own values to is left alone.`,
 	cmd.Flags().Uint16Var(&vipPort, xlatFlagPort, 0, "binding spec.port")
 	cmd.Flags().StringVar(&backendAddr, xlatFlagBackend, "", "binding spec.backendAddress")
 	cmd.Flags().Uint16Var(&backendPort, xlatFlagBackendPort, 0, "binding spec.backendPort")
+	cmd.Flags().Uint16Var(&slot, xlatFlagSlot, 0,
+		"ingress row's backend slot; 0 derives it from --backend and --backend-port")
 	for _, name := range []string{xlatFlagProtocol, xlatFlagVIP, xlatFlagPort, xlatFlagBackend, xlatFlagBackendPort} {
 		_ = cmd.MarkFlagRequired(name) // only fails for an undefined flag name
 	}
@@ -150,6 +159,7 @@ const (
 	xlatFlagPort        = "port"
 	xlatFlagBackend     = "backend"
 	xlatFlagBackendPort = "backend-port"
+	xlatFlagSlot        = "slot"
 )
 
 // parseXlatProtocol maps a ServiceVIPBinding protocol name to the IANA number
@@ -168,12 +178,12 @@ func parseXlatProtocol(protocol string) (uint8, error) {
 // writeVipXlatEntries prints entries as an aligned table.
 func writeVipXlatEntries(w io.Writer, entries []vipxlatmap.Entry) error {
 	tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
-	if _, err := fmt.Fprintln(tw, "DIRECTION\tBLOCK\tARGUMENT\tPROTO\tMATCH\tREWRITE"); err != nil {
+	if _, err := fmt.Fprintln(tw, "DIRECTION\tBLOCK\tARGUMENT\tSLOT\tPROTO\tMATCH\tREWRITE"); err != nil {
 		return err
 	}
 	for _, e := range entries {
-		if _, err := fmt.Fprintf(tw, "%s\t%#x\t%d\t%s\t%s\t%s\n",
-			e.Direction, e.Block, e.Argument, xlatProtocolName(e.Proto),
+		if _, err := fmt.Fprintf(tw, "%s\t%#x\t%d\t%#04x\t%s\t%s\t%s\n",
+			e.Direction, e.Block, e.Argument, e.Slot, xlatProtocolName(e.Proto),
 			netip.AddrPortFrom(e.Addr, e.Port),
 			net.JoinHostPort(e.RewriteAddr.String(), strconv.Itoa(int(e.RewritePort)))); err != nil {
 			return err

cmd/galactic-router/vipxlat_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 
 	"go.datum.net/galactic/internal/plumbing/ebpf/vipxlatmap"
+	"go.datum.net/galactic/internal/plumbing/srv6"
 )
 
 // fakeXlatTable records UnregisterBinding calls and serves fixed rows.
@@ -24,9 +25,10 @@ type fakeXlatTable struct {
 
 func (f *fakeXlatTable) List() ([]vipxlatmap.Entry, error) { return f.entries, nil }
 
-func (f *fakeXlatTable) UnregisterBinding(proto uint8, vipAddr net.IP, vipPort uint16,
+func (f *fakeXlatTable) UnregisterBinding(slot uint16, proto uint8, vipAddr net.IP, vipPort uint16,
 	backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error) {
 	f.calls = append(f.calls, strings.Join([]string{
+		"slot=" + itoa(slot),
 		xlatProtocolName(proto),
 		net.JoinHostPort(vipAddr.String(), itoa(vipPort)),
 		net.JoinHostPort(backendAddr.String(), itoa(backendPort)),
@@ -60,7 +62,7 @@ func testXlatEntries() []vipxlatmap.Entry {
 	return []vipxlatmap.Entry{
 		{
 			Key: vipxlatmap.Key{
-				Block: 0x20010db8ff01, Argument: 70, Proto: vipxlatmap.ProtoTCP,
+				Block: 0x20010db8ff01, Argument: 70, Slot: 0xbeef, Proto: vipxlatmap.ProtoTCP,
 				Addr: netip.MustParseAddr("2001:db8::1"), Port: 8080,
 			},
 			Direction:   vipxlatmap.DirectionIngress,
@@ -85,7 +87,8 @@ func TestVIPXlatList(t *testing.T) {
 		t.Fatalf("list: unexpected error: %v", err)
 	}
 	for _, want := range []string{
-		"DIRECTION", "MATCH", "ingress", "egress", "0x20010db8ff01", "[fd20:60::1]:30080", "[2001:db8::1]:8080",
+		"DIRECTION", "SLOT", "0xbeef", "MATCH", "ingress", "egress",
+		"0x20010db8ff01", "[fd20:60::1]:30080", "[2001:db8::1]:8080",
 	} {
 		if !strings.Contains(out, want) {
 			t.Errorf("list output missing %q:\n%s", want, out)
@@ -101,15 +104,29 @@ func TestVIPXlatRemove(t *testing.T) {
 	if err != nil {
 		t.Fatalf("remove: unexpected error: %v", err)
 	}
-	want := "tcp [2001:db8::1]:8080 [fd20:60::1]:30080"
+	slot := srv6.BackendSlot(netip.MustParseAddr("fd20:60::1"), 30080)
+	want := "slot=" + itoa(slot) + " tcp [2001:db8::1]:8080 [fd20:60::1]:30080"
 	if len(table.calls) != 1 || table.calls[0] != want {
-		t.Errorf("UnregisterBinding calls = %q, want [%q]", table.calls, want)
+		t.Errorf("UnregisterBinding calls = %q, want [%q] (slot derived from the backend)", table.calls, want)
 	}
 	if !strings.Contains(out, "egress") {
 		t.Errorf("remove output does not list the removed rows:\n%s", out)
 	}
 }
 
+func TestVIPXlatRemoveExplicitSlot(t *testing.T) {
+	table := &fakeXlatTable{entries: testXlatEntries()}
+	if _, err := runXlat(t, table, "remove",
+		"--protocol", "tcp", "--vip", "2001:db8::1", "--port", "8080",
+		"--backend", "fd20:60::1", "--backend-port", "30080", "--slot", "7"); err != nil {
+		t.Fatalf("remove: unexpected error: %v", err)
+	}
+	want := "slot=7 tcp [2001:db8::1]:8080 [fd20:60::1]:30080"
+	if len(table.calls) != 1 || table.calls[0] != want {
+		t.Errorf("UnregisterBinding calls = %q, want [%q]", table.calls, want)
+	}
+}
+
 func TestVIPXlatRemoveNoMatch(t *testing.T) {
 	out, err := runXlat(t, &fakeXlatTable{}, "remove",
 		"--protocol", "udp", "--vip", "2001:db8::1", "--port", "53",

config/galactic-gateway/rbac.yaml

@@ -62,6 +62,14 @@ rules:
     resources:
       - bgpvrfinstances
     verbs: ["get", "list", "watch"]
+  # A NetworkRule's backends are the VPCAttachments its backendSelector
+  # picks, read cluster-wide since they live in tenant namespaces, and
+  # watched so a backend appearing, moving node or changing address
+  # reconverges the rule.
+  - apiGroups: ["cloud.datumapis.com"]
+    resources:
+      - vpcattachments
+    verbs: ["get", "list", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

config/galactic-router/rbac.yaml

@@ -23,16 +23,22 @@ rules:
       - bgpadvertisements
       - bgpvrfinstances
     verbs: ["get", "list", "watch", "delete"]
-  # servicevipbindings needs update/patch on the base resource (not just
-  # delete, unlike bgpadvertisements/bgpvrfinstances above):
-  # ServiceVIPBindingReconciler adds/removes its own teardown finalizer via
-  # a Patch call on the object itself (mirroring NetworkRuleReconciler's
-  # identical finalizer pattern on NetworkRule, config/galactic-gateway/
-  # rbac.yaml), not just its /status subresource.
+  # servicevipbindings needs full CRUD on the base resource:
+  # NetworkRuleBindingReconciler creates, updates and deletes this node's
+  # generated bindings, and ServiceVIPBindingReconciler adds and removes its
+  # own teardown finalizer with a Patch on the object itself, not just its
+  # /status subresource.
   - apiGroups: ["network.datumapis.com"]
     resources:
       - servicevipbindings
-    verbs: ["get", "list", "watch", "update", "patch"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  # NetworkRuleBindingReconciler reads every NetworkRule to find the backends
+  # on this node, and writes its per-node BackendsBound condition to rule
+  # status. It never writes the rule itself.
+  - apiGroups: ["network.datumapis.com"]
+    resources:
+      - networkrules
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["network.datumapis.com"]
     resources:
       - serviceendpoints
@@ -46,6 +52,7 @@ rules:
       - bgppolicies/status
       - bgpvrfinstances/status
       - servicevipbindings/status
+      - networkrules/status
       - serviceroutepolicies/status
     verbs: ["get", "update", "patch"]
   - apiGroups: ["cloud.datumapis.com"]

deploy/containerlab/AGENTS.md

@@ -35,7 +35,7 @@
 - **iad-control** — `iad-worker3`, the lab's single EVPN route reflector; every galactic-router in every cluster peers with it. galactic-router + BGP CRDs in `resources/galactic-control/iad/` (one `bgppeer-<nodename>.yaml` per client), fabric in `resources/fabric-router/iad/` (`frr.conf.iad-worker3`).
 - **Edge** — the four `nat=enabled` workers (`dfw-worker2`, `dfw-worker3`, `sjc-worker2`, `iad-worker2`): each site's transit-facing nodes, the uplinks its compute node routes through, the egress shards and the VIP gateways. Their `BGPRouter` and route-reflector `BGPPeer` live alongside the compute node's in `resources/galactic-router/<site>/` (`bgprouter-<nodename>.yaml`, `bgppeer-<nodename>.yaml`).
 - **Egress** — `galactic-nat`, one egress shard per edge node, running from that node's XDP dispatcher on `bond0`/`bond1`. One per-site overlay under `resources/galactic-nat/<site>/` (uplink override + one `EgressShard` per edge node; dfw has two). Each site's compute node lists only its own site's shard SIDs (`resources/galactic-cni/<site>/egress-shards-patch.yaml`), and each edge node's `frr.conf` originates its shard's SID `/64` and masquerade addresses.
-- **Ingress** — `galactic-gateway`, one DaemonSet per edge node (`resources/galactic-gateway/<node>/`: hostname pin, `PUBLIC_INTERFACE=bond0`, `INTERNAL_INTERFACES=bond1`, `SRV6_ADDRESS` left unset so the gateway derives the node's own uSID from its `BGPRouter`, and its `NetworkGateway`). Each site directory (`resources/galactic-gateway/<site>/`) adds the `ns60-tcp`/`ns60-udp` `NetworkRule`s for that site's own VIP and one `ServiceVIPBinding` per rule for the site's single `ns60` backend (`resources/tenants/ns60/`, fixed address from its NAD). Each edge node's `frr.conf` originates its site's VIP `/64` (`2001:db8:6060:{1,2,3}::/64` for dfw, sjc, iad) to `Null0`. One backend per site, because a node's VIP rewrite (`usid.c`'s `vip_xlat_key`) holds no VIP or backend address, so two backends of one VIP port in one VPC on one node would collide (#755).
+- **Ingress** — `galactic-gateway`, one DaemonSet per edge node (`resources/galactic-gateway/<node>/`: hostname pin, `PUBLIC_INTERFACE=bond0`, `INTERNAL_INTERFACES=bond1`, `SRV6_ADDRESS` left unset so the gateway derives the node's own uSID from its `BGPRouter`, and its `NetworkGateway`). Each site directory (`resources/galactic-gateway/<site>/`) adds the `ns60-tcp`/`ns60-udp` `NetworkRule`s for that site's own VIP, which select the site's two `ns60` backends by label through their `VPCAttachment`s (`resources/tenants/ns60/<site>/vpcattachments.yaml`, fixed addresses from their NADs; `scripts/publish-ns60-attachments.sh` writes their status, the attachment controller's job in production). The compute node's `galactic-router` writes one `ServiceVIPBinding` per rule per backend; none is hand-written. Each edge node's `frr.conf` originates its site's VIP `/64` (`2001:db8:6060:{1,2,3}::/64` for dfw, sjc, iad) to `Null0`. Both backends share the site's one compute node on purpose: the gateway carries Maglev's choice to the node in the SRv6 destination's slot, so each gets its own flows (#799).
 
 ## References