diff --git a/cmd/galactic-gateway/main.go b/cmd/galactic-gateway/main.go index df2d9b7d..974be15f 100644 --- a/cmd/galactic-gateway/main.go +++ b/cmd/galactic-gateway/main.go @@ -30,6 +30,11 @@ const ( resourceNetworkRules = "networkrules" resourceBGPAdvertisements = "bgpadvertisements" resourceBGPRouters = "bgprouters" + resourceBGPVRFInstances = "bgpvrfinstances" + + cloudAPIGroup = "cloud.datumapis.com" + cloudAPIVersion = "v1alpha1" + resourceVPCAttachments = "vpcattachments" ) func main() { @@ -68,6 +73,8 @@ func checkWatchPermissions(mgr ctrl.Manager) { {group: networkAPIGroup, version: networkAPIVersion, resource: resourceNetworkRules}, {group: networkAPIGroup, version: networkAPIVersion, resource: resourceBGPAdvertisements}, {group: networkAPIGroup, version: networkAPIVersion, resource: resourceBGPRouters}, + {group: networkAPIGroup, version: networkAPIVersion, resource: resourceBGPVRFInstances}, + {group: cloudAPIGroup, version: cloudAPIVersion, resource: resourceVPCAttachments}, } for _, r := range resources { diff --git a/cmd/galactic-gateway/root.go b/cmd/galactic-gateway/root.go index 21cbbb54..a93c1cd0 100644 --- a/cmd/galactic-gateway/root.go +++ b/cmd/galactic-gateway/root.go @@ -23,6 +23,7 @@ import ( ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/galactic/internal/config" "go.datum.net/galactic/internal/controller" "go.datum.net/galactic/internal/gateway" @@ -54,6 +55,7 @@ func runCmd(cfg *config.GatewayConfig) error { scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(bgpv1alpha1.AddToScheme(scheme)) + utilruntime.Must(cloudv1alpha1.AddToScheme(scheme)) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, diff --git a/cmd/galactic-router/main.go b/cmd/galactic-router/main.go index 66de1801..cd39acee 100644 --- a/cmd/galactic-router/main.go +++ b/cmd/galactic-router/main.go @@ -26,6 +26,7 @@ const ( resourceBGPPolicies = "bgppolicies" resourceBGPVRFInstances = "bgpvrfinstances" resourceServiceVIPBindings = "servicevipbindings" + resourceNetworkRules = "networkrules" resourceServiceEndpoints = "serviceendpoints" resourceServiceRoutePolicy = "serviceroutepolicies" resourceVPCAttachments = "vpcattachments" @@ -68,6 +69,7 @@ func checkWatchPermissions(mgr ctrl.Manager) { {group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceBGPPolicies}, {group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceBGPVRFInstances}, {group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceServiceVIPBindings}, + {group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceNetworkRules}, {group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceServiceEndpoints}, {group: bgpAPIGroup, version: bgpAPIVersion, resource: resourceServiceRoutePolicy}, {group: "cloud.datumapis.com", version: "v1alpha1", resource: resourceVPCAttachments}, diff --git a/cmd/galactic-router/root.go b/cmd/galactic-router/root.go index d2b02409..cf7db50d 100644 --- a/cmd/galactic-router/root.go +++ b/cmd/galactic-router/root.go @@ -265,14 +265,10 @@ func runCmd(cfg *config.RouterConfig) error { return fmt.Errorf("setup Node controller: %w", err) } - // Register ServiceVIPBinding controller. - if err := (&controller.ServiceVIPBindingReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - NodeName: nodeName, - VIPTranslationTable: vipTranslationTable, - }).SetupWithManager(mgr); err != nil { - return fmt.Errorf("setup ServiceVIPBinding controller: %w", err) + // Register the ServiceVIPBinding controller and the NetworkRule binding + // writer that generates its bindings. + if err := setupVIPBindingControllers(mgr, nodeName, vipTranslationTable); err != nil { + return err } // Register the node-local platform service route controller. It resolves @@ -350,6 +346,30 @@ func runCmd(cfg *config.RouterConfig) error { return nil } +// setupVIPBindingControllers registers the two halves of this node's DSR +// backend side: NetworkRuleBindingReconciler, which writes a ServiceVIPBinding +// for every NetworkRule backend whose VPCAttachment is on this node, and +// ServiceVIPBindingReconciler, which programs vip_xlat_table from those +// bindings. +func setupVIPBindingControllers(mgr ctrl.Manager, nodeName string, table controller.VIPTranslationTable) error { + if err := (&controller.ServiceVIPBindingReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + NodeName: nodeName, + VIPTranslationTable: table, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("setup ServiceVIPBinding controller: %w", err) + } + if err := (&controller.NetworkRuleBindingReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + NodeName: nodeName, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("setup NetworkRule binding controller: %w", err) + } + return nil +} + func setupServiceRouteController( mgr ctrl.Manager, nodeName, bgpNamespace string, frontendEnabled bool, diff --git a/cmd/galactic-router/vipxlat.go b/cmd/galactic-router/vipxlat.go index 11960b08..76b2cb0c 100644 --- a/cmd/galactic-router/vipxlat.go +++ b/cmd/galactic-router/vipxlat.go @@ -18,13 +18,14 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/attach" "go.datum.net/galactic/internal/plumbing/ebpf/vipxlatmap" + "go.datum.net/galactic/internal/plumbing/srv6" ) // vipXlatTable is the part of *vipxlatmap.VipXlatTable the xlat subcommands // use, so tests can run them against a fake. type vipXlatTable interface { List() ([]vipxlatmap.Entry, error) - UnregisterBinding(proto uint8, vipAddr net.IP, vipPort uint16, + UnregisterBinding(slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error) } @@ -56,9 +57,9 @@ func newVIPXlatListCommand(pinDir *string) *cobra.Command { return &cobra.Command{ Use: "list", Short: "Print every vip_xlat_table row", - Long: `list prints every vip_xlat_table row. An ingress row is keyed on the VIP -and its port and rewrites to the backend. An egress row is keyed on the backend -and its port and rewrites to the VIP.`, + Long: `list prints every vip_xlat_table row. An ingress row is keyed on the VIP, +its port and the backend's slot, and rewrites to the backend. An egress row is +keyed on the backend and its port, with slot 0, and rewrites to the VIP.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { table, closer, err := openVipXlatTable(*pinDir) @@ -83,6 +84,7 @@ func newVIPXlatRemoveCommand(pinDir *string) *cobra.Command { vipPort uint16 backendAddr string backendPort uint16 + slot uint16 ) cmd := &cobra.Command{ Use: "remove", @@ -90,7 +92,9 @@ func newVIPXlatRemoveCommand(pinDir *string) *cobra.Command { Long: `remove deletes the two vip_xlat_table rows a ServiceVIPBinding with these spec values wrote, under whatever VRF they were written, and prints them. A row is removed only while it still rewrites to these values. One another binding -has since written its own values to is left alone.`, +has since written its own values to is left alone. The ingress row's slot is +derived from the backend address and port, as galactic-router derives it, unless +--slot names another.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { proto, err := parseXlatProtocol(protocol) @@ -115,7 +119,10 @@ has since written its own values to is left alone.`, } defer closer.Close() //nolint:errcheck // our own fd; the pinned map outlives it - removed, err := table.UnregisterBinding(proto, vip, vipPort, backend, backendPort) + if slot == 0 { + slot = srv6.BackendSlot(netip.AddrFrom16([16]byte(backend.To16())), backendPort) + } + removed, err := table.UnregisterBinding(slot, proto, vip, vipPort, backend, backendPort) if len(removed) > 0 { if werr := writeVipXlatEntries(cmd.OutOrStdout(), removed); werr != nil { err = errors.Join(err, werr) @@ -131,6 +138,8 @@ has since written its own values to is left alone.`, cmd.Flags().Uint16Var(&vipPort, xlatFlagPort, 0, "binding spec.port") cmd.Flags().StringVar(&backendAddr, xlatFlagBackend, "", "binding spec.backendAddress") cmd.Flags().Uint16Var(&backendPort, xlatFlagBackendPort, 0, "binding spec.backendPort") + cmd.Flags().Uint16Var(&slot, xlatFlagSlot, 0, + "ingress row's backend slot; 0 derives it from --backend and --backend-port") for _, name := range []string{xlatFlagProtocol, xlatFlagVIP, xlatFlagPort, xlatFlagBackend, xlatFlagBackendPort} { _ = cmd.MarkFlagRequired(name) // only fails for an undefined flag name } @@ -150,6 +159,7 @@ const ( xlatFlagPort = "port" xlatFlagBackend = "backend" xlatFlagBackendPort = "backend-port" + xlatFlagSlot = "slot" ) // parseXlatProtocol maps a ServiceVIPBinding protocol name to the IANA number @@ -168,12 +178,12 @@ func parseXlatProtocol(protocol string) (uint8, error) { // writeVipXlatEntries prints entries as an aligned table. func writeVipXlatEntries(w io.Writer, entries []vipxlatmap.Entry) error { tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) - if _, err := fmt.Fprintln(tw, "DIRECTION\tBLOCK\tARGUMENT\tPROTO\tMATCH\tREWRITE"); err != nil { + if _, err := fmt.Fprintln(tw, "DIRECTION\tBLOCK\tARGUMENT\tSLOT\tPROTO\tMATCH\tREWRITE"); err != nil { return err } for _, e := range entries { - if _, err := fmt.Fprintf(tw, "%s\t%#x\t%d\t%s\t%s\t%s\n", - e.Direction, e.Block, e.Argument, xlatProtocolName(e.Proto), + if _, err := fmt.Fprintf(tw, "%s\t%#x\t%d\t%#04x\t%s\t%s\t%s\n", + e.Direction, e.Block, e.Argument, e.Slot, xlatProtocolName(e.Proto), netip.AddrPortFrom(e.Addr, e.Port), net.JoinHostPort(e.RewriteAddr.String(), strconv.Itoa(int(e.RewritePort)))); err != nil { return err diff --git a/cmd/galactic-router/vipxlat_test.go b/cmd/galactic-router/vipxlat_test.go index f30e57f1..a529d30f 100644 --- a/cmd/galactic-router/vipxlat_test.go +++ b/cmd/galactic-router/vipxlat_test.go @@ -14,6 +14,7 @@ import ( "testing" "go.datum.net/galactic/internal/plumbing/ebpf/vipxlatmap" + "go.datum.net/galactic/internal/plumbing/srv6" ) // fakeXlatTable records UnregisterBinding calls and serves fixed rows. @@ -24,9 +25,10 @@ type fakeXlatTable struct { func (f *fakeXlatTable) List() ([]vipxlatmap.Entry, error) { return f.entries, nil } -func (f *fakeXlatTable) UnregisterBinding(proto uint8, vipAddr net.IP, vipPort uint16, +func (f *fakeXlatTable) UnregisterBinding(slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error) { f.calls = append(f.calls, strings.Join([]string{ + "slot=" + itoa(slot), xlatProtocolName(proto), net.JoinHostPort(vipAddr.String(), itoa(vipPort)), net.JoinHostPort(backendAddr.String(), itoa(backendPort)), @@ -60,7 +62,7 @@ func testXlatEntries() []vipxlatmap.Entry { return []vipxlatmap.Entry{ { Key: vipxlatmap.Key{ - Block: 0x20010db8ff01, Argument: 70, Proto: vipxlatmap.ProtoTCP, + Block: 0x20010db8ff01, Argument: 70, Slot: 0xbeef, Proto: vipxlatmap.ProtoTCP, Addr: netip.MustParseAddr("2001:db8::1"), Port: 8080, }, Direction: vipxlatmap.DirectionIngress, @@ -85,7 +87,8 @@ func TestVIPXlatList(t *testing.T) { t.Fatalf("list: unexpected error: %v", err) } for _, want := range []string{ - "DIRECTION", "MATCH", "ingress", "egress", "0x20010db8ff01", "[fd20:60::1]:30080", "[2001:db8::1]:8080", + "DIRECTION", "SLOT", "0xbeef", "MATCH", "ingress", "egress", + "0x20010db8ff01", "[fd20:60::1]:30080", "[2001:db8::1]:8080", } { if !strings.Contains(out, want) { t.Errorf("list output missing %q:\n%s", want, out) @@ -101,15 +104,29 @@ func TestVIPXlatRemove(t *testing.T) { if err != nil { t.Fatalf("remove: unexpected error: %v", err) } - want := "tcp [2001:db8::1]:8080 [fd20:60::1]:30080" + slot := srv6.BackendSlot(netip.MustParseAddr("fd20:60::1"), 30080) + want := "slot=" + itoa(slot) + " tcp [2001:db8::1]:8080 [fd20:60::1]:30080" if len(table.calls) != 1 || table.calls[0] != want { - t.Errorf("UnregisterBinding calls = %q, want [%q]", table.calls, want) + t.Errorf("UnregisterBinding calls = %q, want [%q] (slot derived from the backend)", table.calls, want) } if !strings.Contains(out, "egress") { t.Errorf("remove output does not list the removed rows:\n%s", out) } } +func TestVIPXlatRemoveExplicitSlot(t *testing.T) { + table := &fakeXlatTable{entries: testXlatEntries()} + if _, err := runXlat(t, table, "remove", + "--protocol", "tcp", "--vip", "2001:db8::1", "--port", "8080", + "--backend", "fd20:60::1", "--backend-port", "30080", "--slot", "7"); err != nil { + t.Fatalf("remove: unexpected error: %v", err) + } + want := "slot=7 tcp [2001:db8::1]:8080 [fd20:60::1]:30080" + if len(table.calls) != 1 || table.calls[0] != want { + t.Errorf("UnregisterBinding calls = %q, want [%q]", table.calls, want) + } +} + func TestVIPXlatRemoveNoMatch(t *testing.T) { out, err := runXlat(t, &fakeXlatTable{}, "remove", "--protocol", "udp", "--vip", "2001:db8::1", "--port", "53", diff --git a/config/galactic-gateway/rbac.yaml b/config/galactic-gateway/rbac.yaml index b289f768..7e857c6e 100644 --- a/config/galactic-gateway/rbac.yaml +++ b/config/galactic-gateway/rbac.yaml @@ -62,6 +62,14 @@ rules: resources: - bgpvrfinstances verbs: ["get", "list", "watch"] + # A NetworkRule's backends are the VPCAttachments its backendSelector + # picks, read cluster-wide since they live in tenant namespaces, and + # watched so a backend appearing, moving node or changing address + # reconverges the rule. + - apiGroups: ["cloud.datumapis.com"] + resources: + - vpcattachments + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/config/galactic-router/rbac.yaml b/config/galactic-router/rbac.yaml index 656429ac..08f825db 100644 --- a/config/galactic-router/rbac.yaml +++ b/config/galactic-router/rbac.yaml @@ -23,16 +23,22 @@ rules: - bgpadvertisements - bgpvrfinstances verbs: ["get", "list", "watch", "delete"] - # servicevipbindings needs update/patch on the base resource (not just - # delete, unlike bgpadvertisements/bgpvrfinstances above): - # ServiceVIPBindingReconciler adds/removes its own teardown finalizer via - # a Patch call on the object itself (mirroring NetworkRuleReconciler's - # identical finalizer pattern on NetworkRule, config/galactic-gateway/ - # rbac.yaml), not just its /status subresource. + # servicevipbindings needs full CRUD on the base resource: + # NetworkRuleBindingReconciler creates, updates and deletes this node's + # generated bindings, and ServiceVIPBindingReconciler adds and removes its + # own teardown finalizer with a Patch on the object itself, not just its + # /status subresource. - apiGroups: ["network.datumapis.com"] resources: - servicevipbindings - verbs: ["get", "list", "watch", "update", "patch"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # NetworkRuleBindingReconciler reads every NetworkRule to find the backends + # on this node, and writes its per-node BackendsBound condition to rule + # status. It never writes the rule itself. + - apiGroups: ["network.datumapis.com"] + resources: + - networkrules + verbs: ["get", "list", "watch"] - apiGroups: ["network.datumapis.com"] resources: - serviceendpoints @@ -46,6 +52,7 @@ rules: - bgppolicies/status - bgpvrfinstances/status - servicevipbindings/status + - networkrules/status - serviceroutepolicies/status verbs: ["get", "update", "patch"] - apiGroups: ["cloud.datumapis.com"] diff --git a/deploy/containerlab/AGENTS.md b/deploy/containerlab/AGENTS.md index a31af517..68b8d7c6 100644 --- a/deploy/containerlab/AGENTS.md +++ b/deploy/containerlab/AGENTS.md @@ -35,7 +35,7 @@ - **iad-control** — `iad-worker3`, the lab's single EVPN route reflector; every galactic-router in every cluster peers with it. galactic-router + BGP CRDs in `resources/galactic-control/iad/` (one `bgppeer-.yaml` per client), fabric in `resources/fabric-router/iad/` (`frr.conf.iad-worker3`). - **Edge** — the four `nat=enabled` workers (`dfw-worker2`, `dfw-worker3`, `sjc-worker2`, `iad-worker2`): each site's transit-facing nodes, the uplinks its compute node routes through, the egress shards and the VIP gateways. Their `BGPRouter` and route-reflector `BGPPeer` live alongside the compute node's in `resources/galactic-router//` (`bgprouter-.yaml`, `bgppeer-.yaml`). - **Egress** — `galactic-nat`, one egress shard per edge node, running from that node's XDP dispatcher on `bond0`/`bond1`. One per-site overlay under `resources/galactic-nat//` (uplink override + one `EgressShard` per edge node; dfw has two). Each site's compute node lists only its own site's shard SIDs (`resources/galactic-cni//egress-shards-patch.yaml`), and each edge node's `frr.conf` originates its shard's SID `/64` and masquerade addresses. -- **Ingress** — `galactic-gateway`, one DaemonSet per edge node (`resources/galactic-gateway//`: hostname pin, `PUBLIC_INTERFACE=bond0`, `INTERNAL_INTERFACES=bond1`, `SRV6_ADDRESS` left unset so the gateway derives the node's own uSID from its `BGPRouter`, and its `NetworkGateway`). Each site directory (`resources/galactic-gateway//`) adds the `ns60-tcp`/`ns60-udp` `NetworkRule`s for that site's own VIP and one `ServiceVIPBinding` per rule for the site's single `ns60` backend (`resources/tenants/ns60/`, fixed address from its NAD). Each edge node's `frr.conf` originates its site's VIP `/64` (`2001:db8:6060:{1,2,3}::/64` for dfw, sjc, iad) to `Null0`. One backend per site, because a node's VIP rewrite (`usid.c`'s `vip_xlat_key`) holds no VIP or backend address, so two backends of one VIP port in one VPC on one node would collide (#755). +- **Ingress** — `galactic-gateway`, one DaemonSet per edge node (`resources/galactic-gateway//`: hostname pin, `PUBLIC_INTERFACE=bond0`, `INTERNAL_INTERFACES=bond1`, `SRV6_ADDRESS` left unset so the gateway derives the node's own uSID from its `BGPRouter`, and its `NetworkGateway`). Each site directory (`resources/galactic-gateway//`) adds the `ns60-tcp`/`ns60-udp` `NetworkRule`s for that site's own VIP, which select the site's two `ns60` backends by label through their `VPCAttachment`s (`resources/tenants/ns60//vpcattachments.yaml`, fixed addresses from their NADs; `scripts/publish-ns60-attachments.sh` writes their status, the attachment controller's job in production). The compute node's `galactic-router` writes one `ServiceVIPBinding` per rule per backend; none is hand-written. Each edge node's `frr.conf` originates its site's VIP `/64` (`2001:db8:6060:{1,2,3}::/64` for dfw, sjc, iad) to `Null0`. Both backends share the site's one compute node on purpose: the gateway carries Maglev's choice to the node in the SRv6 destination's slot, so each gets its own flows (#799). ## References diff --git a/deploy/containerlab/README.md b/deploy/containerlab/README.md index f22d4413..604a2bce 100644 --- a/deploy/containerlab/README.md +++ b/deploy/containerlab/README.md @@ -132,7 +132,8 @@ nodes, plus `galactic-nat`, the egress shard, and `galactic-gateway`, the VIP lo (`GALACTIC_NAT_XDP_ATTACH=dispatch`; the gateway's default), each in its own slot, so either can restart without detaching the other. Each site has its own VIP `/64` (`2001:db8:6060:1::/64` for dfw, `:2::` for sjc, `:3::` for iad), originated only by that -site's edge nodes and served by one `ns60` backend on the site's compute node. +site's edge nodes and served by two `ns60` backends on the site's compute node, which the +site's `NetworkRule`s select by label through their `VPCAttachment`s. Compute nodes run no shard: each egresses through its own site's edge shards only. The reflector runs neither `galactic-cni` nor a shard — its `galactic=control` label is mutually exclusive with the `galactic=router` value that pulls @@ -416,7 +417,7 @@ deploy/containerlab/ │ ├── bmp-collector/iad/ # the lab's one BMP collector, on the reflector's host network │ ├── galactic-nat/ # egress shards per site, one per edge node (dfw has two) │ ├── galactic-gateway/ # one gateway overlay per edge node, plus each site's -│ │ # ns60 NetworkRules and ServiceVIPBindings +│ │ # ns60 NetworkRules (backends picked by selector) │ └── tenants/ # test VPCs — one shared base/ (Namespace + netshoot │ ├── base/ # Deployment), each tenant patching its namespace and │ ├── ns10/ # default-network annotation; per-site dirs hold each @@ -427,8 +428,9 @@ deploy/containerlab/ │ │ # their own NAD+Deployment (distinct vpcattachment, │ │ # same vpc), not one NAD scaled to replicas: 2 — see │ │ # docs/tenants.md for why. -│ └── ns60/ # the gateway's VIP backend, one per site at a fixed -│ # address, answering TCP and UDP on port 80 +│ └── ns60/ # the gateway's VIP backends, two per site on one node, +│ # each with a fixed address and a VPCAttachment, +│ # answering TCP and UDP on port 80 ├── node_files/ │ ├── dfw/ config.yaml │ ├── iad/ config.yaml @@ -499,7 +501,7 @@ task deploy | `deploy:ns20` | Deploy ns20 test VPC (dual-stack, fd20 ULA + IPv4) | | `deploy:ns30` | Deploy ns30 test VPC (dfw only, 2 pods) | | `deploy:ns40` | Deploy ns40 test VPC (iad only, 2 pods) | -| `deploy:ns60` | Deploy ns60 test VPC (one VIP backend per site) | +| `deploy:ns60` | Deploy ns60 test VPC (two VIP backends per site, on one node) | | `verify:fabric-metrics` | Scrape every fabric-router's frr-exporter; fail on a failed collector or a session not Established | | `verify:bmp` | Check the route reflector and every fabric-router stream to the BMP collector, and that it agrees with each on every session's state | | `verify:underlay` | Ping every underlay loopback from tr1 over both IPv4 and IPv6 | diff --git a/deploy/containerlab/Taskfile.yaml b/deploy/containerlab/Taskfile.yaml index 92b6fd0f..7ab1d5f3 100644 --- a/deploy/containerlab/Taskfile.yaml +++ b/deploy/containerlab/Taskfile.yaml @@ -100,7 +100,7 @@ tasks: - task: "deploy:galactic-router" - task: "deploy:galactic-nat" - task: "deploy:scenarios" - # After the scenarios: the gateway's ServiceVIPBindings bind ns60's + # After the scenarios: the gateway's NetworkRules select ns60's # backends, which deploy:scenarios creates. - task: "deploy:galactic-gateway" - task: "deploy:fabric-api" @@ -363,9 +363,10 @@ tasks: - ./scripts/deploy-ns.sh ns40 iad "deploy:ns60": - desc: Deploy ns60 test VPC (one VIP backend per site) + desc: Deploy ns60 test VPC (two VIP backends per site, on one node) cmds: - ./scripts/deploy-ns.sh ns60 dfw sjc iad + - ./scripts/publish-ns60-attachments.sh dfw sjc iad "deploy:ns71": desc: Deploy ns71 test VPCs (overlapping ULA on one node, same-node collision fixture) @@ -519,7 +520,13 @@ tasks: docker exec ${cp} kubectl -n galactic-system get networkgateways,networkrules,servicevipbindings # Ready: the engine loaded every rule and advertised it. Accepted: # the rule found a gateway. Bound: the backend's node rewrites the - # VIP for it. + # VIP for it. Each backend's node writes one binding per rule per + # ns60 backend: two rules, two backends. + bindings=$(docker exec ${cp} kubectl -n galactic-system get servicevipbindings -o name | wc -l) + if [ "${bindings}" -ne 4 ]; then + echo "FAIL ${site}: ${bindings} ServiceVIPBindings, want 4" >&2 + rc=1 + fi for check in "networkgateway Ready" "networkrule Accepted" "servicevipbinding Bound"; do set -- ${check} if ! docker exec ${cp} kubectl -n galactic-system wait $1 --all \ diff --git a/deploy/containerlab/docs/verification.md b/deploy/containerlab/docs/verification.md index 5b146a95..d48870c8 100644 --- a/deploy/containerlab/docs/verification.md +++ b/deploy/containerlab/docs/verification.md @@ -199,9 +199,11 @@ docker exec dfw-worker2 ip -d link show dev eth1 | grep -o 'prog/xdp id [0-9]*' The same four edge nodes each run `galactic-gateway` (`galactic-gateway-`) and a `NetworkGateway` named after the node. Each site has its own VIP (`2001:db8:6060:1::1` in dfw, `:2::1` in sjc, `:3::1` in iad), in a `/64` only -that site's edge nodes originate, and one `ns60` backend on its compute node -that answers TCP and UDP on port 80 with its pod name and the client address -it saw. +that site's edge nodes originate, and two `ns60` backends on its compute node +that answer TCP and UDP on port 80 with their pod name and the client address +they saw. The site's `NetworkRule`s select both through their +`VPCAttachment`s, and the compute node writes one `ServiceVIPBinding` per rule +per backend. `verify:gateway-ingress` fails unless both backends answer. ```bash # NetworkGateways Ready, NetworkRules Accepted, ServiceVIPBindings Bound diff --git a/deploy/containerlab/resources/galactic-gateway/dfw/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/dfw/kustomization.yaml index e5652c8d..717a8cfa 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw/kustomization.yaml @@ -1,11 +1,10 @@ # dfw's ingress tier: a galactic-gateway and NetworkGateway on each of -# its edge nodes, and the NetworkRules and ServiceVIPBindings for ns60, the -# site's backend tenant (resources/tenants/ns60/dfw/). The VIP is -# dfw's own, and so are its backends: a load balancer belongs to one -# region (datum-cloud/enhancements#884). +# its edge nodes, and the NetworkRules for ns60, the site's backend tenant +# (resources/tenants/ns60/dfw/). The VIP is dfw's own, and so are its +# backends: a load balancer belongs to one region +# (datum-cloud/enhancements#884). namespace: galactic-system resources: - ../dfw-worker2 - ../dfw-worker3 - networkrules.yaml - - servicevipbindings.yaml diff --git a/deploy/containerlab/resources/galactic-gateway/dfw/networkrules.yaml b/deploy/containerlab/resources/galactic-gateway/dfw/networkrules.yaml index ee82f62a..fb0dc9a9 100644 --- a/deploy/containerlab/resources/galactic-gateway/dfw/networkrules.yaml +++ b/deploy/containerlab/resources/galactic-gateway/dfw/networkrules.yaml @@ -1,4 +1,4 @@ -# ns60's port 80 over TCP, served by every ns60 backend in dfw. +# ns60's port 80 over TCP, served by every ns60 backend its selector picks in dfw. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: @@ -11,11 +11,12 @@ spec: - 2001:db8:6060:1::1 protocol: tcp port: 80 - backends: - - address: fd20:60:ff01:a::100 - port: 80 + backendSelector: + matchLabels: + app: backend + backendPort: 80 --- -# ns60's port 80 over UDP, served by every ns60 backend in dfw. +# ns60's port 80 over UDP, served by every ns60 backend its selector picks in dfw. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: @@ -28,6 +29,7 @@ spec: - 2001:db8:6060:1::1 protocol: udp port: 80 - backends: - - address: fd20:60:ff01:a::100 - port: 80 + backendSelector: + matchLabels: + app: backend + backendPort: 80 diff --git a/deploy/containerlab/resources/galactic-gateway/dfw/servicevipbindings.yaml b/deploy/containerlab/resources/galactic-gateway/dfw/servicevipbindings.yaml deleted file mode 100644 index bce4ad78..00000000 --- a/deploy/containerlab/resources/galactic-gateway/dfw/servicevipbindings.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Lets ns60/backend on dfw-worker answer TCP for the VIP: its node -# rewrites the VIP to the backend's address on the way in, and back on the -# way out. -apiVersion: network.datumapis.com/v1alpha1 -kind: ServiceVIPBinding -metadata: - name: ns60-tcp-backend - namespace: galactic-system -spec: - targetRef: - kind: Node - name: dfw-worker - vipAddress: 2001:db8:6060:1::1 - port: 80 - protocol: tcp - backendAddress: fd20:60:ff01:a::100 - backendPort: 80 - egressKind: veth ---- -# Lets ns60/backend on dfw-worker answer UDP for the VIP: its node -# rewrites the VIP to the backend's address on the way in, and back on the -# way out. -apiVersion: network.datumapis.com/v1alpha1 -kind: ServiceVIPBinding -metadata: - name: ns60-udp-backend - namespace: galactic-system -spec: - targetRef: - kind: Node - name: dfw-worker - vipAddress: 2001:db8:6060:1::1 - port: 80 - protocol: udp - backendAddress: fd20:60:ff01:a::100 - backendPort: 80 - egressKind: veth diff --git a/deploy/containerlab/resources/galactic-gateway/iad/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/iad/kustomization.yaml index 2f6de20e..8b5c4f8d 100644 --- a/deploy/containerlab/resources/galactic-gateway/iad/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/iad/kustomization.yaml @@ -1,10 +1,9 @@ # iad's ingress tier: a galactic-gateway and NetworkGateway on each of -# its edge nodes, and the NetworkRules and ServiceVIPBindings for ns60, the -# site's backend tenant (resources/tenants/ns60/iad/). The VIP is -# iad's own, and so are its backends: a load balancer belongs to one -# region (datum-cloud/enhancements#884). +# its edge nodes, and the NetworkRules for ns60, the site's backend tenant +# (resources/tenants/ns60/iad/). The VIP is iad's own, and so are its +# backends: a load balancer belongs to one region +# (datum-cloud/enhancements#884). namespace: galactic-system resources: - ../iad-worker2 - networkrules.yaml - - servicevipbindings.yaml diff --git a/deploy/containerlab/resources/galactic-gateway/iad/networkrules.yaml b/deploy/containerlab/resources/galactic-gateway/iad/networkrules.yaml index 0cacfeca..d0278ce4 100644 --- a/deploy/containerlab/resources/galactic-gateway/iad/networkrules.yaml +++ b/deploy/containerlab/resources/galactic-gateway/iad/networkrules.yaml @@ -1,4 +1,4 @@ -# ns60's port 80 over TCP, served by every ns60 backend in iad. +# ns60's port 80 over TCP, served by every ns60 backend its selector picks in iad. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: @@ -11,11 +11,12 @@ spec: - 2001:db8:6060:3::1 protocol: tcp port: 80 - backends: - - address: fd20:60:ff03:a::100 - port: 80 + backendSelector: + matchLabels: + app: backend + backendPort: 80 --- -# ns60's port 80 over UDP, served by every ns60 backend in iad. +# ns60's port 80 over UDP, served by every ns60 backend its selector picks in iad. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: @@ -28,6 +29,7 @@ spec: - 2001:db8:6060:3::1 protocol: udp port: 80 - backends: - - address: fd20:60:ff03:a::100 - port: 80 + backendSelector: + matchLabels: + app: backend + backendPort: 80 diff --git a/deploy/containerlab/resources/galactic-gateway/iad/servicevipbindings.yaml b/deploy/containerlab/resources/galactic-gateway/iad/servicevipbindings.yaml deleted file mode 100644 index 89f3e21a..00000000 --- a/deploy/containerlab/resources/galactic-gateway/iad/servicevipbindings.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Lets ns60/backend on iad-worker answer TCP for the VIP: its node -# rewrites the VIP to the backend's address on the way in, and back on the -# way out. -apiVersion: network.datumapis.com/v1alpha1 -kind: ServiceVIPBinding -metadata: - name: ns60-tcp-backend - namespace: galactic-system -spec: - targetRef: - kind: Node - name: iad-worker - vipAddress: 2001:db8:6060:3::1 - port: 80 - protocol: tcp - backendAddress: fd20:60:ff03:a::100 - backendPort: 80 - egressKind: veth ---- -# Lets ns60/backend on iad-worker answer UDP for the VIP: its node -# rewrites the VIP to the backend's address on the way in, and back on the -# way out. -apiVersion: network.datumapis.com/v1alpha1 -kind: ServiceVIPBinding -metadata: - name: ns60-udp-backend - namespace: galactic-system -spec: - targetRef: - kind: Node - name: iad-worker - vipAddress: 2001:db8:6060:3::1 - port: 80 - protocol: udp - backendAddress: fd20:60:ff03:a::100 - backendPort: 80 - egressKind: veth diff --git a/deploy/containerlab/resources/galactic-gateway/sjc/kustomization.yaml b/deploy/containerlab/resources/galactic-gateway/sjc/kustomization.yaml index cb496a96..e77ce317 100644 --- a/deploy/containerlab/resources/galactic-gateway/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/galactic-gateway/sjc/kustomization.yaml @@ -1,10 +1,9 @@ # sjc's ingress tier: a galactic-gateway and NetworkGateway on each of -# its edge nodes, and the NetworkRules and ServiceVIPBindings for ns60, the -# site's backend tenant (resources/tenants/ns60/sjc/). The VIP is -# sjc's own, and so are its backends: a load balancer belongs to one -# region (datum-cloud/enhancements#884). +# its edge nodes, and the NetworkRules for ns60, the site's backend tenant +# (resources/tenants/ns60/sjc/). The VIP is sjc's own, and so are its +# backends: a load balancer belongs to one region +# (datum-cloud/enhancements#884). namespace: galactic-system resources: - ../sjc-worker2 - networkrules.yaml - - servicevipbindings.yaml diff --git a/deploy/containerlab/resources/galactic-gateway/sjc/networkrules.yaml b/deploy/containerlab/resources/galactic-gateway/sjc/networkrules.yaml index 4244bea4..4f0dc220 100644 --- a/deploy/containerlab/resources/galactic-gateway/sjc/networkrules.yaml +++ b/deploy/containerlab/resources/galactic-gateway/sjc/networkrules.yaml @@ -1,4 +1,4 @@ -# ns60's port 80 over TCP, served by every ns60 backend in sjc. +# ns60's port 80 over TCP, served by every ns60 backend its selector picks in sjc. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: @@ -11,11 +11,12 @@ spec: - 2001:db8:6060:2::1 protocol: tcp port: 80 - backends: - - address: fd20:60:ff02:a::100 - port: 80 + backendSelector: + matchLabels: + app: backend + backendPort: 80 --- -# ns60's port 80 over UDP, served by every ns60 backend in sjc. +# ns60's port 80 over UDP, served by every ns60 backend its selector picks in sjc. apiVersion: network.datumapis.com/v1alpha1 kind: NetworkRule metadata: @@ -28,6 +29,7 @@ spec: - 2001:db8:6060:2::1 protocol: udp port: 80 - backends: - - address: fd20:60:ff02:a::100 - port: 80 + backendSelector: + matchLabels: + app: backend + backendPort: 80 diff --git a/deploy/containerlab/resources/galactic-gateway/sjc/servicevipbindings.yaml b/deploy/containerlab/resources/galactic-gateway/sjc/servicevipbindings.yaml deleted file mode 100644 index 5adc687a..00000000 --- a/deploy/containerlab/resources/galactic-gateway/sjc/servicevipbindings.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Lets ns60/backend on sjc-worker answer TCP for the VIP: its node -# rewrites the VIP to the backend's address on the way in, and back on the -# way out. -apiVersion: network.datumapis.com/v1alpha1 -kind: ServiceVIPBinding -metadata: - name: ns60-tcp-backend - namespace: galactic-system -spec: - targetRef: - kind: Node - name: sjc-worker - vipAddress: 2001:db8:6060:2::1 - port: 80 - protocol: tcp - backendAddress: fd20:60:ff02:a::100 - backendPort: 80 - egressKind: veth ---- -# Lets ns60/backend on sjc-worker answer UDP for the VIP: its node -# rewrites the VIP to the backend's address on the way in, and back on the -# way out. -apiVersion: network.datumapis.com/v1alpha1 -kind: ServiceVIPBinding -metadata: - name: ns60-udp-backend - namespace: galactic-system -spec: - targetRef: - kind: Node - name: sjc-worker - vipAddress: 2001:db8:6060:2::1 - port: 80 - protocol: udp - backendAddress: fd20:60:ff02:a::100 - backendPort: 80 - egressKind: veth diff --git a/deploy/containerlab/resources/galactic-nat/README.md b/deploy/containerlab/resources/galactic-nat/README.md index 9f255515..4b32934b 100644 --- a/deploy/containerlab/resources/galactic-nat/README.md +++ b/deploy/containerlab/resources/galactic-nat/README.md @@ -69,7 +69,7 @@ the rollout: a shard reports ready once attached, before its identity is programmed. `task deploy:galactic-gateway` runs later, after `deploy:scenarios`: the -gateway's `ServiceVIPBinding`s bind the `ns60` backends that step creates. +gateway's `NetworkRule`s select the `ns60` backends that step creates. Because the shard and the gateway share the uplinks through the dispatcher, neither has to be deployed first for the other to attach. diff --git a/deploy/containerlab/resources/tenants/ns60/base/backend.yaml b/deploy/containerlab/resources/tenants/ns60/base/backend.yaml index 1d64a9d2..4d7960f1 100644 --- a/deploy/containerlab/resources/tenants/ns60/base/backend.yaml +++ b/deploy/containerlab/resources/tenants/ns60/base/backend.yaml @@ -1,23 +1,30 @@ -# ns60's VIP backend, one per site, behind that site's NetworkRules -# (resources/galactic-gateway//). It answers TCP and UDP on port 80 +# ns60's VIP backends, two per site, behind that site's NetworkRules +# (resources/galactic-gateway//). Each answers TCP and UDP on port 80 # with its own pod name and the address it saw the client as, which is how # scripts/verify-gateway-ingress.sh tells which backend answered and checks # that the client's address arrived unchanged. # -# One per site rather than two, though two would give Maglev a choice: each -# site has one compute node, and a node can bind a VIP's port to only one -# backend per VPC (#755: usid.c's vip_xlat_key holds no backend address). +# Both land on the site's one compute node, on purpose: the rules select +# them through their VPCAttachments (vpcattachments.yaml), and the gateway +# encodes which backend Maglev chose in the SRv6 destination, so two backends +# of one VIP on one node each get their own flows (#799). # -# Its address is fixed in the site's NAD rather than taken from the pool, -# because the NetworkRule and ServiceVIPBinding name it. Recreate, not -# RollingUpdate, so a new pod never asks for the address while the old -# one still holds it. +# Each has its own NAD with its own vpcattachment value, because +# galactic-veth names a pod's veth after (vpc, vpcattachment) alone, and two +# pods sharing one on a node would collide (docs/tenants.md, ns30/ns40). Each +# also has its own /96, as each attachment gets its own subnet in production: +# the VRF holds one route per subnet, so two attachments sharing one would +# send both backends' traffic down one veth. The address is fixed in the NAD +# rather than taken from the pool, because the backend's VPCAttachment names +# it. Recreate, not RollingUpdate, so a new pod never +# asks for the address while the old one still holds it. apiVersion: apps/v1 kind: Deployment metadata: - name: backend + name: backend-a labels: app: backend + instance: a spec: replicas: 1 strategy: @@ -25,12 +32,67 @@ spec: selector: matchLabels: app: backend + instance: a template: metadata: labels: app: backend + instance: a annotations: - v1.multus-cni.io/default-network: ns60/backend + v1.multus-cni.io/default-network: ns60/backend-a + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: galactic.datumapis.com/node + operator: In + values: + - compute + containers: + - name: backend + image: ghcr.io/datum-cloud/galactic-debug:latest + imagePullPolicy: IfNotPresent + # The image sets HOSTNAME itself, so the pod's name comes from the + # downward API instead. + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # socat sets SOCAT_PEERADDR to the client's address for each + # connection or datagram. + command: + - sh + - -c + - | + reply='echo "$POD_NAME $SOCAT_PEERADDR"' + socat TCP6-LISTEN:80,reuseaddr,fork SYSTEM:"$reply" & + exec socat UDP6-RECVFROM:80,reuseaddr,fork SYSTEM:"$reply" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend-b + labels: + app: backend + instance: b +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: backend + instance: b + template: + metadata: + labels: + app: backend + instance: b + annotations: + v1.multus-cni.io/default-network: ns60/backend-b spec: affinity: nodeAffinity: diff --git a/deploy/containerlab/resources/tenants/ns60/dfw/kustomization.yaml b/deploy/containerlab/resources/tenants/ns60/dfw/kustomization.yaml index ee81d25b..4973a065 100644 --- a/deploy/containerlab/resources/tenants/ns60/dfw/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns60/dfw/kustomization.yaml @@ -1,3 +1,4 @@ resources: - ../base - nad.yaml + - vpcattachments.yaml diff --git a/deploy/containerlab/resources/tenants/ns60/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns60/dfw/nad.yaml index 6bb80d84..d9f10b12 100644 --- a/deploy/containerlab/resources/tenants/ns60/dfw/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns60/dfw/nad.yaml @@ -1,18 +1,18 @@ apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition metadata: - name: backend + name: backend-a namespace: ns60 spec: config: |- { "cniVersion": "1.0.0", - "name": "backend", + "name": "backend-a", "plugins": [ { "type": "galactic-veth", "vpc": "60", - "vpcattachment": "60", + "vpcattachment": "60a", "namespace": "galactic-system", "ipam": { "type": "galactic-ipam", @@ -24,7 +24,39 @@ spec: { "type": "galactic-bgp", "vpc": "60", - "vpcattachment": "60", + "vpcattachment": "60a", + "namespace": "galactic-system" + } + ] + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: backend-b + namespace: ns60 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "backend-b", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "60", + "vpcattachment": "60b", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { "address": "fd20:60:ff01:b::100/96", "gateway": "fd20:60:ff01:b::1" } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "60", + "vpcattachment": "60b", "namespace": "galactic-system" } ] diff --git a/deploy/containerlab/resources/tenants/ns60/dfw/vpcattachments.yaml b/deploy/containerlab/resources/tenants/ns60/dfw/vpcattachments.yaml new file mode 100644 index 00000000..16088656 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns60/dfw/vpcattachments.yaml @@ -0,0 +1,39 @@ +# The VPCAttachments for ns60's two backends in dfw, which the site's +# NetworkRules select by label (resources/galactic-gateway/dfw/). +# In production the attachment controller writes these and their status. The +# lab has no such controller, so deploy:ns60 applies them here and then +# writes each one's status from where its pod actually runs +# (scripts/publish-ns60-attachments.sh). +apiVersion: cloud.datumapis.com/v1alpha1 +kind: VPCAttachment +metadata: + name: backend-a + namespace: ns60 + labels: + app: backend + instance: a +spec: + vpc: + name: "60" + interface: + name: eth0 + mode: Netns + addresses: + - fd20:60:ff01:a::100/96 +--- +apiVersion: cloud.datumapis.com/v1alpha1 +kind: VPCAttachment +metadata: + name: backend-b + namespace: ns60 + labels: + app: backend + instance: b +spec: + vpc: + name: "60" + interface: + name: eth0 + mode: Netns + addresses: + - fd20:60:ff01:b::100/96 diff --git a/deploy/containerlab/resources/tenants/ns60/iad/kustomization.yaml b/deploy/containerlab/resources/tenants/ns60/iad/kustomization.yaml index ee81d25b..4973a065 100644 --- a/deploy/containerlab/resources/tenants/ns60/iad/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns60/iad/kustomization.yaml @@ -1,3 +1,4 @@ resources: - ../base - nad.yaml + - vpcattachments.yaml diff --git a/deploy/containerlab/resources/tenants/ns60/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns60/iad/nad.yaml index 998df576..c8250f24 100644 --- a/deploy/containerlab/resources/tenants/ns60/iad/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns60/iad/nad.yaml @@ -1,18 +1,18 @@ apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition metadata: - name: backend + name: backend-a namespace: ns60 spec: config: |- { "cniVersion": "1.0.0", - "name": "backend", + "name": "backend-a", "plugins": [ { "type": "galactic-veth", "vpc": "60", - "vpcattachment": "60", + "vpcattachment": "60a", "namespace": "galactic-system", "ipam": { "type": "galactic-ipam", @@ -24,7 +24,39 @@ spec: { "type": "galactic-bgp", "vpc": "60", - "vpcattachment": "60", + "vpcattachment": "60a", + "namespace": "galactic-system" + } + ] + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: backend-b + namespace: ns60 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "backend-b", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "60", + "vpcattachment": "60b", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { "address": "fd20:60:ff03:b::100/96", "gateway": "fd20:60:ff03:b::1" } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "60", + "vpcattachment": "60b", "namespace": "galactic-system" } ] diff --git a/deploy/containerlab/resources/tenants/ns60/iad/vpcattachments.yaml b/deploy/containerlab/resources/tenants/ns60/iad/vpcattachments.yaml new file mode 100644 index 00000000..177906a5 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns60/iad/vpcattachments.yaml @@ -0,0 +1,39 @@ +# The VPCAttachments for ns60's two backends in iad, which the site's +# NetworkRules select by label (resources/galactic-gateway/iad/). +# In production the attachment controller writes these and their status. The +# lab has no such controller, so deploy:ns60 applies them here and then +# writes each one's status from where its pod actually runs +# (scripts/publish-ns60-attachments.sh). +apiVersion: cloud.datumapis.com/v1alpha1 +kind: VPCAttachment +metadata: + name: backend-a + namespace: ns60 + labels: + app: backend + instance: a +spec: + vpc: + name: "60" + interface: + name: eth0 + mode: Netns + addresses: + - fd20:60:ff03:a::100/96 +--- +apiVersion: cloud.datumapis.com/v1alpha1 +kind: VPCAttachment +metadata: + name: backend-b + namespace: ns60 + labels: + app: backend + instance: b +spec: + vpc: + name: "60" + interface: + name: eth0 + mode: Netns + addresses: + - fd20:60:ff03:b::100/96 diff --git a/deploy/containerlab/resources/tenants/ns60/sjc/kustomization.yaml b/deploy/containerlab/resources/tenants/ns60/sjc/kustomization.yaml index ee81d25b..4973a065 100644 --- a/deploy/containerlab/resources/tenants/ns60/sjc/kustomization.yaml +++ b/deploy/containerlab/resources/tenants/ns60/sjc/kustomization.yaml @@ -1,3 +1,4 @@ resources: - ../base - nad.yaml + - vpcattachments.yaml diff --git a/deploy/containerlab/resources/tenants/ns60/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns60/sjc/nad.yaml index 9cf0fb1d..1dcf90b7 100644 --- a/deploy/containerlab/resources/tenants/ns60/sjc/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns60/sjc/nad.yaml @@ -1,18 +1,18 @@ apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition metadata: - name: backend + name: backend-a namespace: ns60 spec: config: |- { "cniVersion": "1.0.0", - "name": "backend", + "name": "backend-a", "plugins": [ { "type": "galactic-veth", "vpc": "60", - "vpcattachment": "60", + "vpcattachment": "60a", "namespace": "galactic-system", "ipam": { "type": "galactic-ipam", @@ -24,7 +24,39 @@ spec: { "type": "galactic-bgp", "vpc": "60", - "vpcattachment": "60", + "vpcattachment": "60a", + "namespace": "galactic-system" + } + ] + } +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: backend-b + namespace: ns60 +spec: + config: |- + { + "cniVersion": "1.0.0", + "name": "backend-b", + "plugins": [ + { + "type": "galactic-veth", + "vpc": "60", + "vpcattachment": "60b", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "addresses": [ + { "address": "fd20:60:ff02:b::100/96", "gateway": "fd20:60:ff02:b::1" } + ] + } + }, + { + "type": "galactic-bgp", + "vpc": "60", + "vpcattachment": "60b", "namespace": "galactic-system" } ] diff --git a/deploy/containerlab/resources/tenants/ns60/sjc/vpcattachments.yaml b/deploy/containerlab/resources/tenants/ns60/sjc/vpcattachments.yaml new file mode 100644 index 00000000..835edea5 --- /dev/null +++ b/deploy/containerlab/resources/tenants/ns60/sjc/vpcattachments.yaml @@ -0,0 +1,39 @@ +# The VPCAttachments for ns60's two backends in sjc, which the site's +# NetworkRules select by label (resources/galactic-gateway/sjc/). +# In production the attachment controller writes these and their status. The +# lab has no such controller, so deploy:ns60 applies them here and then +# writes each one's status from where its pod actually runs +# (scripts/publish-ns60-attachments.sh). +apiVersion: cloud.datumapis.com/v1alpha1 +kind: VPCAttachment +metadata: + name: backend-a + namespace: ns60 + labels: + app: backend + instance: a +spec: + vpc: + name: "60" + interface: + name: eth0 + mode: Netns + addresses: + - fd20:60:ff02:a::100/96 +--- +apiVersion: cloud.datumapis.com/v1alpha1 +kind: VPCAttachment +metadata: + name: backend-b + namespace: ns60 + labels: + app: backend + instance: b +spec: + vpc: + name: "60" + interface: + name: eth0 + mode: Netns + addresses: + - fd20:60:ff02:b::100/96 diff --git a/deploy/containerlab/scripts/deploy-galactic-gateway.sh b/deploy/containerlab/scripts/deploy-galactic-gateway.sh index 097aaaf0..971b19be 100755 --- a/deploy/containerlab/scripts/deploy-galactic-gateway.sh +++ b/deploy/containerlab/scripts/deploy-galactic-gateway.sh @@ -1,11 +1,12 @@ #!/bin/bash # deploy-galactic-gateway.sh — Install galactic-gateway on every edge node, one -# NetworkGateway per node, and each site's ns60 NetworkRules and -# ServiceVIPBindings (resources/galactic-gateway/). Requires deploy:system +# NetworkGateway per node, and each site's ns60 NetworkRules +# (resources/galactic-gateway/). Requires deploy:system # (the gateway's RBAC/ServiceAccount), deploy:images (galactic-gateway:latest # on every edge node), deploy:galactic-router (each node's BGPRouter, which -# the gateway advertises its VIPs through), and deploy:ns60 (the backends the -# ServiceVIPBindings bind). +# the gateway advertises its VIPs through), and deploy:ns60 (the backends and +# VPCAttachments the rules select). Each backend's node writes its own +# ServiceVIPBindings from the rules. # # The gateway shares each edge node's uplinks with the egress shard through # the node's XDP dispatcher, so the two can be deployed in either order. @@ -37,6 +38,25 @@ copy_gateway_config() { # from such a run, so the transit forwards by BGP again. "${SCRIPT_DIR}/verify-gateway-ingress.sh" --unpin +# NS60_BINDINGS is how many ServiceVIPBindings each site's backend node +# writes: one per rule (TCP and UDP) per ns60 backend. +NS60_BINDINGS=4 + +# wait_bindings NODE COUNT TIMEOUT waits up to TIMEOUT seconds for COUNT +# ServiceVIPBindings to exist in galactic-system. +wait_bindings() { + local node="$1" want="$2" deadline=$((SECONDS + $3)) have + while :; do + have=$(docker exec "${node}" kubectl -n galactic-system get servicevipbindings -o name | wc -l) + [ "${have}" -ge "${want}" ] && return 0 + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "error: ${node}: ${have} of ${want} ServiceVIPBindings after $3s" >&2 + return 1 + fi + sleep 2 + done +} + for site in dfw sjc iad; do node=$(control_plane "${site}") echo "Applying galactic-gateway/${site} to ${node}..." @@ -48,11 +68,13 @@ for site in dfw sjc iad; do done # Ready means the engine loaded every rule and advertised it, Accepted that # the rule found a gateway, and Bound that the backend's node rewrites the - # VIP for it. + # VIP for it. The bindings appear only once each backend's node has seen + # its accepted rule, so wait for all of them to exist first. docker exec "${node}" kubectl -n galactic-system wait networkgateway --all \ --for=condition=Ready --timeout=120s docker exec "${node}" kubectl -n galactic-system wait networkrule --all \ --for=condition=Accepted --timeout=60s + wait_bindings "${node}" "${NS60_BINDINGS}" 60 docker exec "${node}" kubectl -n galactic-system wait servicevipbinding --all \ --for=condition=Bound --timeout=60s done diff --git a/deploy/containerlab/scripts/deploy-system.sh b/deploy/containerlab/scripts/deploy-system.sh index 282fe3e2..5e412046 100755 --- a/deploy/containerlab/scripts/deploy-system.sh +++ b/deploy/containerlab/scripts/deploy-system.sh @@ -29,11 +29,10 @@ echo "Using datum-cloud/network CRDs at ${NETWORK_SHA} (${NETWORK_REF})" NETWORK_CRD_URL="https://raw.githubusercontent.com/datum-cloud/network/${NETWORK_SHA}/config/crd" # VPC/VPCAttachment CRDs come from the separate companion VPC operator, -# datum-cloud/cloud. Nothing in this repo's Go code imports it (the CNI -# plugin only reads VPC/VPCAttachment identifiers as plain JSON fields off -# the NAD), so there's no go.mod pseudo-version to derive a SHA from — pin -# one explicitly here and bump it by hand when the VPC CRD schema changes. -CLOUD_SHA="71a4f0f9c12166a758da4e2b90c80a17709804f2" +# datum-cloud/cloud. galactic-router and galactic-gateway read VPCAttachments +# through go.datum.net/cloud, so pin the same commit go.mod's pseudo-version +# names, and bump both together. +CLOUD_SHA="260f797bbc5c3f7b2ae323f041c185b544f79dbd" CLOUD_CRD_URL="https://raw.githubusercontent.com/datum-cloud/cloud/${CLOUD_SHA}/config/crd" # Install whatever network's own config/crd/kustomization.yaml lists at diff --git a/deploy/containerlab/scripts/publish-ns60-attachments.sh b/deploy/containerlab/scripts/publish-ns60-attachments.sh new file mode 100755 index 00000000..6a67af04 --- /dev/null +++ b/deploy/containerlab/scripts/publish-ns60-attachments.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# publish-ns60-attachments.sh — write the status of ns60's VPCAttachments, +# the job the attachment controller does in production. +# +# Each site's NetworkRules select ns60's backends through these attachments +# (resources/tenants/ns60//vpcattachments.yaml). An attachment is a +# backend only once its status names its VPC and its node, so this waits for +# each attachment's pod (same "instance" label) to be scheduled and copies its +# node into the attachment's status. +# +# Usage: publish-ns60-attachments.sh [site...] +set -euo pipefail + +if [ "$#" -lt 1 ]; then + echo "usage: $(basename "$0") [site...]" >&2 + exit 1 +fi + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +source "${SCRIPT_DIR}/lib.sh" + +NS=ns60 +VPC=60 + +for site in "$@"; do + cp=$(control_plane "${site}") + for attachment in $(docker exec "${cp}" kubectl get vpcattachments -n "${NS}" -l app=backend \ + -o jsonpath='{.items[*].metadata.name}'); do + instance=$(docker exec "${cp}" kubectl get vpcattachment -n "${NS}" "${attachment}" \ + -o jsonpath='{.metadata.labels.instance}') + docker exec "${cp}" kubectl wait -n "${NS}" --for=condition=PodScheduled pod \ + -l "app=backend,instance=${instance}" --timeout=120s >/dev/null + pod=$(docker exec "${cp}" kubectl get pods -n "${NS}" -l "app=backend,instance=${instance}" \ + -o jsonpath='{.items[0].metadata.name}') + node=$(docker exec "${cp}" kubectl get pod -n "${NS}" "${pod}" -o jsonpath='{.spec.nodeName}') + docker exec "${cp}" kubectl patch vpcattachment -n "${NS}" "${attachment}" --subresource=status --type=merge \ + -p "{\"status\":{\"vpc\":\"${VPC}\",\"vpcAttachment\":\"${VPC}${instance}\",\"node\":\"${node}\",\"podName\":\"${pod}\"}}" >/dev/null + echo "${site}: ${NS}/${attachment} -> ${node} (${pod})" + done +done diff --git a/deploy/containerlab/scripts/verify-gateway-ingress.sh b/deploy/containerlab/scripts/verify-gateway-ingress.sh index 93c78414..ac7129dd 100755 --- a/deploy/containerlab/scripts/verify-gateway-ingress.sh +++ b/deploy/containerlab/scripts/verify-gateway-ingress.sh @@ -15,6 +15,11 @@ # DSR's defining property: the gateway encapsulates the client's packet # and changes nothing in it. # +# Across every flow to a site's VIP, each of the site's ns60 backends answers +# at least one. Both run on the site's one compute node, so this is what +# proves the gateway's per-backend slot reaches that node and selects the +# backend Maglev chose (#799); without it every flow lands on one backend. +# # Across the flows through each node: # # - the node's own galactic_edge_rule_packets_total for the VIP rises. A @@ -158,6 +163,7 @@ for site in "${SITES[@]}"; do fail "${site}: no ns60 backend pod (run deploy:ns60)" continue fi + declare -A served=() for gw in ${GATEWAYS[${site}]}; do [ -z "${only}" ] || [ "${only}" = "${gw}" ] || continue @@ -189,6 +195,7 @@ for site in "${SITES[@]}"; do fail "${proto} from port ${sport}: ${pod} saw the client as '${peer}', want ${CLIENT}" else answered=$((answered + 1)) + served[${pod}]=1 fi done echo " ${proto}: ${answered}/${FLOWS} flows answered by ${site}'s backends, client address intact" @@ -224,6 +231,15 @@ for site in "${SITES[@]}"; do transit "${tr}" ip -6 route del "${vip}/128" done + + for pod in ${backends}; do + if [ -n "${served[${pod}]:-}" ]; then + echo " ok ${pod} answered flows to ${vip}" + else + fail "${pod} answered none of the flows to ${vip}: the VIP is not spread across ${site}'s backends" + fi + done + unset served done if [ "${checked}" -eq 0 ]; then diff --git a/docs/agents/ARCHITECTURE-GATEWAY.md b/docs/agents/ARCHITECTURE-GATEWAY.md index 915b37f2..29f1e995 100644 --- a/docs/agents/ARCHITECTURE-GATEWAY.md +++ b/docs/agents/ARCHITECTURE-GATEWAY.md @@ -222,9 +222,17 @@ that converges this node's whole gateway engine: 1. **Assemble desired state.** Lists every accepted, non-deleting `NetworkRule` in the namespace — under the DSR anycast model every gateway node in a PoP serves every accepted rule identically, so there - is no primary/secondary subset to filter on — resolves each backend's - SRv6 uSID via `usidresolver.go`'s `buildBackendSIDIndex`, and converges - `gateway.Engine` toward the result. A deleting rule stays in that state, + is no primary/secondary subset to filter on — expands each rule's + `backendSelector` into backends, the IPv6 interface addresses of the + `VPCAttachment`s it selects in the rule's VPC (`rulebackends.go`'s + `selectRuleBackends`), resolves each to the SRv6 uSID of the node its + attachment reports via `usidresolver.go`'s `buildBackendSIDIndex`, writes + the backend's slot (`srv6.BackendSlot`, a hash of its address and port) + into bits 81–96 of that uSID so a node hosting several of the rule's + backends can tell which one Maglev chose (#799), and converges + `gateway.Engine` toward the result. The reconciler watches + `VPCAttachment`s cluster-wide, so a backend appearing, moving node or + changing address reconverges every gateway without a rule edit. A deleting rule stays in that state, as *draining*, while it still carries the teardown finalizer and any `BGPAdvertisement` labelled with its name exists: it is not advertised and its `/Programmed` condition is left alone, but the datapath @@ -450,7 +458,8 @@ in `vip_return_stats_table` and surface as `galactic_edge_return_*`. binary's own resource set: `networkgateways`, `networkrules`, `bgpadvertisements`, and — even though this binary has no BGP client of its own — `bgprouters`, since `usidresolver.go` reads `BGPRouter` CRDs directly -to resolve backend uSIDs). +to resolve backend uSIDs). The scheme also registers `cloud.datumapis.com`, +since a rule's backends are `VPCAttachment`s. `root.go`'s `runCmd`: @@ -614,7 +623,9 @@ separate pod spec entirely, not part of this manifest. `NetworkGatewayReconciler`/`NetworkRuleReconciler` touch: `networkgateways`/`networkrules` (+ `/status`) read-write, `bgpadvertisements` full CRUD, and read-only `get`/`list`/`watch` on both -`bgprouters` (for `usidresolver.go` — see above) and `bgpvrfinstances` +`bgprouters` (for `usidresolver.go` — see above), `vpcattachments` in +`cloud.datumapis.com` (a rule's backends, selected and watched +cluster-wide) and `bgpvrfinstances` (`NetworkGatewayReconciler.SetupWithManager` also watches `BGPVRFInstance` to re-trigger reconciliation once a backend's owning VRF/advertisement data actually exists; omitting this rule left the manager's informer cache never @@ -639,25 +650,25 @@ pod on that node is not a supported configuration. ## Module / Package Reference -| Package | Binary | Responsibility | Owns state | -| ------------------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | -| `internal/config` (`gateway.go`) | galactic-gateway | `GatewayConfig`: node name, ports, public interface, SRv6 encap-source address; three-tier CLI/env/default precedence via viper | No | -| `internal/controller` (`networkgateway_controller.go`) | galactic-gateway | `NetworkGatewayReconciler`: desired-state assembly (with the deleted-rule drain in `networkgateway_drain.go`), BGP wiring, orphan-crash recovery | No | -| `internal/controller` (`networkrule_controller.go`) | galactic-gateway | `NetworkRuleReconciler`: finalizer-guarded teardown ordering, `Accepted`-condition maintenance (`updateAcceptedCondition`) | No | -| `internal/controller` (`usidresolver.go`) | galactic-gateway | `backendSIDIndex`: resolves a `NetworkRule` backend address to the worker node's SRv6 uSID by matching against `BGPAdvertisement`/`BGPRouter`/`BGPVRFInstance` CRDs, verifying tenant ownership | No | -| `internal/gateway` (`engine.go`) | galactic-gateway | `Engine`: mutex-guarded convergence loop ("apply everything in desired, remove everything not in desired"), mirroring `GoBGPRuntime`'s shape | Yes (active-rule map) | -| `internal/gateway` (`types.go`) | galactic-gateway | `DesiredRule`/`DesiredBackend`/`EngineState`/`EngineStatus`/`RuleStatus` — the engine's own representation, assembled by the controllers above; `DesiredBackend` implements `internal/maglev.Backend` | No | -| `internal/gateway` (`datapath.go`, `kerneldatapath.go`) | galactic-gateway | `Datapath`/`QuotaEnforcer`/`TelemetryEmitter` interfaces; `KernelDatapath`, the real `Datapath` backed by `edgemap.VIPTable` over a loaded `edgeprog.EdgedsrObjects`, building a `internal/maglev.Table` per rule; `NoopDatapath` for tests | Yes (`vipKeysByName` bookkeeping) | -| `internal/gateway` (`quota.go`) | galactic-gateway | `NodeQuotaEnforcer` — real, coarse node-level admission caps (max rules/tenant, max total `vip_table` entries); `NoopQuotaEnforcer` for tests | Yes (in-memory reservation counters) | -| `internal/gateway` (`telemetry.go`) | galactic-gateway | `PrometheusTelemetryEmitter` — control-plane-drop counter only; `NoopTelemetryEmitter` for tests | Yes (Prometheus metric state) | -| `internal/gateway` (`recovery.go`, `diff.go`) | galactic-gateway | `Engine.ReconcileOrphans` (crash recovery) and `diffRuleKeys` (the pure key-set diff both `Reconcile`/`ReconcileOrphans` build on) | No | -| `internal/maglev` (`table.go`) | galactic-gateway | Pure-Go Maglev consistent-hash lookup table (`New`/`Lookup`/`Backends`) — one `*Table` built per ring; this binary's only importer today (`galactic-nat` has no analogous shard-placement ring — see Known Constraints) | No | -| `internal/plumbing/ebpf/edgeprog` | galactic-gateway | Compiled XDP program (`edgedsr.c`, program `edge_lb`) + bpf2go-generated Go bindings (`EdgedsrObjects`) | No | -| `internal/plumbing/ebpf/edgemap` | galactic-gateway | `VIPTable`: `vip_table`/`vip_stats_table`/`vip_addr_table` read/write API, `Generation`/`Reconcile` crash-safety mechanism | Yes (via `KernelTable`) | -| `internal/plumbing/ebpf/edgeattach` | galactic-gateway | Load + native-XDP-only attach of the compiled program to one interface | Yes (pinned maps, held link) | -| `internal/plumbing/ebpf/edgemetrics` | galactic-gateway | Pull-based `prometheus.Collector` reading `vip_table`/`vip_stats_table`/`drop_reasons` live at every scrape | No | -| `internal/plumbing/ebpf/edgepreflight` | galactic-gateway | Startup kernel-capability check (`BPF_PROG_TYPE_XDP`, `BPF_MAP_TYPE_HASH`, kernel BTF, `bpf_xdp_adjust_head`) — no partial pass, no degraded fallback | No | -| `internal/plumbing/ebpf/xdpattach` | gateway, nat | Native-XDP attach across a target list, shared with `natattach`: per-NIC support check before any attach, each bond slave waited back into its aggregate before the next | No | +| Package | Binary | Responsibility | Owns state | +| ------------------------------------------------------- | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | +| `internal/config` (`gateway.go`) | galactic-gateway | `GatewayConfig`: node name, ports, public interface, SRv6 encap-source address; three-tier CLI/env/default precedence via viper | No | +| `internal/controller` (`networkgateway_controller.go`) | galactic-gateway | `NetworkGatewayReconciler`: desired-state assembly (with the deleted-rule drain in `networkgateway_drain.go`), BGP wiring, orphan-crash recovery | No | +| `internal/controller` (`networkrule_controller.go`) | galactic-gateway | `NetworkRuleReconciler`: finalizer-guarded teardown ordering, `Accepted`-condition maintenance (`updateAcceptedCondition`) | No | +| `internal/controller` (`usidresolver.go`) | galactic-gateway | `backendSIDIndex`: resolves a `NetworkRule` backend address to the SRv6 uSID of the node its `VPCAttachment` reports by matching against `BGPAdvertisement`/`BGPRouter`/`BGPVRFInstance` CRDs, verifying tenant ownership; `backendSID` adds the backend's slot | No | +| `internal/gateway` (`engine.go`) | galactic-gateway | `Engine`: mutex-guarded convergence loop ("apply everything in desired, remove everything not in desired"), mirroring `GoBGPRuntime`'s shape | Yes (active-rule map) | +| `internal/gateway` (`types.go`) | galactic-gateway | `DesiredRule`/`DesiredBackend`/`EngineState`/`EngineStatus`/`RuleStatus` — the engine's own representation, assembled by the controllers above; `DesiredBackend` implements `internal/maglev.Backend` | No | +| `internal/gateway` (`datapath.go`, `kerneldatapath.go`) | galactic-gateway | `Datapath`/`QuotaEnforcer`/`TelemetryEmitter` interfaces; `KernelDatapath`, the real `Datapath` backed by `edgemap.VIPTable` over a loaded `edgeprog.EdgedsrObjects`, building a `internal/maglev.Table` per rule; `NoopDatapath` for tests | Yes (`vipKeysByName` bookkeeping) | +| `internal/gateway` (`quota.go`) | galactic-gateway | `NodeQuotaEnforcer` — real, coarse node-level admission caps (max rules/tenant, max total `vip_table` entries); `NoopQuotaEnforcer` for tests | Yes (in-memory reservation counters) | +| `internal/gateway` (`telemetry.go`) | galactic-gateway | `PrometheusTelemetryEmitter` — control-plane-drop counter only; `NoopTelemetryEmitter` for tests | Yes (Prometheus metric state) | +| `internal/gateway` (`recovery.go`, `diff.go`) | galactic-gateway | `Engine.ReconcileOrphans` (crash recovery) and `diffRuleKeys` (the pure key-set diff both `Reconcile`/`ReconcileOrphans` build on) | No | +| `internal/maglev` (`table.go`) | galactic-gateway | Pure-Go Maglev consistent-hash lookup table (`New`/`Lookup`/`Backends`) — one `*Table` built per ring; this binary's only importer today (`galactic-nat` has no analogous shard-placement ring — see Known Constraints) | No | +| `internal/plumbing/ebpf/edgeprog` | galactic-gateway | Compiled XDP program (`edgedsr.c`, program `edge_lb`) + bpf2go-generated Go bindings (`EdgedsrObjects`) | No | +| `internal/plumbing/ebpf/edgemap` | galactic-gateway | `VIPTable`: `vip_table`/`vip_stats_table`/`vip_addr_table` read/write API, `Generation`/`Reconcile` crash-safety mechanism | Yes (via `KernelTable`) | +| `internal/plumbing/ebpf/edgeattach` | galactic-gateway | Load + native-XDP-only attach of the compiled program to one interface | Yes (pinned maps, held link) | +| `internal/plumbing/ebpf/edgemetrics` | galactic-gateway | Pull-based `prometheus.Collector` reading `vip_table`/`vip_stats_table`/`drop_reasons` live at every scrape | No | +| `internal/plumbing/ebpf/edgepreflight` | galactic-gateway | Startup kernel-capability check (`BPF_PROG_TYPE_XDP`, `BPF_MAP_TYPE_HASH`, kernel BTF, `bpf_xdp_adjust_head`) — no partial pass, no degraded fallback | No | +| `internal/plumbing/ebpf/xdpattach` | gateway, nat | Native-XDP attach across a target list, shared with `natattach`: per-NIC support check before any attach, each bond slave waited back into its aggregate before the next | No | --- @@ -856,6 +867,8 @@ that tag into `config/galactic-gateway/base`. | Node-scoped aggregate reconcile (desired-state assembly, BGP wiring, crash recovery) | `internal/controller/networkgateway_controller.go:Reconcile` | | Per-object lifecycle (finalizer teardown ordering, `Accepted`-condition maintenance) | `internal/controller/networkrule_controller.go:Reconcile`, `updateAcceptedCondition`, `reconcileDelete` | | Backend address → SRv6 uSID resolution (with tenant-ownership verification) | `internal/controller/usidresolver.go:buildBackendSIDIndex`, `resolveUSID`, `verifyTenantOwnership` | +| Backend selection from `backendSelector` (shared with the binding writer) | `internal/controller/rulebackends.go:selectRuleBackends` | +| Backend-side `ServiceVIPBinding`s, generated per node | `internal/controller/networkrule_binding_controller.go` (runs in `galactic-router`) | | SRv6 encap source (configured, or derived from this node's `BGPRouter`) | `cmd/galactic-gateway/encapsource.go:resolveEncapSource`, `srv6.NodeLocatorAddress` | | Engine convergence loop | `internal/gateway/engine.go:Reconcile`, `applyRuleLocked`, `removeRuleLocked` | | Real datapath implementation (vip_table read/write, Maglev table construction) | `internal/gateway/kerneldatapath.go:ApplyRule`, `RemoveRule`, `buildMaglevTable` | diff --git a/docs/agents/ARCHITECTURE-ROUTER.md b/docs/agents/ARCHITECTURE-ROUTER.md index 929a2aad..5c56fdb3 100644 --- a/docs/agents/ARCHITECTURE-ROUTER.md +++ b/docs/agents/ARCHITECTURE-ROUTER.md @@ -119,19 +119,19 @@ See [docs/agent-startup.md](../agent-startup.md) for the router startup sequence ## Components -| Component | Binary | Role | -| ------------------------ | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `internal/controller` | `galactic-router` | controller-runtime reconcilers (BGPRouter, BGPPeer, BGPAdvertisement, BGPVRFInstance, BGPPolicy, Secret, Node, ServiceVIPBinding, GC — not NetworkGateway/NetworkRule, see the note above); field index registration; CRD status helpers | -| `internal/webhook` | `galactic-router` | `NetworkRuleValidator` — the `NetworkRule` admission webhook, gated behind `GALACTIC_ROUTER_WEBHOOK_ENABLED` (default off); registered on *this* binary's manager despite validating a gateway-domain CRD — see [Known Constraints](#known-constraints) | -| `internal/reconcile` | `galactic-router` | CRD → DesiredRouter translation | -| `internal/runtime/gobgp` | `galactic-router` | Embedded GoBGP server (the only backend) | -| `internal/model` | `galactic-router` | Internal BGP model types | -| `internal/hash` | `galactic-router` | Change detection | -| `internal/metadata` | every binary in the repo | Build-time version info stamped via `-ldflags` | -| `internal/gc` | `galactic-router` | Orphaned CRD/VRF/eBPF-entry cleanup, driven by the GC controller's ticker | -| `internal/plumbing/srv6` | `galactic-router` | SID computation (`ComputeSID`) for the BGP Prefix-SID path attribute; plain-route table, policy rules and route writes (`plainroute.go`) | -| `internal/plumbing/intf` | `galactic-router` + every CNI-chain binary | Interface naming, base62↔hex encoding | -| `internal/plumbing/vrf` | `galactic-router` + every CNI-chain binary | Linux VRF create/delete/lookup | +| Component | Binary | Role | +| ------------------------ | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `internal/controller` | `galactic-router` | controller-runtime reconcilers (BGPRouter, BGPPeer, BGPAdvertisement, BGPVRFInstance, BGPPolicy, Secret, Node, ServiceVIPBinding, NetworkRuleBinding, GC — not NetworkGateway/NetworkRule, see the note above); field index registration; CRD status helpers | +| `internal/webhook` | `galactic-router` | `NetworkRuleValidator` — the `NetworkRule` admission webhook, gated behind `GALACTIC_ROUTER_WEBHOOK_ENABLED` (default off); registered on *this* binary's manager despite validating a gateway-domain CRD — see [Known Constraints](#known-constraints) | +| `internal/reconcile` | `galactic-router` | CRD → DesiredRouter translation | +| `internal/runtime/gobgp` | `galactic-router` | Embedded GoBGP server (the only backend) | +| `internal/model` | `galactic-router` | Internal BGP model types | +| `internal/hash` | `galactic-router` | Change detection | +| `internal/metadata` | every binary in the repo | Build-time version info stamped via `-ldflags` | +| `internal/gc` | `galactic-router` | Orphaned CRD/VRF/eBPF-entry cleanup, driven by the GC controller's ticker | +| `internal/plumbing/srv6` | `galactic-router` | SID computation (`ComputeSID`) for the BGP Prefix-SID path attribute; plain-route table, policy rules and route writes (`plainroute.go`) | +| `internal/plumbing/intf` | `galactic-router` + every CNI-chain binary | Interface naming, base62↔hex encoding | +| `internal/plumbing/vrf` | `galactic-router` + every CNI-chain binary | Linux VRF create/delete/lookup | --- @@ -175,10 +175,11 @@ lives in `root.go`'s `runCmd`: BGPAdvertisement→router, BGPVRFInstance→router, BGPRouter→node. 9. Register the peer-state event emitter (`controller.NewPeerStateEventEmitter`) with the manager — see the peer-FSM design decision below. -10. Register nine controllers: BGPRouter, BGPPeer, BGPAdvertisement, - BGPVRFInstance, BGPPolicy, Secret, Node, ServiceVIPBinding, and GC (the - GC controller also starts a ticker goroutine that waits for cache sync, - then runs on `--gc-interval`, default 5m). +10. Register the controllers: BGPRouter, BGPPeer, BGPAdvertisement, + BGPVRFInstance, BGPPolicy, Secret, Node, ServiceVIPBinding, + NetworkRuleBinding, ServiceRoutePolicy, and GC (the GC controller also + starts a ticker goroutine that waits for cache sync, then runs on + `--gc-interval`, default 5m). 11. `mgr.Start(ctx)` — blocks until the signal-handler context is cancelled. This is identical whether the pod is running in the plain role @@ -231,7 +232,7 @@ co-located `galactic-gateway` container's own health port on the same | Package | Binary | Responsibility | Owns state | | ------------------------ | ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | -| `internal/controller` | galactic-router | controller-runtime reconcilers (BGPRouter, BGPPeer, BGPAdvertisement, BGPVRFInstance, BGPPolicy, Node, Secret, ServiceVIPBinding, GC); field index registration; CRD status helpers | No | +| `internal/controller` | galactic-router | controller-runtime reconcilers (BGPRouter, BGPPeer, BGPAdvertisement, BGPVRFInstance, BGPPolicy, Node, Secret, ServiceVIPBinding, NetworkRuleBinding, GC); field index registration; CRD status helpers | No | | `internal/webhook` | galactic-router | `NetworkRuleValidator`: the `NetworkRule` admission webhook (opt-in via `GALACTIC_ROUTER_WEBHOOK_ENABLED`); `AllowAllAuthorizer` is the only `Authorizer` implementation today | No | | `internal/reconcile` | galactic-router | Translates BGPRouter + related CRDs into `model.DesiredRouter`; enforces node targeting, timer validation, AFI validation | No | | `internal/runtime` | galactic-router | `RouterRuntime` interface; `RuntimeManager` (keyed map of live runtimes, double-checked lock create) | Yes (runtime map) | @@ -276,7 +277,8 @@ co-located `galactic-gateway` container's own health port on the same - **Peer FSM transitions are logged via a dedicated GoBGP event watcher, not GoBGP's own logger — and also drive real-time BGPPeer Kubernetes Events.** `GALACTIC_ROUTER`'s `LogLevel` config only gates GoBGP's *internal* `slog` logger, whose handler is wired to `io.Discard` regardless of level (`internal/runtime/gobgp/server.go`'s `buildServerOptions`) — so none of GoBGP's own peer state log lines ever reach this process's output. Instead, `internal/runtime/gobgp/peer_monitor.go` runs a shared `BgpServer.WatchEvent(..., WatchPeer())` goroutine (the same one-goroutine-per-runtime pattern `monitor.go`'s `watchEVPNRIB` uses for EVPN best-path) that diffs each peer's FSM state against `lastPeerState` and logs a `slog.Info` line on every real transition, including `disconnectReason`/`disconnectMessage` when a peer drops out of Established. The same transition is also handed to an optional `model.PeerStateObserver` (set via `gobgp.NewRuntimeFactory`'s `observer` parameter): in production this is `internal/controller/peer_events.go`'s `PeerStateEventEmitter`, a `manager.Runnable` that resolves the peer's address to its `BGPPeer` object and raises a `SessionEstablished`/`SessionDown` Kubernetes Event on it — but only for transitions into/out of Established, not every intermediate FSM hop (see the emitter's own doc comments for why). This is independent of, and does not replace, the periodic (`peerStatusRequeue`, 30s) `BGPPeer.Status` update in `internal/controller/bgprouter_controller.go`'s `updatePeerStatuses`: that keeps the CRD's status current on a poll and can miss a transition that reverts between two polls; the `slog` line and the Event are both real-time and don't have that gap. See `docs/plans/router-bgp-peer-session-events.md` for the full design. - **Routes for route-target-less EVPN paths defer to the underlay.** A path with no route target (the gateway's anycast VIPs, and each egress shard's SID `/64` and masquerade address) becomes a plain kernel route toward its BGP next hop, with no encapsulation. These routes live in their own table, `srv6.PlainRouteTable` (`0xFFFE0000`), marked `proto 250`, and are reached through two policy rules ahead of the main table's own: `32000: lookup main suppress_prefixlength 0`, then `32001: lookup 0xFFFE0000`. An underlay route for the same destination, or for anything more specific than the default route, therefore always wins. Keeping them out of the main table is required, not cosmetic: zebra imports every main-table route it did not install as a `kernel` route with distance 0, so a plain route there stopped FRR installing its own BGP route for the same prefix (#670). The kernel cannot resolve an IPv6 next hop recursively, so each route is written with the link and next hop the gateway resolves to at the time. `internal/runtime/gobgp/plainroutes.go` re-resolves every route whenever the main table changes and every 30s, and retries one that cannot resolve yet. It removes owned routes no path asks for once a two-minute startup grace has passed, and deletes the unmarked main-table route an earlier version left for each prefix. `ip route show table all proto 250` lists every route it owns. XDP datapaths that call `bpf_fib_lookup` with `BPF_FIB_LOOKUP_DIRECT` (`natprog`, `edgeprog`) read the main table only and do not see these routes. - **A tenant VRF route that fails to install is retried, not dropped.** `processEVPNPath` writes a VRF-scoped path's route into `egress_route_table` through `internal/runtime/gobgp/vrfroutes.go`. `Register` resolves the SID's uplink and neighbor MAC at write time, which fails while an uplink is still coming up after a restart, and BGP never redelivers an unchanged best path, so before #676 such a route stayed missing until the path changed upstream. `vrfRoutes` keeps each failed route in a pending set and retries it whenever the main table changes (1s debounce) and every 30s, until it installs or its path is withdrawn; removing a VRF drops its pending routes. One mutex serializes every write to the map from this runtime, taken per route during a retry so an unresolvable neighbor (up to 2s) does not hold up the RIB watcher for the whole pass. `EgressRouteTable.Refresh` cannot cover this: it walks existing entries only. The pending count, and the plain-route reconciler's failing count, are exported as `galactic_router_route_install_failing{kind="vrf"|"plain"}`, and `GalacticRouterRoutesNotInstalled` fires when either stays above 0 for 10 minutes. -- **Deleting a `ServiceVIPBinding` does not need its VRF on the node.** A binding's two `vip_xlat_table` rows are keyed on its VRF's uSID Argument, which the binding does not record. `ServiceVIPBindingReconciler` derives it from this node's `BGPVRFInstance`s and `BGPAdvertisement`s, and those are gone once the VPC leaves the node. Before #750, deletion re-derived it, failed, held the finalizer forever and left both rows in the map. Teardown now finds the rows by value with `vipxlatmap.UnregisterBinding`: since #764 each key carries the VIP or backend address, so the binding's ingress row (its VIP and port, rewriting to its backend) and egress row (its backend and port, rewriting to its VIP) are found exactly, under any Argument. Each row is removed only while it still holds the binding's values, so a row another binding has since written stays. When the VRF context still resolves, `UnregisterBindingAt` applies the same checks at that location too. Neither pass deletes a row without first reading it, and a process-wide lock keeps a bind from landing between the read and the delete. A value check cannot tell two bindings with identical values apart, so while such a twin is live on the node, deleting one removes nothing. Rows exist only for IPv6 addresses, so an IPv4 or invalid address skips both passes instead of blocking deletion. Only a failed kernel delete keeps the finalizer. Nothing sweeps `vip_xlat_table` periodically, so rows left behind by a finalizer cleared by hand stay until removed with `galactic-router vip xlat list` and `vip xlat remove --protocol --vip --port --backend --backend-port`, run in that node's `galactic-router` pod (`kubectl exec -- /galactic-router vip xlat list`). +- **`ServiceVIPBinding`s are generated, one per node per backend.** `NetworkRuleBindingReconciler` (`internal/controller/networkrule_binding_controller.go`) runs on every node and writes a binding for each IPv6 VIP of each accepted `NetworkRule` and each backend the rule's `backendSelector` places on this node, through the same `selectRuleBackends` the gateway uses, so the gateway never sends a backend's flows to a node with no binding planned for it. Each node writes and deletes only its own bindings (labelled `galactic.datum.net/binding-node` and `app.kubernetes.io/managed-by: galactic-router`), so there is one writer per object and no leader election. A binding is owned by its rule, records the rule's `vpcRef` (which names the VRF its rows go in) and takes `egressKind` from the attachment's interface mode. The node summarizes its bindings on the rule as `/BackendsBound`. Several backends of one VIP can share a node: the gateway writes each backend's slot into the uSID and `usid.c` keys the ingress row on it (#799). +- **Deleting a `ServiceVIPBinding` does not need its VRF on the node.** A binding's two `vip_xlat_table` rows are keyed on its VRF's uSID Argument. `ServiceVIPBindingReconciler` reads it from the `BGPVRFInstance` its `vpcRef` names on this node, and that is gone once the VPC leaves the node. Before #750, deletion re-derived it, failed, held the finalizer forever and left both rows in the map. Teardown now finds the rows by value with `vipxlatmap.UnregisterBinding`: since #764 each key carries the VIP or backend address, and since #799 the ingress key also carries the backend's slot, so the binding's ingress row (its VIP, port and slot, rewriting to its backend) and egress row (its backend and port, rewriting to its VIP) are found exactly, under any Argument. Each row is removed only while it still holds the binding's values, so a row another binding has since written stays. When the VRF context still resolves, `UnregisterBindingAt` applies the same checks at that location too. Neither pass deletes a row without first reading it, and a process-wide lock keeps a bind from landing between the read and the delete. A value check cannot tell two bindings with identical values apart, so while such a twin is live on the node, deleting one removes nothing. Rows exist only for IPv6 addresses, so an IPv4 or invalid address skips both passes instead of blocking deletion. Only a failed kernel delete keeps the finalizer. Nothing sweeps `vip_xlat_table` periodically, so rows left behind by a finalizer cleared by hand stay until removed with `galactic-router vip xlat list` and `vip xlat remove --protocol --vip --port --backend --backend-port` (`--slot` defaults to the backend's own), run in that node's `galactic-router` pod (`kubectl exec -- /galactic-router vip xlat list`). --- @@ -327,21 +329,22 @@ own publish/image details. **Where to start for each concern:** -| Concern | Start here | -| -------------------------------------------- | ---------------------------------------------------------------------------------------- | -| CRD → BGP translation | `internal/reconcile/reconcile.go:BuildDesiredRouter` | -| BGP runtime application (GoBGP) | `internal/runtime/gobgp/runtime.go:Apply` | -| BGP peer / VRF / advertisement / policy CRUD | `internal/runtime/gobgp/peers.go`, `runtime.go` (`applyVRFs`), `paths.go`, `policies.go` | -| Controller watch graph | `internal/controller/bgprouter_controller.go:SetupWithManager` | -| CRD status update logic | `internal/controller/status.go`, `bgprouter_controller.go:updateRouterStatus` | -| Orphaned CRD/VRF garbage collection | `internal/controller/gc_controller.go`, `internal/gc/gc.go` | -| Plain routes for RT-less EVPN paths | `internal/runtime/gobgp/plainroutes.go`, `internal/plumbing/srv6/plainroute.go` | -| RBAC pre-flight self-check | `cmd/galactic-router/main.go:checkWatchPermissions` | -| Hash-based no-op suppression | `internal/hash/hash.go`; annotation `galactic.datum.net/config-hash` on BGPRouter | -| GoBGP server lifecycle (start/reconfigure) | `internal/runtime/gobgp/server.go` | -| SRv6 SID computation for BGP Prefix-SID | `internal/plumbing/srv6/usid.go:ComputeSID` | -| Tap-kind ServiceVIPBinding / vip_xlat_table | `internal/controller/servicevipbinding_controller.go`, `internal/plumbing/ebpf/vipxlatmap` | -| `NetworkRule` admission webhook | `internal/webhook/networkrule_webhook.go:NetworkRuleValidator` | +| Concern | Start here | +| ---------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| CRD → BGP translation | `internal/reconcile/reconcile.go:BuildDesiredRouter` | +| BGP runtime application (GoBGP) | `internal/runtime/gobgp/runtime.go:Apply` | +| BGP peer / VRF / advertisement / policy CRUD | `internal/runtime/gobgp/peers.go`, `runtime.go` (`applyVRFs`), `paths.go`, `policies.go` | +| Controller watch graph | `internal/controller/bgprouter_controller.go:SetupWithManager` | +| CRD status update logic | `internal/controller/status.go`, `bgprouter_controller.go:updateRouterStatus` | +| Orphaned CRD/VRF garbage collection | `internal/controller/gc_controller.go`, `internal/gc/gc.go` | +| Plain routes for RT-less EVPN paths | `internal/runtime/gobgp/plainroutes.go`, `internal/plumbing/srv6/plainroute.go` | +| RBAC pre-flight self-check | `cmd/galactic-router/main.go:checkWatchPermissions` | +| Hash-based no-op suppression | `internal/hash/hash.go`; annotation `galactic.datum.net/config-hash` on BGPRouter | +| GoBGP server lifecycle (start/reconfigure) | `internal/runtime/gobgp/server.go` | +| SRv6 SID computation for BGP Prefix-SID | `internal/plumbing/srv6/usid.go:ComputeSID` | +| Tap-kind ServiceVIPBinding / vip_xlat_table | `internal/controller/servicevipbinding_controller.go`, `internal/plumbing/ebpf/vipxlatmap` | +| Generated ServiceVIPBindings from NetworkRules | `internal/controller/networkrule_binding_controller.go`, `internal/controller/rulebackends.go` | +| `NetworkRule` admission webhook | `internal/webhook/networkrule_webhook.go:NetworkRuleValidator` | **Stable vs. frequently changed:** - Stable: `internal/plumbing/` (pure kernel primitives), `internal/model/types.go`, `internal/runtime/runtime.go` (interface) @@ -357,5 +360,5 @@ own publish/image details. - GoBGP `Reconfigure()` calls `old.Stop()` then creates a fresh `BgpServer` — it does NOT call the BGP-level `StopBgp`/`StartBgp` on the old server, avoiding the v4 "Serve loop permanently dead" problem. - No binary's `cmdDel` deletes the VRF, veth/tap, routes, the eBPF `vrf_table` entry, or `BGPAdvertisement`/`BGPVRFInstance` CRDs — each CNI-chain binary's own DEL only handles its own per-container bookkeeping. Shared-resource cleanup is entirely this GC controller's job (`internal/gc`), to avoid racing a concurrent ADD during pod restarts. - Production images are published by `.github/workflows/publish.yaml` as separate per-binary images (`galactic-cni`, `galactic-router`, `galactic-gateway`), not one shared image — see CI/CD above. -- `internal/controller` also hosts `NetworkGatewayReconciler`/`NetworkRuleReconciler`/`usidresolver.go` — these register with `galactic-gateway`'s manager, not `galactic-router`'s (compare `cmd/galactic-router/root.go`'s controller registration list in Entry Points above against `cmd/galactic-gateway/root.go`'s — see [ARCHITECTURE-GATEWAY.md](ARCHITECTURE-GATEWAY.md)). Don't assume every reconciler type in this package runs in this binary. +- `internal/controller` also hosts `NetworkGatewayReconciler`/`NetworkRuleReconciler` — these register with `galactic-gateway`'s manager, not `galactic-router`'s (`usidresolver.go` and `rulebackends.go` are shared: `NetworkRuleBindingReconciler` and `ServiceVIPBindingReconciler` use them here too) (compare `cmd/galactic-router/root.go`'s controller registration list in Entry Points above against `cmd/galactic-gateway/root.go`'s — see [ARCHITECTURE-GATEWAY.md](ARCHITECTURE-GATEWAY.md)). Don't assume every reconciler type in this package runs in this binary. - The reverse also holds: `internal/webhook.NetworkRuleValidator` — the admission webhook for `galactic-gateway`'s own `NetworkRule` CRD — registers with *this* binary's manager (`cmd/galactic-router/root.go`), not `galactic-gateway`'s. Don't assume every `NetworkRule`-related type runs wherever `NetworkRuleReconciler` does. diff --git a/docs/gateway/configuration.md b/docs/gateway/configuration.md index da346e96..3ad35e88 100644 --- a/docs/gateway/configuration.md +++ b/docs/gateway/configuration.md @@ -273,15 +273,16 @@ deploy/containerlab/resources/galactic-gateway/ ├── iad-worker2/ # " ├── dfw/ │ ├── kustomization.yaml # composes this site's edge nodes + its rules -│ ├── networkrules.yaml # ns60-tcp and ns60-udp, port 80 -│ └── servicevipbindings.yaml # one ServiceVIPBinding per rule for the backend +│ └── networkrules.yaml # ns60-tcp and ns60-udp, port 80, selecting ns60's backends ├── sjc/ # same shape, one edge node └── iad/ # " ``` Each site has its own VIP, in a `/64` only that site's edge nodes originate -(`2001:db8:6060:1::/64` for dfw, `:2::` for sjc, `:3::` for iad), and one -`ns60` backend of its own, so the per-site directories differ in both. +(`2001:db8:6060:1::/64` for dfw, `:2::` for sjc, `:3::` for iad), and two +`ns60` backends of its own on its compute node, so the per-site directories +differ in both. The backends' `ServiceVIPBinding`s are generated by the +compute node's `galactic-router`, so no site directory carries any. ### `kustomization.yaml` and `node-patch.yaml` — pin to one node, set the per-node values @@ -446,15 +447,16 @@ gateway node, named after that node (see the worked example above). ### `NetworkRule` — tenant-writable ingress load-balancing spec -| Field | Required | Type | Description | -| ----------------------- | -------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| `spec.vpcRef` | Yes | `string` | Opaque VPC identifier (owned by the companion operator, not validated here beyond non-emptiness). | -| `spec.vpcAttachmentRef` | Yes | `string` | Opaque VPC attachment identifier, paired with `vpcRef`. | -| `spec.vipAddresses` | Yes | `[]string` (1–8) | Ingress VIP addresses (IPv4 and/or IPv6) this rule provisions. | -| `spec.protocol` | Yes | `tcp` \| `udp` | Transport protocol matched by `vipAddresses`/`port`. | -| `spec.port` | Yes | `int32` (1–65535) | Ingress port on `vipAddresses` this rule load-balances. | -| `spec.backends` | Yes | `[]{address, port}` (1–64) | Backend `address:port` targets traffic is load-balanced to. | -| `status.conditions` | — | — | `Accepted` condition — currently set `True` unconditionally once gateway nodes exist for the namespace (see the admission-webhook caveat below). | +| Field | Required | Type | Description | +| ----------------------- | -------- | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `spec.vpcRef` | Yes | `string` | Opaque VPC identifier (owned by the companion operator, not validated here beyond non-emptiness). Only attachments whose `status.vpc` equals it can be backends. | +| `spec.vpcAttachmentRef` | Yes | `string` | Opaque VPC attachment identifier, paired with `vpcRef`. | +| `spec.vipAddresses` | Yes | `[]string` (1–8) | Ingress VIP addresses this rule provisions. The backend nodes translate IPv6 VIPs only (#705). | +| `spec.protocol` | Yes | `tcp` \| `udp` | Transport protocol matched by `vipAddresses`/`port`. | +| `spec.port` | Yes | `int32` (1–65535) | Ingress port on `vipAddresses` this rule load-balances. | +| `spec.backendSelector` | Yes | `metav1.LabelSelector` | Selects the `VPCAttachment`s (`cloud.datumapis.com`, any namespace) that serve the rule. Each contributes its IPv6 interface addresses as backends. Must not be empty. | +| `spec.backendPort` | Yes | `int32` (1–65535) | Destination port on every selected backend. | +| `status.conditions` | — | — | `Accepted` (set `True` once gateway nodes exist; see the admission-webhook caveat below), `/Programmed` per gateway node, and `/BackendsBound` per node hosting a backend. | Example, the TCP rule from `deploy/containerlab/resources/galactic-gateway/iad/networkrules.yaml`: @@ -471,21 +473,42 @@ spec: - 2001:db8:6060:3::1 protocol: tcp port: 80 - backends: - - address: fd20:60:ff03:a::100 - port: 80 + backendSelector: + matchLabels: + app: backend + backendPort: 80 ``` +A rule may carry at most one IPv6 VIP: a backend node rewrites a reply's +source back to a single VIP, so a backend cannot answer for a second one. A +rule with two fails to load on the gateways and reports `InvalidRule` on its +backend nodes; split it into one rule per IPv6 VIP. IPv4 VIPs are not +translated on the backend side (#705). + +A selected attachment is a backend once its status names a node and its +interface has an IPv6 address. Two attachments claiming one address, or two +backends on one node whose slots (below) collide, keep only the first in +address order; the other is reported as unresolved rather than given flows +it could not answer. Until then it appears in the rule's +`/Programmed` message with reason `BackendsUnresolved`, as does +a backend whose node advertises no route for it yet. A rule with no backend +at all is not loaded. + Every accepted, non-deleting `NetworkRule` in the namespace is served by **every** `NetworkGateway` node identically — there is no `status.primaryNode`/placement field to set, and no per-node subset to -target. `galactic-gateway` resolves each backend address's SRv6 uSID by -matching it against `BGPAdvertisement`/`BGPRouter`/`BGPVRFInstance` CRDs -(read-only, not watched) and verifies the requesting rule's own `vpcRef` -owns that match before trusting it. Placement across gateway nodes is -Maglev's consistent-hash ring (built identically on every node from the -same `(VIP, backend list)` input) plus anycast BGP at equal preference — -not a control-plane assignment step you configure here. See +target. `galactic-gateway` resolves each backend to the SRv6 uSID of the +node its attachment reports, through that node's +`BGPAdvertisement`/`BGPRouter`/`BGPVRFInstance` CRDs, and verifies the rule's +own `vpcRef` owns the match before trusting it. It then writes the backend's +slot, a hash of its address and port, into bits 81–96 of that uSID, so a node +hosting several of the rule's backends can tell which one Maglev chose. +Placement across gateway nodes is Maglev's consistent-hash ring (built +identically on every node from the same `(VIP, backend list)` input) plus +anycast BGP at equal preference — not a control-plane assignment step you +configure here. The gateways watch `VPCAttachment`s, so a backend that +appears, moves to another node or changes address reconverges the rule +without an edit. See [ARCHITECTURE-GATEWAY.md's Data Flow section](../agents/ARCHITECTURE-GATEWAY.md#data-flow) for the full mechanics. @@ -500,28 +523,37 @@ drain delay, so traffic already on its way to the VIP is not dropped. > `galactic-system` can currently provision ingress for any `vpcRef`. See > ARCHITECTURE-GATEWAY.md's Known Constraints for detail. -### `ServiceVIPBinding` — the backend-side half, not reconciled by `galactic-gateway` +### `ServiceVIPBinding` — the backend-side half, generated by `galactic-router` -A `NetworkRule` alone configures the gateway-side half of DSR. For a -backend to actually **answer the client directly** on the VIP (DSR's whole -premise), its own worker node also needs a `ServiceVIPBinding` object -naming that `(node, VIP, backend)` triple — this is reconciled by -`ServiceVIPBindingReconciler` running inside **`galactic-router`** (not -`galactic-gateway`), so it's out of this document's direct scope, but -worth knowing about since it's required for the datapath to work -end-to-end. Example, the TCP binding from -`deploy/containerlab/resources/galactic-gateway/iad/servicevipbindings.yaml`: +A `NetworkRule` configures the gateway side of DSR. For a backend to +**answer the client directly** on the VIP, its own node also needs a +`ServiceVIPBinding` for each `(VIP, backend)` pair. Nobody writes these by +hand: `galactic-router` on every node runs `NetworkRuleBindingReconciler` +(`internal/controller/networkrule_binding_controller.go`), which writes one +for each IPv6 VIP of each accepted rule and each selected backend whose +attachment reports that node, and deletes the ones that no longer apply. +`ServiceVIPBindingReconciler`, in the same process, then programs the +node's `vip_xlat_table` from it. Example of a generated binding: ```yaml apiVersion: network.datumapis.com/v1alpha1 kind: ServiceVIPBinding metadata: - name: ns60-tcp-backend + name: ns60-tcp-iad-worker-3f0c9a1d6b2e4f57 namespace: galactic-system + labels: + app.kubernetes.io/managed-by: galactic-router + galactic.datum.net/network-rule: ns60-tcp + galactic.datum.net/binding-node: iad-worker + ownerReferences: + - apiVersion: network.datumapis.com/v1alpha1 + kind: NetworkRule + name: ns60-tcp spec: targetRef: kind: Node name: iad-worker + vpcRef: "60" vipAddress: 2001:db8:6060:3::1 port: 80 protocol: tcp @@ -530,21 +562,36 @@ spec: egressKind: veth ``` -`targetRef.name` is the **backend's own worker node**, not a gateway node. -`egressKind` is `veth` for a plain container backend or `tap` for a VM -backend, mirroring the same fork the SRv6 uSID datapath's `vrf_table` -already has. As of this writing there is no controller in this repo that -derives `ServiceVIPBinding` objects automatically from a `NetworkRule`'s -backend list — the lab example above is hand-authored, and a real -deployment must currently do the same for each backend. See -`internal/controller/servicevipbinding_controller.go`'s package doc -comment for the full mechanics of what happens once this object exists. - -Bindings on one node may share a port. Two bindings conflict only when they -claim the same VIP, port and protocol, or the same backend address, port and -protocol, in the same VPC. The older binding keeps serving. The newer one is -not programmed and reports `Bound=False` with reason `Conflict`, naming the -binding that holds the row. It takes over once that binding is deleted. +- `targetRef.name` is the **backend's own node**, from its attachment's + `status.node`, not a gateway node. +- `vpcRef` is the rule's. It names the VRF the binding's rows go into + (`BGPVRFInstance` `-`), so two tenants using the same backend + address on one node never share a row. +- `egressKind` comes from the attachment's interface mode: `veth` for + `Netns` (a container), `tap` for `Hypervisor` or `HypervisorDeclared` (a + VM). +- The binding is owned by the rule, so deleting the rule removes it, after + the gateways have drained the VIP. A rule that loses `Accepted`, as it + does whenever its namespace briefly has no `NetworkGateway`, keeps its + bindings, so the backend nodes' rows survive until the gateways return. + A backend that moves to another node or changes address has its binding + deleted on the old node and a new one written where it now runs. + +The ingress row is keyed on the VIP, port and the backend's slot, the same +hash of the backend's address and port the gateway writes into the uSID. So +several backends of one VIP can share a node, each answering the flows +Maglev sent it. Two bindings conflict only when they claim the same row: +the same VIP, port, protocol and slot, or the same backend address, port +and protocol, in the same VPC. The older binding keeps serving. The newer +one is not programmed and reports `Bound=False` with reason `Conflict`, +naming the binding that holds the row. + +Each node summarizes its bindings on the rule as +`/BackendsBound`: `True` with reason `Bound` once every one +reports `Bound`, `False` with reason `BindingsNotBound` naming each that +does not, or `InvalidRule` if the selector cannot be parsed. A rule can be +`Programmed` on every gateway and still fail to reach a backend; this +condition is where that shows. ## Verifying the deployment diff --git a/go.mod b/go.mod index d9c618d3..35caa284 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/spf13/viper v1.21.0 github.com/vishvananda/netlink v1.3.2-0.20261002201547-1d1c62507e3c go.datum.net/cloud v0.1.1-0.20260918213150-260f797bbc5c - go.datum.net/network v0.3.1-0.20261008185758-e391e93cf3de + go.datum.net/network v0.3.1-0.20261008204016-fc3154bde4d8 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 diff --git a/go.sum b/go.sum index 158f1bf8..81446c52 100644 --- a/go.sum +++ b/go.sum @@ -204,10 +204,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.datum.net/cloud v0.1.1-0.20260918213150-260f797bbc5c h1:k0P5980+kS90zCa0M4MohyS0woJL3bh/uEarRrW9PmY= go.datum.net/cloud v0.1.1-0.20260918213150-260f797bbc5c/go.mod h1:HTeaepXL0EqiqNsQ13K4/kGEwv+VLJ//BGrUfVIFfEA= -go.datum.net/network v0.3.1-0.20261006185716-0f099eb3b0ea h1:q/8ZPV2fjLwFTpTepNn/9HigADL9c9nQHWKKGWMRGHQ= -go.datum.net/network v0.3.1-0.20261006185716-0f099eb3b0ea/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= -go.datum.net/network v0.3.1-0.20261008185758-e391e93cf3de h1:vGtnfrEwHrnH2YipcqgobhfPRFaA6OEtDFlvKYC+z9E= -go.datum.net/network v0.3.1-0.20261008185758-e391e93cf3de/go.mod h1:LawI34k+ddtv3kRbmjiss0Lb6QKpSr60YP3fX1IG5RU= +go.datum.net/network v0.3.1-0.20261008204016-fc3154bde4d8 h1:SN5aP+EC7S1JRrW57BQ8d9VzQvE9duXZAyMDgGn1pRg= +go.datum.net/network v0.3.1-0.20261008204016-fc3154bde4d8/go.mod h1:LawI34k+ddtv3kRbmjiss0Lb6QKpSr60YP3fX1IG5RU= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0= diff --git a/internal/controller/networkgateway_controller.go b/internal/controller/networkgateway_controller.go index 8627d161..f8c55493 100644 --- a/internal/controller/networkgateway_controller.go +++ b/internal/controller/networkgateway_controller.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/galactic/internal/gateway" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -713,19 +714,23 @@ func pruneRuleAdvertisements( return errors.Join(errs...) } -// buildDesiredRule converts rule into a gateway.DesiredRule, resolving each -// backend's SRv6 uSID through sidIndex. There is no kernel VRF or FIB -// dependency. +// buildDesiredRule converts rule into a gateway.DesiredRule. Its backends are +// the IPv6 addresses of the VPCAttachments rule's BackendSelector picks from +// sidIndex's attachments (selectRuleBackends), each resolved through sidIndex to the +// SRv6 uSID of the node its attachment reports, with that backend's own slot +// (backendSID), so a node hosting several of the rule's backends can tell which +// one Maglev chose. There is no kernel VRF or FIB dependency. // -// A backend whose uSID does not resolve is left out and returned in -// unresolved as "address:port", so the rule keeps serving through the rest. -// A backend pod being recreated is enough to cause this, since its -// BGPAdvertisement is gone until the new pod is attached. Every gateway node -// resolves from the same API objects and the Maglev table depends only on the -// backend set, so nodes that see the same objects build the same table. Only -// when no backend resolves does the rule fail, so it is withdrawn rather than -// advertised with nothing behind it. A malformed address fails the rule -// outright, since that is a spec error rather than a passing state. +// A selected attachment that is not a backend yet, or a backend whose uSID +// does not resolve, is left out and returned in unresolved, so the rule keeps +// serving through the rest. A backend pod being recreated is enough to cause +// this, since its BGPAdvertisement is gone until the new pod is attached. +// Every gateway node resolves from the same API objects and the Maglev table +// depends only on the backend set, so nodes that see the same objects build +// the same table. Only when nothing resolves does the rule fail, so it is +// withdrawn rather than advertised with nothing behind it. An invalid VIP or +// selector fails the rule outright, since that is a spec error rather than a +// passing state. func buildDesiredRule( rule *bgpv1alpha1.NetworkRule, sidIndex *backendSIDIndex, ) (dr gateway.DesiredRule, unresolved []string, err error) { @@ -738,27 +743,42 @@ func buildDesiredRule( vips = append(vips, addr) } - backends := make([]gateway.DesiredBackend, 0, len(rule.Spec.Backends)) + if _, _, err := ruleTranslatedVIP(rule); err != nil { + return gateway.DesiredRule{}, nil, err + } + + selected, unresolved, err := selectRuleBackends(rule, sidIndex.attachments) + if err != nil { + return gateway.DesiredRule{}, nil, err + } + + backends := make([]gateway.DesiredBackend, 0, len(selected)) var firstResolveErr error - for _, b := range rule.Spec.Backends { - addr, err := netip.ParseAddr(b.Address) - if err != nil { - return gateway.DesiredRule{}, nil, fmt.Errorf("invalid backend address %q: %w", b.Address, err) + for _, b := range selected { + usid, err := sidIndex.resolveUSID(b.addr, rule.Spec.VPCRef, b.node) + if err == nil { + usid, err = backendSID(usid, b.addr, b.port) } - //nolint:gosec // b.Port is CRD-validated to [1,65535] (Minimum/Maximum markers on NetworkRuleBackend.Port) - port := uint16(b.Port) - usid, err := sidIndex.resolveUSID(addr, rule.Spec.VPCRef) if err != nil { if firstResolveErr == nil { - firstResolveErr = fmt.Errorf("resolve backend %s: %w", addr, err) + firstResolveErr = fmt.Errorf("resolve backend %s: %w", b, err) } - unresolved = append(unresolved, netip.AddrPortFrom(addr, port).String()) + unresolved = append(unresolved, b.String()) continue } - backends = append(backends, gateway.DesiredBackend{Address: addr, Port: port, USID: usid}) + backends = append(backends, gateway.DesiredBackend{Address: b.addr, Port: b.port, USID: usid}) } - if len(backends) == 0 && len(unresolved) > 0 { - return gateway.DesiredRule{}, nil, fmt.Errorf("none of %d backends resolves: %w", len(unresolved), firstResolveErr) + if len(backends) == 0 { + switch { + case firstResolveErr != nil: + return gateway.DesiredRule{}, nil, fmt.Errorf("none of %d backends resolves: %w", len(unresolved), firstResolveErr) + case len(unresolved) > 0: + return gateway.DesiredRule{}, nil, fmt.Errorf("no selected attachment is a backend yet: %s", + cappedList(unresolved, maxReadyFailures)) + default: + return gateway.DesiredRule{}, nil, fmt.Errorf("backendSelector matches no VPCAttachment in VPC %s", + rule.Spec.VPCRef) + } } return gateway.DesiredRule{ @@ -989,6 +1009,18 @@ func (r *NetworkGatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { }), builder.WithPredicates(predicate.GenerationChangedPredicate{}), ). + // A rule's backends are the VPCAttachments its selector picks, so an + // attachment appearing, moving node, changing address or changing + // labels changes some rule's backend set. Attachments live in tenant + // namespaces, not the gateway's, so every NetworkGateway is + // re-queued. Their address and node come from spec and status alike, + // so status updates are not filtered out; the engine skips every rule + // whose resolved state is unchanged. + Watches(&cloudv1alpha1.VPCAttachment{}, handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, obj client.Object) []ctrlreconcile.Request { + return allGatewayRequests(ctx, r.Client, "VPCAttachment", obj.GetNamespace()+"/"+obj.GetName()) + }), + ). Named("networkgateway"). Complete(r) } @@ -1074,6 +1106,13 @@ func broadcastToGatewayRequests( return reqs } +// allGatewayRequests returns a reconcile request for every NetworkGateway in +// every namespace, for a change to an object that lives outside the gateways' +// namespace, such as a tenant's VPCAttachment. +func allGatewayRequests(ctx context.Context, c client.Client, sourceKind, sourceName string) []ctrlreconcile.Request { + return broadcastToGatewayRequests(ctx, c, metav1.NamespaceAll, sourceKind, sourceName) +} + // gatewayNodeNames returns the targetRef.name of every NetworkGateway in // namespace, the pool of gateway nodes for this PoP. func gatewayNodeNames(ctx context.Context, c client.Client, namespace string) ([]string, error) { diff --git a/internal/controller/networkgateway_controller_test.go b/internal/controller/networkgateway_controller_test.go index 1158fae9..dec927c1 100644 --- a/internal/controller/networkgateway_controller_test.go +++ b/internal/controller/networkgateway_controller_test.go @@ -25,7 +25,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/galactic/internal/gateway" + "go.datum.net/galactic/internal/plumbing/srv6" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -291,6 +293,7 @@ func TestNetworkGatewayReconciler_BuildsDesiredStateForAcceptedRules(t *testing. fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). WithObjects(gwA, gwB, backendRouter, backendAdv, backendVRF, backendAdv2, backendVRF2, + newBackendAttachment("vpc-1"), newBackendAttachment("vpc-2"), ruleA, ruleB, notAccepted, deleting). Build() @@ -363,13 +366,14 @@ func TestNetworkGatewayReconciler_ServesThroughResolvedBackends(t *testing.T) { scheme := newRuleTestScheme(t) backendRouter, backendAdv, backendVRF := newBackendFixtures("vpc-1") rule := newTestRule(testRuleName, "vpc-1", testVIP) - // Outside testBackendPrefix, so it resolves against nothing. - rule.Spec.Backends = append(rule.Spec.Backends, bgpv1alpha1.NetworkRuleBackend{Address: "192.0.2.99", Port: 8443}) acceptRule(rule) + // The attachment's second address is outside testBackendPrefix, so it + // resolves against nothing. + attachment := newBackendAttachment("vpc-1", testBackendAddr, "fd00:99::1") fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, rule). + WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, attachment, rule). Build() engine := newFakeGatewayEngine() @@ -394,43 +398,82 @@ func TestNetworkGatewayReconciler_ServesThroughResolvedBackends(t *testing.T) { assertRuleProgrammed(t, fakeClient, metav1.ConditionTrue, reasonBackendsUnresolved) cond := ruleProgrammedCondition(t, fakeClient, testRuleName) - if !strings.Contains(cond.Message, "1 of 2 backends") || !strings.Contains(cond.Message, "192.0.2.99:8443") { - t.Errorf("Programmed message = %q, want it to count 1 of 2 backends and name 192.0.2.99:8443", cond.Message) + if !strings.Contains(cond.Message, "1 of 2 backends") || !strings.Contains(cond.Message, "[fd00:99::1]:8443") { + t.Errorf("Programmed message = %q, want it to count 1 of 2 backends and name [fd00:99::1]:8443", cond.Message) } } -// TestBuildDesiredRule_BackendResolution covers which backend failures leave -// a backend out and which fail the whole rule. +// TestBuildDesiredRule_BackendResolution covers which selected attachments +// become backends, which are left out as unresolved, and which outcomes fail +// the whole rule. func TestBuildDesiredRule_BackendResolution(t *testing.T) { backendRouter, backendAdv, backendVRF := newBackendFixtures("vpc-1") - idx := &backendSIDIndex{ - routers: map[string]*bgpv1alpha1.BGPRouter{backendRouter.Name: backendRouter}, - advs: []*bgpv1alpha1.BGPAdvertisement{backendAdv}, - vrfInstances: map[string]*bgpv1alpha1.BGPVRFInstance{backendVRF.Name: backendVRF}, - } + + pendingNode := newBackendAttachment("vpc-1", "fd00:10::3") + pendingNode.Name = "no-node" + pendingNode.Status.Node = "" + ipv4Only := newBackendAttachment("vpc-1", "10.0.0.9") + ipv4Only.Name = "ipv4-only" + otherVPC := newBackendAttachment("vpc-2", "fd00:10::4") + unlabelled := newBackendAttachment("vpc-1", "fd00:10::5") + unlabelled.Name = "unlabelled" + unlabelled.Labels = nil + otherNode := newBackendAttachment("vpc-1", "fd00:10::6") + otherNode.Name = "other-node" + otherNode.Status.Node = testOtherNode tests := []struct { name string - backends []string + attachments []*cloudv1alpha1.VPCAttachment wantErr bool - wantBackends int + wantBackends []string wantUnresolved []string }{ - {name: "all resolve", backends: []string{testBackendAddr, "10.0.0.2"}, wantBackends: 2}, { - name: "some unresolved", backends: []string{testBackendAddr, "198.51.100.1", "2001:db8::1"}, - wantBackends: 1, wantUnresolved: []string{"198.51.100.1:8443", "[2001:db8::1]:8443"}, + name: "all resolve", + attachments: []*cloudv1alpha1.VPCAttachment{newBackendAttachment("vpc-1", testBackendAddr, "fd00:10::2")}, + wantBackends: []string{testBackendAddr, "fd00:10::2"}, + }, + { + name: "some unresolved", + attachments: []*cloudv1alpha1.VPCAttachment{newBackendAttachment("vpc-1", testBackendAddr, "fd00:99::1")}, + wantBackends: []string{testBackendAddr}, + wantUnresolved: []string{"[fd00:99::1]:8443"}, + }, + { + name: "attachments not ready are pending", + attachments: []*cloudv1alpha1.VPCAttachment{newBackendAttachment("vpc-1"), pendingNode, ipv4Only}, + wantBackends: []string{testBackendAddr}, + wantUnresolved: []string{"tenant-vpc-1/ipv4-only: no IPv6 address", "tenant-vpc-1/no-node: no node"}, + }, + { + name: "other VPCs and unmatched labels are not selected", + attachments: []*cloudv1alpha1.VPCAttachment{newBackendAttachment("vpc-1"), otherVPC, unlabelled}, + wantBackends: []string{testBackendAddr}, }, - {name: "none resolve", backends: []string{"198.51.100.1", "198.51.100.2"}, wantErr: true}, - {name: "malformed address", backends: []string{testBackendAddr, "not-an-ip"}, wantErr: true}, + { + name: "a backend resolves only on its attachment's node", + attachments: []*cloudv1alpha1.VPCAttachment{newBackendAttachment("vpc-1"), otherNode}, + wantBackends: []string{testBackendAddr}, + wantUnresolved: []string{"[fd00:10::6]:8443"}, + }, + { + name: "none resolve", + attachments: []*cloudv1alpha1.VPCAttachment{newBackendAttachment("vpc-1", "fd00:99::1")}, + wantErr: true, + }, + {name: "selector matches nothing", wantErr: true}, + {name: "only pending attachments", attachments: []*cloudv1alpha1.VPCAttachment{pendingNode}, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rule := newTestRule(testRuleName, "vpc-1", testVIP) - rule.Spec.Backends = nil - for _, b := range tt.backends { - rule.Spec.Backends = append(rule.Spec.Backends, bgpv1alpha1.NetworkRuleBackend{Address: b, Port: 8443}) + idx := &backendSIDIndex{ + routers: map[string]*bgpv1alpha1.BGPRouter{backendRouter.Name: backendRouter}, + advs: []*bgpv1alpha1.BGPAdvertisement{backendAdv}, + vrfInstances: map[string]*bgpv1alpha1.BGPVRFInstance{backendVRF.Name: backendVRF}, + attachments: tt.attachments, } + rule := newTestRule(testRuleName, "vpc-1", testVIP) dr, unresolved, err := buildDesiredRule(rule, idx) if tt.wantErr { @@ -442,8 +485,12 @@ func TestBuildDesiredRule_BackendResolution(t *testing.T) { if err != nil { t.Fatalf("buildDesiredRule: %v", err) } - if len(dr.Backends) != tt.wantBackends { - t.Errorf("backends = %d, want %d", len(dr.Backends), tt.wantBackends) + var got []string + for _, b := range dr.Backends { + got = append(got, b.Address.String()) + } + if !slices.Equal(got, tt.wantBackends) { + t.Errorf("backends = %v, want %v", got, tt.wantBackends) } if !slices.Equal(unresolved, tt.wantUnresolved) { t.Errorf("unresolved = %v, want %v", unresolved, tt.wantUnresolved) @@ -452,6 +499,72 @@ func TestBuildDesiredRule_BackendResolution(t *testing.T) { } } +// TestBuildDesiredRule_RefusesSecondIPv6VIP covers a rule with two IPv6 VIPs: +// it fails to build, so it is not loaded with a VIP whose flows no backend +// could answer. +func TestBuildDesiredRule_RefusesSecondIPv6VIP(t *testing.T) { + backendRouter, backendAdv, backendVRF := newBackendFixtures("vpc-1") + idx := &backendSIDIndex{ + routers: map[string]*bgpv1alpha1.BGPRouter{backendRouter.Name: backendRouter}, + advs: []*bgpv1alpha1.BGPAdvertisement{backendAdv}, + vrfInstances: map[string]*bgpv1alpha1.BGPVRFInstance{backendVRF.Name: backendVRF}, + attachments: []*cloudv1alpha1.VPCAttachment{newBackendAttachment("vpc-1")}, + } + rule := newTestRule(testRuleName, "vpc-1", "2001:db8:100::32", "2001:db8:100::33", testVIP) + if _, _, err := buildDesiredRule(rule, idx); err == nil { + t.Fatal("buildDesiredRule: err = nil, want an error for two IPv6 VIPs") + } + + rule.Spec.VIPAddresses = []string{"2001:db8:100::32", testVIP} + if _, _, err := buildDesiredRule(rule, idx); err != nil { + t.Fatalf("buildDesiredRule with one IPv6 and one IPv4 VIP: %v", err) + } +} + +// TestBuildDesiredRule_SameNodeBackendsGetTheirOwnSlot covers #799's +// same-node case: two backends of one rule on one node resolve to the same +// node and VRF, and differ only in the slot the gateway writes into the uSID, +// which is the hash the backend's node keys its vip_xlat_table row on. +func TestBuildDesiredRule_SameNodeBackendsGetTheirOwnSlot(t *testing.T) { + backendRouter, backendAdv, backendVRF := newBackendFixtures("vpc-1") + idx := &backendSIDIndex{ + routers: map[string]*bgpv1alpha1.BGPRouter{backendRouter.Name: backendRouter}, + advs: []*bgpv1alpha1.BGPAdvertisement{backendAdv}, + vrfInstances: map[string]*bgpv1alpha1.BGPVRFInstance{backendVRF.Name: backendVRF}, + attachments: []*cloudv1alpha1.VPCAttachment{newBackendAttachment("vpc-1", testBackendAddr, "fd00:10::2")}, + } + + dr, _, err := buildDesiredRule(newTestRule(testRuleName, "vpc-1", testVIP), idx) + if err != nil { + t.Fatalf("buildDesiredRule: %v", err) + } + if len(dr.Backends) != 2 { + t.Fatalf("backends = %+v, want 2", dr.Backends) + } + + nodeSID, err := srv6.ComputeSID( + testBackendLocator, testBackendNodeID, testBackendVRFID, bgpv1alpha1.SRv6FunctionEndDT46) + if err != nil { + t.Fatalf("srv6.ComputeSID: %v", err) + } + for _, b := range dr.Backends { + want, err := srv6.ComputeBackendSID(testBackendLocator, testBackendNodeID, testBackendVRFID, + bgpv1alpha1.SRv6FunctionEndDT46, srv6.BackendSlot(b.Address, b.Port)) + if err != nil { + t.Fatalf("srv6.ComputeBackendSID: %v", err) + } + if b.USID != want { + t.Errorf("backend %s uSID = %s, want %s (the node's uSID with this backend's slot)", b.Address, b.USID, want) + } + if b.USID == nodeSID { + t.Errorf("backend %s uSID = %s carries no slot", b.Address, b.USID) + } + } + if dr.Backends[0].USID == dr.Backends[1].USID { + t.Errorf("both backends share uSID %s; the node could not tell them apart", dr.Backends[0].USID) + } +} + func TestNetworkGatewayReconciler_SkipsBGPAdvertisementWiringWithoutRouter(t *testing.T) { scheme := newRuleTestScheme(t) gwA := newTestGateway(testNodeGWA) @@ -462,7 +575,7 @@ func TestNetworkGatewayReconciler_SkipsBGPAdvertisementWiringWithoutRouter(t *te fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(gwA, backendRouter, backendAdv, backendVRF, rule). + WithObjects(gwA, backendRouter, backendAdv, backendVRF, newBackendAttachment("vpc-1"), rule). Build() engine := newFakeGatewayEngine() @@ -500,7 +613,7 @@ func TestNetworkGatewayReconciler_CreatesBGPAdvertisement(t *testing.T) { fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(gwA, gwB, router, backendRouter, backendAdv, backendVRF, rule). + WithObjects(gwA, gwB, router, backendRouter, backendAdv, backendVRF, newBackendAttachment("vpc-1"), rule). Build() engine := newFakeGatewayEngine() @@ -569,7 +682,7 @@ func TestNetworkGatewayReconciler_BackfillsLabelOnExistingAdvertisement(t *testi fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(gwA, router, backendRouter, backendAdv, backendVRF, rule, preexisting). + WithObjects(gwA, router, backendRouter, backendAdv, backendVRF, newBackendAttachment("vpc-1"), rule, preexisting). Build() engine := newFakeGatewayEngine() @@ -609,7 +722,7 @@ func TestNetworkGatewayReconciler_AdvertisementFailureSurfaces(t *testing.T) { fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(gwA, router, backendRouter, backendAdv, backendVRF, rule). + WithObjects(gwA, router, backendRouter, backendAdv, backendVRF, newBackendAttachment("vpc-1"), rule). WithInterceptorFuncs(interceptor.Funcs{ Create: func( ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption, @@ -660,7 +773,7 @@ func TestNetworkGatewayReconciler_ReportsEngineHealthyOnCleanPass(t *testing.T) fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(gwA, router, backendRouter, backendAdv, backendVRF, rule). + WithObjects(gwA, router, backendRouter, backendAdv, backendVRF, newBackendAttachment("vpc-1"), rule). Build() engine := newFakeGatewayEngine() @@ -701,7 +814,8 @@ func TestNetworkGatewayReconciler_DoesNotAdvertiseUnloadedRule(t *testing.T) { fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, rule). + WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, + newBackendAttachment("vpc-1"), rule). Build() engine := newFakeGatewayEngine() @@ -744,7 +858,8 @@ func TestNetworkGatewayReconciler_WithdrawsRuleThatStopsLoading(t *testing.T) { fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, rule). + WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, + newBackendAttachment("vpc-1"), rule). Build() engine := newFakeGatewayEngine() @@ -1293,7 +1408,8 @@ func TestNetworkGatewayReconciler_RuleDeletedMidPassIsNotAnError(t *testing.T) { fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, rule). + WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, + newBackendAttachment("vpc-1"), rule). WithInterceptorFuncs(interceptor.Funcs{ Get: func( ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption, @@ -1325,7 +1441,8 @@ func TestNetworkGatewayReconciler_UnchangedProgrammedConditionWritesNothing(t *t var ruleWrites atomic.Int32 fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, rule). + WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, + newBackendAttachment("vpc-1"), rule). WithInterceptorFuncs(interceptor.Funcs{ SubResourceUpdate: func( ctx context.Context, c client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption, diff --git a/internal/controller/networkgateway_drain_test.go b/internal/controller/networkgateway_drain_test.go index f3de7377..d4c79e14 100644 --- a/internal/controller/networkgateway_drain_test.go +++ b/internal/controller/networkgateway_drain_test.go @@ -96,7 +96,8 @@ func TestNetworkGatewayReconciler_DrainsDeletedRuleUntilWithdrawn(t *testing.T) var writes advWriteCounter fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, rule). + WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, + newBackendAttachment("vpc-1"), rule). WithInterceptorFuncs(writes.funcs()). Build() @@ -201,7 +202,8 @@ func TestNetworkGatewayReconciler_DrainingRuleKeepsLastLoadedState(t *testing.T) fakeClient := newIndexedClientBuilder(scheme). WithStatusSubresource(&bgpv1alpha1.NetworkGateway{}, &bgpv1alpha1.NetworkRule{}). - WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, rule). + WithObjects(newTestGateway(testNodeGWA), newTestRouter(), backendRouter, backendAdv, backendVRF, + newBackendAttachment("vpc-1"), rule). Build() engine := newFakeGatewayEngine() diff --git a/internal/controller/networkrule_binding_controller.go b/internal/controller/networkrule_binding_controller.go new file mode 100644 index 00000000..86749b9b --- /dev/null +++ b/internal/controller/networkrule_binding_controller.go @@ -0,0 +1,440 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controller + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "sort" + "strconv" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + "go.datum.net/galactic/internal/crdnames" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// bindingNodeLabel is set on every ServiceVIPBinding NetworkRuleBindingReconciler +// writes, to gatewayNodeLabelValue of the node it was written for, so each +// node lists only its own bindings. +const bindingNodeLabel = "galactic.datum.net/binding-node" + +// bindingManagedByLabel and bindingManagedByValue mark a ServiceVIPBinding as +// generated. NetworkRuleBindingReconciler deletes only bindings carrying them, +// so it never removes one it did not write. +const ( + bindingManagedByLabel = "app.kubernetes.io/managed-by" + bindingManagedByValue = "galactic-router" +) + +// conditionTypeBackendsBound is the per-node NetworkRule condition, as +// "/BackendsBound", recording whether every binding node needs for the +// rule exists and reports Bound. Reasons are the reasonBindings* constants. +const conditionTypeBackendsBound = "BackendsBound" + +// bindingTargetKind is the targetRef kind of every generated binding. +const bindingTargetKind = "Node" + +const ( + // reasonBindingsBound: every backend of the rule on this node is bound. + reasonBindingsBound = "Bound" + + // reasonBindingsPending: at least one binding is not Bound yet, or reports + // why it cannot be. + reasonBindingsPending = "BindingsNotBound" + + // reasonBindingsInvalid: the rule's selector is invalid, so no backend + // could be selected. + reasonBindingsInvalid = "InvalidRule" +) + +// NetworkRuleBindingReconciler writes, on its own node, the ServiceVIPBindings +// that a NetworkRule's backends need: one per IPv6 VIP per backend whose +// VPCAttachment reports this node. It runs in galactic-router on every node. +// +// It is the only writer of generated bindings, and each node writes only its +// own, so gateway replicas never race and a gateway rollout never touches +// them. A binding is owned by its rule, so deleting the rule garbage-collects +// it, and the binding's own finalizer still lets ServiceVIPBindingReconciler +// remove its vip_xlat_table rows first. A backend that moves to another node, +// changes address, or stops matching the selector has its binding deleted +// here and a new one written by whichever node now hosts it. +// +// The binding's egressKind comes from the attachment's interface mode, which +// is known before the backend's interface exists, and its vpcRef from the +// rule, which names the VRF the binding's rows go into. Both sides of the +// datapath derive the backend set from selectRuleBackends, and the binding's +// slot from its backend address and port, so the slot the gateway encodes and +// the row this node writes always agree. +// +// The rule's "/BackendsBound" condition summarizes this node's bindings, +// so a rule that is Programmed on the gateways but cannot reach a backend on +// some node says so on the rule itself. +type NetworkRuleBindingReconciler struct { + client.Client + Scheme *runtime.Scheme + NodeName string +} + +// Reconcile converges this node's generated bindings for one NetworkRule. +func (r *NetworkRuleBindingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + rule := &bgpv1alpha1.NetworkRule{} + if err := r.Get(ctx, req.NamespacedName, rule); err != nil { + if apierrors.IsNotFound(err) { + // Owner references garbage-collect the rule's bindings. + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("get NetworkRule %s: %w", req.NamespacedName, err) + } + if !rule.DeletionTimestamp.IsZero() { + // The gateways keep serving a deleting rule until its routes are + // withdrawn and drained, so its bindings stay until the rule is gone + // and garbage collection removes them. + return ctrl.Result{}, nil + } + + if !meta.IsStatusConditionTrue(rule.Status.Conditions, bgpv1alpha1.ConditionTypeAccepted) { + // Not accepted yet, or no longer: Accepted goes False whenever no + // NetworkGateway exists in the namespace, which a delete and re-apply + // of the gateways passes through. Deleting the bindings then would + // remove every backend node's vip_xlat_table rows for a moment that + // costs nothing to wait out, so existing bindings stay. The gateways + // do not serve an unaccepted rule, so an idle binding draws no traffic. + return ctrl.Result{}, nil + } + + attachments, err := listVPCAttachments(ctx, r.Client) + if err != nil { + return ctrl.Result{}, err + } + desired, buildErr := r.desiredBindings(rule, attachments) + + if err := r.applyBindings(ctx, rule, desired); err != nil { + return ctrl.Result{}, err + } + if err := r.publishCondition(ctx, rule, desired, buildErr); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +// desiredBindings returns the bindings this node needs for rule: one for its +// IPv6 VIP per selected backend on this node. The datapath translates IPv6 +// only, so a rule with no IPv6 VIP needs none, and ruleTranslatedVIP refuses a +// rule with more than one. +func (r *NetworkRuleBindingReconciler) desiredBindings( + rule *bgpv1alpha1.NetworkRule, attachments []*cloudv1alpha1.VPCAttachment, +) ([]*bgpv1alpha1.ServiceVIPBinding, error) { + vip, ok, err := ruleTranslatedVIP(rule) + if err != nil || !ok { + return nil, err + } + backends, _, err := selectRuleBackends(rule, attachments) + if err != nil { + return nil, err + } + + var out []*bgpv1alpha1.ServiceVIPBinding + for _, b := range backends { + if b.node == r.NodeName { + out = append(out, r.newBinding(rule, vip, b)) + } + } + return out, nil +} + +// newBinding builds the binding for one VIP and one backend on this node. +func (r *NetworkRuleBindingReconciler) newBinding( + rule *bgpv1alpha1.NetworkRule, vip netip.Addr, b ruleBackend, +) *bgpv1alpha1.ServiceVIPBinding { + return &bgpv1alpha1.ServiceVIPBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: rule.Namespace, + Name: crdnames.ServiceVIPBindingName( + rule.Name, r.NodeName, vip.String(), b.addr.String(), int32(b.port)), + Labels: map[string]string{ + networkRuleLabel: rule.Name, + bindingNodeLabel: gatewayNodeLabelValue(r.NodeName), + bindingManagedByLabel: bindingManagedByValue, + }, + }, + Spec: bgpv1alpha1.ServiceVIPBindingSpec{ + TargetRef: bgpv1alpha1.TargetRef{Kind: bindingTargetKind, Name: r.NodeName}, + VPCRef: rule.Spec.VPCRef, + VIPAddress: vip.String(), + Port: rule.Spec.Port, + Protocol: rule.Spec.Protocol, + BackendAddress: b.addr.String(), + BackendPort: int32(b.port), + EgressKind: b.egressKind, + }, + } +} + +// applyBindings creates or updates every binding in desired and deletes this +// node's other generated bindings for rule. +func (r *NetworkRuleBindingReconciler) applyBindings( + ctx context.Context, rule *bgpv1alpha1.NetworkRule, desired []*bgpv1alpha1.ServiceVIPBinding, +) error { + keep := make(map[string]bool, len(desired)) + var errs []error + for _, want := range desired { + keep[want.Name] = true + binding := &bgpv1alpha1.ServiceVIPBinding{ObjectMeta: metav1.ObjectMeta{ + Namespace: want.Namespace, Name: want.Name, + }} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, binding, func() error { + if binding.Labels == nil { + binding.Labels = make(map[string]string, len(want.Labels)) + } + for k, v := range want.Labels { + binding.Labels[k] = v + } + binding.Spec = want.Spec + // Not a controller reference, and not blocking owner deletion: + // the binding's own finalizer orders its teardown. + return controllerutil.SetOwnerReference(rule, binding, r.Scheme) + }) + if err != nil { + errs = append(errs, fmt.Errorf("apply ServiceVIPBinding %s: %w", want.Name, err)) + } + } + + existing, err := r.nodeBindings(ctx, rule) + if err != nil { + return errors.Join(append(errs, err)...) + } + for i := range existing { + binding := &existing[i] + if keep[binding.Name] { + continue + } + if err := r.Delete(ctx, binding); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("delete stale ServiceVIPBinding %s: %w", binding.Name, err)) + } + } + return errors.Join(errs...) +} + +// nodeBindings lists the generated bindings this node wrote for rule. +func (r *NetworkRuleBindingReconciler) nodeBindings( + ctx context.Context, rule *bgpv1alpha1.NetworkRule, +) ([]bgpv1alpha1.ServiceVIPBinding, error) { + list := &bgpv1alpha1.ServiceVIPBindingList{} + if err := r.List(ctx, list, + client.InNamespace(rule.Namespace), + client.MatchingLabels{ + networkRuleLabel: rule.Name, + bindingNodeLabel: gatewayNodeLabelValue(r.NodeName), + bindingManagedByLabel: bindingManagedByValue, + }, + ); err != nil { + return nil, fmt.Errorf("list ServiceVIPBindings for NetworkRule %s: %w", rule.Name, err) + } + return list.Items, nil +} + +// publishCondition sets this node's BackendsBound condition on rule, or removes +// it when the node hosts none of the rule's backends and the rule is valid, +// so a rule's status names only the nodes that serve it. +func (r *NetworkRuleBindingReconciler) publishCondition( + ctx context.Context, rule *bgpv1alpha1.NetworkRule, desired []*bgpv1alpha1.ServiceVIPBinding, buildErr error, +) error { + condType := backendsBoundConditionType(r.NodeName) + if buildErr == nil && len(desired) == 0 { + return r.updateRuleCondition(ctx, rule, func(conds *[]metav1.Condition) bool { + return meta.RemoveStatusCondition(conds, condType) + }) + } + + cond := metav1.Condition{Type: condType} + switch { + case buildErr != nil: + cond.Status = metav1.ConditionFalse + cond.Reason = reasonBindingsInvalid + cond.Message = buildErr.Error() + default: + unbound, err := r.unboundBindings(ctx, rule, desired) + if err != nil { + return err + } + if len(unbound) == 0 { + cond.Status = metav1.ConditionTrue + cond.Reason = reasonBindingsBound + cond.Message = fmt.Sprintf("%d bindings bound on node %s", len(desired), r.NodeName) + } else { + cond.Status = metav1.ConditionFalse + cond.Reason = reasonBindingsPending + cond.Message = fmt.Sprintf("%d of %d bindings on node %s not bound: %s", + len(unbound), len(desired), r.NodeName, cappedList(unbound, maxReadyFailures)) + } + } + return r.updateRuleCondition(ctx, rule, func(conds *[]metav1.Condition) bool { + c := cond + c.ObservedGeneration = rule.Generation + return meta.SetStatusCondition(conds, c) + }) +} + +// unboundBindings returns "vip -> backend: reason" for every desired binding +// that is not Bound yet, reading each one's current status. +func (r *NetworkRuleBindingReconciler) unboundBindings( + ctx context.Context, rule *bgpv1alpha1.NetworkRule, desired []*bgpv1alpha1.ServiceVIPBinding, +) ([]string, error) { + existing, err := r.nodeBindings(ctx, rule) + if err != nil { + return nil, err + } + byName := make(map[string]*bgpv1alpha1.ServiceVIPBinding, len(existing)) + for i := range existing { + byName[existing[i].Name] = &existing[i] + } + + var unbound []string + for _, want := range desired { + label := net.JoinHostPort(want.Spec.VIPAddress, strconv.Itoa(int(want.Spec.Port))) + " -> " + + net.JoinHostPort(want.Spec.BackendAddress, strconv.Itoa(int(want.Spec.BackendPort))) + got, ok := byName[want.Name] + if !ok { + unbound = append(unbound, label+": not created yet") + continue + } + bound := meta.FindStatusCondition(got.Status.Conditions, bgpv1alpha1.ConditionTypeBound) + switch { + case bound == nil || bound.ObservedGeneration != got.Generation: + unbound = append(unbound, label+": not reconciled yet") + case bound.Status != metav1.ConditionTrue: + unbound = append(unbound, label+": "+bound.Reason) + } + } + sort.Strings(unbound) + return unbound, nil +} + +// updateRuleCondition applies mutate to rule's current conditions and writes +// them only if mutate reports a change. Every status write fans out to every +// reconciler watching NetworkRules, so an unconditional write would never +// settle. +func (r *NetworkRuleBindingReconciler) updateRuleCondition( + ctx context.Context, rule *bgpv1alpha1.NetworkRule, mutate func(*[]metav1.Condition) bool, +) error { + key := client.ObjectKeyFromObject(rule) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + current := &bgpv1alpha1.NetworkRule{} + if err := r.Get(ctx, key, current); err != nil { + return err + } + if !mutate(¤t.Status.Conditions) { + return nil + } + return r.Status().Update(ctx, current) + }) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("update %s condition on NetworkRule %s: %w", conditionTypeBackendsBound, key, err) + } + return nil +} + +// backendsBoundConditionType returns node's BackendsBound condition type. Rule +// status is shared by every node, so each node owns its own condition type and +// never overwrites another node's result. +func backendsBoundConditionType(node string) string { + return node + "/" + conditionTypeBackendsBound +} + +// SetupWithManager registers the NetworkRuleBindingReconciler with the manager. +func (r *NetworkRuleBindingReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + // A rule's bindings depend on its spec and on whether it is + // Accepted. Other status writes, including this reconciler's own + // conditions and the gateways' Programmed conditions, change neither. + For(&bgpv1alpha1.NetworkRule{}, builder.WithPredicates(ruleBindingInputsChanged())). + // Any attachment can enter or leave a rule's selection, or move to + // or from this node, so every rule is re-queued. A rule with no + // backend on this node and none before costs one list. + Watches(&cloudv1alpha1.VPCAttachment{}, handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, _ client.Object) []ctrlreconcile.Request { + return allRuleRequests(ctx, r.Client) + }), + ). + // A binding's Bound condition feeds the rule's BackendsBound + // condition, and a deleted binding must be rewritten. + Watches(&bgpv1alpha1.ServiceVIPBinding{}, handler.EnqueueRequestsFromMapFunc( + func(_ context.Context, obj client.Object) []ctrlreconcile.Request { + return r.bindingRuleRequests(obj) + }), + ). + Named("networkrule-binding"). + Complete(r) +} + +// ruleBindingInputsChanged passes NetworkRule events that can change the +// bindings it needs: a create or delete, a spec change, or a change to its +// Accepted condition. +func ruleBindingInputsChanged() predicate.Predicate { + return predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldRule, okOld := e.ObjectOld.(*bgpv1alpha1.NetworkRule) + newRule, okNew := e.ObjectNew.(*bgpv1alpha1.NetworkRule) + if !okOld || !okNew { + return true + } + if oldRule.Generation != newRule.Generation { + return true + } + return meta.IsStatusConditionTrue(oldRule.Status.Conditions, bgpv1alpha1.ConditionTypeAccepted) != + meta.IsStatusConditionTrue(newRule.Status.Conditions, bgpv1alpha1.ConditionTypeAccepted) + }, + } +} + +// allRuleRequests returns a reconcile request for every NetworkRule. +func allRuleRequests(ctx context.Context, c client.Client) []ctrlreconcile.Request { + list := &bgpv1alpha1.NetworkRuleList{} + if err := c.List(ctx, list); err != nil { + log.FromContext(ctx).Error(err, "list NetworkRules for VPCAttachment change") + return nil + } + reqs := make([]ctrlreconcile.Request, 0, len(list.Items)) + for i := range list.Items { + reqs = append(reqs, ctrlreconcile.Request{NamespacedName: client.ObjectKeyFromObject(&list.Items[i])}) + } + return reqs +} + +// bindingRuleRequests maps a generated binding written for this node to its +// rule. Other nodes' bindings and hand-written ones map to nothing. +func (r *NetworkRuleBindingReconciler) bindingRuleRequests(obj client.Object) []ctrlreconcile.Request { + labels := obj.GetLabels() + if labels[bindingManagedByLabel] != bindingManagedByValue || + labels[bindingNodeLabel] != gatewayNodeLabelValue(r.NodeName) { + return nil + } + rule := labels[networkRuleLabel] + if rule == "" { + return nil + } + return []ctrlreconcile.Request{{NamespacedName: client.ObjectKey{Namespace: obj.GetNamespace(), Name: rule}}} +} diff --git a/internal/controller/networkrule_binding_controller_test.go b/internal/controller/networkrule_binding_controller_test.go new file mode 100644 index 00000000..f5072f2d --- /dev/null +++ b/internal/controller/networkrule_binding_controller_test.go @@ -0,0 +1,401 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controller + +import ( + "context" + "sort" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + "go.datum.net/galactic/internal/crdnames" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// testBindingVIP is the IPv6 VIP the binding writer's rules use: the backend +// nodes translate IPv6 VIPs only. +const testBindingVIP = "2001:db8:100::32" + +// newBindingTestRule returns an accepted rule with one IPv6 and one IPv4 VIP, +// selecting newBackendAttachment's attachments in testVPCRef. +func newBindingTestRule() *bgpv1alpha1.NetworkRule { + rule := newTestRule(testRuleName, testVPCRef, testBindingVIP, testVIP) + rule.UID = "rule-uid" + acceptRule(rule) + return rule +} + +// newBindingWriter returns a NetworkRuleBindingReconciler for +// testComputeNodeName over a fake client holding objs. +func newBindingWriter(t *testing.T, objs ...client.Object) (*NetworkRuleBindingReconciler, client.Client) { + t.Helper() + scheme := newRuleTestScheme(t) + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&bgpv1alpha1.NetworkRule{}, &bgpv1alpha1.ServiceVIPBinding{}, &cloudv1alpha1.VPCAttachment{}). + WithObjects(objs...). + Build() + return &NetworkRuleBindingReconciler{Client: c, Scheme: scheme, NodeName: testComputeNodeName}, c +} + +func reconcileBindings(t *testing.T, r *NetworkRuleBindingReconciler) { + t.Helper() + req := ctrl.Request{NamespacedName: client.ObjectKey{Namespace: testNamespace, Name: testRuleName}} + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } +} + +// listBindings returns every ServiceVIPBinding in the test namespace, sorted +// by backend address. +func listBindings(t *testing.T, c client.Client) []bgpv1alpha1.ServiceVIPBinding { + t.Helper() + list := &bgpv1alpha1.ServiceVIPBindingList{} + if err := c.List(context.Background(), list, client.InNamespace(testNamespace)); err != nil { + t.Fatalf("list ServiceVIPBindings: %v", err) + } + sort.Slice(list.Items, func(i, j int) bool { + return list.Items[i].Spec.BackendAddress < list.Items[j].Spec.BackendAddress + }) + return list.Items +} + +func backendAddresses(bindings []bgpv1alpha1.ServiceVIPBinding) []string { + out := make([]string, 0, len(bindings)) + for _, b := range bindings { + out = append(out, b.Spec.BackendAddress) + } + return out +} + +func getBindingRule(t *testing.T, c client.Client) *bgpv1alpha1.NetworkRule { + t.Helper() + rule := &bgpv1alpha1.NetworkRule{} + key := client.ObjectKey{Namespace: testNamespace, Name: testRuleName} + if err := c.Get(context.Background(), key, rule); err != nil { + t.Fatalf("get NetworkRule: %v", err) + } + return rule +} + +// TestNetworkRuleBindingReconciler_WritesBindingForLocalBackend is #799's +// regression test: applying only a NetworkRule produces a working binding on +// the backend's node, with nothing written by hand. +func TestNetworkRuleBindingReconciler_WritesBindingForLocalBackend(t *testing.T) { + rule := newBindingTestRule() + r, c := newBindingWriter(t, rule, newBackendAttachment(testVPCRef)) + + reconcileBindings(t, r) + + bindings := listBindings(t, c) + if len(bindings) != 1 { + t.Fatalf("bindings = %d, want 1 (one IPv6 VIP, one backend; the IPv4 VIP gets none)", len(bindings)) + } + b := bindings[0] + want := bgpv1alpha1.ServiceVIPBindingSpec{ + TargetRef: bgpv1alpha1.TargetRef{Kind: testTargetRefKind, Name: testComputeNodeName}, + VPCRef: testVPCRef, + VIPAddress: testBindingVIP, + Port: 443, + Protocol: bgpv1alpha1.NetworkRuleProtocolTCP, + BackendAddress: testBackendAddr, + BackendPort: testBackendPort, + EgressKind: bgpv1alpha1.ServiceVIPBindingEgressKindVeth, + } + if b.Spec != want { + t.Errorf("binding spec = %+v, want %+v", b.Spec, want) + } + if wantName := crdnames.ServiceVIPBindingName( + testRuleName, testComputeNodeName, testBindingVIP, testBackendAddr, testBackendPort); b.Name != wantName { + t.Errorf("binding name = %q, want %q", b.Name, wantName) + } + if b.Labels[bindingManagedByLabel] != bindingManagedByValue || b.Labels[networkRuleLabel] != testRuleName || + b.Labels[bindingNodeLabel] != gatewayNodeLabelValue(testComputeNodeName) { + t.Errorf("binding labels = %v, want managed-by, rule and node labels", b.Labels) + } + owners := b.OwnerReferences + if len(owners) != 1 || owners[0].Kind != "NetworkRule" || owners[0].Name != testRuleName { + t.Errorf("binding owner references = %+v, want the NetworkRule", b.OwnerReferences) + } +} + +// TestNetworkRuleBindingReconciler_SameNodeBackendsGetOwnBindings covers two +// backends of one VIP on one node: each gets its own binding, which the +// per-backend slot lets the node program side by side. +func TestNetworkRuleBindingReconciler_SameNodeBackendsGetOwnBindings(t *testing.T) { + rule := newBindingTestRule() + r, c := newBindingWriter(t, rule, newBackendAttachment(testVPCRef, testBackendAddr, "fd00:10::2")) + + reconcileBindings(t, r) + + got := backendAddresses(listBindings(t, c)) + if strings.Join(got, ",") != testBackendAddr+",fd00:10::2" { + t.Errorf("binding backends = %v, want both %s and fd00:10::2", got, testBackendAddr) + } +} + +// TestNetworkRuleBindingReconciler_OnlyThisNodesBackends covers a backend on +// another node and one in another VPC carrying the selected label: neither +// gets a binding here. +func TestNetworkRuleBindingReconciler_OnlyThisNodesBackends(t *testing.T) { + rule := newBindingTestRule() + elsewhere := newBackendAttachment(testVPCRef, "fd00:10::7") + elsewhere.Name = "elsewhere" + elsewhere.Status.Node = testOtherNode + otherVPC := newBackendAttachment("vpc-2", "fd00:10::8") + r, c := newBindingWriter(t, rule, newBackendAttachment(testVPCRef), elsewhere, otherVPC) + + reconcileBindings(t, r) + + if got := backendAddresses(listBindings(t, c)); len(got) != 1 || got[0] != testBackendAddr { + t.Errorf("binding backends = %v, want only %s", got, testBackendAddr) + } +} + +// TestNetworkRuleBindingReconciler_FollowsBackend covers a backend moving to +// another node, changing address, and leaving the selector: each time the +// stale binding goes and, where the backend is still local, a new one comes. +func TestNetworkRuleBindingReconciler_FollowsBackend(t *testing.T) { + tests := []struct { + name string + change func(*cloudv1alpha1.VPCAttachment) + want []string + }{ + { + name: "moves to another node", + change: func(a *cloudv1alpha1.VPCAttachment) { a.Status.Node = testOtherNode }, + }, + { + name: "changes address", + change: func(a *cloudv1alpha1.VPCAttachment) { + a.Spec.Interface.Addresses = []cloudv1alpha1.IPAddress{"fd00:10::9/64"} + }, + want: []string{"fd00:10::9"}, + }, + { + name: "leaves the selector", + change: func(a *cloudv1alpha1.VPCAttachment) { a.Labels = map[string]string{testBackendLabel: "other"} }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + attachment := newBackendAttachment(testVPCRef) + r, c := newBindingWriter(t, newBindingTestRule(), attachment) + reconcileBindings(t, r) + if got := listBindings(t, c); len(got) != 1 { + t.Fatalf("bindings before the change = %d, want 1", len(got)) + } + + current := &cloudv1alpha1.VPCAttachment{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(attachment), current); err != nil { + t.Fatalf("get attachment: %v", err) + } + // Spec and status are written separately, and each write reads + // the other half back from the server, so the change is applied + // before each. + tt.change(current) + if err := c.Update(context.Background(), current); err != nil { + t.Fatalf("update attachment: %v", err) + } + tt.change(current) + if err := c.Status().Update(context.Background(), current); err != nil { + t.Fatalf("update attachment status: %v", err) + } + reconcileBindings(t, r) + + got := backendAddresses(listBindings(t, c)) + if strings.Join(got, ",") != strings.Join(tt.want, ",") { + t.Errorf("binding backends after the change = %v, want %v", got, tt.want) + } + }) + } +} + +// TestNetworkRuleBindingReconciler_KeepsBindingsWhileUnaccepted covers a rule +// that loses Accepted, as it does whenever the namespace briefly has no +// NetworkGateway: its bindings stay, so the backend nodes keep their +// vip_xlat_table rows for when the gateways come back. +func TestNetworkRuleBindingReconciler_KeepsBindingsWhileUnaccepted(t *testing.T) { + rule := newBindingTestRule() + r, c := newBindingWriter(t, rule, newBackendAttachment(testVPCRef)) + reconcileBindings(t, r) + + current := getBindingRule(t, c) + meta.SetStatusCondition(¤t.Status.Conditions, metav1.Condition{ + Type: bgpv1alpha1.ConditionTypeAccepted, Status: metav1.ConditionFalse, Reason: "NoGatewayNodes", + }) + if err := c.Status().Update(context.Background(), current); err != nil { + t.Fatalf("update rule status: %v", err) + } + reconcileBindings(t, r) + + if got := listBindings(t, c); len(got) != 1 { + t.Errorf("bindings = %d, want the 1 written while accepted kept", len(got)) + } +} + +// TestNetworkRuleBindingReconciler_NeverAcceptedRuleHasNoBindings covers a +// rule the gateways have not accepted yet: nothing is written for it. +func TestNetworkRuleBindingReconciler_NeverAcceptedRuleHasNoBindings(t *testing.T) { + rule := newBindingTestRule() + rule.Status.Conditions = nil + r, c := newBindingWriter(t, rule, newBackendAttachment(testVPCRef)) + + reconcileBindings(t, r) + + if got := listBindings(t, c); len(got) != 0 { + t.Errorf("bindings = %d, want 0 for a rule that was never accepted", len(got)) + } +} + +// TestNetworkRuleBindingReconciler_RefusesSecondIPv6VIP covers a rule with two +// IPv6 VIPs: a backend's reply can be rewritten to only one VIP, so the rule +// gets no bindings and says why, rather than a binding per VIP of which all +// but one could only report Conflict. +func TestNetworkRuleBindingReconciler_RefusesSecondIPv6VIP(t *testing.T) { + rule := newBindingTestRule() + rule.Spec.VIPAddresses = append(rule.Spec.VIPAddresses, "2001:db8:100::33") + r, c := newBindingWriter(t, rule, newBackendAttachment(testVPCRef)) + + reconcileBindings(t, r) + + if got := listBindings(t, c); len(got) != 0 { + t.Errorf("bindings = %d, want 0 for a rule with two IPv6 VIPs", len(got)) + } + condType := backendsBoundConditionType(testComputeNodeName) + cond := meta.FindStatusCondition(getBindingRule(t, c).Status.Conditions, condType) + if cond == nil || cond.Reason != reasonBindingsInvalid || !strings.Contains(cond.Message, "2 IPv6 VIPs") { + t.Errorf("BackendsBound = %+v, want False/%s naming the 2 IPv6 VIPs", cond, reasonBindingsInvalid) + } +} + +// TestNetworkRuleBindingReconciler_LeavesOtherBindingsAlone covers bindings +// this node did not write: one written by hand, and one another node +// generated for the same rule. Neither is deleted. +func TestNetworkRuleBindingReconciler_LeavesOtherBindingsAlone(t *testing.T) { + rule := newBindingTestRule() + handWritten := newTestServiceVIPBinding(bgpv1alpha1.ServiceVIPBindingEgressKindVeth, testComputeNodeName) + handWritten.Labels = map[string]string{networkRuleLabel: testRuleName} + otherNode := newTestServiceVIPBinding(bgpv1alpha1.ServiceVIPBindingEgressKindVeth, testOtherNode) + otherNode.Name = "other-node-binding" + otherNode.Labels = map[string]string{ + networkRuleLabel: testRuleName, + bindingNodeLabel: gatewayNodeLabelValue(testOtherNode), + bindingManagedByLabel: bindingManagedByValue, + } + r, c := newBindingWriter(t, rule, handWritten, otherNode) + + reconcileBindings(t, r) + + names := map[string]bool{} + for _, b := range listBindings(t, c) { + names[b.Name] = true + } + if !names[handWritten.Name] || !names[otherNode.Name] { + t.Errorf("bindings = %v, want the hand-written and other node's bindings kept", names) + } +} + +// TestNetworkRuleBindingReconciler_TapBackend covers a VM backend: its +// attachment's Hypervisor mode selects the tap mechanism. +func TestNetworkRuleBindingReconciler_TapBackend(t *testing.T) { + attachment := newBackendAttachment(testVPCRef) + attachment.Spec.Interface.Mode = cloudv1alpha1.VPCAttachmentInterfaceModeHypervisor + r, c := newBindingWriter(t, newBindingTestRule(), attachment) + + reconcileBindings(t, r) + + bindings := listBindings(t, c) + if len(bindings) != 1 || bindings[0].Spec.EgressKind != bgpv1alpha1.ServiceVIPBindingEgressKindTap { + t.Errorf("bindings = %+v, want one tap-kind binding", bindings) + } +} + +// TestNetworkRuleBindingReconciler_BackendsBoundCondition covers the rule's +// per-node summary: False naming each binding not yet Bound, then True once +// every binding reports Bound for its current generation. +func TestNetworkRuleBindingReconciler_BackendsBoundCondition(t *testing.T) { + r, c := newBindingWriter(t, newBindingTestRule(), newBackendAttachment(testVPCRef)) + condType := backendsBoundConditionType(testComputeNodeName) + + reconcileBindings(t, r) + cond := meta.FindStatusCondition(getBindingRule(t, c).Status.Conditions, condType) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != reasonBindingsPending { + t.Fatalf("BackendsBound before the binding is bound = %+v, want False/%s", cond, reasonBindingsPending) + } + if !strings.Contains(cond.Message, "[fd00:10::1]:8443") { + t.Errorf("BackendsBound message = %q, want it to name the unbound backend", cond.Message) + } + + binding := listBindings(t, c)[0] + meta.SetStatusCondition(&binding.Status.Conditions, metav1.Condition{ + Type: bgpv1alpha1.ConditionTypeBound, Status: metav1.ConditionTrue, Reason: "Bound", + ObservedGeneration: binding.Generation, + }) + if err := c.Status().Update(context.Background(), &binding); err != nil { + t.Fatalf("update binding status: %v", err) + } + + reconcileBindings(t, r) + cond = meta.FindStatusCondition(getBindingRule(t, c).Status.Conditions, condType) + if cond == nil || cond.Status != metav1.ConditionTrue || cond.Reason != reasonBindingsBound { + t.Errorf("BackendsBound once bound = %+v, want True/%s", cond, reasonBindingsBound) + } +} + +// TestNetworkRuleBindingReconciler_InvalidSelector covers a selector the +// writer cannot parse: no bindings, and the rule says why. +func TestNetworkRuleBindingReconciler_InvalidSelector(t *testing.T) { + rule := newBindingTestRule() + rule.Spec.BackendSelector = metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: testBackendLabel, Operator: "Bogus"}, + }} + r, c := newBindingWriter(t, rule, newBackendAttachment(testVPCRef)) + + reconcileBindings(t, r) + + if got := listBindings(t, c); len(got) != 0 { + t.Errorf("bindings = %d, want 0 for an invalid selector", len(got)) + } + condType := backendsBoundConditionType(testComputeNodeName) + cond := meta.FindStatusCondition(getBindingRule(t, c).Status.Conditions, condType) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != reasonBindingsInvalid { + t.Errorf("BackendsBound = %+v, want False/%s", cond, reasonBindingsInvalid) + } +} + +// TestNetworkRuleBindingReconciler_BindingRuleRequests covers the watch +// mapping: only this node's generated bindings requeue their rule. +func TestNetworkRuleBindingReconciler_BindingRuleRequests(t *testing.T) { + r := &NetworkRuleBindingReconciler{NodeName: testComputeNodeName} + mine := &bgpv1alpha1.ServiceVIPBinding{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Labels: map[string]string{ + networkRuleLabel: testRuleName, + bindingNodeLabel: gatewayNodeLabelValue(testComputeNodeName), + bindingManagedByLabel: bindingManagedByValue, + }, + }} + if reqs := r.bindingRuleRequests(mine); len(reqs) != 1 || reqs[0].Name != testRuleName { + t.Errorf("requests for this node's binding = %v, want its rule", reqs) + } + + other := mine.DeepCopy() + other.Labels[bindingNodeLabel] = gatewayNodeLabelValue(testOtherNode) + handWritten := mine.DeepCopy() + delete(handWritten.Labels, bindingManagedByLabel) + for name, obj := range map[string]*bgpv1alpha1.ServiceVIPBinding{"other node": other, "hand-written": handWritten} { + if reqs := r.bindingRuleRequests(obj); len(reqs) != 0 { + t.Errorf("requests for a %s binding = %v, want none", name, reqs) + } + } +} diff --git a/internal/controller/networkrule_controller.go b/internal/controller/networkrule_controller.go index 5703fbde..038c4078 100644 --- a/internal/controller/networkrule_controller.go +++ b/internal/controller/networkrule_controller.go @@ -175,7 +175,12 @@ func (r *NetworkRuleReconciler) updateAcceptedCondition(ctx context.Context, rul return r.Status().Update(ctx, ruleCopy) } - if meta.IsStatusConditionTrue(rule.Status.Conditions, bgpv1alpha1.ConditionTypeAccepted) { + // Already accepted for this generation: nothing to write. A spec edit + // bumps the generation, and the condition has to follow it, or anything + // waiting on Accepted for the current spec (kubectl wait among them) + // treats it as stale. + if cond := meta.FindStatusCondition(rule.Status.Conditions, bgpv1alpha1.ConditionTypeAccepted); cond != nil && + cond.Status == metav1.ConditionTrue && cond.ObservedGeneration == rule.Generation { return nil } setRuleCondition(ruleCopy, metav1.Condition{ diff --git a/internal/controller/networkrule_controller_test.go b/internal/controller/networkrule_controller_test.go index 8941378d..a77ebf6e 100644 --- a/internal/controller/networkrule_controller_test.go +++ b/internal/controller/networkrule_controller_test.go @@ -18,6 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/galactic/internal/crdnames" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -39,8 +40,8 @@ const ( // BGPRouter/BGPAdvertisement newBackendFixtures returns, which resolves // it to a real SRv6 uSID — required for buildDesiredRule to succeed // (design plan decision #5; see usidresolver.go). - testBackendAddr = "10.0.0.1" - testBackendPrefix = "10.0.0.0/24" + testBackendAddr = "fd00:10::1" + testBackendPrefix = "fd00:10::/64" testBackendRouterName = "backend-router" testBackendLocator = "2001:db8:ff01::/48" testBackendNodeID = 7 @@ -70,6 +71,9 @@ func newRuleTestScheme(t *testing.T) *runtime.Scheme { if err := bgpv1alpha1.AddToScheme(scheme); err != nil { t.Fatalf("AddToScheme: %v", err) } + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("AddToScheme cloud: %v", err) + } return scheme } @@ -89,10 +93,51 @@ func newTestRule(name, vpcRef string, vips ...string) *bgpv1alpha1.NetworkRule { VIPAddresses: vips, Protocol: bgpv1alpha1.NetworkRuleProtocolTCP, Port: 443, - Backends: []bgpv1alpha1.NetworkRuleBackend{ - {Address: testBackendAddr, Port: 8443}, + BackendSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{testBackendLabel: testBackendLabelValue}, + }, + BackendPort: testBackendPort, + }, + } +} + +// testBackendLabel and testBackendLabelValue are the label newTestRule's +// selector matches and newBackendAttachment sets; testBackendPort is the +// rule's backend port. +const ( + testBackendLabel = "app" + testOtherNode = "node-elsewhere" + testBackendLabelValue = "backend" + testBackendPort = 8443 +) + +// newBackendAttachment returns the VPCAttachment newTestRule's selector picks +// for vpcRef: addr on testComputeNodeName, the node newBackendFixtures' +// router targets, so the two together resolve one backend. With no addr it +// carries testBackendAddr. +func newBackendAttachment(vpcRef string, addrs ...string) *cloudv1alpha1.VPCAttachment { + if len(addrs) == 0 { + addrs = []string{testBackendAddr} + } + interfaceAddrs := make([]cloudv1alpha1.IPAddress, 0, len(addrs)) + for _, a := range addrs { + interfaceAddrs = append(interfaceAddrs, cloudv1alpha1.IPAddress(a+"/64")) + } + return &cloudv1alpha1.VPCAttachment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant-" + vpcRef, + Name: "backend-" + vpcRef, + Labels: map[string]string{testBackendLabel: testBackendLabelValue}, + }, + Spec: cloudv1alpha1.VPCAttachmentSpec{ + VPC: cloudv1alpha1.VPCRef{Name: vpcRef}, + Interface: cloudv1alpha1.VPCAttachmentInterface{ + Name: "eth0", + Mode: cloudv1alpha1.VPCAttachmentInterfaceModeNetns, + Addresses: interfaceAddrs, }, }, + Status: cloudv1alpha1.VPCAttachmentStatus{VPC: vpcRef, Node: testComputeNodeName}, } } @@ -204,6 +249,42 @@ func TestNetworkRuleReconciler_SetsFinalizerAndAccepted(t *testing.T) { } } +// TestNetworkRuleReconciler_AcceptedFollowsSpecGeneration covers a spec edit to +// an accepted rule: Accepted stays True but records the new generation, so +// anything waiting on Accepted for the current spec, kubectl wait among them, +// does not see a stale condition. +func TestNetworkRuleReconciler_AcceptedFollowsSpecGeneration(t *testing.T) { + scheme := newRuleTestScheme(t) + rule := newTestRule(testRuleName, "vpc-1", testVIP) + rule.Generation = 2 + rule.Finalizers = []string{networkRuleFinalizer} + meta.SetStatusCondition(&rule.Status.Conditions, metav1.Condition{ + Type: bgpv1alpha1.ConditionTypeAccepted, Status: metav1.ConditionTrue, + Reason: "GatewayNodesRegistered", ObservedGeneration: 1, + }) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource(&bgpv1alpha1.NetworkRule{}). + WithObjects(newTestGateway(testNodeGWA), rule). + Build() + + r := &NetworkRuleReconciler{Client: fakeClient, APIReader: fakeClient, Scheme: scheme, NodeName: testNodeGWA} + req := ctrl.Request{NamespacedName: testRuleKey(testRuleName)} + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: unexpected error: %v", err) + } + + got := &bgpv1alpha1.NetworkRule{} + if err := fakeClient.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("get rule: %v", err) + } + cond := meta.FindStatusCondition(got.Status.Conditions, bgpv1alpha1.ConditionTypeAccepted) + if cond == nil || cond.Status != metav1.ConditionTrue || cond.ObservedGeneration != got.Generation { + t.Errorf("Accepted = %+v, want True at generation %d", cond, got.Generation) + } +} + // TestNetworkRuleReconciler_UpdateAcceptedCondition_FalseWithNoGatewayNodes // covers the converse of the above directly against updateAcceptedCondition: // with no NetworkGateway registered in the namespace, Accepted must be diff --git a/internal/controller/rulebackends.go b/internal/controller/rulebackends.go new file mode 100644 index 00000000..cd0ef0da --- /dev/null +++ b/internal/controller/rulebackends.go @@ -0,0 +1,240 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controller + +import ( + "context" + "errors" + "fmt" + "net/netip" + "sort" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + "go.datum.net/galactic/internal/plumbing/srv6" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// ruleBackend is one backend a NetworkRule's BackendSelector picks: one IPv6 +// interface address of one selected VPCAttachment, on the rule's BackendPort. +// +// galactic-gateway load-balances to it and galactic-router on node writes its +// ServiceVIPBinding, and both derive it from the same objects through +// selectRuleBackends, so the gateway never sends a flow to a node that has no +// binding planned for it. +type ruleBackend struct { + attachment types.NamespacedName + node string + addr netip.Addr + port uint16 + egressKind bgpv1alpha1.ServiceVIPBindingEgressKind +} + +// String returns the backend as "address:port", the form status messages use. +func (b ruleBackend) String() string { + return netip.AddrPortFrom(b.addr, b.port).String() +} + +// listVPCAttachments lists every VPCAttachment in the cluster. Attachments live +// in tenant namespaces while NetworkRules live in galactic-system, so the list +// is cluster-wide; selectRuleBackends scopes it to one rule's VPC. +func listVPCAttachments(ctx context.Context, c client.Client) ([]*cloudv1alpha1.VPCAttachment, error) { + list := &cloudv1alpha1.VPCAttachmentList{} + if err := c.List(ctx, list); err != nil { + return nil, fmt.Errorf("list VPCAttachments: %w", err) + } + out := make([]*cloudv1alpha1.VPCAttachment, 0, len(list.Items)) + for i := range list.Items { + out = append(out, &list.Items[i]) + } + return out, nil +} + +// selectRuleBackends returns the backends rule's BackendSelector picks from +// attachments, sorted by address, and the selected attachments that are not +// backends yet, as "namespace/name: reason". +// +// Only an attachment whose observed VPC equals rule.Spec.VPCRef is a +// candidate, whatever its labels, so a selector cannot reach another tenant's +// VPC. A selected attachment with no observed node, or no IPv6 interface +// address, is pending rather than a backend: the DSR datapath carries only +// IPv6 backends. An attachment with several IPv6 addresses contributes each of +// them. +// +// An invalid selector is a spec error and fails the rule. +func selectRuleBackends( + rule *bgpv1alpha1.NetworkRule, attachments []*cloudv1alpha1.VPCAttachment, +) (backends []ruleBackend, pending []string, err error) { + selector, err := ruleBackendSelector(rule) + if err != nil { + return nil, nil, err + } + //nolint:gosec // BackendPort is CRD-validated to [1,65535] (Minimum/Maximum markers on NetworkRuleSpec.BackendPort) + port := uint16(rule.Spec.BackendPort) + + for _, attachment := range attachments { + if attachment.Status.VPC != rule.Spec.VPCRef || !selector.Matches(labels.Set(attachment.Labels)) { + continue + } + key := types.NamespacedName{Namespace: attachment.Namespace, Name: attachment.Name} + if attachment.Status.Node == "" { + pending = append(pending, key.String()+": no node") + continue + } + addrs := attachmentIPv6Addresses(attachment) + if len(addrs) == 0 { + pending = append(pending, key.String()+": no IPv6 address") + continue + } + kind := attachmentEgressKind(attachment.Spec.Interface.Mode) + for _, addr := range addrs { + backends = append(backends, ruleBackend{ + attachment: key, + node: attachment.Status.Node, + addr: addr, + port: port, + egressKind: kind, + }) + } + } + + // The list order is the cache's, so sort for a stable Maglev input and a + // stable status message. Two attachments claiming one address keep their + // relative order by name. + sort.SliceStable(backends, func(i, j int) bool { + if c := backends[i].addr.Compare(backends[j].addr); c != 0 { + return c < 0 + } + return backends[i].attachment.String() < backends[j].attachment.String() + }) + backends, rejected := dropConflictingBackends(backends) + pending = append(pending, rejected...) + sort.Strings(pending) + return backends, pending, nil +} + +// dropConflictingBackends removes, from backends in their sorted order, each +// backend that cannot get its own vip_xlat_table rows, and returns why as +// "namespace/name: reason". +// +// - An address a backend earlier in the order already claims: two +// attachments reporting one address would put two Maglev entries, maybe +// on two nodes, behind one real backend. +// - A slot (srv6.BackendSlot) an earlier backend on the same node already +// uses: the node keys its ingress row on the slot, so the later backend's +// binding could only report Conflict while the gateway kept sending it +// flows. +// +// The order is the same on every node, so the gateway and every backend node +// drop the same backend. +func dropConflictingBackends(backends []ruleBackend) (kept []ruleBackend, rejected []string) { + type nodeSlot struct { + node string + slot uint16 + } + byAddr := make(map[netip.Addr]ruleBackend, len(backends)) + bySlot := make(map[nodeSlot]ruleBackend, len(backends)) + for _, b := range backends { + if first, ok := byAddr[b.addr]; ok { + rejected = append(rejected, fmt.Sprintf("%s: address %s is also claimed by %s", + b.attachment, b.addr, first.attachment)) + continue + } + key := nodeSlot{node: b.node, slot: srv6.BackendSlot(b.addr, b.port)} + if first, ok := bySlot[key]; ok { + rejected = append(rejected, fmt.Sprintf("%s: backend %s shares slot %#04x on node %s with %s", + b.attachment, b, key.slot, b.node, first)) + continue + } + byAddr[b.addr] = b + bySlot[key] = b + kept = append(kept, b) + } + return kept, rejected +} + +// ruleTranslatedVIP returns the one IPv6 VIP of rule that backend nodes +// translate, or false if it has none. A rule may carry at most one: a backend +// node rewrites a reply's source from the backend's address and port back to +// a single VIP, so a backend cannot answer for a second IPv6 VIP and that +// VIP's flows would be lost. IPv4 VIPs are not translated (#705). +func ruleTranslatedVIP(rule *bgpv1alpha1.NetworkRule) (netip.Addr, bool, error) { + var found []netip.Addr + for _, v := range rule.Spec.VIPAddresses { + vip, err := netip.ParseAddr(v) + if err != nil { + return netip.Addr{}, false, fmt.Errorf("invalid VIP address %q: %w", v, err) + } + if vip.Is6() && !vip.Is4In6() { + found = append(found, vip) + } + } + switch len(found) { + case 0: + return netip.Addr{}, false, nil + case 1: + return found[0], true, nil + default: + return netip.Addr{}, false, fmt.Errorf( + "rule has %d IPv6 VIPs but a backend can answer for only one; split it into one rule per IPv6 VIP", + len(found)) + } +} + +// ruleBackendSelector converts rule's BackendSelector to a labels.Selector. +// The CRD rejects an empty selector; this rejects one anyway, since an empty +// selector would otherwise match every attachment in the VPC. +func ruleBackendSelector(rule *bgpv1alpha1.NetworkRule) (labels.Selector, error) { + sel := rule.Spec.BackendSelector + if len(sel.MatchLabels) == 0 && len(sel.MatchExpressions) == 0 { + return nil, errors.New("backendSelector is empty") + } + selector, err := metav1.LabelSelectorAsSelector(&sel) + if err != nil { + return nil, fmt.Errorf("invalid backendSelector: %w", err) + } + return selector, nil +} + +// attachmentIPv6Addresses returns attachment's IPv6 interface addresses, with +// their prefix lengths dropped. An entry that does not parse is skipped: the +// Cloud API validates them, and one bad entry should not hide the others. +func attachmentIPv6Addresses(attachment *cloudv1alpha1.VPCAttachment) []netip.Addr { + var out []netip.Addr + for _, a := range attachment.Spec.Interface.Addresses { + addr, err := parseInterfaceAddress(string(a)) + if err != nil || !addr.Is6() || addr.Is4In6() { + continue + } + out = append(out, addr) + } + return out +} + +// parseInterfaceAddress parses an interface address in CIDR notation, as the +// Cloud API stores it, or a bare address. +func parseInterfaceAddress(s string) (netip.Addr, error) { + if prefix, err := netip.ParsePrefix(s); err == nil { + return prefix.Addr(), nil + } + return netip.ParseAddr(s) +} + +// attachmentEgressKind maps an attachment's interface mode to the binding +// mechanism its node uses: a container's interface is a veth in its netns, and +// a VM's is a tap handed to the hypervisor. An empty mode is the Cloud API's +// default, Netns. +func attachmentEgressKind(mode cloudv1alpha1.VPCAttachmentInterfaceMode) bgpv1alpha1.ServiceVIPBindingEgressKind { + switch mode { + case cloudv1alpha1.VPCAttachmentInterfaceModeHypervisor, cloudv1alpha1.VPCAttachmentInterfaceModeHypervisorDeclared: + return bgpv1alpha1.ServiceVIPBindingEgressKindTap + default: + return bgpv1alpha1.ServiceVIPBindingEgressKindVeth + } +} diff --git a/internal/controller/rulebackends_test.go b/internal/controller/rulebackends_test.go new file mode 100644 index 00000000..72d4a937 --- /dev/null +++ b/internal/controller/rulebackends_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package controller + +import ( + "fmt" + "net/netip" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/types" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + "go.datum.net/galactic/internal/plumbing/srv6" +) + +// collidingAddrs returns two addresses in fd00:10::/64 whose srv6.BackendSlot +// on port is equal, found by search: a 16-bit slot space makes one turn up +// within a few hundred candidates. +func collidingAddrs(t *testing.T, port uint16) (netip.Addr, netip.Addr) { + t.Helper() + seen := make(map[uint16]netip.Addr) + for i := 1; i < 1<<20; i++ { + addr := netip.MustParseAddr(fmt.Sprintf("fd00:10::%x:%x", i>>16, i&0xffff)) + slot := srv6.BackendSlot(addr, port) + if first, ok := seen[slot]; ok { + return first, addr + } + seen[slot] = addr + } + t.Fatal("no two addresses share a slot") + return netip.Addr{}, netip.Addr{} +} + +func TestDropConflictingBackends(t *testing.T) { + const port = 8443 + att := func(name string) types.NamespacedName { return types.NamespacedName{Namespace: "tenant", Name: name} } + a, b := collidingAddrs(t, port) + other := netip.MustParseAddr("fd00:10::ffff:1") + + backends := []ruleBackend{ + {attachment: att("first"), node: testNAT66NodeA, addr: a, port: port}, + {attachment: att("dup"), node: testNAT66NodeB, addr: a, port: port}, + {attachment: att("slot"), node: testNAT66NodeA, addr: b, port: port}, + {attachment: att("slot-elsewhere"), node: testNAT66NodeB, addr: b, port: port}, + {attachment: att("plain"), node: testNAT66NodeA, addr: other, port: port}, + } + // The slot entry is dropped for its slot on node-a, so it claims no + // address, and slot-elsewhere, the same address on node-b where the slot + // is free, is kept. + + kept, rejected := dropConflictingBackends(backends) + + keptNames := make([]string, 0, len(kept)) + for _, k := range kept { + keptNames = append(keptNames, k.attachment.Name) + } + if strings.Join(keptNames, ",") != "first,slot-elsewhere,plain" { + t.Errorf("kept = %v, want [first slot-elsewhere plain]", keptNames) + } + if len(rejected) != 2 { + t.Fatalf("rejected = %v, want 2 entries", rejected) + } + if !strings.Contains(rejected[0], "tenant/dup: address") { + t.Errorf("rejected[0] = %q, want the duplicate address", rejected[0]) + } + if !strings.Contains(rejected[1], "tenant/slot: backend") || !strings.Contains(rejected[1], "shares slot") { + t.Errorf("rejected[1] = %q, want the slot collision on the first node", rejected[1]) + } +} + +// TestDropConflictingBackends_SlotOnOtherNode covers two backends whose slots +// match on different nodes: each node keys its own rows, so both are kept. +func TestDropConflictingBackends_SlotOnOtherNode(t *testing.T) { + const port = 8443 + a, b := collidingAddrs(t, port) + kept, rejected := dropConflictingBackends([]ruleBackend{ + {attachment: types.NamespacedName{Name: "a"}, node: testNAT66NodeA, addr: a, port: port}, + {attachment: types.NamespacedName{Name: "b"}, node: testNAT66NodeB, addr: b, port: port}, + }) + if len(kept) != 2 || len(rejected) != 0 { + t.Errorf("kept = %d, rejected = %v, want both kept", len(kept), rejected) + } +} + +// TestSelectRuleBackends_DropsConflictingBackends covers the selection both +// the gateway and the binding writer use: an attachment repeating another's +// address, and one whose backend shares a slot with another on its node, are +// left out and reported as pending, so neither side plans flows or bindings +// for them. +func TestSelectRuleBackends_DropsConflictingBackends(t *testing.T) { + rule := newTestRule(testRuleName, testVPCRef, testVIP) + a, b := collidingAddrs(t, testBackendPort) + + first := newBackendAttachment(testVPCRef, a.String()) + first.Name = "first" + dup := newBackendAttachment(testVPCRef, a.String()) + dup.Name = "dup" + slot := newBackendAttachment(testVPCRef, b.String()) + slot.Name = "slot" + + backends, pending, err := selectRuleBackends(rule, []*cloudv1alpha1.VPCAttachment{slot, dup, first}) + if err != nil { + t.Fatalf("selectRuleBackends: %v", err) + } + if len(backends) != 1 || backends[0].addr != a || backends[0].attachment.Name != "dup" { + t.Errorf("backends = %+v, want only %s from the first attachment by name", backends, a) + } + joined := strings.Join(pending, "\n") + if !strings.Contains(joined, "/first: address") || !strings.Contains(joined, "/slot: backend") { + t.Errorf("pending = %v, want the repeated address and the slot collision reported", pending) + } +} diff --git a/internal/controller/servicevipbinding_conflict_test.go b/internal/controller/servicevipbinding_conflict_test.go index 5efad153..c7c8593d 100644 --- a/internal/controller/servicevipbinding_conflict_test.go +++ b/internal/controller/servicevipbinding_conflict_test.go @@ -7,6 +7,7 @@ package controller import ( "context" "net" + "net/netip" "testing" "time" @@ -19,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "go.datum.net/galactic/internal/plumbing/srv6" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -86,8 +88,11 @@ func boundCondition(t *testing.T, c client.Client, name string) *metav1.Conditio return meta.FindStatusCondition(got.Status.Conditions, bgpv1alpha1.ConditionTypeBound) } -func ingressRow(vip string) fakeRowKey { - return fakeRowKey{addr: net.ParseIP(vip).String(), port: 443} +// ingressRow is the ingress row a conflict-test binding for vip and backend +// claims, keyed on the backend's slot. +func ingressRow(vip, backend string) fakeRowKey { + slot := srv6.BackendSlot(netip.MustParseAddr(backend), 8443) + return fakeRowKey{slot: slot, addr: net.ParseIP(vip).String(), port: 443} } func egressRow(backend string) fakeRowKey { @@ -111,10 +116,10 @@ func TestServiceVIPBinding_SharedPortDistinctVIPsBothBind(t *testing.T) { } } want := map[fakeRowKey]string{ - ingressRow(testConflictVIPA): net.ParseIP(testConflictBackendA).String(), - ingressRow(testConflictVIPB): net.ParseIP(testConflictBackendB).String(), - egressRow(testConflictBackendA): net.ParseIP(testConflictVIPA).String(), - egressRow(testConflictBackendB): net.ParseIP(testConflictVIPB).String(), + ingressRow(testConflictVIPA, testConflictBackendA): net.ParseIP(testConflictBackendA).String(), + ingressRow(testConflictVIPB, testConflictBackendB): net.ParseIP(testConflictBackendB).String(), + egressRow(testConflictBackendA): net.ParseIP(testConflictVIPA).String(), + egressRow(testConflictBackendB): net.ParseIP(testConflictVIPB).String(), } if len(table.rows) != len(want) { t.Fatalf("rows = %v, want %v", table.rows, want) @@ -126,14 +131,40 @@ func TestServiceVIPBinding_SharedPortDistinctVIPsBothBind(t *testing.T) { } } -// TestServiceVIPBinding_SameVIPPortConflicts: a younger binding claiming an -// older binding's VIP and port is not programmed and reports Conflict, and the -// older binding's rows are untouched, whichever reconciles first. -func TestServiceVIPBinding_SameVIPPortConflicts(t *testing.T) { +// TestServiceVIPBinding_SameVIPDistinctBackendsBothBind: two backends of one +// VIP and port in the same VRF on one node each get their own ingress row, +// keyed on their own slot, so neither conflicts with the other. +func TestServiceVIPBinding_SameVIPDistinctBackendsBothBind(t *testing.T) { + a := newConflictBinding("a", testConflictVIPA, testConflictBackendA, 20) + b := newConflictBinding("b", testConflictVIPA, testConflictBackendB, 10) + table := &fakeVIPTable{} + r, c := newConflictReconciler(t, table, a, b) + + reconcileBinding(t, r, "a") + reconcileBinding(t, r, "b") + + for _, name := range []string{"a", "b"} { + if cond := boundCondition(t, c, name); cond == nil || cond.Status != metav1.ConditionTrue { + t.Errorf("binding %s Bound = %+v, want True", name, cond) + } + } + assertRowRewrite(t, table, ingressRow(testConflictVIPA, testConflictBackendA), testConflictBackendA, + "backend A's ingress row") + assertRowRewrite(t, table, ingressRow(testConflictVIPA, testConflictBackendB), testConflictBackendB, + "backend B's ingress row") + assertRowRewrite(t, table, egressRow(testConflictBackendA), testConflictVIPA, "backend A's egress row") + assertRowRewrite(t, table, egressRow(testConflictBackendB), testConflictVIPA, "backend B's egress row") +} + +// TestServiceVIPBinding_SameBackendConflicts: a younger binding claiming an +// older binding's backend address and port, for another VIP, is not programmed +// and reports Conflict, and the older binding's rows are untouched, whichever +// reconciles first. The backend's reply can be rewritten to only one VIP. +func TestServiceVIPBinding_SameBackendConflicts(t *testing.T) { for _, order := range [][]string{{testConflictOld, testConflictYoung}, {testConflictYoung, testConflictOld}} { t.Run(order[0]+"-first", func(t *testing.T) { old := newConflictBinding(testConflictOld, testConflictVIPA, testConflictBackendA, 20) - young := newConflictBinding(testConflictYoung, testConflictVIPA, testConflictBackendB, 10) + young := newConflictBinding(testConflictYoung, testConflictVIPB, testConflictBackendA, 10) table := &fakeVIPTable{} r, c := newConflictReconciler(t, table, old, young) @@ -148,12 +179,12 @@ func TestServiceVIPBinding_SameVIPPortConflicts(t *testing.T) { if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "Conflict" { t.Fatalf("young Bound = %+v, want False/Conflict", cond) } - assertRowRewrite(t, table, ingressRow(testConflictVIPA), testConflictBackendA, "VIP A ingress row") - if _, ok := table.rows[egressRow(testConflictBackendB)]; ok { - t.Errorf("young's egress row is present, want absent: a binding in conflict writes neither row") + assertRowRewrite(t, table, egressRow(testConflictBackendA), testConflictVIPA, "backend A egress row") + if _, ok := table.rows[ingressRow(testConflictVIPB, testConflictBackendA)]; ok { + t.Errorf("young's ingress row is present, want absent: a binding in conflict writes neither row") } for _, call := range table.ingressCalls { - if call.addr2.Equal(net.ParseIP(testConflictBackendB)) { + if call.addr1.Equal(net.ParseIP(testConflictVIPB)) { t.Errorf("young registered an ingress row: %+v", call) } } @@ -165,7 +196,7 @@ func TestServiceVIPBinding_SameVIPPortConflicts(t *testing.T) { // binding in conflict leaves the owner's shared row in place. func TestServiceVIPBinding_DeletingConflictedBindingKeepsOwnerRow(t *testing.T) { old := newConflictBinding(testConflictOld, testConflictVIPA, testConflictBackendA, 20) - young := newConflictBinding(testConflictYoung, testConflictVIPA, testConflictBackendB, 10) + young := newConflictBinding(testConflictYoung, testConflictVIPB, testConflictBackendA, 10) table := &fakeVIPTable{} r, c := newConflictReconciler(t, table, old, young) reconcileBinding(t, r, testConflictOld) @@ -180,9 +211,9 @@ func TestServiceVIPBinding_DeletingConflictedBindingKeepsOwnerRow(t *testing.T) } reconcileBinding(t, r, testConflictYoung) - assertRowRewrite(t, table, ingressRow(testConflictVIPA), testConflictBackendA, "VIP A ingress row") - if _, ok := table.rows[egressRow(testConflictBackendA)]; !ok { - t.Errorf("old's egress row is gone after deleting young") + assertRowRewrite(t, table, egressRow(testConflictBackendA), testConflictVIPA, "backend A egress row") + if _, ok := table.rows[ingressRow(testConflictVIPA, testConflictBackendA)]; !ok { + t.Errorf("old's ingress row is gone after deleting young") } err := c.Get(context.Background(), conflictKey(testConflictYoung), got) if !apierrors.IsNotFound(err) { @@ -194,7 +225,7 @@ func TestServiceVIPBinding_DeletingConflictedBindingKeepsOwnerRow(t *testing.T) // its rows, and the binding in conflict then takes the shared row over. func TestServiceVIPBinding_DeletingOwnerHandsRowOver(t *testing.T) { old := newConflictBinding(testConflictOld, testConflictVIPA, testConflictBackendA, 20) - young := newConflictBinding(testConflictYoung, testConflictVIPA, testConflictBackendB, 10) + young := newConflictBinding(testConflictYoung, testConflictVIPB, testConflictBackendA, 10) table := &fakeVIPTable{} r, c := newConflictReconciler(t, table, old, young) reconcileBinding(t, r, testConflictOld) @@ -209,16 +240,17 @@ func TestServiceVIPBinding_DeletingOwnerHandsRowOver(t *testing.T) { } reconcileBinding(t, r, testConflictOld) - if _, ok := table.rows[egressRow(testConflictBackendA)]; ok { - t.Errorf("old's own egress row survived its deletion") + if _, ok := table.rows[ingressRow(testConflictVIPA, testConflictBackendA)]; ok { + t.Errorf("old's own ingress row survived its deletion") } reconcileBinding(t, r, testConflictYoung) if cond := boundCondition(t, c, testConflictYoung); cond == nil || cond.Status != metav1.ConditionTrue { t.Errorf("young Bound = %+v after old's deletion, want True", cond) } - assertRowRewrite(t, table, ingressRow(testConflictVIPA), testConflictBackendB, "VIP A ingress row") - assertRowRewrite(t, table, egressRow(testConflictBackendB), testConflictVIPA, "young's egress row") + assertRowRewrite(t, table, egressRow(testConflictBackendA), testConflictVIPB, "backend A egress row") + assertRowRewrite(t, table, ingressRow(testConflictVIPB, testConflictBackendA), testConflictBackendA, + "young's ingress row") } // TestServiceVIPBinding_VethUnbindKeepsSharedVIP: deleting a veth binding @@ -298,7 +330,7 @@ func TestServiceVIPBinding_DeletingTwinKeepsRows(t *testing.T) { t.Errorf("unregBinding=%d unregAt=%d, want 0: the twin still needs the rows", len(table.unregBinding), len(table.unregAt)) } - if _, ok := table.rows[ingressRow(testConflictVIPA)]; !ok { + if _, ok := table.rows[ingressRow(testConflictVIPA, testConflictBackendA)]; !ok { t.Errorf("VIP A ingress row is gone after deleting one of two identical bindings") } if _, ok := table.rows[egressRow(testConflictBackendA)]; !ok { diff --git a/internal/controller/servicevipbinding_controller.go b/internal/controller/servicevipbinding_controller.go index e8e51cec..e5d0ccfd 100644 --- a/internal/controller/servicevipbinding_controller.go +++ b/internal/controller/servicevipbinding_controller.go @@ -23,8 +23,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" + "go.datum.net/galactic/internal/crdnames" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" "go.datum.net/galactic/internal/plumbing/ebpf/vipxlatmap" + "go.datum.net/galactic/internal/plumbing/srv6" "go.datum.net/galactic/internal/plumbing/vip" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -54,15 +56,15 @@ var ( // both egress kinds, satisfied by *vipxlatmap.VipXlatTable in production and a // fake in tests. type VIPTranslationTable interface { - RegisterIngress(block uint64, argument uint16, proto uint8, + RegisterIngress(block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) error RegisterEgress(block uint64, argument uint16, proto uint8, backendAddr net.IP, backendPort uint16, vipAddr net.IP, vipPort uint16) error - UnregisterIngress(block uint64, argument uint16, proto uint8, vipAddr net.IP, vipPort uint16) error + UnregisterIngress(block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16) error UnregisterEgress(block uint64, argument uint16, proto uint8, backendAddr net.IP, backendPort uint16) error - UnregisterBinding(proto uint8, vipAddr net.IP, vipPort uint16, + UnregisterBinding(slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error) - UnregisterBindingAt(block uint64, argument uint16, proto uint8, vipAddr net.IP, vipPort uint16, + UnregisterBindingAt(block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error) } @@ -264,7 +266,7 @@ func (r *ServiceVIPBindingReconciler) registerVIPTranslation( } if err := r.VIPTranslationTable.RegisterIngress( - self.ingress.block, self.ingress.argument, self.ingress.proto, + self.ingress.block, self.ingress.argument, self.ingress.slot, self.ingress.proto, self.vipAddr, self.vipPort, self.backendAddr, self.backendPort); err != nil { return fmt.Errorf("register vip_xlat_table ingress row: %w", err) } @@ -389,19 +391,22 @@ func (r *ServiceVIPBindingReconciler) unregisterVIPTranslation( return nil } + backendAddrIP, _ := netip.AddrFromSlice(backendAddr) // a 16-byte slice ipv6Address returned + slot := srv6.BackendSlot(backendAddrIP.Unmap(), backendPort) + var errs []error - if _, err := r.VIPTranslationTable.UnregisterBinding(proto, vipAddr, vipPort, backendAddr, backendPort); err != nil { + if _, err := r.VIPTranslationTable.UnregisterBinding( + slot, proto, vipAddr, vipPort, backendAddr, backendPort); err != nil { errs = append(errs, fmt.Errorf("unregister vip_xlat_table rows by value: %w", err)) } - backendAddrIP, _ := netip.AddrFromSlice(backendAddr) // a 16-byte slice ipv6Address returned - block, argument, err := resolveVIPBindingContext(ctx, r.Client, binding.Namespace, r.NodeName, backendAddrIP.Unmap()) + block, argument, err := resolveVIPBindingContext(ctx, r.Client, binding.Namespace, r.NodeName, binding.Spec.VPCRef) if err != nil { logger.Info("VRF context no longer resolves; skipped checking its vip_xlat_table location", "reason", err.Error()) return errors.Join(errs...) } if _, err := r.VIPTranslationTable.UnregisterBindingAt( - block, argument, proto, vipAddr, vipPort, backendAddr, backendPort); err != nil { + block, argument, slot, proto, vipAddr, vipPort, backendAddr, backendPort); err != nil { errs = append(errs, fmt.Errorf("unregister vip_xlat_table rows at the resolved VRF: %w", err)) } return errors.Join(errs...) @@ -451,29 +456,32 @@ func (r *ServiceVIPBindingReconciler) unregisterRow(row vipRow) error { } return nil } - if err := r.VIPTranslationTable.UnregisterIngress(row.block, row.argument, row.proto, addr, row.port); err != nil { + if err := r.VIPTranslationTable.UnregisterIngress( + row.block, row.argument, row.slot, row.proto, addr, row.port); err != nil { return fmt.Errorf("unregister vip_xlat_table ingress row: %w", err) } return nil } // vipRow identifies one vip_xlat_table row. addr and port are the VIP's for an -// ingress row and the backend's for an egress row. +// ingress row and the backend's for an egress row. slot is the backend slot an +// ingress row is keyed on, so two backends of one VIP claim different ingress +// rows; it is always 0 on an egress row. type vipRow struct { egress bool block uint64 argument uint16 + slot uint16 proto uint8 addr netip.Addr port uint16 } func (r vipRow) String() string { - dir := "ingress" if r.egress { - dir = "egress" + return fmt.Sprintf("egress row for VRF %d %s", r.argument, netip.AddrPortFrom(r.addr, r.port)) } - return fmt.Sprintf("%s row for VRF %d %s", dir, r.argument, netip.AddrPortFrom(r.addr, r.port)) + return fmt.Sprintf("ingress row for VRF %d slot %#04x %s", r.argument, r.slot, netip.AddrPortFrom(r.addr, r.port)) } // vipBindingRows is one binding's claim on vip_xlat_table, with the addresses @@ -505,7 +513,7 @@ func resolveVIPBindingRows( return vipBindingRows{}, err } - block, argument, err := resolveVIPBindingContextFromIndex(idx, nodeName, backendIP) + block, argument, err := resolveVIPBindingContextFromIndex(idx, nodeName, binding.Spec.VPCRef) if err != nil { return vipBindingRows{}, fmt.Errorf("resolve VRF context for VIP binding: %w", err) } @@ -513,8 +521,10 @@ func resolveVIPBindingRows( vipPort := uint16(binding.Spec.Port) //nolint:gosec // kubebuilder-validated 1-65535 backendPort := uint16(binding.Spec.BackendPort) //nolint:gosec // kubebuilder-validated 1-65535 + slot := srv6.BackendSlot(backendIP, backendPort) + return vipBindingRows{ - ingress: vipRow{block: block, argument: argument, proto: proto, addr: vipIP, port: vipPort}, + ingress: vipRow{block: block, argument: argument, slot: slot, proto: proto, addr: vipIP, port: vipPort}, egress: vipRow{egress: true, block: block, argument: argument, proto: proto, addr: backendIP, port: backendPort}, vipAddr: net.IP(vipIP.AsSlice()), backendAddr: net.IP(backendIP.AsSlice()), @@ -653,45 +663,31 @@ func ipProtocolNumber(proto bgpv1alpha1.NetworkRuleProtocol) (uint8, error) { // resolveVIPBindingContext resolves the (block, argument) pair vip_xlat_table's // key needs for a binding on this node: the Block from this node's BGPRouter -// locator, and the Argument from the BGPVRFInstance whose advertised prefix -// contains backendAddr. -// -// # A documented ambiguity -// -// ServiceVIPBinding carries no VPC or VRF reference of its own. The writer that -// creates these objects has the owning rule's VPC reference at creation time, -// but does not record it here. Absent that field, ownership is resolved by -// matching backendAddr against the advertised prefixes of BGPVRFInstances -// targeting this node's own BGPRouter, since a binding's VRF context is always -// local to the node it was written for. +// locator, and the Argument from vpcRef's BGPVRFInstance on this node, named +// crdnames.BGPVRFInstanceName(vpcRef, nodeName). // -// That is unambiguous only while no two VRFs on this node advertise overlapping -// prefixes containing backendAddr, such as two tenants choosing the same ULA -// range. The ownership guard used elsewhere needs a known VPC reference to check -// against, which this function does not have, so when more than one candidate -// matches it fails with an explicit error rather than guessing: picking one -// would translate a backend's traffic into the wrong tenant's VRF. -// -// TODO(dsr-maglev): once ServiceVIPBinding carries a VPC or VRF reference, or -// the writer records the resolved (block, argument) on the object, replace this -// address-containment heuristic with a direct lookup by -// crdnames.BGPVRFInstanceName. +// The binding names its VPC, so this is a direct lookup. Two tenants on one +// node using the same backend address resolve to their own VRFs, and a VPC +// with no VRF on this node yet fails rather than borrowing another's. func resolveVIPBindingContext( - ctx context.Context, c client.Client, namespace, nodeName string, backendAddr netip.Addr, + ctx context.Context, c client.Client, namespace, nodeName, vpcRef string, ) (block uint64, argument uint16, err error) { idx, err := buildBackendSIDIndex(ctx, c, namespace) if err != nil { return 0, 0, fmt.Errorf("build backend SID index: %w", err) } - return resolveVIPBindingContextFromIndex(idx, nodeName, backendAddr) + return resolveVIPBindingContextFromIndex(idx, nodeName, vpcRef) } // resolveVIPBindingContextFromIndex is resolveVIPBindingContext over an index // the caller already built, so resolving every binding on a node lists each // resource once. func resolveVIPBindingContextFromIndex( - idx *backendSIDIndex, nodeName string, backendAddr netip.Addr, + idx *backendSIDIndex, nodeName, vpcRef string, ) (block uint64, argument uint16, err error) { + if vpcRef == "" { + return 0, 0, errors.New("binding has no vpcRef") + } var router *bgpv1alpha1.BGPRouter for _, rt := range idx.routers { if rt.Spec.TargetRef.Name == nodeName { @@ -715,53 +711,15 @@ func resolveVIPBindingContextFromIndex( return 0, 0, fmt.Errorf("derive uSID Block from BGPRouter %s's SRv6Locator: %w", router.Name, err) } - localVRFIDs := make(map[int32]struct{}) - for _, instance := range idx.vrfInstances { - if vrfInstanceTargetsRouter(instance, router) { - localVRFIDs[instance.Spec.VRFID] = struct{}{} - } + name := crdnames.BGPVRFInstanceName(vpcRef, nodeName) + instance, ok := idx.vrfInstances[name] + if !ok { + return 0, 0, fmt.Errorf("VPC %s has no BGPVRFInstance %s on node %q", vpcRef, name, nodeName) } - - seen := make(map[int32]struct{}) - var matches []int32 - for _, adv := range idx.advs { - if adv.Spec.RouterRef.Name != router.Name || adv.Spec.VRFID == nil { - continue - } - vrfID := *adv.Spec.VRFID - if _, ok := localVRFIDs[vrfID]; !ok { - continue // not one of this node's own local VRFs - } - if _, already := seen[vrfID]; already { - continue - } - for _, p := range adv.Spec.Prefixes { - pfx, perr := netip.ParsePrefix(string(p)) - if perr != nil { - continue - } - if pfx.Contains(backendAddr) { - seen[vrfID] = struct{}{} - matches = append(matches, vrfID) - break - } - } - } - - switch len(matches) { - case 0: - return 0, 0, fmt.Errorf( - "no local BGPVRFInstance on node %q has an advertised BGPAdvertisement prefix containing backend address %s", - nodeName, backendAddr) - case 1: - return block, uint16(matches[0]), nil //nolint:gosec // VRFID is kubebuilder-validated 1-65535 - default: - return 0, 0, fmt.Errorf( - "ambiguous VRF ownership for backend address %s on node %q: %d candidate VRFIDs %v all advertise a "+ - "containing prefix, and ServiceVIPBinding carries no VPCRef/VRFRef to disambiguate "+ - "(see resolveVIPBindingContext's doc comment); refusing to guess", - backendAddr, nodeName, len(matches), matches) + if !vrfInstanceTargetsRouter(instance, router) { + return 0, 0, fmt.Errorf("BGPVRFInstance %s does not target node %q's BGPRouter %s", name, nodeName, router.Name) } + return block, uint16(instance.Spec.VRFID), nil //nolint:gosec // VRFID is kubebuilder-validated 1-65535 } // vrfInstanceTargetsRouter reports whether vrf's router target, by reference or diff --git a/internal/controller/servicevipbinding_controller_test.go b/internal/controller/servicevipbinding_controller_test.go index ccee7b67..b72939c0 100644 --- a/internal/controller/servicevipbinding_controller_test.go +++ b/internal/controller/servicevipbinding_controller_test.go @@ -8,7 +8,6 @@ import ( "context" "errors" "net" - "net/netip" "testing" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -37,6 +36,7 @@ func newTestServiceVIPBinding( ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testVIPBindingName}, Spec: bgpv1alpha1.ServiceVIPBindingSpec{ TargetRef: bgpv1alpha1.TargetRef{Kind: testTargetRefKind, Name: nodeName}, + VPCRef: testVPCRef, VIPAddress: testVIPBindingVIPAddr, Port: 8080, Protocol: bgpv1alpha1.NetworkRuleProtocolTCP, @@ -89,9 +89,10 @@ type unregisterCall struct { } // fakeRowKey is one row of fakeVIPTable's simulated vip_xlat_table, ignoring -// the VRF, which every test here shares. +// the VRF, which every test here shares. slot is 0 on an egress row. type fakeRowKey struct { egress bool + slot uint16 addr string port uint16 } @@ -125,11 +126,11 @@ func (f *fakeVIPTable) put(k fakeRowKey, addr net.IP, port uint16) { f.rows[k] = fakeRowValue{addr.String(), port} } -func (f *fakeVIPTable) RegisterIngress(block uint64, argument uint16, proto uint8, +func (f *fakeVIPTable) RegisterIngress(block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) error { f.ingressCalls = append(f.ingressCalls, vipCall{block, argument, proto, vipAddr, vipPort, backendAddr, backendPort}) if f.registerErr == nil { - f.put(fakeRowKey{addr: vipAddr.String(), port: vipPort}, backendAddr, backendPort) + f.put(fakeRowKey{slot: slot, addr: vipAddr.String(), port: vipPort}, backendAddr, backendPort) } return f.registerErr } @@ -144,11 +145,11 @@ func (f *fakeVIPTable) RegisterEgress(block uint64, argument uint16, proto uint8 } func (f *fakeVIPTable) UnregisterIngress( - block uint64, argument uint16, proto uint8, vipAddr net.IP, vipPort uint16, + block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, ) error { f.unregIngress = append(f.unregIngress, unregisterCall{block, argument, proto, vipAddr, vipPort}) if f.unregisterErr == nil { - delete(f.rows, fakeRowKey{addr: vipAddr.String(), port: vipPort}) + delete(f.rows, fakeRowKey{slot: slot, addr: vipAddr.String(), port: vipPort}) } return f.unregisterErr } @@ -165,8 +166,10 @@ func (f *fakeVIPTable) UnregisterEgress( // removeOwnRows deletes the binding's two rows where they still hold its // values, as vipxlatmap's removal by value does. -func (f *fakeVIPTable) removeOwnRows(vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) { - in := fakeRowKey{addr: vipAddr.String(), port: vipPort} +func (f *fakeVIPTable) removeOwnRows( + slot uint16, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16, +) { + in := fakeRowKey{slot: slot, addr: vipAddr.String(), port: vipPort} if f.rows[in] == (fakeRowValue{backendAddr.String(), backendPort}) { delete(f.rows, in) } @@ -176,21 +179,21 @@ func (f *fakeVIPTable) removeOwnRows(vipAddr net.IP, vipPort uint16, backendAddr } } -func (f *fakeVIPTable) UnregisterBindingAt(block uint64, argument uint16, proto uint8, vipAddr net.IP, vipPort uint16, - backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error) { +func (f *fakeVIPTable) UnregisterBindingAt(block uint64, argument, slot uint16, proto uint8, + vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error) { f.unregAt = append(f.unregAt, unregisterAtCall{ block, argument, unregisterBindingCall{proto, vipAddr, vipPort, backendAddr, backendPort}}) if f.unregisterErr == nil { - f.removeOwnRows(vipAddr, vipPort, backendAddr, backendPort) + f.removeOwnRows(slot, vipAddr, vipPort, backendAddr, backendPort) } return nil, f.unregisterErr } -func (f *fakeVIPTable) UnregisterBinding(proto uint8, vipAddr net.IP, vipPort uint16, +func (f *fakeVIPTable) UnregisterBinding(slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) ([]vipxlatmap.Entry, error) { f.unregBinding = append(f.unregBinding, unregisterBindingCall{proto, vipAddr, vipPort, backendAddr, backendPort}) if f.unregisterErr == nil { - f.removeOwnRows(vipAddr, vipPort, backendAddr, backendPort) + f.removeOwnRows(slot, vipAddr, vipPort, backendAddr, backendPort) } return nil, f.unregisterErr } @@ -587,7 +590,7 @@ func TestServiceVIPBindingReconciler_FinalizerKeptWhenRowRemovalFails(t *testing // accepts and vip_xlat_table does not, or an unset one. With the VRF gone too, // deletion must still finish without looking for rows by value. func TestServiceVIPBindingReconciler_DeleteSkipsAddressesWithNoRows(t *testing.T) { - for name, backend := range map[string]string{"ipv4": testBackendAddr, "unset": ""} { + for name, backend := range map[string]string{"ipv4": "10.0.0.1", "unset": ""} { t.Run(name, func(t *testing.T) { scheme := newRuleTestScheme(t) router, _, _ := newBackendFixtures(testVPCRef) @@ -645,7 +648,7 @@ func TestResolveVIPBindingContext_Success(t *testing.T) { fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(router, adv, vrf).Build() block, argument, err := resolveVIPBindingContext( - context.Background(), fakeClient, testNamespace, testComputeNodeName, netip.MustParseAddr(testBackendAddr)) + context.Background(), fakeClient, testNamespace, testComputeNodeName, testVPCRef) if err != nil { t.Fatalf("resolveVIPBindingContext: unexpected error: %v", err) } @@ -657,52 +660,40 @@ func TestResolveVIPBindingContext_Success(t *testing.T) { } } -func TestResolveVIPBindingContext_NoMatchingVRF(t *testing.T) { +func TestResolveVIPBindingContext_NoVRFForVPC(t *testing.T) { scheme := newRuleTestScheme(t) router, adv, vrf := newBackendFixtures(testVPCRef) fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(router, adv, vrf).Build() - if _, _, err := resolveVIPBindingContext( - context.Background(), fakeClient, testNamespace, testComputeNodeName, netip.MustParseAddr("192.0.2.1")); err == nil { - t.Fatal("resolveVIPBindingContext: expected an error for an address with no matching advertised prefix") + for _, vpcRef := range []string{"vpc-absent", ""} { + if _, _, err := resolveVIPBindingContext( + context.Background(), fakeClient, testNamespace, testComputeNodeName, vpcRef); err == nil { + t.Errorf("resolveVIPBindingContext(vpcRef %q): expected an error for a VPC with no VRF on this node", vpcRef) + } } } -// TestResolveVIPBindingContext_AmbiguousFailsClosed covers the documented -// ambiguity: two local BGPVRFInstances on the same node both advertise a -// prefix containing the same backend address (e.g. two tenants using the -// same ULA range) -- resolveVIPBindingContext must fail rather than guess. -func TestResolveVIPBindingContext_AmbiguousFailsClosed(t *testing.T) { +// TestResolveVIPBindingContext_CollidingTenantsResolveToTheirOwnVRF covers two +// tenants on one node advertising the same prefix, such as two VPCs choosing +// the same ULA range. Each binding names its VPC, so each resolves to its own +// VRF rather than failing as ambiguous or picking the other tenant's. +func TestResolveVIPBindingContext_CollidingTenantsResolveToTheirOwnVRF(t *testing.T) { scheme := newRuleTestScheme(t) router, adv, vrf := newBackendFixtures(testVPCRef) - - vrfID2 := int32(testBackendVRFID + 1) - function := bgpv1alpha1.SRv6FunctionEndDT46 - adv2 := &bgpv1alpha1.BGPAdvertisement{ - ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: "second-adv"}, - Spec: bgpv1alpha1.BGPAdvertisementSpec{ - RouterRef: bgpv1alpha1.RouterRef{Name: testBackendRouterName}, - AddressFamily: bgpv1alpha1.AddressFamily{AFI: bgpv1alpha1.AFIL2VPN, SAFI: bgpv1alpha1.SAFIEVPN}, - Prefixes: []bgpv1alpha1.Prefix{testBackendPrefix}, // same prefix, colliding - VRFID: &vrfID2, - Function: &function, - }, - } - vrf2 := &bgpv1alpha1.BGPVRFInstance{ - ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: "second-vrf"}, - Spec: bgpv1alpha1.BGPVRFInstanceSpec{ - RouterTarget: bgpv1alpha1.RouterTarget{RouterRef: &bgpv1alpha1.RouterRef{Name: testBackendRouterName}}, - VRFID: vrfID2, - ImportRouteTargets: []bgpv1alpha1.RouteTarget{{Value: testRouteTargetValue}}, - ExportRouteTargets: []bgpv1alpha1.RouteTarget{{Value: testRouteTargetValue}}, - }, - } + _, adv2, vrf2 := newBackendFixtures("vpc-2") + vrf2.Spec.VRFID = testBackendVRFID + 1 + adv2.Spec.VRFID = ptr(int32(testBackendVRFID + 1)) fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(router, adv, vrf, adv2, vrf2).Build() - _, _, err := resolveVIPBindingContext( - context.Background(), fakeClient, testNamespace, testComputeNodeName, netip.MustParseAddr(testBackendAddr)) - if err == nil { - t.Fatal("resolveVIPBindingContext: expected a fail-closed error for ambiguous VRF ownership") + for vpcRef, want := range map[string]uint16{testVPCRef: testBackendVRFID, "vpc-2": testBackendVRFID + 1} { + _, argument, err := resolveVIPBindingContext( + context.Background(), fakeClient, testNamespace, testComputeNodeName, vpcRef) + if err != nil { + t.Fatalf("resolveVIPBindingContext(%s): %v", vpcRef, err) + } + if argument != want { + t.Errorf("resolveVIPBindingContext(%s) argument = %d, want %d", vpcRef, argument, want) + } } } diff --git a/internal/controller/servicevipbinding_xlat_test.go b/internal/controller/servicevipbinding_xlat_test.go index ceca4fb0..32497bc4 100644 --- a/internal/controller/servicevipbinding_xlat_test.go +++ b/internal/controller/servicevipbinding_xlat_test.go @@ -20,6 +20,7 @@ import ( "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" "go.datum.net/galactic/internal/plumbing/ebpf/vipxlatmap" + "go.datum.net/galactic/internal/plumbing/srv6" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -108,7 +109,7 @@ func deleteBindingWithVRFPresent(t *testing.T, backends ...string) *vipxlatmap.V Build() block, argument, err := resolveVIPBindingContext(context.Background(), fakeClient, testNamespace, - testComputeNodeName, netip.MustParseAddr(testVIPBindingBackendAddr)) + testComputeNodeName, testVPCRef) if err != nil { t.Fatalf("resolveVIPBindingContext: %v", err) } @@ -117,7 +118,8 @@ func deleteBindingWithVRFPresent(t *testing.T, backends ...string) *vipxlatmap.V vip := net.ParseIP(testVIPBindingVIPAddr) for _, b := range backends { backend := net.ParseIP(b) - if err := table.RegisterIngress(block, argument, ipProtoTCP, vip, 8080, backend, 30080); err != nil { + slot := srv6.BackendSlot(netip.MustParseAddr(b), 30080) + if err := table.RegisterIngress(block, argument, slot, ipProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: %v", err) } if err := table.RegisterEgress(block, argument, ipProtoTCP, backend, 30080, vip, 8080); err != nil { diff --git a/internal/controller/usidresolver.go b/internal/controller/usidresolver.go index 9a78d9d2..82961bc9 100644 --- a/internal/controller/usidresolver.go +++ b/internal/controller/usidresolver.go @@ -11,7 +11,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/galactic/internal/crdnames" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" "go.datum.net/galactic/internal/plumbing/srv6" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -42,10 +44,11 @@ type backendSIDIndex struct { routers map[string]*bgpv1alpha1.BGPRouter advs []*bgpv1alpha1.BGPAdvertisement vrfInstances map[string]*bgpv1alpha1.BGPVRFInstance // keyed by name: crdnames.BGPVRFInstanceName(vpc, nodeName) + attachments []*cloudv1alpha1.VPCAttachment // every VPCAttachment in the cluster; see selectRuleBackends } // buildBackendSIDIndex lists every BGPRouter, BGPAdvertisement, and -// BGPVRFInstance in namespace once, so resolving many backends across a +// BGPVRFInstance in namespace, and every VPCAttachment in the cluster, once, so resolving many backends across a // reconcile costs one set of list calls rather than one per backend. // // Advertisements with no VRFID or function set are excluded up front: those @@ -81,7 +84,12 @@ func buildBackendSIDIndex(ctx context.Context, c client.Client, namespace string vrfInstances[vrfList.Items[i].Name] = &vrfList.Items[i] } - return &backendSIDIndex{routers: routers, advs: advs, vrfInstances: vrfInstances}, nil + attachments, err := listVPCAttachments(ctx, c) + if err != nil { + return nil, err + } + + return &backendSIDIndex{routers: routers, advs: advs, vrfInstances: vrfInstances, attachments: attachments}, nil } // verifyTenantOwnership reports whether adv, matched against an address by @@ -100,15 +108,17 @@ func (idx *backendSIDIndex) verifyTenantOwnership(vpcRef string, router *bgpv1al return ok && vrf.Spec.VRFID == vrfID } -// resolveUSID returns the SRv6 uSID of the worker node the backend address addr, -// belonging to vpcRef, is reachable through. vpcRef scopes the match to -// advertisements that tenant's own BGPVRFInstance owns: a prefix-containment hit -// alone is never sufficient, so two tenants with colliding address space cannot -// resolve into each other's VRF. -func (idx *backendSIDIndex) resolveUSID(addr netip.Addr, vpcRef string) (netip.Addr, error) { +// resolveUSID returns the SRv6 uSID of node for the backend address addr, +// belonging to vpcRef. vpcRef scopes the match to advertisements that tenant's +// own BGPVRFInstance owns: a prefix-containment hit alone is never sufficient, +// so two tenants with colliding address space cannot resolve into each other's +// VRF. node is the node the backend's VPCAttachment reports, and only that +// node's routers are candidates, so the gateway never sends a backend's flows +// to a node other than the one galactic-router writes its ServiceVIPBinding on. +func (idx *backendSIDIndex) resolveUSID(addr netip.Addr, vpcRef, node string) (netip.Addr, error) { for _, adv := range idx.advs { router, ok := idx.routers[adv.Spec.RouterRef.Name] - if !ok || router.Spec.SRv6Locator == "" || router.Spec.NodeID == 0 { + if !ok || router.Spec.SRv6Locator == "" || router.Spec.NodeID == 0 || router.Spec.TargetRef.Name != node { continue } if !idx.verifyTenantOwnership(vpcRef, router, *adv.Spec.VRFID) { @@ -132,6 +142,22 @@ func (idx *backendSIDIndex) resolveUSID(addr netip.Addr, vpcRef string) (netip.A } } return netip.Addr{}, fmt.Errorf( - "no BGPAdvertisement owned by VPC %s with a matching prefix and SRv6 VRFID/Function found for backend address %s", - vpcRef, addr) + "no BGPAdvertisement owned by VPC %s on node %s with a matching prefix and SRv6 VRFID/Function "+ + "found for backend address %s", + vpcRef, node, addr) +} + +// backendSID returns nodeSID, a node's End.DT46 uSID for the backend's VRF, +// with the backend's slot (srv6.BackendSlot) set. The gateway encapsulates a +// flow to this address, and the backend's node reads the slot back to pick +// the vip_xlat_table row for the backend Maglev chose, so several backends of +// one VIP can share a node. The node's ServiceVIPBinding reconciler derives +// the same slot from the binding's backend address and port. +func backendSID(nodeSID netip.Addr, addr netip.Addr, port uint16) (netip.Addr, error) { + fields, err := uformat.Decode(nodeSID) + if err != nil { + return netip.Addr{}, fmt.Errorf("decode node uSID %s: %w", nodeSID, err) + } + fields.Slot = srv6.BackendSlot(addr, port) + return uformat.Encode(fields) } diff --git a/internal/controller/usidresolver_test.go b/internal/controller/usidresolver_test.go index 00b20edf..eb921cd3 100644 --- a/internal/controller/usidresolver_test.go +++ b/internal/controller/usidresolver_test.go @@ -32,7 +32,7 @@ func TestBackendSIDIndex_ResolvesMatchingPrefix(t *testing.T) { t.Fatalf("buildBackendSIDIndex: %v", err) } - got, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef) + got, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef, testComputeNodeName) if err != nil { t.Fatalf("resolveUSID: %v", err) } @@ -46,6 +46,24 @@ func TestBackendSIDIndex_ResolvesMatchingPrefix(t *testing.T) { } } +// TestBackendSIDIndex_OtherNodeIsError verifies that a backend resolves only +// on the node its attachment reports: an advertisement from another node's +// router is never a candidate, even with a matching prefix and VPC. +func TestBackendSIDIndex_OtherNodeIsError(t *testing.T) { + scheme := newRuleTestScheme(t) + router, adv, vrf := newBackendFixtures(testVPCRef) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(router, adv, vrf).Build() + + idx, err := buildBackendSIDIndex(context.Background(), fakeClient, testNamespace) + if err != nil { + t.Fatalf("buildBackendSIDIndex: %v", err) + } + + if _, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef, testOtherNode); err == nil { + t.Fatal("resolveUSID: expected error for a node whose router does not advertise the backend") + } +} + func TestBackendSIDIndex_NoMatchingPrefixIsError(t *testing.T) { scheme := newRuleTestScheme(t) router, adv, vrf := newBackendFixtures(testVPCRef) @@ -56,7 +74,7 @@ func TestBackendSIDIndex_NoMatchingPrefixIsError(t *testing.T) { t.Fatalf("buildBackendSIDIndex: %v", err) } - if _, err := idx.resolveUSID(netip.MustParseAddr("192.0.2.1"), testVPCRef); err == nil { + if _, err := idx.resolveUSID(netip.MustParseAddr("192.0.2.1"), testVPCRef, testComputeNodeName); err == nil { t.Fatal("resolveUSID: expected error for an address with no matching BGPAdvertisement prefix") } } @@ -88,7 +106,7 @@ func TestBackendSIDIndex_ExcludesAdvertisementsWithoutVRFIDOrFunction(t *testing t.Fatalf("advs = %d, want 0 (VRFID/Function-less advertisement must be excluded)", len(idx.advs)) } - if _, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef); err == nil { + if _, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef, testComputeNodeName); err == nil { t.Fatal("resolveUSID: expected error since the only matching advertisement has no VRFID/Function") } } @@ -126,7 +144,7 @@ func TestBackendSIDIndex_SkipsRouterWithoutSRv6Config(t *testing.T) { if err != nil { t.Fatalf("buildBackendSIDIndex: %v", err) } - if _, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef); err == nil { + if _, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef, testComputeNodeName); err == nil { t.Fatal("resolveUSID: expected error since the matching router has no SRv6Locator/NodeID") } } @@ -231,7 +249,7 @@ func TestBackendSIDIndex_TenantOwnershipDisambiguatesCollidingPrefixes(t *testin addr := netip.MustParseAddr(testBackendAddr) // falls within both advA's and advB's prefix - gotA, err := idx.resolveUSID(addr, vpcA) + gotA, err := idx.resolveUSID(addr, vpcA, testNAT66NodeA) if err != nil { t.Fatalf("resolveUSID(vpc-1): %v", err) } @@ -243,7 +261,7 @@ func TestBackendSIDIndex_TenantOwnershipDisambiguatesCollidingPrefixes(t *testin t.Errorf("resolveUSID(vpc-1) = %s, want %s (vpc-1's own SID, not vpc-2's)", gotA, wantA) } - gotB, err := idx.resolveUSID(addr, vpcB) + gotB, err := idx.resolveUSID(addr, vpcB, "node-b") if err != nil { t.Fatalf("resolveUSID(vpc-2): %v", err) } @@ -283,7 +301,7 @@ func TestBackendSIDIndex_UnrelatedTenantVRFIsNotTrusted(t *testing.T) { if err != nil { t.Fatalf("buildBackendSIDIndex: %v", err) } - if _, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef); err == nil { + if _, err := idx.resolveUSID(netip.MustParseAddr(testBackendAddr), testVPCRef, testComputeNodeName); err == nil { t.Fatal("resolveUSID: expected error -- no BGPVRFInstance named for testVPCRef exists, " + "a same-node VRF instance belonging to a different VPC must not be trusted") } diff --git a/internal/crdnames/crdnames.go b/internal/crdnames/crdnames.go index 61be894d..4686b696 100644 --- a/internal/crdnames/crdnames.go +++ b/internal/crdnames/crdnames.go @@ -194,17 +194,27 @@ func BGPAdvertisementName(vpc, vpcAttachment, nodeName string) string { return fmt.Sprintf("%s-%s-%s", VPCSegment(vpc), nameSegment(vpcAttachment), nodeName) } -// vipNameReplacer sanitizes an IP address for use inside a Kubernetes object -// name, which must be an RFC 1123 DNS subdomain. Neither ':' nor '.' is valid -// there. -var vipNameReplacer = strings.NewReplacer(":", "-", ".", "-") - -// ServiceVIPBindingName returns the deterministic name for a ServiceVIPBinding, -// one per (node, VIP, protocol, port). It leads with nodeName, which is always -// alphanumeric, so an IPv6 address's leading "::" never produces a name -// starting with '-'. -func ServiceVIPBindingName(nodeName, vip string, port int32, proto string) string { - return fmt.Sprintf("%s-%s-%s-%d", nodeName, vipNameReplacer.Replace(vip), proto, port) +// serviceVIPBindingHashLen is how many hex characters of the binding identity's +// SHA-256 ServiceVIPBindingName keeps. +const serviceVIPBindingHashLen = 16 + +// ServiceVIPBindingName returns the deterministic name of the ServiceVIPBinding +// galactic-router writes for one backend of one NetworkRule VIP on nodeName: +// "--", where the hash covers every part of the identity, so +// two backends of one VIP on one node, the case a selector makes routine, get +// distinct names. The readable prefix is cut to keep the result within an +// object name's 253 characters and trimmed to end in an alphanumeric. +func ServiceVIPBindingName(rule, nodeName, vip, backend string, backendPort int32) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s/%s/%s/%s/%d", rule, nodeName, vip, backend, backendPort))) + suffix := "-" + hex.EncodeToString(sum[:])[:serviceVIPBindingHashLen] + prefix := rule + "-" + nodeName + if maxPrefix := 253 - len(suffix); len(prefix) > maxPrefix { + prefix = prefix[:maxPrefix] + } + prefix = strings.TrimRightFunc(prefix, func(c rune) bool { + return (c < 'a' || c > 'z') && (c < '0' || c > '9') + }) + return prefix + suffix } // TenantIdentifier returns the value identifying a VPC attachment across the diff --git a/internal/crdnames/crdnames_test.go b/internal/crdnames/crdnames_test.go index f5df7b1e..7dc9b641 100644 --- a/internal/crdnames/crdnames_test.go +++ b/internal/crdnames/crdnames_test.go @@ -12,27 +12,35 @@ import ( ) func TestServiceVIPBindingName(t *testing.T) { - tests := []struct { - name string - nodeName, vip, proto string - port int32 - want string - }{ - {"IPv6", "iad-worker", "2001:db8::1", "tcp", 443, "iad-worker-2001-db8--1-tcp-443"}, - {"IPv4", "iad-worker", "203.0.113.5", "udp", 53, "iad-worker-203-0-113-5-udp-53"}, + base := ServiceVIPBindingName("web", "node-a", "2001:db8::1", "fd00:10::1", 8443) + if errs := validation.IsDNS1123Subdomain(base); len(errs) > 0 { + t.Fatalf("ServiceVIPBindingName = %q is not a valid object name: %v", base, errs) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := ServiceVIPBindingName(tt.nodeName, tt.vip, tt.port, tt.proto) - if got != tt.want { - t.Errorf("ServiceVIPBindingName(%q, %q, %d, %q) = %q, want %q", - tt.nodeName, tt.vip, tt.port, tt.proto, got, tt.want) - } - if strings.HasPrefix(got, "-") { - t.Errorf("ServiceVIPBindingName(%q, %q, %d, %q) = %q starts with '-' (invalid k8s name)", - tt.nodeName, tt.vip, tt.port, tt.proto, got) - } - }) + if !strings.HasPrefix(base, "web-node-a-") { + t.Errorf("ServiceVIPBindingName = %q, want prefix %q", base, "web-node-a-") + } + if again := ServiceVIPBindingName("web", "node-a", "2001:db8::1", "fd00:10::1", 8443); again != base { + t.Errorf("ServiceVIPBindingName is not deterministic: %q then %q", base, again) + } + + // Every part of the identity changes the name: in particular two backends + // of one VIP on one node must never share an object. + for name, other := range map[string]string{ + "rule": ServiceVIPBindingName("api", "node-a", "2001:db8::1", "fd00:10::1", 8443), + "node": ServiceVIPBindingName("web", "node-b", "2001:db8::1", "fd00:10::1", 8443), + "vip": ServiceVIPBindingName("web", "node-a", "2001:db8::2", "fd00:10::1", 8443), + "backend": ServiceVIPBindingName("web", "node-a", "2001:db8::1", "fd00:10::2", 8443), + "port": ServiceVIPBindingName("web", "node-a", "2001:db8::1", "fd00:10::1", 8444), + } { + if other == base { + t.Errorf("changing the %s left the name unchanged: %q", name, base) + } + } + + longRule, longNode := strings.Repeat("r", 200), strings.Repeat("n", 200)+".example" + long := ServiceVIPBindingName(longRule, longNode, "2001:db8::1", "fd00:10::1", 1) + if errs := validation.IsDNS1123Subdomain(long); len(errs) > 0 { + t.Errorf("ServiceVIPBindingName with long names = %q is not a valid object name: %v", long, errs) } } diff --git a/internal/plumbing/ebpf/prog/usid.c b/internal/plumbing/ebpf/prog/usid.c index 3117b164..89d2a82b 100644 --- a/internal/plumbing/ebpf/prog/usid.c +++ b/internal/plumbing/ebpf/prog/usid.c @@ -481,9 +481,13 @@ struct nptv6_value { // and undeliverable at the backend. usid_ingress always builds its key with the // ingress direction and usid_egress with the egress one. // -// pad2 is an explicit, always-zero field so lookup_vip_xlat can zero it -// explicitly. The lookup hashes all 32 key bytes, and a stale value there -// would miss a row the control plane wrote with zeros. +// slot names which backend of a VIP the row belongs to, so several backends of +// one VIP can share a node and VRF. galactic-gateway's Maglev picks a backend +// and writes that backend's slot into bits 81-96 of the uSID it encapsulates +// toward, and usid_ingress reads it back from the outer destination for the +// ingress lookup. Egress rows always carry slot 0: a reply's source address and +// port already identify its backend. slot sits where an always-zero pad field +// used to, so the key keeps its 32-byte size and layout. #define USID_VIP_XLAT_DIR_INGRESS 0 #define USID_VIP_XLAT_DIR_EGRESS 1 @@ -493,7 +497,7 @@ struct vip_xlat_key { __u8 proto; __u8 direction; __be16 port; - __u16 pad2; + __u16 slot; __u8 addr[16]; }; @@ -1426,13 +1430,16 @@ static USID_ALWAYS_INLINE void apply_nptv6(__u8 *addr, const struct nptv6_value *word = __builtin_bswap16((__u16) sum); } -// lookup_vip_xlat returns the vip_xlat_table row for (block, argument, proto, -// direction, port, addr), or a null pointer if there is none. addr and port +// lookup_vip_xlat returns the vip_xlat_table row for (block, argument, slot, +// proto, direction, port, addr), or a null pointer if there is none. slot is +// the outer destination's backend slot for an ingress lookup and 0 for an +// egress lookup. addr and port // are the packet's destination for an ingress lookup and its source for an // egress lookup, both in wire order. See vip_xlat_key_scratch for why the key // is not built on the stack. -static USID_ALWAYS_INLINE struct vip_xlat_value *lookup_vip_xlat(__u64 block, __u16 argument, __u8 proto, - __u8 direction, __be16 port, const __u8 *addr) +static USID_ALWAYS_INLINE struct vip_xlat_value *lookup_vip_xlat(__u64 block, __u16 argument, __u16 slot, + __u8 proto, __u8 direction, __be16 port, + const __u8 *addr) { __u32 zero = 0; struct vip_xlat_key *key = bpf_map_lookup_elem(&vip_xlat_key_scratch, &zero); @@ -1445,7 +1452,7 @@ static USID_ALWAYS_INLINE struct vip_xlat_value *lookup_vip_xlat(__u64 block, __ key->proto = proto; key->direction = direction; key->port = port; - key->pad2 = 0; + key->slot = slot; __builtin_memcpy(key->addr, addr, sizeof(key->addr)); return bpf_map_lookup_elem(&vip_xlat_table, key); @@ -2325,6 +2332,11 @@ int usid_ingress(struct __sk_buff *skb) // low nibble of destination byte 8 plus byte 9. __u16 argument = ((__u16) (fn_arg_byte & 0x0F) << 8) | ip6->daddr[9]; + // The backend slot, bytes 10-11, read now for the same reason: the outer + // header is gone by the time the VIP lookup below needs it. Zero on every + // destination except galactic-gateway's, which names one backend of a VIP. + __u16 backend_slot = ((__u16) ip6->daddr[10] << 8) | ip6->daddr[11]; + // Step 6: exact-match (Block, Argument) against vrf_table. Argument 0x000 // is reserved and never registered, so it always misses here and needs no // special case. @@ -2543,9 +2555,9 @@ int usid_ingress(struct __sk_buff *skb) struct usid_l4ports *ports = (void *) (inner6 + 1); if ((void *) (ports + 1) <= data_end) { - struct vip_xlat_value *vv = lookup_vip_xlat(block, argument, inner6->nexthdr, - USID_VIP_XLAT_DIR_INGRESS, ports->dest, - inner6->daddr); + struct vip_xlat_value *vv = lookup_vip_xlat(block, argument, backend_slot, + inner6->nexthdr, USID_VIP_XLAT_DIR_INGRESS, + ports->dest, inner6->daddr); if (vv) { vip_hit = 1; @@ -3362,9 +3374,9 @@ int usid_egress(struct __sk_buff *skb) struct usid_l4ports *ports = (void *) (ip6 + 1); if ((void *) (ports + 1) <= data_end) { - struct vip_xlat_value *vv = lookup_vip_xlat(iv->block, iv->argument, ip6->nexthdr, - USID_VIP_XLAT_DIR_EGRESS, ports->source, - ip6->saddr); + struct vip_xlat_value *vv = lookup_vip_xlat(iv->block, iv->argument, 0, + ip6->nexthdr, USID_VIP_XLAT_DIR_EGRESS, + ports->source, ip6->saddr); if (vv) { vip_hit = 1; diff --git a/internal/plumbing/ebpf/prog/usid_test.go b/internal/plumbing/ebpf/prog/usid_test.go index 284a2add..9d6d7238 100644 --- a/internal/plumbing/ebpf/prog/usid_test.go +++ b/internal/plumbing/ebpf/prog/usid_test.go @@ -79,12 +79,13 @@ type testUSID struct { nodeID uint16 function uint8 argument uint16 + slot uint16 } func (u testUSID) addr(t *testing.T) netip.Addr { t.Helper() addr, err := uformat.Encode(uformat.Fields{ - Block: u.block, NodeID: u.nodeID, Function: u.function, Argument: u.argument, + Block: u.block, NodeID: u.nodeID, Function: u.function, Argument: u.argument, Slot: u.slot, }) if err != nil { t.Fatalf("uformat.Encode(%+v): %v", u, err) diff --git a/internal/plumbing/ebpf/prog/usid_vipxlat_test.go b/internal/plumbing/ebpf/prog/usid_vipxlat_test.go index 899817bc..b3afc193 100644 --- a/internal/plumbing/ebpf/prog/usid_vipxlat_test.go +++ b/internal/plumbing/ebpf/prog/usid_vipxlat_test.go @@ -35,11 +35,12 @@ func setUpVIPIngress(t *testing.T, objs *UsidObjects, usid testUSID) { } // putVIPIngressRow writes the ingress row RegisterIngress would for a UDP -// binding of vip:port to backend:port. -func putVIPIngressRow(t *testing.T, objs *UsidObjects, usid testUSID, vip, backend netip.Addr, port uint16) { +// binding of vip:vipTestPort to backend:vipTestPort, keyed on usid's slot. +func putVIPIngressRow(t *testing.T, objs *UsidObjects, usid testUSID, vip, backend netip.Addr) { t.Helper() + const port = vipTestPort key := UsidVipXlatKey{ - Block: usid.block, Argument: usid.argument, Proto: ipProtoUDP, + Block: usid.block, Argument: usid.argument, Slot: usid.slot, Proto: ipProtoUDP, Direction: usidVIPXlatDirIngress, Port: bswap16(port), Addr: vip.As16(), } if err := objs.VipXlatTable.Put(key, UsidVipXlatValue{Addr: backend.As16(), Port: bswap16(port)}); err != nil { @@ -104,11 +105,10 @@ func TestUsidIngress_VIPXlatSelectsRowByVIPAddress(t *testing.T) { usid := testUSID{block: baseUSID.block, nodeID: baseUSID.nodeID, function: uformat.FunctionEndDT46, argument: 0x764} setUpVIPIngress(t, objs, usid) - const port = vipTestPort vipA, backendA := netip.MustParseAddr("2001:db8:6060::a"), netip.MustParseAddr("fd20:60::a") vipB, backendB := netip.MustParseAddr("2001:db8:6060::b"), netip.MustParseAddr("fd20:60::b") - putVIPIngressRow(t, objs, usid, vipA, backendA, port) - putVIPIngressRow(t, objs, usid, vipB, backendB, port) + putVIPIngressRow(t, objs, usid, vipA, backendA) + putVIPIngressRow(t, objs, usid, vipB, backendB) if got := runIngressToDaddr(t, objs, usid, vipA); got != backendA { t.Errorf("packet to %s left for %s, want %s", vipA, got, backendA) @@ -124,6 +124,41 @@ func TestUsidIngress_VIPXlatSelectsRowByVIPAddress(t *testing.T) { } } +// TestUsidIngress_VIPXlatSelectsRowBySlot: two backends of one VIP and port in +// one VRF each have their own row, and the slot in the outer destination picks +// which one a packet is rewritten to. A slot with no row leaves the packet +// alone. +func TestUsidIngress_VIPXlatSelectsRowBySlot(t *testing.T) { + requireRoot(t) + objs := loadObjects(t) + usid := testUSID{block: baseUSID.block, nodeID: baseUSID.nodeID, function: uformat.FunctionEndDT46, argument: 0x799} + setUpVIPIngress(t, objs, usid) + + vip := netip.MustParseAddr("2001:db8:6060::a") + backendA, backendB := netip.MustParseAddr("fd20:60::a"), netip.MustParseAddr("fd20:60::b") + toA, toB := usid, usid + toA.slot, toB.slot = 0x1111, 0x2222 + putVIPIngressRow(t, objs, toA, vip, backendA) + putVIPIngressRow(t, objs, toB, vip, backendB) + + if got := runIngressToDaddr(t, objs, toA, vip); got != backendA { + t.Errorf("packet with slot %#x left for %s, want %s", toA.slot, got, backendA) + } + if got := runIngressToDaddr(t, objs, toB, vip); got != backendB { + t.Errorf("packet with slot %#x left for %s, want %s", toB.slot, got, backendB) + } + + unbound := usid + unbound.slot = 0x3333 + if got := runIngressToDaddr(t, objs, unbound, vip); got != vip { + t.Errorf("packet with unbound slot %#x left for %s, want it unchanged", unbound.slot, got) + } + // Slot 0, a destination that names no backend, has no fallback row either. + if got := runIngressToDaddr(t, objs, usid, vip); got != vip { + t.Errorf("packet with slot 0 left for %s, want it unchanged", got) + } +} + // TestUsidIngress_VIPXlatRunsBeforeNPTv6: in a VRF with NPTv6, a packet to a // VIP is rewritten to its backend rather than having its prefix translated // first, and a packet to any other address still gets NPTv6. @@ -134,9 +169,8 @@ func TestUsidIngress_VIPXlatRunsBeforeNPTv6(t *testing.T) { setUpVIPIngress(t, objs, usid) putNPTv6(t, objs, usid.block, usid.argument) - const port = vipTestPort vip, backend := netip.MustParseAddr("2001:db8:6060::a"), netip.MustParseAddr("fd01:203:405:1::a") - putVIPIngressRow(t, objs, usid, vip, backend, port) + putVIPIngressRow(t, objs, usid, vip, backend) if got := runIngressToDaddr(t, objs, usid, vip); got != backend { t.Errorf("packet to VIP %s left for %s, want %s", vip, got, backend) diff --git a/internal/plumbing/ebpf/uformat/uformat.go b/internal/plumbing/ebpf/uformat/uformat.go index 8153a3f9..c92291db 100644 --- a/internal/plumbing/ebpf/uformat/uformat.go +++ b/internal/plumbing/ebpf/uformat/uformat.go @@ -9,8 +9,8 @@ // // Bit layout, in the RFC 9800 REPLACE-CSID flavor: // -// bit 1 48 49 64 65 68 69 80 81 128 -// |------ uSID Block (48) ------|-- Node-ID (16) --|-Fn(4)-|-- Argument (12) --|------ Padding (48, zero) ------| +// | uSID Block (48) | Node-ID (16) | Fn (4) | Argument (12) | Slot (16) | Padding (32, zero) | +// bits 1-48 bits 49-64 65-68 69-80 81-96 97-128 // // Byte layout of the 16-byte address, bit 1 being the most significant bit of // byte 0: @@ -19,7 +19,13 @@ // bytes 6-7 (16 bits) Node-ID // byte 8 hi ( 4 bits) Function // byte 8 lo + byte 9 (12 bits) Argument -// bytes 10-15 (48 bits) Padding (must be zero) +// bytes 10-11 (16 bits) Slot +// bytes 12-15 (32 bits) Padding (must be zero) +// +// Slot is zero on every SID a node advertises. Only a gateway-to-backend +// destination sets it: it names which backend of a load-balanced VIP the +// gateway chose, so several backends of one VIP can share a node, and the +// backend node's vip_xlat_table lookup keys on it. // // Every accessor reads or writes its field at that field's fixed offset only. // Nothing here shifts one field's bits into another field's frame: all four @@ -41,7 +47,8 @@ const ( NodeIDBits = 16 FunctionBits = 4 ArgumentBits = 12 - PaddingBits = 48 + SlotBits = 16 + PaddingBits = 32 // LocatorBits is the width of the locator itself: Block and Node-ID // together, the leading run every uSID on one node shares whatever its @@ -91,12 +98,14 @@ const ( // Fields is the decoded set of uFMT 48+16 fields carried by one uSID address. // Block and NodeID occupy their full width; Function and Argument are stored // right-justified in wider types with only their low 4 and 12 bits -// significant. +// significant. Slot is zero except on a gateway-to-backend destination; see the +// package doc comment. type Fields struct { Block uint64 NodeID uint16 Function uint8 Argument uint16 + Slot uint16 } // Validate returns an error if any field is out of range: Block above BlockMax, @@ -207,8 +216,18 @@ func Argument(addr netip.Addr) (uint16, error) { return uint16(b[8]&0x0F)<<8 | uint16(b[9]), nil } +// Slot returns the 16-bit backend Slot at bits 81-96 of addr, bytes 10-11 read +// from the unmutated address. Zero means the address names no backend slot. +func Slot(addr netip.Addr) (uint16, error) { + b, err := as16(addr) + if err != nil { + return 0, err + } + return binary.BigEndian.Uint16(b[10:12]), nil +} + // Decode extracts every uFMT 48+16 field from addr at its fixed offset. It -// returns an error if addr is not a 16-byte IPv6 address, or if the 48-bit +// returns an error if addr is not a 16-byte IPv6 address, or if the 32-bit // padding tail is non-zero. That is a structural check, distinct from the // semantic range checks Validate performs and Decode does not. func Decode(addr netip.Addr) (Fields, error) { @@ -216,9 +235,9 @@ func Decode(addr netip.Addr) (Fields, error) { if err != nil { return Fields{}, err } - for i := 10; i < 16; i++ { + for i := 12; i < 16; i++ { if b[i] != 0 { - return Fields{}, fmt.Errorf("uformat: %s has non-zero padding at byte %d (bits 81-128 must be zero)", addr, i) + return Fields{}, fmt.Errorf("uformat: %s has non-zero padding at byte %d (bits 97-128 must be zero)", addr, i) } } return Fields{ @@ -226,6 +245,7 @@ func Decode(addr netip.Addr) (Fields, error) { NodeID: binary.BigEndian.Uint16(b[6:8]), Function: b[8] >> 4, Argument: uint16(b[8]&0x0F)<<8 | uint16(b[9]), + Slot: binary.BigEndian.Uint16(b[10:12]), }, nil } @@ -250,7 +270,8 @@ func Encode(f Fields) (netip.Addr, error) { binary.BigEndian.PutUint64(b[:8], f.Block<>8) & 0x0F) b[9] = byte(f.Argument) - // bytes 10-15 remain zero (padding). + binary.BigEndian.PutUint16(b[10:12], f.Slot) + // bytes 12-15 remain zero (padding). return netip.AddrFrom16(b), nil } diff --git a/internal/plumbing/ebpf/uformat/uformat_test.go b/internal/plumbing/ebpf/uformat/uformat_test.go index 1dab58c2..a66c7ce6 100644 --- a/internal/plumbing/ebpf/uformat/uformat_test.go +++ b/internal/plumbing/ebpf/uformat/uformat_test.go @@ -38,6 +38,14 @@ func TestEncodeDecodeRoundTrip(t *testing.T) { "NodeIDAboveReservedRangeStillEncodes", Fields{Block: testBlock, NodeID: 0xFFFF, Function: FunctionEndDT46, Argument: 0x001}, }, + { + "BackendSlot", + Fields{Block: testBlock, NodeID: 0x0001, Function: FunctionEndDT46, Argument: 0x001, Slot: 0xBEEF}, + }, + { + "MaxSlot", + Fields{Block: testBlock, NodeID: 0x0001, Function: FunctionEndDT46, Argument: 0x001, Slot: 0xFFFF}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -88,6 +96,31 @@ func TestDecodeRejectsNonZeroPadding(t *testing.T) { } } +// TestSlotAtBytes10And11 pins Slot to bytes 10-11 of the address, big-endian, +// where usid_ingress reads it, with the padding after it left zero. +func TestSlotAtBytes10And11(t *testing.T) { + addr, err := Encode(Fields{Block: testBlock, NodeID: 1, Function: FunctionEndDT46, Argument: 1, Slot: 0x1234}) + if err != nil { + t.Fatalf("Encode: unexpected error: %v", err) + } + b := addr.As16() + if b[10] != 0x12 || b[11] != 0x34 { + t.Errorf("bytes 10-11 = %#x %#x, want 0x12 0x34", b[10], b[11]) + } + for i := 12; i < 16; i++ { + if b[i] != 0 { + t.Errorf("byte %d = %#x, want 0 (padding)", i, b[i]) + } + } + slot, err := Slot(addr) + if err != nil { + t.Fatalf("Slot: unexpected error: %v", err) + } + if slot != 0x1234 { + t.Errorf("Slot = %#x, want 0x1234", slot) + } +} + func TestDecodeRejectsNonV6Address(t *testing.T) { if _, err := Decode(netip.MustParseAddr("192.0.2.1")); err == nil { t.Errorf("Decode(IPv4) = nil error, want error") diff --git a/internal/plumbing/ebpf/vipxlatmap/vipxlat.go b/internal/plumbing/ebpf/vipxlatmap/vipxlat.go index 7337518d..5eb23218 100644 --- a/internal/plumbing/ebpf/vipxlatmap/vipxlat.go +++ b/internal/plumbing/ebpf/vipxlatmap/vipxlat.go @@ -75,9 +75,16 @@ const ( // Argument identify the tenant VRF and Proto is the transport protocol. Addr and // Port are direction-dependent: the VIP's for an ingress row, the backend's for // an egress row. Port is in host order here and converted internally. +// +// Slot is the backend slot an ingress row is keyed on, the value +// srv6.BackendSlot derives from the row's backend and galactic-gateway writes +// into the uSID it encapsulates toward that backend. It lets several backends of +// one VIP share a node and VRF, each with its own ingress row. Egress rows +// always carry Slot 0. type Key struct { Block uint64 Argument uint16 + Slot uint16 Proto uint8 Addr netip.Addr Port uint16 @@ -218,7 +225,7 @@ func (d Direction) String() string { // composed with, which is direction-dependent; valAddr and valPort are the // rewrite target the row substitutes in. func (t *VipXlatTable) register( - direction uint8, block uint64, argument uint16, proto uint8, + direction uint8, block uint64, argument, slot uint16, proto uint8, keyAddr net.IP, keyPort uint16, valAddr net.IP, valPort uint16, ) error { t.writeMu.Lock() @@ -239,7 +246,7 @@ func (t *VipXlatTable) register( if err := validatePort(valPort); err != nil { return fmt.Errorf("vipxlatmap: vip_xlat_table: register: rewrite port: %w", err) } - key, err := kernelKey(direction, block, argument, proto, keyAddr, keyPort) + key, err := kernelKey(direction, block, argument, slot, proto, keyAddr, keyPort) if err != nil { return fmt.Errorf("vipxlatmap: vip_xlat_table: register: key address: %w", err) } @@ -253,8 +260,8 @@ func (t *VipXlatTable) register( Port: hostToNetwork16(valPort), } if err := t.table.Put(key, value); err != nil { - return fmt.Errorf("vipxlatmap: vip_xlat_table: register block=%#x argument=%#x proto=%d addr=%s port=%d: %w", - block, argument, proto, keyAddr, keyPort, err) + return fmt.Errorf("vipxlatmap: vip_xlat_table: register block=%#x argument=%#x slot=%#x proto=%d addr=%s port=%d: %w", + block, argument, slot, proto, keyAddr, keyPort, err) } t.mu.Lock() @@ -264,14 +271,15 @@ func (t *VipXlatTable) register( } // RegisterIngress writes the ingress-direction row for one binding, keyed on -// (block, argument, proto, vipAddr, vipPort), the packet's destination on an -// inbound request, and rewriting to backendAddr and backendPort. +// (block, argument, slot, proto, vipAddr, vipPort), the packet's outer +// destination slot and inner destination on an inbound request, and rewriting +// to backendAddr and backendPort. slot is srv6.BackendSlot of the backend. func (t *VipXlatTable) RegisterIngress( - block uint64, argument uint16, proto uint8, + block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16, ) error { - return t.register(directionIngress, block, argument, proto, vipAddr, vipPort, backendAddr, backendPort) + return t.register(directionIngress, block, argument, slot, proto, vipAddr, vipPort, backendAddr, backendPort) } // RegisterEgress writes the egress-direction row for one binding, keyed on @@ -282,13 +290,13 @@ func (t *VipXlatTable) RegisterEgress( backendAddr net.IP, backendPort uint16, vipAddr net.IP, vipPort uint16, ) error { - return t.register(directionEgress, block, argument, proto, backendAddr, backendPort, vipAddr, vipPort) + return t.register(directionEgress, block, argument, 0, proto, backendAddr, backendPort, vipAddr, vipPort) } -// kernelKey composes the kernel key for (direction, block, argument, proto, -// addr, port), returning an error if addr is not an IPv6 address. +// kernelKey composes the kernel key for (direction, block, argument, slot, +// proto, addr, port), returning an error if addr is not an IPv6 address. func kernelKey( - direction uint8, block uint64, argument uint16, proto uint8, addr net.IP, port uint16, + direction uint8, block uint64, argument, slot uint16, proto uint8, addr net.IP, port uint16, ) (prog.UsidVipXlatKey, error) { raw, err := addrTo16(addr) if err != nil { @@ -297,6 +305,7 @@ func kernelKey( return prog.UsidVipXlatKey{ Block: block, Argument: argument, + Slot: slot, Proto: proto, Direction: direction, Port: hostToNetwork16(port), @@ -304,13 +313,13 @@ func kernelKey( }, nil } -// unregister removes the entry keyed by (direction, block, argument, proto, -// addr, port) if present. An entry that is already absent is not an error. -// The caller holds writeMu. +// unregister removes the entry keyed by (direction, block, argument, slot, +// proto, addr, port) if present. An entry that is already absent is not an +// error. The caller holds writeMu. func (t *VipXlatTable) unregister( - direction uint8, block uint64, argument uint16, proto uint8, addr net.IP, port uint16, + direction uint8, block uint64, argument, slot uint16, proto uint8, addr net.IP, port uint16, ) error { - key, err := kernelKey(direction, block, argument, proto, addr, port) + key, err := kernelKey(direction, block, argument, slot, proto, addr, port) if err != nil { return fmt.Errorf("vipxlatmap: vip_xlat_table: unregister: %w", err) } @@ -318,8 +327,9 @@ func (t *VipXlatTable) unregister( if errors.Is(err, ebpf.ErrKeyNotExist) { return nil } - return fmt.Errorf("vipxlatmap: vip_xlat_table: unregister block=%#x argument=%#x proto=%d addr=%s port=%d: %w", - block, argument, proto, addr, port, err) + return fmt.Errorf( + "vipxlatmap: vip_xlat_table: unregister block=%#x argument=%#x slot=%#x proto=%d addr=%s port=%d: %w", + block, argument, slot, proto, addr, port, err) } t.mu.Lock() delete(t.generations, key) @@ -328,13 +338,13 @@ func (t *VipXlatTable) unregister( } // UnregisterIngress removes the ingress-direction row RegisterIngress -// would have written for (block, argument, proto, vipAddr, vipPort). +// would have written for (block, argument, slot, proto, vipAddr, vipPort). func (t *VipXlatTable) UnregisterIngress( - block uint64, argument uint16, proto uint8, vipAddr net.IP, vipPort uint16, + block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, ) error { t.writeMu.Lock() defer t.writeMu.Unlock() - return t.unregister(directionIngress, block, argument, proto, vipAddr, vipPort) + return t.unregister(directionIngress, block, argument, slot, proto, vipAddr, vipPort) } // UnregisterEgress removes the egress-direction row RegisterEgress would @@ -344,12 +354,13 @@ func (t *VipXlatTable) UnregisterEgress( ) error { t.writeMu.Lock() defer t.writeMu.Unlock() - return t.unregister(directionEgress, block, argument, proto, backendAddr, backendPort) + return t.unregister(directionEgress, block, argument, 0, proto, backendAddr, backendPort) } // binding identifies the two rows one binding writes, validated and in the -// raw form the map stores. +// raw form the map stores. slot is the ingress row's backend slot. type binding struct { + slot uint16 proto uint8 vip [16]byte vipPort uint16 @@ -358,7 +369,9 @@ type binding struct { } // newBinding validates a binding's protocol and addresses. -func newBinding(proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16) (binding, error) { +func newBinding( + slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16, +) (binding, error) { if err := validateProto(proto); err != nil { return binding{}, err } @@ -370,13 +383,15 @@ func newBinding(proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, if err != nil { return binding{}, fmt.Errorf("vipxlatmap: vip_xlat_table: unregister binding: backend address: %w", err) } - return binding{proto: proto, vip: rawVIP, vipPort: vipPort, backend: rawBackend, backendPort: backendPort}, nil + return binding{ + slot: slot, proto: proto, vip: rawVIP, vipPort: vipPort, backend: rawBackend, backendPort: backendPort, + }, nil } -// isIngressRow reports whether e is the ingress row b writes: keyed on b's VIP -// and port, and rewriting to b's backend and port. +// isIngressRow reports whether e is the ingress row b writes: keyed on b's +// slot, VIP and port, and rewriting to b's backend and port. func (b binding) isIngressRow(e Entry) bool { - return e.Direction == DirectionIngress && e.Proto == b.proto && + return e.Direction == DirectionIngress && e.Slot == b.slot && e.Proto == b.proto && e.Addr == netip.AddrFrom16(b.vip) && e.Port == b.vipPort && [16]byte(e.RewriteAddr) == b.backend && e.RewritePort == b.backendPort } @@ -384,7 +399,7 @@ func (b binding) isIngressRow(e Entry) bool { // isEgressRow reports whether e is the egress row b writes: keyed on b's // backend and port, and rewriting to b's VIP and port. func (b binding) isEgressRow(e Entry) bool { - return e.Direction == DirectionEgress && e.Proto == b.proto && + return e.Direction == DirectionEgress && e.Slot == 0 && e.Proto == b.proto && e.Addr == netip.AddrFrom16(b.backend) && e.Port == b.backendPort && [16]byte(e.RewriteAddr) == b.vip && e.RewritePort == b.vipPort } @@ -401,23 +416,24 @@ func (b binding) isEgressRow(e Entry) bool { func (t *VipXlatTable) removeAt(block uint64, argument uint16, b binding) ([]Entry, error) { var removed []Entry - ingress, found, err := t.get(directionIngress, block, argument, b.proto, net.IP(b.vip[:]), b.vipPort) + ingress, found, err := t.get(directionIngress, block, argument, b.slot, b.proto, net.IP(b.vip[:]), b.vipPort) if err != nil { return nil, err } if found && b.isIngressRow(ingress) { - if err := t.unregister(directionIngress, block, argument, b.proto, net.IP(b.vip[:]), b.vipPort); err != nil { + if err := t.unregister(directionIngress, block, argument, b.slot, b.proto, net.IP(b.vip[:]), b.vipPort); err != nil { return nil, err } removed = append(removed, ingress) } - egress, found, err := t.get(directionEgress, block, argument, b.proto, net.IP(b.backend[:]), b.backendPort) + egress, found, err := t.get(directionEgress, block, argument, 0, b.proto, net.IP(b.backend[:]), b.backendPort) if err != nil { return removed, err } if found && b.isEgressRow(egress) { - if err := t.unregister(directionEgress, block, argument, b.proto, net.IP(b.backend[:]), b.backendPort); err != nil { + backend := net.IP(b.backend[:]) + if err := t.unregister(directionEgress, block, argument, 0, b.proto, backend, b.backendPort); err != nil { return removed, err } removed = append(removed, egress) @@ -434,9 +450,9 @@ func (t *VipXlatTable) removeAt(block uint64, argument uint16, b binding) ([]Ent // Every row that is exactly one of the binding's two rows, by key and by value, // marks a location, and removeAt removes the binding's rows there. func (t *VipXlatTable) UnregisterBinding( - proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16, + slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16, ) ([]Entry, error) { - b, err := newBinding(proto, vipAddr, vipPort, backendAddr, backendPort) + b, err := newBinding(slot, proto, vipAddr, vipPort, backendAddr, backendPort) if err != nil { return nil, err } @@ -480,10 +496,10 @@ func (t *VipXlatTable) UnregisterBinding( // argument), with the same value checks as UnregisterBinding. It returns the // rows it removed. func (t *VipXlatTable) UnregisterBindingAt( - block uint64, argument uint16, + block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, backendAddr net.IP, backendPort uint16, ) ([]Entry, error) { - b, err := newBinding(proto, vipAddr, vipPort, backendAddr, backendPort) + b, err := newBinding(slot, proto, vipAddr, vipPort, backendAddr, backendPort) if err != nil { return nil, err } @@ -507,6 +523,7 @@ func (t *VipXlatTable) decodeEntry(key prog.UsidVipXlatKey, value prog.UsidVipXl Key: Key{ Block: key.Block, Argument: key.Argument, + Slot: key.Slot, Proto: key.Proto, Addr: netip.AddrFrom16(key.Addr), Port: hostToNetwork16(key.Port), // self-inverse: wire -> host @@ -518,13 +535,13 @@ func (t *VipXlatTable) decodeEntry(key prog.UsidVipXlatKey, value prog.UsidVipXl } } -// GetIngress reads the ingress-direction entry for (block, argument, proto, -// vipAddr, vipPort), the row RegisterIngress writes, and reports whether it -// exists. +// GetIngress reads the ingress-direction entry for (block, argument, slot, +// proto, vipAddr, vipPort), the row RegisterIngress writes, and reports whether +// it exists. func (t *VipXlatTable) GetIngress( - block uint64, argument uint16, proto uint8, vipAddr net.IP, vipPort uint16, + block uint64, argument, slot uint16, proto uint8, vipAddr net.IP, vipPort uint16, ) (Entry, bool, error) { - return t.get(directionIngress, block, argument, proto, vipAddr, vipPort) + return t.get(directionIngress, block, argument, slot, proto, vipAddr, vipPort) } // GetEgress reads the egress-direction entry for (block, argument, proto, @@ -533,14 +550,14 @@ func (t *VipXlatTable) GetIngress( func (t *VipXlatTable) GetEgress( block uint64, argument uint16, proto uint8, backendAddr net.IP, backendPort uint16, ) (Entry, bool, error) { - return t.get(directionEgress, block, argument, proto, backendAddr, backendPort) + return t.get(directionEgress, block, argument, 0, proto, backendAddr, backendPort) } // get is GetIngress/GetEgress's shared primitive. func (t *VipXlatTable) get( - direction uint8, block uint64, argument uint16, proto uint8, addr net.IP, port uint16, + direction uint8, block uint64, argument, slot uint16, proto uint8, addr net.IP, port uint16, ) (Entry, bool, error) { - key, err := kernelKey(direction, block, argument, proto, addr, port) + key, err := kernelKey(direction, block, argument, slot, proto, addr, port) if err != nil { return Entry{}, false, fmt.Errorf("vipxlatmap: vip_xlat_table: get: %w", err) } @@ -550,8 +567,8 @@ func (t *VipXlatTable) get( return Entry{}, false, nil } return Entry{}, false, fmt.Errorf( - "vipxlatmap: vip_xlat_table: get block=%#x argument=%#x proto=%d addr=%s port=%d: %w", - block, argument, proto, addr, port, err) + "vipxlatmap: vip_xlat_table: get block=%#x argument=%#x slot=%#x proto=%d addr=%s port=%d: %w", + block, argument, slot, proto, addr, port, err) } return t.decodeEntry(key, value), true, nil } @@ -607,8 +624,8 @@ func (t *VipXlatTable) Reconcile(live map[Key]struct{}, cutoff uint64) (removed // never deletes anything that did not already match e. addr := net.IP(e.Addr.AsSlice()) delErr := errors.Join( - t.unregister(directionIngress, e.Block, e.Argument, e.Proto, addr, e.Port), - t.unregister(directionEgress, e.Block, e.Argument, e.Proto, addr, e.Port), + t.unregister(directionIngress, e.Block, e.Argument, e.Slot, e.Proto, addr, e.Port), + t.unregister(directionEgress, e.Block, e.Argument, e.Slot, e.Proto, addr, e.Port), ) if delErr != nil { errs = append(errs, fmt.Errorf("vipxlatmap: vip_xlat_table: reconcile: delete stale entry %+v: %w", e.Key, delErr)) diff --git a/internal/plumbing/ebpf/vipxlatmap/vipxlat_test.go b/internal/plumbing/ebpf/vipxlatmap/vipxlat_test.go index c3908e8c..9f82ed32 100644 --- a/internal/plumbing/ebpf/vipxlatmap/vipxlat_test.go +++ b/internal/plumbing/ebpf/vipxlatmap/vipxlat_test.go @@ -8,6 +8,7 @@ import ( "net" "net/netip" "testing" + "unsafe" "go.datum.net/galactic/internal/plumbing/ebpf/prog" ) @@ -26,11 +27,11 @@ func TestVipXlatTable_RegisterIngressAndGet(t *testing.T) { backend := net.ParseIP("fd20:60::5:5") const vipPort, backendPort = 8080, 30080 - if err := vt.RegisterIngress(testBlock, 0x654, ProtoUDP, vip, vipPort, backend, backendPort); err != nil { + if err := vt.RegisterIngress(testBlock, 0x654, 0, ProtoUDP, vip, vipPort, backend, backendPort); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } - entry, ok, err := vt.GetIngress(testBlock, 0x654, ProtoUDP, vip, vipPort) + entry, ok, err := vt.GetIngress(testBlock, 0x654, 0, ProtoUDP, vip, vipPort) if err != nil { t.Fatalf("Get: unexpected error: %v", err) } @@ -94,7 +95,7 @@ func TestVipXlatTable_IngressAndEgressRowsAreSwapped(t *testing.T) { backend := net.ParseIP("fd20:60::5:5") const vipPort, backendPort = 8080, 30080 - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, vipPort, backend, backendPort); err != nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, vipPort, backend, backendPort); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } if err := vt.RegisterEgress(testBlock, 0x1, ProtoTCP, backend, backendPort, vip, vipPort); err != nil { @@ -105,7 +106,7 @@ func TestVipXlatTable_IngressAndEgressRowsAreSwapped(t *testing.T) { t.Fatalf("table has %d entries after both registrations, want %d (two independent rows)", got, want) } - ingress, ok, err := vt.GetIngress(testBlock, 0x1, ProtoTCP, vip, vipPort) + ingress, ok, err := vt.GetIngress(testBlock, 0x1, 0, ProtoTCP, vip, vipPort) if err != nil || !ok { t.Fatalf("Get(ingress key): ok=%v err=%v", ok, err) } @@ -153,7 +154,7 @@ func TestVipXlatTable_IngressAndEgressSurviveIdenticalPort(t *testing.T) { backend := net.ParseIP("fd20:60:ff03::100:0") const port = 80 // deliberately identical on both sides - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, port, backend, port); err != nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, port, backend, port); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } if err := vt.RegisterEgress(testBlock, 0x1, ProtoTCP, backend, port, vip, port); err != nil { @@ -165,7 +166,7 @@ func TestVipXlatTable_IngressAndEgressSurviveIdenticalPort(t *testing.T) { "(RegisterEgress must not overwrite RegisterIngress's row)", ft.len(), want) } - ingress, ok, err := vt.GetIngress(testBlock, 0x1, ProtoTCP, vip, port) + ingress, ok, err := vt.GetIngress(testBlock, 0x1, 0, ProtoTCP, vip, port) if err != nil || !ok { t.Fatalf("GetIngress: ok=%v err=%v, want the row RegisterIngress wrote to still be present", ok, err) } @@ -183,7 +184,7 @@ func TestVipXlatTable_IngressAndEgressSurviveIdenticalPort(t *testing.T) { } // Unregistering one direction must not remove the other. - if err := vt.UnregisterIngress(testBlock, 0x1, ProtoTCP, vip, port); err != nil { + if err := vt.UnregisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, port); err != nil { t.Fatalf("UnregisterIngress: unexpected error: %v", err) } if got, want := ft.len(), 1; got != want { @@ -197,7 +198,7 @@ func TestVipXlatTable_IngressAndEgressSurviveIdenticalPort(t *testing.T) { func TestVipXlatTable_GetMissingEntry(t *testing.T) { vt, _ := newTestTable(constClock(1)) - _, ok, err := vt.GetIngress(testBlock, 0x1, ProtoTCP, net.ParseIP("2001:db8::1"), 443) + _, ok, err := vt.GetIngress(testBlock, 0x1, 0, ProtoTCP, net.ParseIP("2001:db8::1"), 443) if err != nil { t.Fatalf("Get: unexpected error: %v", err) } @@ -211,7 +212,7 @@ func TestVipXlatTable_RegisterRejectsInvalidProto(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x1, 1 /* ICMP */, vip, 8080, backend, 30080); err == nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, 1 /* ICMP */, vip, 8080, backend, 30080); err == nil { t.Errorf("RegisterIngress(proto=ICMP) = nil error, want rejection") } if ft.len() != 0 { @@ -224,7 +225,7 @@ func TestVipXlatTable_RegisterRejectsIPv4Address(t *testing.T) { vip := net.ParseIP("2001:db8::1") backendV4 := net.ParseIP("10.0.0.5") - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080, backendV4, 30080); err == nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080, backendV4, 30080); err == nil { t.Errorf("RegisterIngress(backend=IPv4) = nil error, want rejection (IPv6-only substitution)") } if ft.len() != 0 { @@ -237,7 +238,7 @@ func TestVipXlatTable_RegisterRejectsReservedArgumentZero(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x000, ProtoTCP, vip, 8080, backend, 30080); err == nil { + if err := vt.RegisterIngress(testBlock, 0x000, 0, ProtoTCP, vip, 8080, backend, 30080); err == nil { t.Errorf("RegisterIngress(argument=0x000) = nil error, want rejection (reserved value)") } if ft.len() != 0 { @@ -250,17 +251,17 @@ func TestVipXlatTable_UnregisterIsIdempotent(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080, backend, 30080); err != nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } - if err := vt.UnregisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080); err != nil { + if err := vt.UnregisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080); err != nil { t.Fatalf("UnregisterIngress: unexpected error: %v", err) } if ft.len() != 0 { t.Errorf("table has %d entries after unregister, want 0", ft.len()) } // Second call: already absent, still not an error. - if err := vt.UnregisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080); err != nil { + if err := vt.UnregisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080); err != nil { t.Errorf("UnregisterIngress on an already-absent entry: unexpected error: %v", err) } } @@ -270,7 +271,7 @@ func TestVipXlatTable_List(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080, backend, 30080); err != nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } if err := vt.RegisterEgress(testBlock, 0x1, ProtoTCP, backend, 30080, vip, 8080); err != nil { @@ -301,7 +302,7 @@ func TestVipXlatTable_List(t *testing.T) { func registerBinding(t *testing.T, vt *VipXlatTable, argument uint16, vip, backend net.IP) { t.Helper() const vipPort, backendPort = 8080, 30080 - if err := vt.RegisterIngress(testBlock, argument, ProtoTCP, vip, vipPort, backend, backendPort); err != nil { + if err := vt.RegisterIngress(testBlock, argument, 0, ProtoTCP, vip, vipPort, backend, backendPort); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } if err := vt.RegisterEgress(testBlock, argument, ProtoTCP, backend, backendPort, vip, vipPort); err != nil { @@ -316,7 +317,7 @@ func TestVipXlatTable_UnregisterBindingRemovesBothRows(t *testing.T) { backend := net.ParseIP("fd20:60::1") registerBinding(t, vt, 0x46, vip, backend) - removed, err := vt.UnregisterBinding(ProtoTCP, vip, 8080, backend, 30080) + removed, err := vt.UnregisterBinding(0, ProtoTCP, vip, 8080, backend, 30080) if err != nil { t.Fatalf("UnregisterBinding: unexpected error: %v", err) } @@ -328,7 +329,7 @@ func TestVipXlatTable_UnregisterBindingRemovesBothRows(t *testing.T) { } // Second call: nothing left to find, still not an error. - removed, err = vt.UnregisterBinding(ProtoTCP, vip, 8080, backend, 30080) + removed, err = vt.UnregisterBinding(0, ProtoTCP, vip, 8080, backend, 30080) if err != nil || len(removed) != 0 { t.Errorf("second UnregisterBinding = (%d removed, %v), want (0, nil)", len(removed), err) } @@ -344,13 +345,13 @@ func TestVipXlatTable_UnregisterBindingKeepsOtherVIPs(t *testing.T) { // Same backend address and ports in another tenant's VRF, behind another VIP. registerBinding(t, vt, 0x47, otherVIP, backend) - if _, err := vt.UnregisterBinding(ProtoTCP, vip, 8080, backend, 30080); err != nil { + if _, err := vt.UnregisterBinding(0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("UnregisterBinding: unexpected error: %v", err) } if ft.len() != 2 { t.Fatalf("table has %d entries, want 2 (the other VIP's rows)", ft.len()) } - if _, ok, _ := vt.GetIngress(testBlock, 0x47, ProtoTCP, otherVIP, 8080); !ok { + if _, ok, _ := vt.GetIngress(testBlock, 0x47, 0, ProtoTCP, otherVIP, 8080); !ok { t.Error("other VIP's ingress row was removed") } if _, ok, _ := vt.GetEgress(testBlock, 0x47, ProtoTCP, backend, 30080); !ok { @@ -371,7 +372,7 @@ func TestVipXlatTable_UnregisterBindingKeepsRowsOfAnotherBinding(t *testing.T) { registerBinding(t, vt, 0x46, vip, backend) registerBinding(t, vt, 0x46, vip, otherBackend) - removed, err := vt.UnregisterBinding(ProtoTCP, vip, 8080, backend, 30080) + removed, err := vt.UnregisterBinding(0, ProtoTCP, vip, 8080, backend, 30080) if err != nil { t.Fatalf("UnregisterBinding: unexpected error: %v", err) } @@ -379,7 +380,7 @@ func TestVipXlatTable_UnregisterBindingKeepsRowsOfAnotherBinding(t *testing.T) { removed[0].Addr != netip.MustParseAddr("fd20:60::1") { t.Errorf("UnregisterBinding removed %+v, want only this binding's egress row", removed) } - if e, ok, _ := vt.GetIngress(testBlock, 0x46, ProtoTCP, vip, 8080); !ok || !e.RewriteAddr.Equal(otherBackend) { + if e, ok, _ := vt.GetIngress(testBlock, 0x46, 0, ProtoTCP, vip, 8080); !ok || !e.RewriteAddr.Equal(otherBackend) { t.Errorf("ingress row = %+v (found=%v), want the other binding's row kept", e, ok) } if _, ok, _ := vt.GetEgress(testBlock, 0x46, ProtoTCP, otherBackend, 30080); !ok { @@ -396,7 +397,7 @@ func TestVipXlatTable_UnregisterBindingRemovesEgressWithoutIngress(t *testing.T) t.Fatalf("RegisterEgress: unexpected error: %v", err) } - removed, err := vt.UnregisterBinding(ProtoTCP, vip, 8080, backend, 30080) + removed, err := vt.UnregisterBinding(0, ProtoTCP, vip, 8080, backend, 30080) if err != nil { t.Fatalf("UnregisterBinding: unexpected error: %v", err) } @@ -413,13 +414,13 @@ func TestVipXlatTable_UnregisterBindingRejectsInvalidInput(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if _, err := vt.UnregisterBinding(1, vip, 8080, backend, 30080); err == nil { + if _, err := vt.UnregisterBinding(0, 1, vip, 8080, backend, 30080); err == nil { t.Error("UnregisterBinding with proto 1: expected an error") } - if _, err := vt.UnregisterBinding(ProtoTCP, net.ParseIP("192.0.2.1"), 8080, backend, 30080); err == nil { + if _, err := vt.UnregisterBinding(0, ProtoTCP, net.ParseIP("192.0.2.1"), 8080, backend, 30080); err == nil { t.Error("UnregisterBinding with an IPv4 VIP: expected an error") } - if _, err := vt.UnregisterBinding(ProtoTCP, vip, 8080, nil, 30080); err == nil { + if _, err := vt.UnregisterBinding(0, ProtoTCP, vip, 8080, nil, 30080); err == nil { t.Error("UnregisterBinding with a nil backend: expected an error") } } @@ -429,10 +430,10 @@ func TestVipXlatTable_ReconcileRemovesStaleEntries(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080, backend, 30080); err != nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } - if err := vt.RegisterIngress(testBlock, 0x2, ProtoTCP, vip, 9090, backend, 40040); err != nil { + if err := vt.RegisterIngress(testBlock, 0x2, 0, ProtoTCP, vip, 9090, backend, 40040); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } @@ -466,7 +467,7 @@ func TestVipXlatTable_ReconcileKeepsEntriesAtOrAfterCutoff(t *testing.T) { // Registered at generation 5, i.e. at-or-after a cutoff of 5 -- must be // kept even though its key is absent from live (it is simply too new // to judge against a live set snapshotted before it existed). - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080, backend, 30080); err != nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } @@ -487,7 +488,7 @@ func TestVipXlatTable_ReconcileKeepsLiveEntries(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080, backend, 30080); err != nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } @@ -524,7 +525,7 @@ func TestVipXlatTable_UnregisterBindingAtRemovesOwnRows(t *testing.T) { backend := net.ParseIP("fd20:60::1") registerBinding(t, vt, 0x46, vip, backend) - removed, err := vt.UnregisterBindingAt(testBlock, 0x46, ProtoTCP, vip, 8080, backend, 30080) + removed, err := vt.UnregisterBindingAt(testBlock, 0x46, 0, ProtoTCP, vip, 8080, backend, 30080) if err != nil { t.Fatalf("UnregisterBindingAt: unexpected error: %v", err) } @@ -538,11 +539,11 @@ func TestVipXlatTable_UnregisterBindingAtRemovesIngressWithoutEgress(t *testing. vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x46, ProtoTCP, vip, 8080, backend, 30080); err != nil { + if err := vt.RegisterIngress(testBlock, 0x46, 0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } - removed, err := vt.UnregisterBindingAt(testBlock, 0x46, ProtoTCP, vip, 8080, backend, 30080) + removed, err := vt.UnregisterBindingAt(testBlock, 0x46, 0, ProtoTCP, vip, 8080, backend, 30080) if err != nil { t.Fatalf("UnregisterBindingAt: unexpected error: %v", err) } @@ -559,7 +560,7 @@ func TestVipXlatTable_UnregisterBindingAtKeepsRowsOfAnotherBinding(t *testing.T) otherBackend := net.ParseIP("fd20:60::2") registerBinding(t, vt, 0x46, vip, otherBackend) - removed, err := vt.UnregisterBindingAt(testBlock, 0x46, ProtoTCP, vip, 8080, backend, 30080) + removed, err := vt.UnregisterBindingAt(testBlock, 0x46, 0, ProtoTCP, vip, 8080, backend, 30080) if err != nil { t.Fatalf("UnregisterBindingAt: unexpected error: %v", err) } @@ -580,7 +581,7 @@ func TestVipXlatTable_UnregisterBindingAtKeepsEgressOfAnotherVIP(t *testing.T) { t.Fatalf("RegisterEgress: unexpected error: %v", err) } - removed, err := vt.UnregisterBindingAt(testBlock, 0x46, ProtoTCP, vip, 8080, backend, 30080) + removed, err := vt.UnregisterBindingAt(testBlock, 0x46, 0, ProtoTCP, vip, 8080, backend, 30080) if err != nil { t.Fatalf("UnregisterBindingAt: unexpected error: %v", err) } @@ -608,7 +609,7 @@ func TestVipXlatTable_SharedPortBindingsKeepSeparateRows(t *testing.T) { } for _, c := range []struct{ vip, backend net.IP }{{vipA, backendA}, {vipB, backendB}} { - in, ok, err := vt.GetIngress(testBlock, 0x1, ProtoTCP, c.vip, 8080) + in, ok, err := vt.GetIngress(testBlock, 0x1, 0, ProtoTCP, c.vip, 8080) if err != nil || !ok { t.Fatalf("GetIngress(%s): ok=%v err=%v", c.vip, ok, err) } @@ -624,10 +625,10 @@ func TestVipXlatTable_SharedPortBindingsKeepSeparateRows(t *testing.T) { } } - if _, err := vt.UnregisterBindingAt(testBlock, 0x1, ProtoTCP, vipA, 8080, backendA, 30080); err != nil { + if _, err := vt.UnregisterBindingAt(testBlock, 0x1, 0, ProtoTCP, vipA, 8080, backendA, 30080); err != nil { t.Fatalf("UnregisterBindingAt(A): %v", err) } - if _, ok, _ := vt.GetIngress(testBlock, 0x1, ProtoTCP, vipB, 8080); !ok { + if _, ok, _ := vt.GetIngress(testBlock, 0x1, 0, ProtoTCP, vipB, 8080); !ok { t.Errorf("binding B's ingress row is gone after removing binding A's") } if _, ok, _ := vt.GetEgress(testBlock, 0x1, ProtoTCP, backendB, 30080); !ok { @@ -646,11 +647,11 @@ func TestVipXlatTable_UnregisterBindingFindsIngressWithoutEgress(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x46, ProtoTCP, vip, 8080, backend, 30080); err != nil { + if err := vt.RegisterIngress(testBlock, 0x46, 0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } - removed, err := vt.UnregisterBinding(ProtoTCP, vip, 8080, backend, 30080) + removed, err := vt.UnregisterBinding(0, ProtoTCP, vip, 8080, backend, 30080) if err != nil { t.Fatalf("UnregisterBinding: unexpected error: %v", err) } @@ -664,7 +665,7 @@ func TestVipXlatTable_ListDecodesKeyAddress(t *testing.T) { vip := net.ParseIP("2001:db8::1") backend := net.ParseIP("fd20:60::1") - if err := vt.RegisterIngress(testBlock, 0x1, ProtoTCP, vip, 8080, backend, 30080); err != nil { + if err := vt.RegisterIngress(testBlock, 0x1, 0, ProtoTCP, vip, 8080, backend, 30080); err != nil { t.Fatalf("RegisterIngress: unexpected error: %v", err) } entries, err := vt.List() @@ -675,3 +676,68 @@ func TestVipXlatTable_ListDecodesKeyAddress(t *testing.T) { t.Errorf("List key addr = %s, want %s", entries[0].Addr, want) } } + +// TestKernelKeySizeUnchanged pins vip_xlat_table's key at 32 bytes. Slot took +// the place of an always-zero pad field so the key kept its size: a size change +// makes the loader treat the pinned map as incompatible and recreate it, which +// drops every row a node has registered. +func TestKernelKeySizeUnchanged(t *testing.T) { + if got := unsafe.Sizeof(prog.UsidVipXlatKey{}); got != 32 { + t.Errorf("sizeof(UsidVipXlatKey) = %d, want 32", got) + } +} + +// TestVipXlatTable_SlotsShareVIP registers two backends of one VIP and port in +// the same VRF under different slots, and checks each keeps its own ingress +// row, and that removing one binding leaves the other's rows alone. +func TestVipXlatTable_SlotsShareVIP(t *testing.T) { + vt, ft := newTestTable(constClock(1)) + + vip := net.ParseIP("2001:db8:5:5::100") + backendA := net.ParseIP("fd20:60::5:a") + backendB := net.ParseIP("fd20:60::5:b") + const port = 80 + const slotA, slotB = 0x1111, 0x2222 + + for _, b := range []struct { + slot uint16 + backend net.IP + }{{slotA, backendA}, {slotB, backendB}} { + if err := vt.RegisterIngress(testBlock, 0x1, b.slot, ProtoTCP, vip, port, b.backend, port); err != nil { + t.Fatalf("RegisterIngress(slot %#x): unexpected error: %v", b.slot, err) + } + if err := vt.RegisterEgress(testBlock, 0x1, ProtoTCP, b.backend, port, vip, port); err != nil { + t.Fatalf("RegisterEgress(%s): unexpected error: %v", b.backend, err) + } + } + if got, want := ft.len(), 4; got != want { + t.Fatalf("table has %d entries, want %d (one ingress and one egress row per backend)", got, want) + } + + for _, b := range []struct { + slot uint16 + backend net.IP + }{{slotA, backendA}, {slotB, backendB}} { + e, ok, err := vt.GetIngress(testBlock, 0x1, b.slot, ProtoTCP, vip, port) + if err != nil || !ok { + t.Fatalf("GetIngress(slot %#x): ok=%v err=%v, want the row present", b.slot, ok, err) + } + if !e.RewriteAddr.Equal(b.backend) || e.Slot != b.slot { + t.Errorf("GetIngress(slot %#x) = %s slot %#x, want %s", b.slot, e.RewriteAddr, e.Slot, b.backend) + } + } + + removed, err := vt.UnregisterBinding(slotA, ProtoTCP, vip, port, backendA, port) + if err != nil { + t.Fatalf("UnregisterBinding(A): unexpected error: %v", err) + } + if len(removed) != 2 { + t.Errorf("UnregisterBinding(A) removed %d rows, want 2", len(removed)) + } + if _, ok, _ := vt.GetIngress(testBlock, 0x1, slotB, ProtoTCP, vip, port); !ok { + t.Errorf("backend B's ingress row is gone after removing backend A's binding") + } + if _, ok, _ := vt.GetEgress(testBlock, 0x1, ProtoTCP, backendB, port); !ok { + t.Errorf("backend B's egress row is gone after removing backend A's binding") + } +} diff --git a/internal/plumbing/srv6/usid.go b/internal/plumbing/srv6/usid.go index 87cee598..5a9afe5d 100644 --- a/internal/plumbing/srv6/usid.go +++ b/internal/plumbing/srv6/usid.go @@ -6,6 +6,7 @@ package srv6 import ( "fmt" + "hash/fnv" "net/netip" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" @@ -35,12 +36,25 @@ func functionNibble(fn bgpv1alpha1.SRv6Function) (uint8, error) { // local Linux VRF this SID's decapped traffic // resolves to -- allocated per node, not derived // from the VPCAttachment identifier) -// bits 81-128 Padding (always zero) +// bits 81-96 Slot (zero here; see ComputeBackendSID) +// bits 97-128 Padding (always zero) // // locator must be an IPv6 /48; nodeID and argument must fit their fields. // Returns the full 128-bit SID, or an error naming the field that is out of // range. See internal/plumbing/ebpf/uformat for the encode primitives. func ComputeSID(locator string, nodeID, argument int32, function bgpv1alpha1.SRv6Function) (netip.Addr, error) { + return ComputeBackendSID(locator, nodeID, argument, function, 0) +} + +// ComputeBackendSID is ComputeSID with slot written into bits 81-96, the uSID +// galactic-gateway encapsulates toward one specific backend of a load-balanced +// VIP. The backend node's usid_ingress reads slot back and keys its +// vip_xlat_table lookup on it, so the backend Maglev chose is the one the VIP +// is rewritten to even when other backends of the same VIP share that node and +// VRF. slot comes from BackendSlot; zero yields exactly ComputeSID's result. +func ComputeBackendSID( + locator string, nodeID, argument int32, function bgpv1alpha1.SRv6Function, slot uint16, +) (netip.Addr, error) { block, err := nodeIdentity(locator, nodeID) if err != nil { return netip.Addr{}, err @@ -59,9 +73,34 @@ func ComputeSID(locator string, nodeID, argument int32, function bgpv1alpha1.SRv NodeID: uint16(nodeID), Function: fn, Argument: uint16(argument), + Slot: slot, }) } +// BackendSlot returns the uSID Slot that names the backend at addr:port. It is +// a pure function of the backend's own address and port, so galactic-gateway, +// which writes the slot into the destination it encapsulates toward, and the +// backend node's galactic-router, which writes the vip_xlat_table row keyed on +// it, agree without exchanging any state. +// +// The hash is 32-bit FNV-1a over the 16-byte address and the big-endian port, +// folded to 16 bits by XOR of its halves. Zero means "no slot" in the uSID +// layout, so a hash of zero becomes 1. Two backends of one VIP on one node that +// hash to the same slot claim the same row; the binding reconciler reports the +// later one as conflicting rather than delivering both to one backend. +func BackendSlot(addr netip.Addr, port uint16) uint16 { + b := addr.As16() + h := fnv.New32a() + _, _ = h.Write(b[:]) + _, _ = h.Write([]byte{byte(port >> 8), byte(port)}) + sum := h.Sum32() + slot := uint16(sum>>16) ^ uint16(sum) + if slot == 0 { + return 1 + } + return slot +} + // nodeIdentity validates locator and nodeID as the node-identifying half of a // uSID -- bits 1-48 and 49-64 -- and returns the Block they encode. Shared by // every SID this package builds, so a SID and the base it is completed from diff --git a/internal/plumbing/srv6/usid_test.go b/internal/plumbing/srv6/usid_test.go index 3ceb6a86..1f74710a 100644 --- a/internal/plumbing/srv6/usid_test.go +++ b/internal/plumbing/srv6/usid_test.go @@ -341,3 +341,64 @@ func TestNodeLocatorAddressNamesSameNodeAsSIDBase(t *testing.T) { t.Errorf("NodeLocatorAddress() %s and NodeSIDBase() %s differ in bits 1-64", addr, base) } } + +// TestBackendSlot checks BackendSlot is deterministic, never zero, and tells +// apart a handful of backends sharing a node, including two that differ only by +// port. +func TestBackendSlot(t *testing.T) { + backends := []netip.AddrPort{ + netip.MustParseAddrPort("[fd20:60:ff03:a::100]:80"), + netip.MustParseAddrPort("[fd20:60:ff03:a::101]:80"), + netip.MustParseAddrPort("[fd20:60:ff03:a::102]:80"), + netip.MustParseAddrPort("[fd20:60:ff03:a::100]:8080"), + netip.MustParseAddrPort("[fd00:10:1::1]:8443"), + } + seen := make(map[uint16]netip.AddrPort, len(backends)) + for _, b := range backends { + slot := BackendSlot(b.Addr(), b.Port()) + if slot == 0 { + t.Errorf("BackendSlot(%s) = 0, want nonzero", b) + } + if again := BackendSlot(b.Addr(), b.Port()); again != slot { + t.Errorf("BackendSlot(%s) not deterministic: %#x then %#x", b, slot, again) + } + if prev, dup := seen[slot]; dup { + t.Errorf("BackendSlot(%s) = %#x, same as %s", b, slot, prev) + } + seen[slot] = b + } +} + +// TestComputeBackendSID checks the slot lands in the SID's Slot field, leaves +// every other field equal to ComputeSID's, and that slot zero is ComputeSID. +func TestComputeBackendSID(t *testing.T) { + plain, err := ComputeSID(testUSIDLocator, 7, 0x123, bgpv1alpha1.SRv6FunctionEndDT46) + if err != nil { + t.Fatalf("ComputeSID: %v", err) + } + zero, err := ComputeBackendSID(testUSIDLocator, 7, 0x123, bgpv1alpha1.SRv6FunctionEndDT46, 0) + if err != nil { + t.Fatalf("ComputeBackendSID(slot 0): %v", err) + } + if zero != plain { + t.Errorf("ComputeBackendSID(slot 0) = %s, want ComputeSID's %s", zero, plain) + } + + slot := BackendSlot(netip.MustParseAddr("fd20:60:ff03:a::100"), 80) + sid, err := ComputeBackendSID(testUSIDLocator, 7, 0x123, bgpv1alpha1.SRv6FunctionEndDT46, slot) + if err != nil { + t.Fatalf("ComputeBackendSID: %v", err) + } + got, err := uformat.Decode(sid) + if err != nil { + t.Fatalf("uformat.Decode(%s): %v", sid, err) + } + want, err := uformat.Decode(plain) + if err != nil { + t.Fatalf("uformat.Decode(%s): %v", plain, err) + } + want.Slot = slot + if got != want { + t.Errorf("Decode(ComputeBackendSID) = %+v, want %+v", got, want) + } +}