From 9d6ed42db569c673d213f1f2ab20f8cdb7c3c6a7 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Fri, 23 Oct 2026 15:59:24 +0000 Subject: [PATCH] feat: Give load balancers an IPv6 address per location Each load balancer gets one public IPv6 address in every location where its backends run, or in the locations the customer chooses, so customers have a stable per-location address and hostname to point DNS at. Key changes: - Reserve one IPv6 address per address request and selected location - Report selected locations, addresses and per-location hostnames in status - Hold an old address until traffic and DNS for it are withdrawn - Refuse selections above the 1,024 endpoint limit before claiming addresses --- config/rbac/role.yaml | 12 + internal/cmd/manager/manager.go | 7 + internal/cmd/manager/manager_test.go | 1 + internal/config/config.go | 5 + .../networkloadbalancer_controller.go | 708 ++++++++++++++++++ .../networkloadbalancer_controller_test.go | 658 ++++++++++++++++ 6 files changed, 1391 insertions(+) create mode 100644 internal/controller/networkloadbalancer_controller.go create mode 100644 internal/controller/networkloadbalancer_controller_test.go diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d75765d3..0f398e70 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -343,6 +343,7 @@ rules: - networkcontexts/finalizers - networkinterfaceclaims/finalizers - networkinterfaces/finalizers + - networkloadbalancers/finalizers - networkpolicies/finalizers - networks/finalizers - subnetclaims/finalizers @@ -361,6 +362,7 @@ rules: - networkcontexts/status - networkinterfaceclaims/status - networkinterfaces/status + - networkloadbalancers/status - networkpolicies/status - networks/status - networkservices/status @@ -380,3 +382,13 @@ rules: - get - list - watch +- apiGroups: + - networking.datumapis.com + resources: + - networkloadbalancers + verbs: + - get + - list + - patch + - update + - watch diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 2b6d9fb0..db0f3e5f 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -802,6 +802,13 @@ func controllerRegistrations( {"networkservice", true, func() error { return (&controller.NetworkServiceReconciler{}).SetupWithManager(mgr) }}, + {"networkloadbalancer", true, func() error { + return (&controller.NetworkLoadBalancerReconciler{ + IPAM: deps.ipamClients, + PublicIPv6Class: serverConfig.IPAM.Classes.PublicIPv6, + ManagedDomain: serverConfig.Gateway.TargetDomain, + }).SetupWithManager(mgr) + }}, // Which workloads are behind a proxy is only answerable in a project // control plane, and only a cell can act on it. The record this writes to // the hub is what carries the answer between them. diff --git a/internal/cmd/manager/manager_test.go b/internal/cmd/manager/manager_test.go index c9e0ea6c..00f7b088 100644 --- a/internal/cmd/manager/manager_test.go +++ b/internal/cmd/manager/manager_test.go @@ -64,6 +64,7 @@ var reconcilerControllerNames = map[string]string{ "NetworkInterfaceProjector": "networkinterfaceprojector", "NetworkInterfaceReconciler": "networkinterface", "NetworkInterfaceWriteBackReconciler": "networkinterfacewriteback", + "NetworkLoadBalancerReconciler": "networkloadbalancer", "NetworkPolicyReconciler": "networkpolicy", "NetworkPresenceReconciler": "networkpresence", "NetworkPresenceSyncReconciler": "networkpresencesync", diff --git a/internal/config/config.go b/internal/config/config.go index 1567029e..ecfcc229 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -314,6 +314,11 @@ type IPAMConfig struct { // environment's classes are different objects, and an operator deployed // against either has to name the ones it is pointed at. type IPAMClasses struct { + // PublicIPv6 is the address class used for per-location public load + // balancer VIPs. Its backing class must share one platform-wide pool with + // uniqueWithin unset, including when projects use source classes. + PublicIPv6 string `json:"publicIPv6,omitempty"` + // Network is the class that hands out the range a network is addressed // from. Unset, no range is claimed and a network is reconciled with no // address space, the same as an unset IPAM connection. diff --git a/internal/controller/networkloadbalancer_controller.go b/internal/controller/networkloadbalancer_controller.go new file mode 100644 index 00000000..4741fc52 --- /dev/null +++ b/internal/controller/networkloadbalancer_controller.go @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "time" + + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/words" +) + +const ( + networkLoadBalancerFinalizer = "networking.datumapis.com/networkloadbalancer-release" + networkLoadBalancerOwnerLabel = "networking.datumapis.com/network-load-balancer-uid" + networkLoadBalancerRetry = time.Minute +) + +type NetworkLoadBalancerReconciler struct { + IPAM IPAMClientFactory + PublicIPv6Class string + ManagedDomain string + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancers,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancers/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=locations.miloapis.com,resources=locations,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch + +func (r *NetworkLoadBalancerReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + cluster, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + ctx = mccontext.WithCluster(ctx, req.ClusterName) + return r.reconcile(ctx, cluster.GetClient(), req.NamespacedName) +} + +func (r *NetworkLoadBalancerReconciler) reconcile(ctx context.Context, cl client.Client, key client.ObjectKey) (ctrl.Result, error) { + var lb networkingv1alpha.NetworkLoadBalancer + if err := cl.Get(ctx, key, &lb); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !lb.DeletionTimestamp.IsZero() { + return r.release(ctx, cl, &lb) + } + + if r.IPAM == nil || r.PublicIPv6Class == "" { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "AddressServiceUnavailable", + "Public IPv6 address allocation is not configured") + } + if controllerutil.AddFinalizer(&lb, networkLoadBalancerFinalizer) { + if err := cl.Update(ctx, &lb); err != nil { + return ctrl.Result{}, fmt.Errorf("add load balancer finalizer: %w", err) + } + return ctrl.Result{Requeue: true}, nil + } + for _, request := range lb.Spec.Exposure.Addresses { + if slices.Contains(request.Families, networkingv1alpha.IPv4Protocol) { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "UnsupportedCapability", + fmt.Sprintf("address request %q: public IPv4 NLB ingress is unavailable", request.Name)) + } + } + + var network networkingv1alpha.Network + if err := cl.Get(ctx, client.ObjectKey{Namespace: lb.Namespace, Name: lb.Spec.NetworkRef.Name}, &network); err != nil { + if apierrors.IsNotFound(err) { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "NetworkNotFound", + fmt.Sprintf("Network %q was not found", lb.Spec.NetworkRef.Name)) + } + return ctrl.Result{}, fmt.Errorf("read load balancer network: %w", err) + } + if !slices.Contains(network.Spec.IPFamilies, networkingv1alpha.IPv6Protocol) { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "IPv6Required", + "The referenced network does not carry IPv6") + } + + routing, err := resolveProjectOrCluster(ctx, cl, lb.Namespace) + if err != nil { + var unresolved *projectUnresolvable + if errors.As(err, &unresolved) { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "ProjectUnresolved", err.Error()) + } + return ctrl.Result{}, err + } + ipamClient, err := r.IPAM.ClientForProject(routing.project) + if err != nil { + return ctrl.Result{}, fmt.Errorf("build project IPAM client: %w", err) + } + + requests, err := r.addressRequests(ctx, cl, &lb) + if err != nil { + return ctrl.Result{}, err + } + if countAddressEndpoints(requests, lb.Status.Endpoints) > 1024 { + return r.report(ctx, cl, &lb, nil, requests, metav1.ConditionFalse, "TooManyEndpoints", + "Selected and withdrawing address endpoints exceed the status limit of 1024; reduce address requests or locations") + } + + wanted := make(map[string]bool, len(requests)) + status := make([]networkingv1alpha.NetworkLoadBalancerEndpointStatus, 0, len(requests)+len(lb.Status.Endpoints)) + allAllocated := len(requests) > 0 + var firstFailure string + for _, request := range requests { + claimName := loadBalancerClaimName(&lb, request) + wanted[claimName] = true + prior := priorAddressStatus(lb.Status.Endpoints, request.requestName, request.location, request.family) + prior = *prior.DeepCopy() + address, claimUID, claimErr := r.holdAddress(ctx, ipamClient, routing.projectNamespace, &lb, request, claimName) + claimRef := &networkingv1alpha.NetworkLoadBalancerIPClaimRef{ + Project: routing.project, Namespace: routing.projectNamespace, Name: claimName, UID: claimUID, + } + changed := prior.Value != "" && prior.Value != address || + prior.ClaimRef != nil && prior.ClaimRef.UID != "" && claimUID != "" && prior.ClaimRef.UID != claimUID + if changed && addressPendingWithdrawal(prior) { + // Keep the old address visible to withdrawal controllers. In particular, + // do not transfer its announcement or publication to a replacement IP. + allAllocated = false + if firstFailure == "" { + firstFailure = fmt.Sprintf("%s: prior address is awaiting withdrawal", claimName) + } + setLoadBalancerCondition(&prior.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + metav1.ConditionFalse, "AddressReplacementPendingWithdrawal", "Prior address must be withdrawn before its replacement can be served") + appendAddressStatus(&status, lb.Status.Endpoints, request.requestName, request.location, prior) + continue + } + entry := prior + if changed || prior.Value == "" && address != "" { + // Programming, announcement, and DNS publication describe one concrete + // address, not the request that happened to produce it. + entry = networkingv1alpha.NetworkLoadBalancerAddressStatus{Family: request.family} + } + entry.Family = request.family + entry.ClaimRef = claimRef + if claimErr != nil { + allAllocated = false + entry.Value = "" + reason := string(classifyAllocationFailure(claimErr)) + setLoadBalancerCondition(&entry.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + metav1.ConditionFalse, reason, claimErr.Error()) + if firstFailure == "" { + firstFailure = fmt.Sprintf("%s: %s", claimName, claimErr) + } + } else if address == "" { + allAllocated = false + entry.Value = "" + setLoadBalancerCondition(&entry.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + metav1.ConditionFalse, "AllocationPending", "IPAM has not assigned an address yet") + } else { + entry.Value = address + setLoadBalancerCondition(&entry.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + metav1.ConditionTrue, "Allocated", "Public IPv6 address is held in IPAM") + } + appendAddressStatus(&status, lb.Status.Endpoints, request.requestName, request.location, entry) + } + + for _, endpoint := range lb.Status.Endpoints { + for _, old := range endpoint.Addresses { + if old.ClaimRef != nil && wanted[old.ClaimRef.Name] { + continue + } + if addressPendingWithdrawal(old) { + appendAddressStatus(&status, lb.Status.Endpoints, endpoint.RequestName, endpoint.Location, old) + allAllocated = false + continue + } + if old.ClaimRef != nil { + if err := r.deleteAddressClaim(ctx, old.ClaimRef, string(lb.UID)); err != nil { + return ctrl.Result{}, err + } + } + } + } + + if err := r.releaseOrphanedClaims(ctx, ipamClient, routing.projectNamespace, &lb, wanted); err != nil { + return ctrl.Result{}, err + } + slices.SortFunc(status, func(a, b networkingv1alpha.NetworkLoadBalancerEndpointStatus) int { + if a.RequestName != b.RequestName { + return strings.Compare(a.RequestName, b.RequestName) + } + return strings.Compare(a.Location, b.Location) + }) + for i := range status { + if status[i].CanonicalHostname == "" && r.ManagedDomain != "" { + label := words.WordsAndEntropy("", string(lb.UID)+"/"+status[i].RequestName) + status[i].CanonicalHostname = status[i].Location + "." + label + "." + r.ManagedDomain + } + } + + if !allAllocated { + if firstFailure == "" { + firstFailure = "One or more requested addresses are pending or awaiting withdrawal" + } + return r.report(ctx, cl, &lb, status, requests, metav1.ConditionFalse, "AllocationPending", firstFailure) + } + return r.report(ctx, cl, &lb, status, requests, metav1.ConditionTrue, "Allocated", "All requested public IPv6 addresses are held") +} + +func countAddressEndpoints(requests []loadBalancerAddressRequest, prior []networkingv1alpha.NetworkLoadBalancerEndpointStatus) int { + type endpointKey struct{ requestName, location string } + selected := make(map[endpointKey]bool, len(requests)) + for _, request := range requests { + selected[endpointKey{request.requestName, request.location}] = true + } + for _, endpoint := range prior { + key := endpointKey{endpoint.RequestName, endpoint.Location} + if selected[key] { + continue + } + for _, address := range endpoint.Addresses { + if addressPendingWithdrawal(address) { + selected[key] = true + break + } + } + } + return len(selected) +} + +type loadBalancerAddressRequest struct { + requestName string + location string + networkName string + family networkingv1alpha.IPFamily +} + +func (r *NetworkLoadBalancerReconciler) addressRequests( + ctx context.Context, cl client.Client, lb *networkingv1alpha.NetworkLoadBalancer, +) ([]loadBalancerAddressRequest, error) { + var available locationsv1alpha1.LocationList + if err := cl.List(ctx, &available); err != nil { + return nil, fmt.Errorf("list available locations: %w", err) + } + known := map[string]bool{} + for _, location := range available.Items { + ready := apimeta.FindStatusCondition(location.Status.Conditions, locationsv1alpha1.LocationConditionReady) + known[location.Name] = ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == location.Generation + } + + var requests []loadBalancerAddressRequest + for _, address := range lb.Spec.Exposure.Addresses { + var names []string + source := networkingv1alpha.NetworkLoadBalancerLocationSourceBackends + if address.Locations != nil { + source = address.Locations.From + } + switch source { + case networkingv1alpha.NetworkLoadBalancerLocationSourceAll: + for name, ready := range known { + if ready { + names = append(names, name) + } + } + case networkingv1alpha.NetworkLoadBalancerLocationSourceNames: + // A location projection may temporarily disappear. An explicit + // request still owns its IP throughout that gap. + names = append(names, address.Locations.Names...) + default: + backendNames, err := backendLocationsForAddress(ctx, cl, lb, address.Name) + if err != nil { + return nil, err + } + names = backendNames + } + slices.Sort(names) + for _, name := range names { + for _, family := range address.Families { + requests = append(requests, loadBalancerAddressRequest{ + requestName: address.Name, location: name, networkName: lb.Spec.NetworkRef.Name, family: family, + }) + } + } + } + return requests, nil +} + +// backendLocationsForAddress selects locations of the effective backends on +// every listener using the address. It uses live, bound membership +// rather than the capped NetworkService status display list. +func backendLocationsForAddress(ctx context.Context, cl client.Client, + lb *networkingv1alpha.NetworkLoadBalancer, addressName string, +) ([]string, error) { + var routes networkingv1alpha.NetworkLoadBalancerRouteList + if err := cl.List(ctx, &routes, client.InNamespace(lb.Namespace)); err != nil { + return nil, fmt.Errorf("list load balancer routes: %w", err) + } + selected := map[string]bool{} + attached := false + for _, listener := range lb.Spec.Listeners { + if !slices.Contains(listener.AddressRefs, addressName) { + continue + } + var winner *networkingv1alpha.NetworkLoadBalancerRoute + for i := range routes.Items { + route := &routes.Items[i] + if !route.DeletionTimestamp.IsZero() || !slices.ContainsFunc(route.Spec.ParentRefs, + func(ref networkingv1alpha.NetworkLoadBalancerRouteParentRef) bool { + return routeParentSelectsListener(lb, ref, listener.Name) + }) { + continue + } + if winner == nil || route.CreationTimestamp.Before(&winner.CreationTimestamp) || + (route.CreationTimestamp.Equal(&winner.CreationTimestamp) && route.Name < winner.Name) { + winner = route + } + } + if winner == nil { + return nil, nil + } + attached = true + for _, backend := range winner.Spec.Backends { + var service networkingv1alpha.NetworkService + key := client.ObjectKey{Namespace: winner.Namespace, Name: backend.NetworkService.Name} + if backend.NetworkService.Namespace != "" { + key.Namespace = backend.NetworkService.Namespace + } + if err := cl.Get(ctx, key, &service); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("get backend NetworkService %s: %w", key, err) + } + selector, err := metav1.LabelSelectorAsSelector(&service.Spec.NetworkInterfaces.Selector) + if err != nil { + return nil, fmt.Errorf("backend NetworkService %s has invalid selector: %w", key, err) + } + members, err := matchingInterfaces(ctx, cl, service.Namespace, selector) + if err != nil { + return nil, err + } + locations := map[string]bool{} + for i := range members { + member := &members[i] + if member.Spec.Network.Name == lb.Spec.NetworkRef.Name { + if location := member.Labels[networkingv1alpha.NetworkInterfaceLocationLabel]; location != "" { + locations[location] = true + } + } + } + for location := range locations { + selected[location] = true + } + } + } + if !attached { + return nil, nil + } + names := make([]string, 0, len(selected)) + for name := range selected { + names = append(names, name) + } + slices.Sort(names) + return names, nil +} + +func routeParentSelectsListener(lb *networkingv1alpha.NetworkLoadBalancer, + ref networkingv1alpha.NetworkLoadBalancerRouteParentRef, listenerName string, +) bool { + if ref.Name != lb.Name || ref.Namespace != "" && ref.Namespace != lb.Namespace { + return false + } + if ref.Listener != "" { + return ref.Listener == listenerName + } + if ref.Port == nil { + return false + } + var matchedName string + for _, candidate := range lb.Spec.Listeners { + if candidate.Port != *ref.Port { + continue + } + if matchedName != "" { + return false + } + matchedName = candidate.Name + } + return matchedName == listenerName +} + +func loadBalancerClaimName(lb *networkingv1alpha.NetworkLoadBalancer, request loadBalancerAddressRequest) string { + identity := request.requestName + "\x00" + request.location + "\x00" + string(request.family) + sum := sha256.Sum256([]byte(identity)) + return ipClaimName("nlb-"+string(lb.UID), request.requestName+"-"+hex.EncodeToString(sum[:8])) +} + +func (r *NetworkLoadBalancerReconciler) holdAddress( + ctx context.Context, cl client.Client, namespace string, lb *networkingv1alpha.NetworkLoadBalancer, + request loadBalancerAddressRequest, claimName string, +) (string, string, error) { + claim := &ipamv1alpha1.IPClaim{} + claim.Namespace = namespace + claim.Name = claimName + claim.Labels = map[string]string{networkLoadBalancerOwnerLabel: string(lb.UID)} + claim.Spec = ipamv1alpha1.IPClaimSpec{ + ClassName: r.PublicIPv6Class, + IPFamily: ipamv1alpha1.IPFamily(networkingv1alpha.IPv6Protocol), + PrefixLength: ptr.To[int32](128), + ReclaimPolicy: ipamv1alpha1.ReclaimDelete, + Scope: map[string]ipamv1alpha1.ScopeRef{ + ipamScopeRoleNetwork: {APIGroup: datumNetworkingAPIGroup, Kind: "Network", Name: request.networkName}, + }, + } + if request.location != "" { + claim.Spec.Scope[ipamScopeRoleLocation] = ipamv1alpha1.ScopeRef{ + APIGroup: datumNetworkingAPIGroup, Kind: "Location", Name: request.location, + } + } + + existing := &ipamv1alpha1.IPClaim{} + err := cl.Get(ctx, client.ObjectKeyFromObject(claim), existing) + if apierrors.IsNotFound(err) { + if err := cl.Create(ctx, claim); err != nil { + if !apierrors.IsAlreadyExists(err) { + return "", "", err + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(claim), existing); err != nil { + return "", "", err + } + } else { + existing = claim + } + } else if err != nil { + return "", "", err + } + if !existing.DeletionTimestamp.IsZero() { + return "", string(existing.UID), fmt.Errorf("IPClaim %q is being deleted", claimName) + } + if existing.Labels[networkLoadBalancerOwnerLabel] != string(lb.UID) || + existing.Spec.ClassName != claim.Spec.ClassName || + existing.Spec.IPFamily != claim.Spec.IPFamily || + existing.Spec.PrefixLength == nil || *existing.Spec.PrefixLength != 128 || + existing.Spec.ReclaimPolicy != claim.Spec.ReclaimPolicy || + !equality.Semantic.DeepEqual(existing.Spec.Scope, claim.Spec.Scope) { + return "", "", fmt.Errorf("IPClaim %q does not match this load balancer's address request", claimName) + } + address := existing.Status.Address + if address == "" && existing.Status.AllocatedCIDR != "" { + prefix, err := netip.ParsePrefix(existing.Status.AllocatedCIDR) + if err != nil || prefix.Bits() != 128 { + return "", "", fmt.Errorf("IPClaim %q did not allocate one IPv6 address", claimName) + } + address = prefix.Addr().String() + } + if address != "" { + parsed, err := netip.ParseAddr(address) + if err != nil || !parsed.Is6() { + return "", "", fmt.Errorf("IPClaim %q returned an invalid IPv6 address %q", claimName, address) + } + } + return address, string(existing.UID), nil +} + +func priorAddressStatus( + endpoints []networkingv1alpha.NetworkLoadBalancerEndpointStatus, requestName, location string, family networkingv1alpha.IPFamily, +) networkingv1alpha.NetworkLoadBalancerAddressStatus { + for _, endpoint := range endpoints { + if endpoint.RequestName == requestName && endpoint.Location == location { + for _, address := range endpoint.Addresses { + if address.Family == family { + return address + } + } + } + } + return networkingv1alpha.NetworkLoadBalancerAddressStatus{} +} + +func appendAddressStatus(status *[]networkingv1alpha.NetworkLoadBalancerEndpointStatus, + prior []networkingv1alpha.NetworkLoadBalancerEndpointStatus, requestName, location string, + address networkingv1alpha.NetworkLoadBalancerAddressStatus, +) { + i := slices.IndexFunc(*status, func(endpoint networkingv1alpha.NetworkLoadBalancerEndpointStatus) bool { + return endpoint.RequestName == requestName && endpoint.Location == location + }) + if i < 0 { + entry := networkingv1alpha.NetworkLoadBalancerEndpointStatus{RequestName: requestName, Location: location} + for _, old := range prior { + if old.RequestName == requestName && old.Location == location { + entry.CanonicalHostname = old.CanonicalHostname + break + } + } + *status = append(*status, entry) + i = len(*status) - 1 + } + (*status)[i].Addresses = append((*status)[i].Addresses, address) +} + +func addressAnnounced(address networkingv1alpha.NetworkLoadBalancerAddressStatus) bool { + return address.Announced || apimeta.IsStatusConditionTrue(address.Conditions, networkingv1alpha.NetworkLoadBalancerAnnounced) +} + +func addressPendingWithdrawal(address networkingv1alpha.NetworkLoadBalancerAddressStatus) bool { + return addressAnnounced(address) || address.DNSPublished || + apimeta.IsStatusConditionTrue(address.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed) +} + +func (r *NetworkLoadBalancerReconciler) releaseOrphanedClaims( + ctx context.Context, cl client.Client, namespace string, lb *networkingv1alpha.NetworkLoadBalancer, wanted map[string]bool, +) error { + var claims ipamv1alpha1.IPClaimList + if err := cl.List(ctx, &claims, client.InNamespace(namespace), + client.MatchingLabels{networkLoadBalancerOwnerLabel: string(lb.UID)}); err != nil { + return fmt.Errorf("list load balancer claims: %w", err) + } + for i := range claims.Items { + claim := &claims.Items[i] + if wanted[claim.Name] { + continue + } + if old := priorClaimAddressStatus(lb.Status.Endpoints, claim.Name); old != nil && addressPendingWithdrawal(*old) { + continue + } + if err := cl.Delete(ctx, claim, client.Preconditions{UID: &claim.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { + return fmt.Errorf("release stale IPClaim %q: %w", claim.Name, err) + } + } + return nil +} + +func priorClaimAddressStatus( + endpoints []networkingv1alpha.NetworkLoadBalancerEndpointStatus, claimName string, +) *networkingv1alpha.NetworkLoadBalancerAddressStatus { + for i := range endpoints { + for j := range endpoints[i].Addresses { + address := &endpoints[i].Addresses[j] + if address.ClaimRef != nil && address.ClaimRef.Name == claimName { + return address + } + } + } + return nil +} + +func (r *NetworkLoadBalancerReconciler) deleteAddressClaim( + ctx context.Context, ref *networkingv1alpha.NetworkLoadBalancerIPClaimRef, lbUID string, +) error { + if ref.UID == "" { + // A legacy status entry has no safe identity to delete by name. + return nil + } + cl, err := r.IPAM.ClientForProject(ref.Project) + if err != nil { + return err + } + claim := &ipamv1alpha1.IPClaim{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, claim); apierrors.IsNotFound(err) { + return nil + } else if err != nil { + return fmt.Errorf("read IPClaim %q before release: %w", ref.Name, err) + } + if string(claim.UID) != ref.UID || claim.Labels[networkLoadBalancerOwnerLabel] != lbUID { + return nil + } + if err := cl.Delete(ctx, claim, client.Preconditions{UID: &claim.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { + return fmt.Errorf("release IPClaim %q: %w", ref.Name, err) + } + return nil +} + +func (r *NetworkLoadBalancerReconciler) release( + ctx context.Context, cl client.Client, lb *networkingv1alpha.NetworkLoadBalancer, +) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(lb, networkLoadBalancerFinalizer) { + return ctrl.Result{}, nil + } + for _, endpoint := range lb.Status.Endpoints { + for _, address := range endpoint.Addresses { + if addressPendingWithdrawal(address) { + return ctrl.Result{RequeueAfter: networkLoadBalancerRetry}, nil + } + } + } + if r.IPAM == nil { + return ctrl.Result{}, errors.New("cannot release load balancer addresses without IPAM") + } + routing, err := resolveProjectOrCluster(ctx, cl, lb.Namespace) + if err != nil { + return ctrl.Result{}, err + } + ipamClient, err := r.IPAM.ClientForProject(routing.project) + if err != nil { + return ctrl.Result{}, err + } + if err := r.releaseOrphanedClaims(ctx, ipamClient, routing.projectNamespace, lb, nil); err != nil { + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(lb, networkLoadBalancerFinalizer) + return ctrl.Result{}, cl.Update(ctx, lb) +} + +func (r *NetworkLoadBalancerReconciler) report( + ctx context.Context, cl client.Client, lb *networkingv1alpha.NetworkLoadBalancer, + endpoints []networkingv1alpha.NetworkLoadBalancerEndpointStatus, + selected []loadBalancerAddressRequest, + status metav1.ConditionStatus, reason, message string, +) (ctrl.Result, error) { + before := lb.DeepCopy() + if endpoints != nil { + lb.Status.Endpoints = endpoints + } + requestStatuses := make([]networkingv1alpha.NetworkLoadBalancerAddressRequestStatus, 0, len(lb.Spec.Exposure.Addresses)) + for _, request := range lb.Spec.Exposure.Addresses { + source := networkingv1alpha.NetworkLoadBalancerLocationSourceBackends + if request.Locations != nil { + source = request.Locations.From + } + entry := networkingv1alpha.NetworkLoadBalancerAddressRequestStatus{Name: request.Name, LocationSource: source} + for _, selection := range selected { + if selection.requestName == request.Name && !slices.Contains(entry.ResolvedLocations, selection.location) { + entry.ResolvedLocations = append(entry.ResolvedLocations, selection.location) + } + } + slices.Sort(entry.ResolvedLocations) + requestStatuses = append(requestStatuses, entry) + } + lb.Status.AddressRequests = requestStatuses + setLoadBalancerCondition(&lb.Status.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + status, reason, message) + for _, conditionType := range []string{networkingv1alpha.NetworkLoadBalancerProgrammed, networkingv1alpha.NetworkLoadBalancerReady} { + switch { + case status != metav1.ConditionTrue: + setLoadBalancerCondition(&lb.Status.Conditions, lb.Generation, conditionType, + metav1.ConditionFalse, "AllocationPending", "At least one public address is not ready for serving") + case apimeta.FindStatusCondition(lb.Status.Conditions, conditionType) == nil || + addressIdentityChanged(before.Status.Endpoints, lb.Status.Endpoints): + setLoadBalancerCondition(&lb.Status.Conditions, lb.Generation, conditionType, + metav1.ConditionUnknown, "PendingDataPlane", "Waiting for serving locations to report programming and announcements") + } + } + if equality.Semantic.DeepEqual(before.Status, lb.Status) { + return ctrl.Result{RequeueAfter: networkLoadBalancerRetry}, nil + } + // Location controllers also write nested address conditions. A version + // precondition prevents this allocation pass from overwriting their newer + // announcement result; the manager retries the conflict. + if err := cl.Status().Patch(ctx, lb, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{}, fmt.Errorf("update load balancer status: %w", err) + } + // Periodic reconciliation notices IPAM completion and changes to the + // dynamically selected "All" location set without a cross-cluster watch. + return ctrl.Result{RequeueAfter: networkLoadBalancerRetry}, nil +} + +func addressIdentityChanged(before, after []networkingv1alpha.NetworkLoadBalancerEndpointStatus) bool { + if len(before) != len(after) { + return true + } + for _, endpoint := range after { + for _, address := range endpoint.Addresses { + prior := priorAddressStatus(before, endpoint.RequestName, endpoint.Location, address.Family) + if prior.Value != address.Value { + return true + } + if prior.ClaimRef != nil && address.ClaimRef != nil && + prior.ClaimRef.UID != "" && address.ClaimRef.UID != "" && prior.ClaimRef.UID != address.ClaimRef.UID { + return true + } + } + } + return false +} + +func setLoadBalancerCondition( + conditions *[]metav1.Condition, generation int64, conditionType string, + status metav1.ConditionStatus, reason, message string, +) { + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: conditionType, Status: status, Reason: reason, Message: message, ObservedGeneration: generation, + }) +} + +func (r *NetworkLoadBalancerReconciler) SetupWithManager(mgr mcmanager.Manager) error { + r.mgr = mgr + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkLoadBalancer{}, mcbuilder.WithEngageWithLocalCluster(false)). + Named("networkloadbalancer"). + Complete(r) +} diff --git a/internal/controller/networkloadbalancer_controller_test.go b/internal/controller/networkloadbalancer_controller_test.go new file mode 100644 index 00000000..caebeaba --- /dev/null +++ b/internal/controller/networkloadbalancer_controller_test.go @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/downstreamclient" +) + +type fixedIPAMClient struct{ client.Client } + +func (f fixedIPAMClient) ClientForProject(project string) (client.Client, error) { + if project != testProject { + return nil, fmt.Errorf("unexpected project %q", project) + } + return f.Client, nil +} + +type loadBalancerAllocationScenario struct { + t *testing.T + ctx context.Context + client client.Client + ipam client.Client + lb *networkingv1alpha.NetworkLoadBalancer + control *NetworkLoadBalancerReconciler +} + +func newLoadBalancerAllocationScenario(t *testing.T) *loadBalancerAllocationScenario { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + require.NoError(t, locationsv1alpha1.AddToScheme(scheme)) + require.NoError(t, ipamv1alpha1.AddToScheme(scheme)) + + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: "tenant", + Labels: map[string]string{ + downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-" + testProject, + downstreamclient.UpstreamOwnerNamespaceLabel: testProjectNS, + }, + }} + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: ns.Name}} + network.Spec.IPFamilies = []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol} + lb := &networkingv1alpha.NetworkLoadBalancer{ObjectMeta: metav1.ObjectMeta{ + Name: "public", Namespace: ns.Name, UID: types.UID("f96c4e18-cc50-45a8-aead-b368210f13ca"), + Finalizers: []string{networkLoadBalancerFinalizer}, + }} + lb.Spec.NetworkRef.Name = network.Name + lb.Spec.Exposure.Addresses = []networkingv1alpha.NetworkLoadBalancerAddressRequest{ + {Name: "regional", + Families: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, Locations: &networkingv1alpha.NetworkLoadBalancerLocationSelection{ + From: networkingv1alpha.NetworkLoadBalancerLocationSourceNames, Names: []string{"iad", "fra"}, + }}, + {Name: "edge", + Families: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, Locations: &networkingv1alpha.NetworkLoadBalancerLocationSelection{ + From: networkingv1alpha.NetworkLoadBalancerLocationSourceNames, Names: []string{"iad"}, + }}, + } + lb.Spec.Listeners = []networkingv1alpha.NetworkLoadBalancerListener{{ + Name: "web", Protocol: networkingv1alpha.NetworkLoadBalancerProtocolTCP, Port: 443, + AddressRefs: []string{"regional", "edge"}, + }} + ready := locationsv1alpha1.LocationStatus{Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue, Reason: "Test"}}} + locationIAD := &locationsv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Name: "iad"}, Status: ready} + locationFRA := &locationsv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Name: "fra"}, Status: ready} + cl := fakeclient.NewClientBuilder().WithScheme(scheme). + WithObjects(ns, network, lb, locationIAD, locationFRA). + WithStatusSubresource(lb, &networkingv1alpha.NetworkInterface{}).Build() + ipam := fakeclient.NewClientBuilder().WithScheme(scheme). + WithObjects(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testProjectNS}}). + WithStatusSubresource(&ipamv1alpha1.IPClaim{}).Build() + return &loadBalancerAllocationScenario{ + t: t, ctx: context.Background(), client: cl, ipam: ipam, lb: lb, + control: &NetworkLoadBalancerReconciler{IPAM: fixedIPAMClient{ipam}, PublicIPv6Class: "public-v6", ManagedDomain: "datumproxy.net"}, + } +} + +func (s *loadBalancerAllocationScenario) reconcile() networkingv1alpha.NetworkLoadBalancer { + s.t.Helper() + _, err := s.control.reconcile(s.ctx, s.client, client.ObjectKeyFromObject(s.lb)) + require.NoError(s.t, err) + var lb networkingv1alpha.NetworkLoadBalancer + require.NoError(s.t, s.client.Get(s.ctx, client.ObjectKeyFromObject(s.lb), &lb)) + return lb +} + +func (s *loadBalancerAllocationScenario) claims() []ipamv1alpha1.IPClaim { + s.t.Helper() + var list ipamv1alpha1.IPClaimList + require.NoError(s.t, s.ipam.List(s.ctx, &list, client.InNamespace(testProjectNS))) + return list.Items +} + +func TestNetworkLoadBalancerAllocationHoldsPerLocationAddresses(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + require.Len(t, s.claims(), 3) + require.False(t, apimeta.IsStatusConditionTrue(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated)) + + for i, claim := range s.claims() { + require.Equal(t, "public-v6", claim.Spec.ClassName) + require.NotNil(t, claim.Spec.PrefixLength) + require.Equal(t, int32(128), *claim.Spec.PrefixLength) + require.Equal(t, ipamv1alpha1.ReclaimDelete, claim.Spec.ReclaimPolicy) + require.NotEmpty(t, claim.Spec.Scope[ipamScopeRoleLocation].Name) + claim.Status.AllocatedCIDR = fmt.Sprintf("2001:db8::%x/128", i+1) + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + } + lb = s.reconcile() + require.True(t, apimeta.IsStatusConditionTrue(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated)) + require.Len(t, lb.Status.Endpoints, 3) + require.Len(t, lb.Status.AddressRequests, 2) + require.Equal(t, []string{"fra", "iad"}, lb.Status.AddressRequests[0].ResolvedLocations) + for _, endpoint := range lb.Status.Endpoints { + require.NotEmpty(t, endpoint.Location) + require.Regexp(t, "^"+endpoint.Location+`\.[a-z]+-[a-z]+-[a-z0-9]{5}\.datumproxy\.net$`, endpoint.CanonicalHostname) + for _, address := range endpoint.Addresses { + require.NotEmpty(t, address.Value) + require.True(t, apimeta.IsStatusConditionTrue(address.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated)) + } + } + require.Len(t, s.claims(), 3) + require.Equal(t, lb.Status.Endpoints, s.reconcile().Status.Endpoints, "retry must reuse claims and addresses") +} + +func TestNetworkLoadBalancerClaimNamesKeepAddressIdentityDistinct(t *testing.T) { + lb := &networkingv1alpha.NetworkLoadBalancer{ObjectMeta: metav1.ObjectMeta{UID: types.UID("fixture")}} + a := loadBalancerClaimName(lb, loadBalancerAddressRequest{ + requestName: "a-b", location: "c", + }) + b := loadBalancerClaimName(lb, loadBalancerAddressRequest{ + requestName: "a", location: "b-c", + }) + c := loadBalancerClaimName(lb, loadBalancerAddressRequest{ + requestName: "a-b", location: "d", + }) + require.NotEqual(t, a, b) + require.NotEqual(t, a, c) +} + +func TestNetworkLoadBalancerWithoutIPAMDoesNotLeaveAnUnreleasableFinalizer(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := *s.lb.DeepCopy() + lb.Finalizers = nil + require.NoError(t, s.client.Update(s.ctx, &lb)) + s.control.IPAM = nil + lb = s.reconcile() + require.Empty(t, lb.Finalizers) + require.False(t, apimeta.IsStatusConditionTrue(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated)) +} + +func TestNetworkLoadBalancerExplicitLocationRetainsClaimWhenProjectionDisappears(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + before := s.claims() + require.NoError(t, s.client.Delete(s.ctx, &locationsv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Name: "fra"}})) + s.reconcile() + require.Equal(t, before, s.claims()) +} + +func TestNetworkLoadBalancerAllLocationsAddsRegionalClaimAsLocationsAppear(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + for i := range lb.Spec.Exposure.Addresses { + lb.Spec.Exposure.Addresses[i].Locations = &networkingv1alpha.NetworkLoadBalancerLocationSelection{ + From: networkingv1alpha.NetworkLoadBalancerLocationSourceAll, + } + } + require.NoError(t, s.client.Update(s.ctx, &lb)) + s.reconcile() + require.Len(t, s.claims(), 4) + require.NoError(t, s.client.Create(s.ctx, &locationsv1alpha1.Location{ + ObjectMeta: metav1.ObjectMeta{Name: "sea"}, Status: locationsv1alpha1.LocationStatus{Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue, Reason: "Test"}}}, + })) + lb = s.reconcile() + require.Len(t, s.claims(), 6) + require.Len(t, lb.Status.Endpoints, 6) +} + +func TestNetworkLoadBalancerAllLocationsSkipsUnreadyLocations(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + lb.Spec.Exposure.Addresses[0].Locations = &networkingv1alpha.NetworkLoadBalancerLocationSelection{From: networkingv1alpha.NetworkLoadBalancerLocationSourceAll} + lb.Spec.Exposure.Addresses[1].Locations = &networkingv1alpha.NetworkLoadBalancerLocationSelection{From: networkingv1alpha.NetworkLoadBalancerLocationSourceAll} + require.NoError(t, s.client.Update(s.ctx, &lb)) + require.NoError(t, s.client.Create(s.ctx, &locationsv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Name: "unready"}, + Status: locationsv1alpha1.LocationStatus{Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionFalse, Reason: "Test"}}}})) + lb = s.reconcile() + for _, endpoint := range lb.Status.Endpoints { + require.NotEqual(t, "unready", endpoint.Location) + } +} + +func TestNetworkLoadBalancerClaimReleaseSkipsReplacementUID(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + ref := &networkingv1alpha.NetworkLoadBalancerIPClaimRef{Project: testProject, Namespace: claim.Namespace, Name: claim.Name, UID: "old-claim"} + require.NoError(t, s.control.deleteAddressClaim(s.ctx, ref, string(s.lb.UID))) + var existing ipamv1alpha1.IPClaim + require.NoError(t, s.ipam.Get(s.ctx, client.ObjectKeyFromObject(&claim), &existing)) + ref.UID = string(existing.UID) + require.NoError(t, s.control.deleteAddressClaim(s.ctx, ref, "foreign-lb")) + require.NoError(t, s.ipam.Get(s.ctx, client.ObjectKeyFromObject(&claim), &existing)) +} + +func TestNetworkLoadBalancerBackendPlacementResolvesUniquePortParent(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + lb.Spec.Exposure.Addresses[1].Locations = nil + require.NoError(t, s.client.Update(s.ctx, &lb)) + service := &networkingv1alpha.NetworkService{ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: lb.Namespace}} + service.Spec.NetworkInterfaces.Selector = metav1.LabelSelector{MatchLabels: map[string]string{"app": "backend"}} + service.Spec.Ports = []networkingv1alpha.NetworkServicePort{{Name: "http", Port: 8080}} + require.NoError(t, s.client.Create(s.ctx, service)) + member := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{ + Name: "member", Namespace: lb.Namespace, + Labels: map[string]string{"app": "backend", networkingv1alpha.NetworkInterfaceLocationLabel: "iad"}, + }} + member.Spec.Network.Name = lb.Spec.NetworkRef.Name + member.Spec.ClaimRef = &networkingv1alpha.NetworkInterfaceClaimRef{Name: "member"} + member.Status.Phase = networkingv1alpha.NetworkInterfacePhaseBound + require.NoError(t, s.client.Create(s.ctx, member)) + require.NoError(t, s.client.Status().Update(s.ctx, member)) + route := &networkingv1alpha.NetworkLoadBalancerRoute{ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: lb.Namespace}} + route.Spec.ParentRefs = []networkingv1alpha.NetworkLoadBalancerRouteParentRef{{ + Name: lb.Name, Namespace: lb.Namespace, Port: ptr.To[int32](443), + }} + route.Spec.Backends = []networkingv1alpha.NetworkLoadBalancerRouteBackend{{ + NetworkService: networkingv1alpha.NetworkLoadBalancerNetworkServiceRef{Name: service.Name, Port: "http"}, + }} + require.NoError(t, s.client.Create(s.ctx, route)) + lb = s.reconcile() + require.Equal(t, []string{"iad"}, lb.Status.AddressRequests[1].ResolvedLocations) + require.Len(t, s.claims(), 3) + + lb.Spec.Listeners = append(lb.Spec.Listeners, networkingv1alpha.NetworkLoadBalancerListener{ + Name: "udp", Protocol: networkingv1alpha.NetworkLoadBalancerProtocolUDP, Port: 443, + AddressRefs: []string{"edge"}, + }) + require.NoError(t, s.client.Update(s.ctx, &lb)) + lb = s.reconcile() + require.Empty(t, lb.Status.AddressRequests[1].ResolvedLocations, + "ambiguous port parent must not allocate backend-derived addresses") +} + +func TestNetworkLoadBalancerTooManySelectedEndpointsDoesNotAllocate(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + lb.Spec.Exposure.Addresses = nil + for i := range 16 { + lb.Spec.Exposure.Addresses = append(lb.Spec.Exposure.Addresses, networkingv1alpha.NetworkLoadBalancerAddressRequest{ + Name: fmt.Sprintf("request-%02d", i), Families: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + Locations: &networkingv1alpha.NetworkLoadBalancerLocationSelection{From: networkingv1alpha.NetworkLoadBalancerLocationSourceAll}, + }) + } + for i := range 63 { + require.NoError(t, s.client.Create(s.ctx, &locationsv1alpha1.Location{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("location-%02d", i)}, + Status: locationsv1alpha1.LocationStatus{Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue, Reason: "Test"}}}, + })) + } + require.NoError(t, s.client.Update(s.ctx, &lb)) + before := s.claims() + lb = s.reconcile() + require.Equal(t, before, s.claims(), "overflow must not create any additional claims") + condition := apimeta.FindStatusCondition(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated) + require.NotNil(t, condition) + require.Equal(t, "TooManyEndpoints", condition.Reason) + require.Contains(t, condition.Message, "1024") + require.LessOrEqual(t, len(lb.Status.Endpoints), 1024) +} + +func TestNetworkLoadBalancerEndpointLimitIncludesProgrammedWithdrawal(t *testing.T) { + selected := make([]loadBalancerAddressRequest, 1024) + for i := range selected { + selected[i] = loadBalancerAddressRequest{requestName: "a", location: fmt.Sprintf("location-%04d", i)} + } + prior := []networkingv1alpha.NetworkLoadBalancerEndpointStatus{{ + RequestName: "retired", Location: "old", + Addresses: []networkingv1alpha.NetworkLoadBalancerAddressStatus{{ + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue}}, + }}, + }} + require.Equal(t, 1025, countAddressEndpoints(selected, prior)) +} + +func TestNetworkLoadBalancerBackendLocationsIncludeDisjointBackends(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + lb.Spec.Exposure.Addresses = lb.Spec.Exposure.Addresses[:1] + lb.Spec.Exposure.Addresses[0].Locations = nil + lb.Spec.Listeners[0].AddressRefs = []string{"regional"} + require.NoError(t, s.client.Update(s.ctx, &lb)) + for _, entry := range []struct{ name, location string }{{"east", "iad"}, {"west", "fra"}} { + service := &networkingv1alpha.NetworkService{ObjectMeta: metav1.ObjectMeta{Name: entry.name, Namespace: lb.Namespace}} + service.Spec.NetworkInterfaces.Selector.MatchLabels = map[string]string{"app": entry.name} + require.NoError(t, s.client.Create(s.ctx, service)) + iface := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{ + Name: entry.name, Namespace: lb.Namespace, + Labels: map[string]string{"app": entry.name, networkingv1alpha.NetworkInterfaceLocationLabel: entry.location}, + }} + iface.Spec.Network.Name = lb.Spec.NetworkRef.Name + iface.Status.Phase = networkingv1alpha.NetworkInterfacePhaseBound + require.NoError(t, s.client.Create(s.ctx, iface)) + } + route := &networkingv1alpha.NetworkLoadBalancerRoute{ObjectMeta: metav1.ObjectMeta{Name: "backend-route", Namespace: lb.Namespace}} + route.Spec.ParentRefs = []networkingv1alpha.NetworkLoadBalancerRouteParentRef{{Name: lb.Name, Listener: "web"}} + for _, name := range []string{"east", "west"} { + route.Spec.Backends = append(route.Spec.Backends, networkingv1alpha.NetworkLoadBalancerRouteBackend{ + NetworkService: networkingv1alpha.NetworkLoadBalancerNetworkServiceRef{Name: name, Port: "web"}, + }) + } + require.NoError(t, s.client.Create(s.ctx, route)) + locations, err := backendLocationsForAddress(s.ctx, s.client, &lb, "regional") + require.NoError(t, err) + require.Equal(t, []string{"fra", "iad"}, locations) + +} + +func TestNetworkLoadBalancerRetainsAnnouncedAddressUntilWithdrawn(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + setEdgeAnnouncement(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb.Spec.Exposure.Addresses = lb.Spec.Exposure.Addresses[:1] + lb.Spec.Listeners[0].AddressRefs = []string{"regional"} + require.NoError(t, s.client.Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, s.claims(), 3, "announced IP must remain held") + require.Len(t, lb.Status.Endpoints, 3, "withdrawal must see the old address") + require.Len(t, lb.Status.AddressRequests, 1) + require.Equal(t, []string{"fra", "iad"}, lb.Status.AddressRequests[0].ResolvedLocations) + + setEdgeAnnouncement(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, lb.Status.Endpoints, 2) + require.Len(t, s.claims(), 2) +} + +func TestNetworkLoadBalancerRetainsProgrammedAddressUntilWithdrawn(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + setEdgeProgramming(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb.Spec.Exposure.Addresses = lb.Spec.Exposure.Addresses[:1] + lb.Spec.Listeners[0].AddressRefs = []string{"regional"} + require.NoError(t, s.client.Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, s.claims(), 3, "programmed IP must remain held before announcement") + require.Len(t, lb.Status.Endpoints, 3, "withdrawal must see the programmed address") + require.False(t, edgeAddress(t, &lb).Announced) + require.True(t, apimeta.IsStatusConditionTrue(edgeAddress(t, &lb).Conditions, + networkingv1alpha.NetworkLoadBalancerProgrammed)) + + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, lb.Status.Endpoints, 2) + require.Len(t, s.claims(), 2) +} + +func TestNetworkLoadBalancerOrphanCleanupRetainsProgrammedClaim(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + claim := edgeClaim(t, s) + setEdgeProgramming(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + wanted := map[string]bool{} + require.NoError(t, s.control.releaseOrphanedClaims(s.ctx, s.ipam, testProjectNS, &lb, wanted)) + require.NoError(t, s.ipam.Get(s.ctx, client.ObjectKeyFromObject(&claim), &ipamv1alpha1.IPClaim{})) + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.control.releaseOrphanedClaims(s.ctx, s.ipam, testProjectNS, &lb, wanted)) + var remaining ipamv1alpha1.IPClaimList + require.NoError(t, s.ipam.List(s.ctx, &remaining, client.InNamespace(testProjectNS))) + require.Empty(t, remaining.Items) +} + +func TestNetworkLoadBalancerRecreatedClaimDoesNotInheritPublication(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + claim.UID = types.UID("original-claim") + require.NoError(t, s.ipam.Update(s.ctx, &claim)) + claim.Status.Address = "2001:db8::1" + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + lb := s.reconcile() + old := edgeAddress(t, &lb) + require.Equal(t, "2001:db8::1", old.Value) + require.Equal(t, "original-claim", old.ClaimRef.UID) + old.Announced = true + old.DNSPublished = true + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue, + Reason: "Observed", Message: "programmed", + }) + apimeta.SetStatusCondition(&lb.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue, + Reason: "Observed", Message: "programmed", + }) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + + require.NoError(t, s.ipam.Delete(s.ctx, &claim)) + lb = s.reconcile() + require.Equal(t, "AddressReplacementPendingWithdrawal", + apimeta.FindStatusCondition(edgeAddress(t, &lb).Conditions, networkingv1alpha.NetworkLoadBalancerAllocated).Reason) + replacement := edgeClaim(t, s) + replacement.UID = types.UID("replacement-claim") + require.NoError(t, s.ipam.Update(s.ctx, &replacement)) + replacement.Status.Address = "2001:db8::2" + require.NoError(t, s.ipam.Status().Update(s.ctx, &replacement)) + lb = s.reconcile() + old = edgeAddress(t, &lb) + require.Equal(t, "2001:db8::1", old.Value, "old address must remain visible for withdrawal") + require.Equal(t, "original-claim", old.ClaimRef.UID) + require.True(t, old.Announced) + require.True(t, old.DNSPublished) + require.Equal(t, "AddressReplacementPendingWithdrawal", + apimeta.FindStatusCondition(old.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated).Reason) + require.False(t, apimeta.IsStatusConditionTrue(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed)) + + old.Announced = false + old.DNSPublished = false + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionFalse, + Reason: "Withdrawn", Message: "withdrawn", + }) + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerAnnounced, Status: metav1.ConditionFalse, + Reason: "Withdrawn", Message: "withdrawn", + }) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + current := edgeAddress(t, &lb) + require.Equal(t, "2001:db8::2", current.Value) + require.Equal(t, "replacement-claim", current.ClaimRef.UID) + require.False(t, current.Announced) + require.False(t, current.DNSPublished) + require.Nil(t, apimeta.FindStatusCondition(current.Conditions, networkingv1alpha.NetworkLoadBalancerAnnounced)) + require.Nil(t, apimeta.FindStatusCondition(current.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed)) +} + +func TestNetworkLoadBalancerReassignedAddressResetsServingConditions(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + claim.Status.Address = "2001:db8::1" + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + lb := s.reconcile() + old := edgeAddress(t, &lb) + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue, + Reason: "Observed", Message: "programmed", + }) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + claim.Status.Address = "2001:db8::2" + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + lb = s.reconcile() + require.Equal(t, "2001:db8::1", edgeAddress(t, &lb).Value) + require.True(t, apimeta.IsStatusConditionTrue(edgeAddress(t, &lb).Conditions, + networkingv1alpha.NetworkLoadBalancerProgrammed)) + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + current := edgeAddress(t, &lb) + require.Equal(t, "2001:db8::2", current.Value) + require.Nil(t, apimeta.FindStatusCondition(current.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed)) +} + +func TestNetworkLoadBalancerRecreatedClaimWithSameAddressResetsServingConditions(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + claim.UID = types.UID("first-claim") + require.NoError(t, s.ipam.Update(s.ctx, &claim)) + claim.Status.Address = "2001:db8::1" + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + lb := s.reconcile() + old := edgeAddress(t, &lb) + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue, + Reason: "Observed", Message: "programmed", + }) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + require.NoError(t, s.ipam.Delete(s.ctx, &claim)) + s.reconcile() + replacement := edgeClaim(t, s) + replacement.UID = types.UID("second-claim") + require.NoError(t, s.ipam.Update(s.ctx, &replacement)) + replacement.Status.Address = "2001:db8::1" + require.NoError(t, s.ipam.Status().Update(s.ctx, &replacement)) + lb = s.reconcile() + require.Equal(t, "first-claim", edgeAddress(t, &lb).ClaimRef.UID) + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + current := edgeAddress(t, &lb) + require.Equal(t, "second-claim", current.ClaimRef.UID) + require.Nil(t, apimeta.FindStatusCondition(current.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed)) +} + +func edgeClaim(t *testing.T, s *loadBalancerAllocationScenario) ipamv1alpha1.IPClaim { + t.Helper() + for _, claim := range s.claims() { + if claim.Spec.Scope[ipamScopeRoleLocation].Name == "iad" && + claim.Name == loadBalancerClaimName(s.lb, loadBalancerAddressRequest{ + requestName: "edge", location: "iad", family: networkingv1alpha.IPv6Protocol, + }) { + return claim + } + } + t.Fatal("edge IPClaim in iad not found") + return ipamv1alpha1.IPClaim{} +} + +func edgeAddress(t *testing.T, lb *networkingv1alpha.NetworkLoadBalancer) *networkingv1alpha.NetworkLoadBalancerAddressStatus { + t.Helper() + for i := range lb.Status.Endpoints { + if lb.Status.Endpoints[i].RequestName == "edge" { + return &lb.Status.Endpoints[i].Addresses[0] + } + } + t.Fatal("edge address not found") + return nil +} + +func TestNetworkLoadBalancerResolvedLocationsExcludeWithdrawingLocation(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + for i := range lb.Status.Endpoints { + endpoint := &lb.Status.Endpoints[i] + if endpoint.RequestName == "regional" && endpoint.Location == "fra" { + for j := range endpoint.Addresses { + endpoint.Addresses[j].Announced = true + } + } + } + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb.Spec.Exposure.Addresses[0].Locations.Names = []string{"iad"} + require.NoError(t, s.client.Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, lb.Status.Endpoints, 3, "the removed location remains until withdrawal") + require.Equal(t, []string{"iad"}, lb.Status.AddressRequests[0].ResolvedLocations) + require.Len(t, s.claims(), 3) + +} + +func TestNetworkLoadBalancerDeletionWaitsForWithdrawal(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + setEdgeAnnouncement(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + require.NoError(t, s.client.Delete(s.ctx, &lb)) + lb = s.reconcile() + require.NotNil(t, lb.DeletionTimestamp) + require.Len(t, s.claims(), 3) + + setEdgeAnnouncement(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + _, err := s.control.reconcile(s.ctx, s.client, client.ObjectKeyFromObject(s.lb)) + require.NoError(t, err) + require.Empty(t, s.claims()) +} + +func TestNetworkLoadBalancerDeletionWaitsForProgrammingWithdrawal(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + setEdgeProgramming(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + require.NoError(t, s.client.Delete(s.ctx, &lb)) + lb = s.reconcile() + require.NotNil(t, lb.DeletionTimestamp) + require.Len(t, s.claims(), 3) + + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + _, err := s.control.reconcile(s.ctx, s.client, client.ObjectKeyFromObject(s.lb)) + require.NoError(t, err) + require.Empty(t, s.claims()) +} + +func TestNetworkLoadBalancerRejectsDriftedIPClaimSpec(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*ipamv1alpha1.IPClaim) + }{ + {name: "family", mutate: func(claim *ipamv1alpha1.IPClaim) { + claim.Spec.IPFamily = ipamv1alpha1.IPFamily(networkingv1alpha.IPv4Protocol) + }}, + {name: "reclaim policy", mutate: func(claim *ipamv1alpha1.IPClaim) { + claim.Spec.ReclaimPolicy = "Retain" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + tc.mutate(&claim) + require.NoError(t, s.ipam.Update(s.ctx, &claim)) + lb := s.reconcile() + condition := apimeta.FindStatusCondition(edgeAddress(t, &lb).Conditions, + networkingv1alpha.NetworkLoadBalancerAllocated) + require.NotNil(t, condition) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Contains(t, condition.Message, "does not match") + require.Len(t, s.claims(), 3, "drifted claim must not be replaced or released") + }) + } +} + +func setEdgeProgramming(lb *networkingv1alpha.NetworkLoadBalancer, status metav1.ConditionStatus) { + for i := range lb.Status.Endpoints { + if lb.Status.Endpoints[i].RequestName != "edge" { + continue + } + for j := range lb.Status.Endpoints[i].Addresses { + apimeta.SetStatusCondition(&lb.Status.Endpoints[i].Addresses[j].Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: status, + Reason: "Observed", Message: "data plane programming status", ObservedGeneration: lb.Generation, + }) + } + } +} + +func setEdgeAnnouncement(lb *networkingv1alpha.NetworkLoadBalancer, status metav1.ConditionStatus) { + for i := range lb.Status.Endpoints { + if lb.Status.Endpoints[i].RequestName != "edge" { + continue + } + for j := range lb.Status.Endpoints[i].Addresses { + address := &lb.Status.Endpoints[i].Addresses[j] + address.Announced = status == metav1.ConditionTrue + apimeta.SetStatusCondition(&address.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerAnnounced, Status: status, + Reason: "Observed", Message: "BGP announcement status", ObservedGeneration: lb.Generation, + }) + } + } +}