diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d75765d3..0f398e70 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -343,6 +343,7 @@ rules: - networkcontexts/finalizers - networkinterfaceclaims/finalizers - networkinterfaces/finalizers + - networkloadbalancers/finalizers - networkpolicies/finalizers - networks/finalizers - subnetclaims/finalizers @@ -361,6 +362,7 @@ rules: - networkcontexts/status - networkinterfaceclaims/status - networkinterfaces/status + - networkloadbalancers/status - networkpolicies/status - networks/status - networkservices/status @@ -380,3 +382,13 @@ rules: - get - list - watch +- apiGroups: + - networking.datumapis.com + resources: + - networkloadbalancers + verbs: + - get + - list + - patch + - update + - watch diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 2b6d9fb0..db0f3e5f 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -802,6 +802,13 @@ func controllerRegistrations( {"networkservice", true, func() error { return (&controller.NetworkServiceReconciler{}).SetupWithManager(mgr) }}, + {"networkloadbalancer", true, func() error { + return (&controller.NetworkLoadBalancerReconciler{ + IPAM: deps.ipamClients, + PublicIPv6Class: serverConfig.IPAM.Classes.PublicIPv6, + ManagedDomain: serverConfig.Gateway.TargetDomain, + }).SetupWithManager(mgr) + }}, // Which workloads are behind a proxy is only answerable in a project // control plane, and only a cell can act on it. The record this writes to // the hub is what carries the answer between them. diff --git a/internal/cmd/manager/manager_test.go b/internal/cmd/manager/manager_test.go index c9e0ea6c..00f7b088 100644 --- a/internal/cmd/manager/manager_test.go +++ b/internal/cmd/manager/manager_test.go @@ -64,6 +64,7 @@ var reconcilerControllerNames = map[string]string{ "NetworkInterfaceProjector": "networkinterfaceprojector", "NetworkInterfaceReconciler": "networkinterface", "NetworkInterfaceWriteBackReconciler": "networkinterfacewriteback", + "NetworkLoadBalancerReconciler": "networkloadbalancer", "NetworkPolicyReconciler": "networkpolicy", "NetworkPresenceReconciler": "networkpresence", "NetworkPresenceSyncReconciler": "networkpresencesync", diff --git a/internal/config/config.go b/internal/config/config.go index 1567029e..ecfcc229 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -314,6 +314,11 @@ type IPAMConfig struct { // environment's classes are different objects, and an operator deployed // against either has to name the ones it is pointed at. type IPAMClasses struct { + // PublicIPv6 is the address class used for per-location public load + // balancer VIPs. Its backing class must share one platform-wide pool with + // uniqueWithin unset, including when projects use source classes. + PublicIPv6 string `json:"publicIPv6,omitempty"` + // Network is the class that hands out the range a network is addressed // from. Unset, no range is claimed and a network is reconciled with no // address space, the same as an unset IPAM connection. diff --git a/internal/controller/networkloadbalancer_controller.go b/internal/controller/networkloadbalancer_controller.go new file mode 100644 index 00000000..4741fc52 --- /dev/null +++ b/internal/controller/networkloadbalancer_controller.go @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "time" + + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mccontext "sigs.k8s.io/multicluster-runtime/pkg/context" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/words" +) + +const ( + networkLoadBalancerFinalizer = "networking.datumapis.com/networkloadbalancer-release" + networkLoadBalancerOwnerLabel = "networking.datumapis.com/network-load-balancer-uid" + networkLoadBalancerRetry = time.Minute +) + +type NetworkLoadBalancerReconciler struct { + IPAM IPAMClientFactory + PublicIPv6Class string + ManagedDomain string + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancers,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancers/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancers/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=locations.miloapis.com,resources=locations,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch + +func (r *NetworkLoadBalancerReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + cluster, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + ctx = mccontext.WithCluster(ctx, req.ClusterName) + return r.reconcile(ctx, cluster.GetClient(), req.NamespacedName) +} + +func (r *NetworkLoadBalancerReconciler) reconcile(ctx context.Context, cl client.Client, key client.ObjectKey) (ctrl.Result, error) { + var lb networkingv1alpha.NetworkLoadBalancer + if err := cl.Get(ctx, key, &lb); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if !lb.DeletionTimestamp.IsZero() { + return r.release(ctx, cl, &lb) + } + + if r.IPAM == nil || r.PublicIPv6Class == "" { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "AddressServiceUnavailable", + "Public IPv6 address allocation is not configured") + } + if controllerutil.AddFinalizer(&lb, networkLoadBalancerFinalizer) { + if err := cl.Update(ctx, &lb); err != nil { + return ctrl.Result{}, fmt.Errorf("add load balancer finalizer: %w", err) + } + return ctrl.Result{Requeue: true}, nil + } + for _, request := range lb.Spec.Exposure.Addresses { + if slices.Contains(request.Families, networkingv1alpha.IPv4Protocol) { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "UnsupportedCapability", + fmt.Sprintf("address request %q: public IPv4 NLB ingress is unavailable", request.Name)) + } + } + + var network networkingv1alpha.Network + if err := cl.Get(ctx, client.ObjectKey{Namespace: lb.Namespace, Name: lb.Spec.NetworkRef.Name}, &network); err != nil { + if apierrors.IsNotFound(err) { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "NetworkNotFound", + fmt.Sprintf("Network %q was not found", lb.Spec.NetworkRef.Name)) + } + return ctrl.Result{}, fmt.Errorf("read load balancer network: %w", err) + } + if !slices.Contains(network.Spec.IPFamilies, networkingv1alpha.IPv6Protocol) { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "IPv6Required", + "The referenced network does not carry IPv6") + } + + routing, err := resolveProjectOrCluster(ctx, cl, lb.Namespace) + if err != nil { + var unresolved *projectUnresolvable + if errors.As(err, &unresolved) { + return r.report(ctx, cl, &lb, nil, nil, metav1.ConditionFalse, "ProjectUnresolved", err.Error()) + } + return ctrl.Result{}, err + } + ipamClient, err := r.IPAM.ClientForProject(routing.project) + if err != nil { + return ctrl.Result{}, fmt.Errorf("build project IPAM client: %w", err) + } + + requests, err := r.addressRequests(ctx, cl, &lb) + if err != nil { + return ctrl.Result{}, err + } + if countAddressEndpoints(requests, lb.Status.Endpoints) > 1024 { + return r.report(ctx, cl, &lb, nil, requests, metav1.ConditionFalse, "TooManyEndpoints", + "Selected and withdrawing address endpoints exceed the status limit of 1024; reduce address requests or locations") + } + + wanted := make(map[string]bool, len(requests)) + status := make([]networkingv1alpha.NetworkLoadBalancerEndpointStatus, 0, len(requests)+len(lb.Status.Endpoints)) + allAllocated := len(requests) > 0 + var firstFailure string + for _, request := range requests { + claimName := loadBalancerClaimName(&lb, request) + wanted[claimName] = true + prior := priorAddressStatus(lb.Status.Endpoints, request.requestName, request.location, request.family) + prior = *prior.DeepCopy() + address, claimUID, claimErr := r.holdAddress(ctx, ipamClient, routing.projectNamespace, &lb, request, claimName) + claimRef := &networkingv1alpha.NetworkLoadBalancerIPClaimRef{ + Project: routing.project, Namespace: routing.projectNamespace, Name: claimName, UID: claimUID, + } + changed := prior.Value != "" && prior.Value != address || + prior.ClaimRef != nil && prior.ClaimRef.UID != "" && claimUID != "" && prior.ClaimRef.UID != claimUID + if changed && addressPendingWithdrawal(prior) { + // Keep the old address visible to withdrawal controllers. In particular, + // do not transfer its announcement or publication to a replacement IP. + allAllocated = false + if firstFailure == "" { + firstFailure = fmt.Sprintf("%s: prior address is awaiting withdrawal", claimName) + } + setLoadBalancerCondition(&prior.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + metav1.ConditionFalse, "AddressReplacementPendingWithdrawal", "Prior address must be withdrawn before its replacement can be served") + appendAddressStatus(&status, lb.Status.Endpoints, request.requestName, request.location, prior) + continue + } + entry := prior + if changed || prior.Value == "" && address != "" { + // Programming, announcement, and DNS publication describe one concrete + // address, not the request that happened to produce it. + entry = networkingv1alpha.NetworkLoadBalancerAddressStatus{Family: request.family} + } + entry.Family = request.family + entry.ClaimRef = claimRef + if claimErr != nil { + allAllocated = false + entry.Value = "" + reason := string(classifyAllocationFailure(claimErr)) + setLoadBalancerCondition(&entry.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + metav1.ConditionFalse, reason, claimErr.Error()) + if firstFailure == "" { + firstFailure = fmt.Sprintf("%s: %s", claimName, claimErr) + } + } else if address == "" { + allAllocated = false + entry.Value = "" + setLoadBalancerCondition(&entry.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + metav1.ConditionFalse, "AllocationPending", "IPAM has not assigned an address yet") + } else { + entry.Value = address + setLoadBalancerCondition(&entry.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + metav1.ConditionTrue, "Allocated", "Public IPv6 address is held in IPAM") + } + appendAddressStatus(&status, lb.Status.Endpoints, request.requestName, request.location, entry) + } + + for _, endpoint := range lb.Status.Endpoints { + for _, old := range endpoint.Addresses { + if old.ClaimRef != nil && wanted[old.ClaimRef.Name] { + continue + } + if addressPendingWithdrawal(old) { + appendAddressStatus(&status, lb.Status.Endpoints, endpoint.RequestName, endpoint.Location, old) + allAllocated = false + continue + } + if old.ClaimRef != nil { + if err := r.deleteAddressClaim(ctx, old.ClaimRef, string(lb.UID)); err != nil { + return ctrl.Result{}, err + } + } + } + } + + if err := r.releaseOrphanedClaims(ctx, ipamClient, routing.projectNamespace, &lb, wanted); err != nil { + return ctrl.Result{}, err + } + slices.SortFunc(status, func(a, b networkingv1alpha.NetworkLoadBalancerEndpointStatus) int { + if a.RequestName != b.RequestName { + return strings.Compare(a.RequestName, b.RequestName) + } + return strings.Compare(a.Location, b.Location) + }) + for i := range status { + if status[i].CanonicalHostname == "" && r.ManagedDomain != "" { + label := words.WordsAndEntropy("", string(lb.UID)+"/"+status[i].RequestName) + status[i].CanonicalHostname = status[i].Location + "." + label + "." + r.ManagedDomain + } + } + + if !allAllocated { + if firstFailure == "" { + firstFailure = "One or more requested addresses are pending or awaiting withdrawal" + } + return r.report(ctx, cl, &lb, status, requests, metav1.ConditionFalse, "AllocationPending", firstFailure) + } + return r.report(ctx, cl, &lb, status, requests, metav1.ConditionTrue, "Allocated", "All requested public IPv6 addresses are held") +} + +func countAddressEndpoints(requests []loadBalancerAddressRequest, prior []networkingv1alpha.NetworkLoadBalancerEndpointStatus) int { + type endpointKey struct{ requestName, location string } + selected := make(map[endpointKey]bool, len(requests)) + for _, request := range requests { + selected[endpointKey{request.requestName, request.location}] = true + } + for _, endpoint := range prior { + key := endpointKey{endpoint.RequestName, endpoint.Location} + if selected[key] { + continue + } + for _, address := range endpoint.Addresses { + if addressPendingWithdrawal(address) { + selected[key] = true + break + } + } + } + return len(selected) +} + +type loadBalancerAddressRequest struct { + requestName string + location string + networkName string + family networkingv1alpha.IPFamily +} + +func (r *NetworkLoadBalancerReconciler) addressRequests( + ctx context.Context, cl client.Client, lb *networkingv1alpha.NetworkLoadBalancer, +) ([]loadBalancerAddressRequest, error) { + var available locationsv1alpha1.LocationList + if err := cl.List(ctx, &available); err != nil { + return nil, fmt.Errorf("list available locations: %w", err) + } + known := map[string]bool{} + for _, location := range available.Items { + ready := apimeta.FindStatusCondition(location.Status.Conditions, locationsv1alpha1.LocationConditionReady) + known[location.Name] = ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == location.Generation + } + + var requests []loadBalancerAddressRequest + for _, address := range lb.Spec.Exposure.Addresses { + var names []string + source := networkingv1alpha.NetworkLoadBalancerLocationSourceBackends + if address.Locations != nil { + source = address.Locations.From + } + switch source { + case networkingv1alpha.NetworkLoadBalancerLocationSourceAll: + for name, ready := range known { + if ready { + names = append(names, name) + } + } + case networkingv1alpha.NetworkLoadBalancerLocationSourceNames: + // A location projection may temporarily disappear. An explicit + // request still owns its IP throughout that gap. + names = append(names, address.Locations.Names...) + default: + backendNames, err := backendLocationsForAddress(ctx, cl, lb, address.Name) + if err != nil { + return nil, err + } + names = backendNames + } + slices.Sort(names) + for _, name := range names { + for _, family := range address.Families { + requests = append(requests, loadBalancerAddressRequest{ + requestName: address.Name, location: name, networkName: lb.Spec.NetworkRef.Name, family: family, + }) + } + } + } + return requests, nil +} + +// backendLocationsForAddress selects locations of the effective backends on +// every listener using the address. It uses live, bound membership +// rather than the capped NetworkService status display list. +func backendLocationsForAddress(ctx context.Context, cl client.Client, + lb *networkingv1alpha.NetworkLoadBalancer, addressName string, +) ([]string, error) { + var routes networkingv1alpha.NetworkLoadBalancerRouteList + if err := cl.List(ctx, &routes, client.InNamespace(lb.Namespace)); err != nil { + return nil, fmt.Errorf("list load balancer routes: %w", err) + } + selected := map[string]bool{} + attached := false + for _, listener := range lb.Spec.Listeners { + if !slices.Contains(listener.AddressRefs, addressName) { + continue + } + var winner *networkingv1alpha.NetworkLoadBalancerRoute + for i := range routes.Items { + route := &routes.Items[i] + if !route.DeletionTimestamp.IsZero() || !slices.ContainsFunc(route.Spec.ParentRefs, + func(ref networkingv1alpha.NetworkLoadBalancerRouteParentRef) bool { + return routeParentSelectsListener(lb, ref, listener.Name) + }) { + continue + } + if winner == nil || route.CreationTimestamp.Before(&winner.CreationTimestamp) || + (route.CreationTimestamp.Equal(&winner.CreationTimestamp) && route.Name < winner.Name) { + winner = route + } + } + if winner == nil { + return nil, nil + } + attached = true + for _, backend := range winner.Spec.Backends { + var service networkingv1alpha.NetworkService + key := client.ObjectKey{Namespace: winner.Namespace, Name: backend.NetworkService.Name} + if backend.NetworkService.Namespace != "" { + key.Namespace = backend.NetworkService.Namespace + } + if err := cl.Get(ctx, key, &service); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("get backend NetworkService %s: %w", key, err) + } + selector, err := metav1.LabelSelectorAsSelector(&service.Spec.NetworkInterfaces.Selector) + if err != nil { + return nil, fmt.Errorf("backend NetworkService %s has invalid selector: %w", key, err) + } + members, err := matchingInterfaces(ctx, cl, service.Namespace, selector) + if err != nil { + return nil, err + } + locations := map[string]bool{} + for i := range members { + member := &members[i] + if member.Spec.Network.Name == lb.Spec.NetworkRef.Name { + if location := member.Labels[networkingv1alpha.NetworkInterfaceLocationLabel]; location != "" { + locations[location] = true + } + } + } + for location := range locations { + selected[location] = true + } + } + } + if !attached { + return nil, nil + } + names := make([]string, 0, len(selected)) + for name := range selected { + names = append(names, name) + } + slices.Sort(names) + return names, nil +} + +func routeParentSelectsListener(lb *networkingv1alpha.NetworkLoadBalancer, + ref networkingv1alpha.NetworkLoadBalancerRouteParentRef, listenerName string, +) bool { + if ref.Name != lb.Name || ref.Namespace != "" && ref.Namespace != lb.Namespace { + return false + } + if ref.Listener != "" { + return ref.Listener == listenerName + } + if ref.Port == nil { + return false + } + var matchedName string + for _, candidate := range lb.Spec.Listeners { + if candidate.Port != *ref.Port { + continue + } + if matchedName != "" { + return false + } + matchedName = candidate.Name + } + return matchedName == listenerName +} + +func loadBalancerClaimName(lb *networkingv1alpha.NetworkLoadBalancer, request loadBalancerAddressRequest) string { + identity := request.requestName + "\x00" + request.location + "\x00" + string(request.family) + sum := sha256.Sum256([]byte(identity)) + return ipClaimName("nlb-"+string(lb.UID), request.requestName+"-"+hex.EncodeToString(sum[:8])) +} + +func (r *NetworkLoadBalancerReconciler) holdAddress( + ctx context.Context, cl client.Client, namespace string, lb *networkingv1alpha.NetworkLoadBalancer, + request loadBalancerAddressRequest, claimName string, +) (string, string, error) { + claim := &ipamv1alpha1.IPClaim{} + claim.Namespace = namespace + claim.Name = claimName + claim.Labels = map[string]string{networkLoadBalancerOwnerLabel: string(lb.UID)} + claim.Spec = ipamv1alpha1.IPClaimSpec{ + ClassName: r.PublicIPv6Class, + IPFamily: ipamv1alpha1.IPFamily(networkingv1alpha.IPv6Protocol), + PrefixLength: ptr.To[int32](128), + ReclaimPolicy: ipamv1alpha1.ReclaimDelete, + Scope: map[string]ipamv1alpha1.ScopeRef{ + ipamScopeRoleNetwork: {APIGroup: datumNetworkingAPIGroup, Kind: "Network", Name: request.networkName}, + }, + } + if request.location != "" { + claim.Spec.Scope[ipamScopeRoleLocation] = ipamv1alpha1.ScopeRef{ + APIGroup: datumNetworkingAPIGroup, Kind: "Location", Name: request.location, + } + } + + existing := &ipamv1alpha1.IPClaim{} + err := cl.Get(ctx, client.ObjectKeyFromObject(claim), existing) + if apierrors.IsNotFound(err) { + if err := cl.Create(ctx, claim); err != nil { + if !apierrors.IsAlreadyExists(err) { + return "", "", err + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(claim), existing); err != nil { + return "", "", err + } + } else { + existing = claim + } + } else if err != nil { + return "", "", err + } + if !existing.DeletionTimestamp.IsZero() { + return "", string(existing.UID), fmt.Errorf("IPClaim %q is being deleted", claimName) + } + if existing.Labels[networkLoadBalancerOwnerLabel] != string(lb.UID) || + existing.Spec.ClassName != claim.Spec.ClassName || + existing.Spec.IPFamily != claim.Spec.IPFamily || + existing.Spec.PrefixLength == nil || *existing.Spec.PrefixLength != 128 || + existing.Spec.ReclaimPolicy != claim.Spec.ReclaimPolicy || + !equality.Semantic.DeepEqual(existing.Spec.Scope, claim.Spec.Scope) { + return "", "", fmt.Errorf("IPClaim %q does not match this load balancer's address request", claimName) + } + address := existing.Status.Address + if address == "" && existing.Status.AllocatedCIDR != "" { + prefix, err := netip.ParsePrefix(existing.Status.AllocatedCIDR) + if err != nil || prefix.Bits() != 128 { + return "", "", fmt.Errorf("IPClaim %q did not allocate one IPv6 address", claimName) + } + address = prefix.Addr().String() + } + if address != "" { + parsed, err := netip.ParseAddr(address) + if err != nil || !parsed.Is6() { + return "", "", fmt.Errorf("IPClaim %q returned an invalid IPv6 address %q", claimName, address) + } + } + return address, string(existing.UID), nil +} + +func priorAddressStatus( + endpoints []networkingv1alpha.NetworkLoadBalancerEndpointStatus, requestName, location string, family networkingv1alpha.IPFamily, +) networkingv1alpha.NetworkLoadBalancerAddressStatus { + for _, endpoint := range endpoints { + if endpoint.RequestName == requestName && endpoint.Location == location { + for _, address := range endpoint.Addresses { + if address.Family == family { + return address + } + } + } + } + return networkingv1alpha.NetworkLoadBalancerAddressStatus{} +} + +func appendAddressStatus(status *[]networkingv1alpha.NetworkLoadBalancerEndpointStatus, + prior []networkingv1alpha.NetworkLoadBalancerEndpointStatus, requestName, location string, + address networkingv1alpha.NetworkLoadBalancerAddressStatus, +) { + i := slices.IndexFunc(*status, func(endpoint networkingv1alpha.NetworkLoadBalancerEndpointStatus) bool { + return endpoint.RequestName == requestName && endpoint.Location == location + }) + if i < 0 { + entry := networkingv1alpha.NetworkLoadBalancerEndpointStatus{RequestName: requestName, Location: location} + for _, old := range prior { + if old.RequestName == requestName && old.Location == location { + entry.CanonicalHostname = old.CanonicalHostname + break + } + } + *status = append(*status, entry) + i = len(*status) - 1 + } + (*status)[i].Addresses = append((*status)[i].Addresses, address) +} + +func addressAnnounced(address networkingv1alpha.NetworkLoadBalancerAddressStatus) bool { + return address.Announced || apimeta.IsStatusConditionTrue(address.Conditions, networkingv1alpha.NetworkLoadBalancerAnnounced) +} + +func addressPendingWithdrawal(address networkingv1alpha.NetworkLoadBalancerAddressStatus) bool { + return addressAnnounced(address) || address.DNSPublished || + apimeta.IsStatusConditionTrue(address.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed) +} + +func (r *NetworkLoadBalancerReconciler) releaseOrphanedClaims( + ctx context.Context, cl client.Client, namespace string, lb *networkingv1alpha.NetworkLoadBalancer, wanted map[string]bool, +) error { + var claims ipamv1alpha1.IPClaimList + if err := cl.List(ctx, &claims, client.InNamespace(namespace), + client.MatchingLabels{networkLoadBalancerOwnerLabel: string(lb.UID)}); err != nil { + return fmt.Errorf("list load balancer claims: %w", err) + } + for i := range claims.Items { + claim := &claims.Items[i] + if wanted[claim.Name] { + continue + } + if old := priorClaimAddressStatus(lb.Status.Endpoints, claim.Name); old != nil && addressPendingWithdrawal(*old) { + continue + } + if err := cl.Delete(ctx, claim, client.Preconditions{UID: &claim.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { + return fmt.Errorf("release stale IPClaim %q: %w", claim.Name, err) + } + } + return nil +} + +func priorClaimAddressStatus( + endpoints []networkingv1alpha.NetworkLoadBalancerEndpointStatus, claimName string, +) *networkingv1alpha.NetworkLoadBalancerAddressStatus { + for i := range endpoints { + for j := range endpoints[i].Addresses { + address := &endpoints[i].Addresses[j] + if address.ClaimRef != nil && address.ClaimRef.Name == claimName { + return address + } + } + } + return nil +} + +func (r *NetworkLoadBalancerReconciler) deleteAddressClaim( + ctx context.Context, ref *networkingv1alpha.NetworkLoadBalancerIPClaimRef, lbUID string, +) error { + if ref.UID == "" { + // A legacy status entry has no safe identity to delete by name. + return nil + } + cl, err := r.IPAM.ClientForProject(ref.Project) + if err != nil { + return err + } + claim := &ipamv1alpha1.IPClaim{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: ref.Namespace, Name: ref.Name}, claim); apierrors.IsNotFound(err) { + return nil + } else if err != nil { + return fmt.Errorf("read IPClaim %q before release: %w", ref.Name, err) + } + if string(claim.UID) != ref.UID || claim.Labels[networkLoadBalancerOwnerLabel] != lbUID { + return nil + } + if err := cl.Delete(ctx, claim, client.Preconditions{UID: &claim.UID}); err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { + return fmt.Errorf("release IPClaim %q: %w", ref.Name, err) + } + return nil +} + +func (r *NetworkLoadBalancerReconciler) release( + ctx context.Context, cl client.Client, lb *networkingv1alpha.NetworkLoadBalancer, +) (ctrl.Result, error) { + if !controllerutil.ContainsFinalizer(lb, networkLoadBalancerFinalizer) { + return ctrl.Result{}, nil + } + for _, endpoint := range lb.Status.Endpoints { + for _, address := range endpoint.Addresses { + if addressPendingWithdrawal(address) { + return ctrl.Result{RequeueAfter: networkLoadBalancerRetry}, nil + } + } + } + if r.IPAM == nil { + return ctrl.Result{}, errors.New("cannot release load balancer addresses without IPAM") + } + routing, err := resolveProjectOrCluster(ctx, cl, lb.Namespace) + if err != nil { + return ctrl.Result{}, err + } + ipamClient, err := r.IPAM.ClientForProject(routing.project) + if err != nil { + return ctrl.Result{}, err + } + if err := r.releaseOrphanedClaims(ctx, ipamClient, routing.projectNamespace, lb, nil); err != nil { + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(lb, networkLoadBalancerFinalizer) + return ctrl.Result{}, cl.Update(ctx, lb) +} + +func (r *NetworkLoadBalancerReconciler) report( + ctx context.Context, cl client.Client, lb *networkingv1alpha.NetworkLoadBalancer, + endpoints []networkingv1alpha.NetworkLoadBalancerEndpointStatus, + selected []loadBalancerAddressRequest, + status metav1.ConditionStatus, reason, message string, +) (ctrl.Result, error) { + before := lb.DeepCopy() + if endpoints != nil { + lb.Status.Endpoints = endpoints + } + requestStatuses := make([]networkingv1alpha.NetworkLoadBalancerAddressRequestStatus, 0, len(lb.Spec.Exposure.Addresses)) + for _, request := range lb.Spec.Exposure.Addresses { + source := networkingv1alpha.NetworkLoadBalancerLocationSourceBackends + if request.Locations != nil { + source = request.Locations.From + } + entry := networkingv1alpha.NetworkLoadBalancerAddressRequestStatus{Name: request.Name, LocationSource: source} + for _, selection := range selected { + if selection.requestName == request.Name && !slices.Contains(entry.ResolvedLocations, selection.location) { + entry.ResolvedLocations = append(entry.ResolvedLocations, selection.location) + } + } + slices.Sort(entry.ResolvedLocations) + requestStatuses = append(requestStatuses, entry) + } + lb.Status.AddressRequests = requestStatuses + setLoadBalancerCondition(&lb.Status.Conditions, lb.Generation, networkingv1alpha.NetworkLoadBalancerAllocated, + status, reason, message) + for _, conditionType := range []string{networkingv1alpha.NetworkLoadBalancerProgrammed, networkingv1alpha.NetworkLoadBalancerReady} { + switch { + case status != metav1.ConditionTrue: + setLoadBalancerCondition(&lb.Status.Conditions, lb.Generation, conditionType, + metav1.ConditionFalse, "AllocationPending", "At least one public address is not ready for serving") + case apimeta.FindStatusCondition(lb.Status.Conditions, conditionType) == nil || + addressIdentityChanged(before.Status.Endpoints, lb.Status.Endpoints): + setLoadBalancerCondition(&lb.Status.Conditions, lb.Generation, conditionType, + metav1.ConditionUnknown, "PendingDataPlane", "Waiting for serving locations to report programming and announcements") + } + } + if equality.Semantic.DeepEqual(before.Status, lb.Status) { + return ctrl.Result{RequeueAfter: networkLoadBalancerRetry}, nil + } + // Location controllers also write nested address conditions. A version + // precondition prevents this allocation pass from overwriting their newer + // announcement result; the manager retries the conflict. + if err := cl.Status().Patch(ctx, lb, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{}, fmt.Errorf("update load balancer status: %w", err) + } + // Periodic reconciliation notices IPAM completion and changes to the + // dynamically selected "All" location set without a cross-cluster watch. + return ctrl.Result{RequeueAfter: networkLoadBalancerRetry}, nil +} + +func addressIdentityChanged(before, after []networkingv1alpha.NetworkLoadBalancerEndpointStatus) bool { + if len(before) != len(after) { + return true + } + for _, endpoint := range after { + for _, address := range endpoint.Addresses { + prior := priorAddressStatus(before, endpoint.RequestName, endpoint.Location, address.Family) + if prior.Value != address.Value { + return true + } + if prior.ClaimRef != nil && address.ClaimRef != nil && + prior.ClaimRef.UID != "" && address.ClaimRef.UID != "" && prior.ClaimRef.UID != address.ClaimRef.UID { + return true + } + } + } + return false +} + +func setLoadBalancerCondition( + conditions *[]metav1.Condition, generation int64, conditionType string, + status metav1.ConditionStatus, reason, message string, +) { + apimeta.SetStatusCondition(conditions, metav1.Condition{ + Type: conditionType, Status: status, Reason: reason, Message: message, ObservedGeneration: generation, + }) +} + +func (r *NetworkLoadBalancerReconciler) SetupWithManager(mgr mcmanager.Manager) error { + r.mgr = mgr + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkLoadBalancer{}, mcbuilder.WithEngageWithLocalCluster(false)). + Named("networkloadbalancer"). + Complete(r) +} diff --git a/internal/controller/networkloadbalancer_controller_test.go b/internal/controller/networkloadbalancer_controller_test.go new file mode 100644 index 00000000..caebeaba --- /dev/null +++ b/internal/controller/networkloadbalancer_controller_test.go @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + ipamv1alpha1 "go.miloapis.com/ipam/pkg/apis/ipam/v1alpha1" + locationsv1alpha1 "go.miloapis.com/locations/api/v1alpha1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/downstreamclient" +) + +type fixedIPAMClient struct{ client.Client } + +func (f fixedIPAMClient) ClientForProject(project string) (client.Client, error) { + if project != testProject { + return nil, fmt.Errorf("unexpected project %q", project) + } + return f.Client, nil +} + +type loadBalancerAllocationScenario struct { + t *testing.T + ctx context.Context + client client.Client + ipam client.Client + lb *networkingv1alpha.NetworkLoadBalancer + control *NetworkLoadBalancerReconciler +} + +func newLoadBalancerAllocationScenario(t *testing.T) *loadBalancerAllocationScenario { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + require.NoError(t, locationsv1alpha1.AddToScheme(scheme)) + require.NoError(t, ipamv1alpha1.AddToScheme(scheme)) + + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: "tenant", + Labels: map[string]string{ + downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-" + testProject, + downstreamclient.UpstreamOwnerNamespaceLabel: testProjectNS, + }, + }} + network := &networkingv1alpha.Network{ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: ns.Name}} + network.Spec.IPFamilies = []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol} + lb := &networkingv1alpha.NetworkLoadBalancer{ObjectMeta: metav1.ObjectMeta{ + Name: "public", Namespace: ns.Name, UID: types.UID("f96c4e18-cc50-45a8-aead-b368210f13ca"), + Finalizers: []string{networkLoadBalancerFinalizer}, + }} + lb.Spec.NetworkRef.Name = network.Name + lb.Spec.Exposure.Addresses = []networkingv1alpha.NetworkLoadBalancerAddressRequest{ + {Name: "regional", + Families: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, Locations: &networkingv1alpha.NetworkLoadBalancerLocationSelection{ + From: networkingv1alpha.NetworkLoadBalancerLocationSourceNames, Names: []string{"iad", "fra"}, + }}, + {Name: "edge", + Families: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, Locations: &networkingv1alpha.NetworkLoadBalancerLocationSelection{ + From: networkingv1alpha.NetworkLoadBalancerLocationSourceNames, Names: []string{"iad"}, + }}, + } + lb.Spec.Listeners = []networkingv1alpha.NetworkLoadBalancerListener{{ + Name: "web", Protocol: networkingv1alpha.NetworkLoadBalancerProtocolTCP, Port: 443, + AddressRefs: []string{"regional", "edge"}, + }} + ready := locationsv1alpha1.LocationStatus{Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue, Reason: "Test"}}} + locationIAD := &locationsv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Name: "iad"}, Status: ready} + locationFRA := &locationsv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Name: "fra"}, Status: ready} + cl := fakeclient.NewClientBuilder().WithScheme(scheme). + WithObjects(ns, network, lb, locationIAD, locationFRA). + WithStatusSubresource(lb, &networkingv1alpha.NetworkInterface{}).Build() + ipam := fakeclient.NewClientBuilder().WithScheme(scheme). + WithObjects(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testProjectNS}}). + WithStatusSubresource(&ipamv1alpha1.IPClaim{}).Build() + return &loadBalancerAllocationScenario{ + t: t, ctx: context.Background(), client: cl, ipam: ipam, lb: lb, + control: &NetworkLoadBalancerReconciler{IPAM: fixedIPAMClient{ipam}, PublicIPv6Class: "public-v6", ManagedDomain: "datumproxy.net"}, + } +} + +func (s *loadBalancerAllocationScenario) reconcile() networkingv1alpha.NetworkLoadBalancer { + s.t.Helper() + _, err := s.control.reconcile(s.ctx, s.client, client.ObjectKeyFromObject(s.lb)) + require.NoError(s.t, err) + var lb networkingv1alpha.NetworkLoadBalancer + require.NoError(s.t, s.client.Get(s.ctx, client.ObjectKeyFromObject(s.lb), &lb)) + return lb +} + +func (s *loadBalancerAllocationScenario) claims() []ipamv1alpha1.IPClaim { + s.t.Helper() + var list ipamv1alpha1.IPClaimList + require.NoError(s.t, s.ipam.List(s.ctx, &list, client.InNamespace(testProjectNS))) + return list.Items +} + +func TestNetworkLoadBalancerAllocationHoldsPerLocationAddresses(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + require.Len(t, s.claims(), 3) + require.False(t, apimeta.IsStatusConditionTrue(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated)) + + for i, claim := range s.claims() { + require.Equal(t, "public-v6", claim.Spec.ClassName) + require.NotNil(t, claim.Spec.PrefixLength) + require.Equal(t, int32(128), *claim.Spec.PrefixLength) + require.Equal(t, ipamv1alpha1.ReclaimDelete, claim.Spec.ReclaimPolicy) + require.NotEmpty(t, claim.Spec.Scope[ipamScopeRoleLocation].Name) + claim.Status.AllocatedCIDR = fmt.Sprintf("2001:db8::%x/128", i+1) + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + } + lb = s.reconcile() + require.True(t, apimeta.IsStatusConditionTrue(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated)) + require.Len(t, lb.Status.Endpoints, 3) + require.Len(t, lb.Status.AddressRequests, 2) + require.Equal(t, []string{"fra", "iad"}, lb.Status.AddressRequests[0].ResolvedLocations) + for _, endpoint := range lb.Status.Endpoints { + require.NotEmpty(t, endpoint.Location) + require.Regexp(t, "^"+endpoint.Location+`\.[a-z]+-[a-z]+-[a-z0-9]{5}\.datumproxy\.net$`, endpoint.CanonicalHostname) + for _, address := range endpoint.Addresses { + require.NotEmpty(t, address.Value) + require.True(t, apimeta.IsStatusConditionTrue(address.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated)) + } + } + require.Len(t, s.claims(), 3) + require.Equal(t, lb.Status.Endpoints, s.reconcile().Status.Endpoints, "retry must reuse claims and addresses") +} + +func TestNetworkLoadBalancerClaimNamesKeepAddressIdentityDistinct(t *testing.T) { + lb := &networkingv1alpha.NetworkLoadBalancer{ObjectMeta: metav1.ObjectMeta{UID: types.UID("fixture")}} + a := loadBalancerClaimName(lb, loadBalancerAddressRequest{ + requestName: "a-b", location: "c", + }) + b := loadBalancerClaimName(lb, loadBalancerAddressRequest{ + requestName: "a", location: "b-c", + }) + c := loadBalancerClaimName(lb, loadBalancerAddressRequest{ + requestName: "a-b", location: "d", + }) + require.NotEqual(t, a, b) + require.NotEqual(t, a, c) +} + +func TestNetworkLoadBalancerWithoutIPAMDoesNotLeaveAnUnreleasableFinalizer(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := *s.lb.DeepCopy() + lb.Finalizers = nil + require.NoError(t, s.client.Update(s.ctx, &lb)) + s.control.IPAM = nil + lb = s.reconcile() + require.Empty(t, lb.Finalizers) + require.False(t, apimeta.IsStatusConditionTrue(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated)) +} + +func TestNetworkLoadBalancerExplicitLocationRetainsClaimWhenProjectionDisappears(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + before := s.claims() + require.NoError(t, s.client.Delete(s.ctx, &locationsv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Name: "fra"}})) + s.reconcile() + require.Equal(t, before, s.claims()) +} + +func TestNetworkLoadBalancerAllLocationsAddsRegionalClaimAsLocationsAppear(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + for i := range lb.Spec.Exposure.Addresses { + lb.Spec.Exposure.Addresses[i].Locations = &networkingv1alpha.NetworkLoadBalancerLocationSelection{ + From: networkingv1alpha.NetworkLoadBalancerLocationSourceAll, + } + } + require.NoError(t, s.client.Update(s.ctx, &lb)) + s.reconcile() + require.Len(t, s.claims(), 4) + require.NoError(t, s.client.Create(s.ctx, &locationsv1alpha1.Location{ + ObjectMeta: metav1.ObjectMeta{Name: "sea"}, Status: locationsv1alpha1.LocationStatus{Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue, Reason: "Test"}}}, + })) + lb = s.reconcile() + require.Len(t, s.claims(), 6) + require.Len(t, lb.Status.Endpoints, 6) +} + +func TestNetworkLoadBalancerAllLocationsSkipsUnreadyLocations(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + lb.Spec.Exposure.Addresses[0].Locations = &networkingv1alpha.NetworkLoadBalancerLocationSelection{From: networkingv1alpha.NetworkLoadBalancerLocationSourceAll} + lb.Spec.Exposure.Addresses[1].Locations = &networkingv1alpha.NetworkLoadBalancerLocationSelection{From: networkingv1alpha.NetworkLoadBalancerLocationSourceAll} + require.NoError(t, s.client.Update(s.ctx, &lb)) + require.NoError(t, s.client.Create(s.ctx, &locationsv1alpha1.Location{ObjectMeta: metav1.ObjectMeta{Name: "unready"}, + Status: locationsv1alpha1.LocationStatus{Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionFalse, Reason: "Test"}}}})) + lb = s.reconcile() + for _, endpoint := range lb.Status.Endpoints { + require.NotEqual(t, "unready", endpoint.Location) + } +} + +func TestNetworkLoadBalancerClaimReleaseSkipsReplacementUID(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + ref := &networkingv1alpha.NetworkLoadBalancerIPClaimRef{Project: testProject, Namespace: claim.Namespace, Name: claim.Name, UID: "old-claim"} + require.NoError(t, s.control.deleteAddressClaim(s.ctx, ref, string(s.lb.UID))) + var existing ipamv1alpha1.IPClaim + require.NoError(t, s.ipam.Get(s.ctx, client.ObjectKeyFromObject(&claim), &existing)) + ref.UID = string(existing.UID) + require.NoError(t, s.control.deleteAddressClaim(s.ctx, ref, "foreign-lb")) + require.NoError(t, s.ipam.Get(s.ctx, client.ObjectKeyFromObject(&claim), &existing)) +} + +func TestNetworkLoadBalancerBackendPlacementResolvesUniquePortParent(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + lb.Spec.Exposure.Addresses[1].Locations = nil + require.NoError(t, s.client.Update(s.ctx, &lb)) + service := &networkingv1alpha.NetworkService{ObjectMeta: metav1.ObjectMeta{Name: "backend", Namespace: lb.Namespace}} + service.Spec.NetworkInterfaces.Selector = metav1.LabelSelector{MatchLabels: map[string]string{"app": "backend"}} + service.Spec.Ports = []networkingv1alpha.NetworkServicePort{{Name: "http", Port: 8080}} + require.NoError(t, s.client.Create(s.ctx, service)) + member := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{ + Name: "member", Namespace: lb.Namespace, + Labels: map[string]string{"app": "backend", networkingv1alpha.NetworkInterfaceLocationLabel: "iad"}, + }} + member.Spec.Network.Name = lb.Spec.NetworkRef.Name + member.Spec.ClaimRef = &networkingv1alpha.NetworkInterfaceClaimRef{Name: "member"} + member.Status.Phase = networkingv1alpha.NetworkInterfacePhaseBound + require.NoError(t, s.client.Create(s.ctx, member)) + require.NoError(t, s.client.Status().Update(s.ctx, member)) + route := &networkingv1alpha.NetworkLoadBalancerRoute{ObjectMeta: metav1.ObjectMeta{Name: "route", Namespace: lb.Namespace}} + route.Spec.ParentRefs = []networkingv1alpha.NetworkLoadBalancerRouteParentRef{{ + Name: lb.Name, Namespace: lb.Namespace, Port: ptr.To[int32](443), + }} + route.Spec.Backends = []networkingv1alpha.NetworkLoadBalancerRouteBackend{{ + NetworkService: networkingv1alpha.NetworkLoadBalancerNetworkServiceRef{Name: service.Name, Port: "http"}, + }} + require.NoError(t, s.client.Create(s.ctx, route)) + lb = s.reconcile() + require.Equal(t, []string{"iad"}, lb.Status.AddressRequests[1].ResolvedLocations) + require.Len(t, s.claims(), 3) + + lb.Spec.Listeners = append(lb.Spec.Listeners, networkingv1alpha.NetworkLoadBalancerListener{ + Name: "udp", Protocol: networkingv1alpha.NetworkLoadBalancerProtocolUDP, Port: 443, + AddressRefs: []string{"edge"}, + }) + require.NoError(t, s.client.Update(s.ctx, &lb)) + lb = s.reconcile() + require.Empty(t, lb.Status.AddressRequests[1].ResolvedLocations, + "ambiguous port parent must not allocate backend-derived addresses") +} + +func TestNetworkLoadBalancerTooManySelectedEndpointsDoesNotAllocate(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + lb.Spec.Exposure.Addresses = nil + for i := range 16 { + lb.Spec.Exposure.Addresses = append(lb.Spec.Exposure.Addresses, networkingv1alpha.NetworkLoadBalancerAddressRequest{ + Name: fmt.Sprintf("request-%02d", i), Families: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + Locations: &networkingv1alpha.NetworkLoadBalancerLocationSelection{From: networkingv1alpha.NetworkLoadBalancerLocationSourceAll}, + }) + } + for i := range 63 { + require.NoError(t, s.client.Create(s.ctx, &locationsv1alpha1.Location{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("location-%02d", i)}, + Status: locationsv1alpha1.LocationStatus{Conditions: []metav1.Condition{{Type: locationsv1alpha1.LocationConditionReady, Status: metav1.ConditionTrue, Reason: "Test"}}}, + })) + } + require.NoError(t, s.client.Update(s.ctx, &lb)) + before := s.claims() + lb = s.reconcile() + require.Equal(t, before, s.claims(), "overflow must not create any additional claims") + condition := apimeta.FindStatusCondition(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated) + require.NotNil(t, condition) + require.Equal(t, "TooManyEndpoints", condition.Reason) + require.Contains(t, condition.Message, "1024") + require.LessOrEqual(t, len(lb.Status.Endpoints), 1024) +} + +func TestNetworkLoadBalancerEndpointLimitIncludesProgrammedWithdrawal(t *testing.T) { + selected := make([]loadBalancerAddressRequest, 1024) + for i := range selected { + selected[i] = loadBalancerAddressRequest{requestName: "a", location: fmt.Sprintf("location-%04d", i)} + } + prior := []networkingv1alpha.NetworkLoadBalancerEndpointStatus{{ + RequestName: "retired", Location: "old", + Addresses: []networkingv1alpha.NetworkLoadBalancerAddressStatus{{ + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue}}, + }}, + }} + require.Equal(t, 1025, countAddressEndpoints(selected, prior)) +} + +func TestNetworkLoadBalancerBackendLocationsIncludeDisjointBackends(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + lb.Spec.Exposure.Addresses = lb.Spec.Exposure.Addresses[:1] + lb.Spec.Exposure.Addresses[0].Locations = nil + lb.Spec.Listeners[0].AddressRefs = []string{"regional"} + require.NoError(t, s.client.Update(s.ctx, &lb)) + for _, entry := range []struct{ name, location string }{{"east", "iad"}, {"west", "fra"}} { + service := &networkingv1alpha.NetworkService{ObjectMeta: metav1.ObjectMeta{Name: entry.name, Namespace: lb.Namespace}} + service.Spec.NetworkInterfaces.Selector.MatchLabels = map[string]string{"app": entry.name} + require.NoError(t, s.client.Create(s.ctx, service)) + iface := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{ + Name: entry.name, Namespace: lb.Namespace, + Labels: map[string]string{"app": entry.name, networkingv1alpha.NetworkInterfaceLocationLabel: entry.location}, + }} + iface.Spec.Network.Name = lb.Spec.NetworkRef.Name + iface.Status.Phase = networkingv1alpha.NetworkInterfacePhaseBound + require.NoError(t, s.client.Create(s.ctx, iface)) + } + route := &networkingv1alpha.NetworkLoadBalancerRoute{ObjectMeta: metav1.ObjectMeta{Name: "backend-route", Namespace: lb.Namespace}} + route.Spec.ParentRefs = []networkingv1alpha.NetworkLoadBalancerRouteParentRef{{Name: lb.Name, Listener: "web"}} + for _, name := range []string{"east", "west"} { + route.Spec.Backends = append(route.Spec.Backends, networkingv1alpha.NetworkLoadBalancerRouteBackend{ + NetworkService: networkingv1alpha.NetworkLoadBalancerNetworkServiceRef{Name: name, Port: "web"}, + }) + } + require.NoError(t, s.client.Create(s.ctx, route)) + locations, err := backendLocationsForAddress(s.ctx, s.client, &lb, "regional") + require.NoError(t, err) + require.Equal(t, []string{"fra", "iad"}, locations) + +} + +func TestNetworkLoadBalancerRetainsAnnouncedAddressUntilWithdrawn(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + setEdgeAnnouncement(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb.Spec.Exposure.Addresses = lb.Spec.Exposure.Addresses[:1] + lb.Spec.Listeners[0].AddressRefs = []string{"regional"} + require.NoError(t, s.client.Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, s.claims(), 3, "announced IP must remain held") + require.Len(t, lb.Status.Endpoints, 3, "withdrawal must see the old address") + require.Len(t, lb.Status.AddressRequests, 1) + require.Equal(t, []string{"fra", "iad"}, lb.Status.AddressRequests[0].ResolvedLocations) + + setEdgeAnnouncement(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, lb.Status.Endpoints, 2) + require.Len(t, s.claims(), 2) +} + +func TestNetworkLoadBalancerRetainsProgrammedAddressUntilWithdrawn(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + setEdgeProgramming(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb.Spec.Exposure.Addresses = lb.Spec.Exposure.Addresses[:1] + lb.Spec.Listeners[0].AddressRefs = []string{"regional"} + require.NoError(t, s.client.Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, s.claims(), 3, "programmed IP must remain held before announcement") + require.Len(t, lb.Status.Endpoints, 3, "withdrawal must see the programmed address") + require.False(t, edgeAddress(t, &lb).Announced) + require.True(t, apimeta.IsStatusConditionTrue(edgeAddress(t, &lb).Conditions, + networkingv1alpha.NetworkLoadBalancerProgrammed)) + + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, lb.Status.Endpoints, 2) + require.Len(t, s.claims(), 2) +} + +func TestNetworkLoadBalancerOrphanCleanupRetainsProgrammedClaim(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + claim := edgeClaim(t, s) + setEdgeProgramming(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + wanted := map[string]bool{} + require.NoError(t, s.control.releaseOrphanedClaims(s.ctx, s.ipam, testProjectNS, &lb, wanted)) + require.NoError(t, s.ipam.Get(s.ctx, client.ObjectKeyFromObject(&claim), &ipamv1alpha1.IPClaim{})) + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.control.releaseOrphanedClaims(s.ctx, s.ipam, testProjectNS, &lb, wanted)) + var remaining ipamv1alpha1.IPClaimList + require.NoError(t, s.ipam.List(s.ctx, &remaining, client.InNamespace(testProjectNS))) + require.Empty(t, remaining.Items) +} + +func TestNetworkLoadBalancerRecreatedClaimDoesNotInheritPublication(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + claim.UID = types.UID("original-claim") + require.NoError(t, s.ipam.Update(s.ctx, &claim)) + claim.Status.Address = "2001:db8::1" + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + lb := s.reconcile() + old := edgeAddress(t, &lb) + require.Equal(t, "2001:db8::1", old.Value) + require.Equal(t, "original-claim", old.ClaimRef.UID) + old.Announced = true + old.DNSPublished = true + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue, + Reason: "Observed", Message: "programmed", + }) + apimeta.SetStatusCondition(&lb.Status.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue, + Reason: "Observed", Message: "programmed", + }) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + + require.NoError(t, s.ipam.Delete(s.ctx, &claim)) + lb = s.reconcile() + require.Equal(t, "AddressReplacementPendingWithdrawal", + apimeta.FindStatusCondition(edgeAddress(t, &lb).Conditions, networkingv1alpha.NetworkLoadBalancerAllocated).Reason) + replacement := edgeClaim(t, s) + replacement.UID = types.UID("replacement-claim") + require.NoError(t, s.ipam.Update(s.ctx, &replacement)) + replacement.Status.Address = "2001:db8::2" + require.NoError(t, s.ipam.Status().Update(s.ctx, &replacement)) + lb = s.reconcile() + old = edgeAddress(t, &lb) + require.Equal(t, "2001:db8::1", old.Value, "old address must remain visible for withdrawal") + require.Equal(t, "original-claim", old.ClaimRef.UID) + require.True(t, old.Announced) + require.True(t, old.DNSPublished) + require.Equal(t, "AddressReplacementPendingWithdrawal", + apimeta.FindStatusCondition(old.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated).Reason) + require.False(t, apimeta.IsStatusConditionTrue(lb.Status.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed)) + + old.Announced = false + old.DNSPublished = false + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionFalse, + Reason: "Withdrawn", Message: "withdrawn", + }) + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerAnnounced, Status: metav1.ConditionFalse, + Reason: "Withdrawn", Message: "withdrawn", + }) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + current := edgeAddress(t, &lb) + require.Equal(t, "2001:db8::2", current.Value) + require.Equal(t, "replacement-claim", current.ClaimRef.UID) + require.False(t, current.Announced) + require.False(t, current.DNSPublished) + require.Nil(t, apimeta.FindStatusCondition(current.Conditions, networkingv1alpha.NetworkLoadBalancerAnnounced)) + require.Nil(t, apimeta.FindStatusCondition(current.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed)) +} + +func TestNetworkLoadBalancerReassignedAddressResetsServingConditions(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + claim.Status.Address = "2001:db8::1" + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + lb := s.reconcile() + old := edgeAddress(t, &lb) + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue, + Reason: "Observed", Message: "programmed", + }) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + claim.Status.Address = "2001:db8::2" + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + lb = s.reconcile() + require.Equal(t, "2001:db8::1", edgeAddress(t, &lb).Value) + require.True(t, apimeta.IsStatusConditionTrue(edgeAddress(t, &lb).Conditions, + networkingv1alpha.NetworkLoadBalancerProgrammed)) + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + current := edgeAddress(t, &lb) + require.Equal(t, "2001:db8::2", current.Value) + require.Nil(t, apimeta.FindStatusCondition(current.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed)) +} + +func TestNetworkLoadBalancerRecreatedClaimWithSameAddressResetsServingConditions(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + claim.UID = types.UID("first-claim") + require.NoError(t, s.ipam.Update(s.ctx, &claim)) + claim.Status.Address = "2001:db8::1" + require.NoError(t, s.ipam.Status().Update(s.ctx, &claim)) + lb := s.reconcile() + old := edgeAddress(t, &lb) + apimeta.SetStatusCondition(&old.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: metav1.ConditionTrue, + Reason: "Observed", Message: "programmed", + }) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + require.NoError(t, s.ipam.Delete(s.ctx, &claim)) + s.reconcile() + replacement := edgeClaim(t, s) + replacement.UID = types.UID("second-claim") + require.NoError(t, s.ipam.Update(s.ctx, &replacement)) + replacement.Status.Address = "2001:db8::1" + require.NoError(t, s.ipam.Status().Update(s.ctx, &replacement)) + lb = s.reconcile() + require.Equal(t, "first-claim", edgeAddress(t, &lb).ClaimRef.UID) + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb = s.reconcile() + current := edgeAddress(t, &lb) + require.Equal(t, "second-claim", current.ClaimRef.UID) + require.Nil(t, apimeta.FindStatusCondition(current.Conditions, networkingv1alpha.NetworkLoadBalancerProgrammed)) +} + +func edgeClaim(t *testing.T, s *loadBalancerAllocationScenario) ipamv1alpha1.IPClaim { + t.Helper() + for _, claim := range s.claims() { + if claim.Spec.Scope[ipamScopeRoleLocation].Name == "iad" && + claim.Name == loadBalancerClaimName(s.lb, loadBalancerAddressRequest{ + requestName: "edge", location: "iad", family: networkingv1alpha.IPv6Protocol, + }) { + return claim + } + } + t.Fatal("edge IPClaim in iad not found") + return ipamv1alpha1.IPClaim{} +} + +func edgeAddress(t *testing.T, lb *networkingv1alpha.NetworkLoadBalancer) *networkingv1alpha.NetworkLoadBalancerAddressStatus { + t.Helper() + for i := range lb.Status.Endpoints { + if lb.Status.Endpoints[i].RequestName == "edge" { + return &lb.Status.Endpoints[i].Addresses[0] + } + } + t.Fatal("edge address not found") + return nil +} + +func TestNetworkLoadBalancerResolvedLocationsExcludeWithdrawingLocation(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + for i := range lb.Status.Endpoints { + endpoint := &lb.Status.Endpoints[i] + if endpoint.RequestName == "regional" && endpoint.Location == "fra" { + for j := range endpoint.Addresses { + endpoint.Addresses[j].Announced = true + } + } + } + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + lb.Spec.Exposure.Addresses[0].Locations.Names = []string{"iad"} + require.NoError(t, s.client.Update(s.ctx, &lb)) + lb = s.reconcile() + require.Len(t, lb.Status.Endpoints, 3, "the removed location remains until withdrawal") + require.Equal(t, []string{"iad"}, lb.Status.AddressRequests[0].ResolvedLocations) + require.Len(t, s.claims(), 3) + +} + +func TestNetworkLoadBalancerDeletionWaitsForWithdrawal(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + setEdgeAnnouncement(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + require.NoError(t, s.client.Delete(s.ctx, &lb)) + lb = s.reconcile() + require.NotNil(t, lb.DeletionTimestamp) + require.Len(t, s.claims(), 3) + + setEdgeAnnouncement(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + _, err := s.control.reconcile(s.ctx, s.client, client.ObjectKeyFromObject(s.lb)) + require.NoError(t, err) + require.Empty(t, s.claims()) +} + +func TestNetworkLoadBalancerDeletionWaitsForProgrammingWithdrawal(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + lb := s.reconcile() + setEdgeProgramming(&lb, metav1.ConditionTrue) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + require.NoError(t, s.client.Delete(s.ctx, &lb)) + lb = s.reconcile() + require.NotNil(t, lb.DeletionTimestamp) + require.Len(t, s.claims(), 3) + + setEdgeProgramming(&lb, metav1.ConditionFalse) + require.NoError(t, s.client.Status().Update(s.ctx, &lb)) + _, err := s.control.reconcile(s.ctx, s.client, client.ObjectKeyFromObject(s.lb)) + require.NoError(t, err) + require.Empty(t, s.claims()) +} + +func TestNetworkLoadBalancerRejectsDriftedIPClaimSpec(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*ipamv1alpha1.IPClaim) + }{ + {name: "family", mutate: func(claim *ipamv1alpha1.IPClaim) { + claim.Spec.IPFamily = ipamv1alpha1.IPFamily(networkingv1alpha.IPv4Protocol) + }}, + {name: "reclaim policy", mutate: func(claim *ipamv1alpha1.IPClaim) { + claim.Spec.ReclaimPolicy = "Retain" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newLoadBalancerAllocationScenario(t) + s.reconcile() + claim := edgeClaim(t, s) + tc.mutate(&claim) + require.NoError(t, s.ipam.Update(s.ctx, &claim)) + lb := s.reconcile() + condition := apimeta.FindStatusCondition(edgeAddress(t, &lb).Conditions, + networkingv1alpha.NetworkLoadBalancerAllocated) + require.NotNil(t, condition) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.Contains(t, condition.Message, "does not match") + require.Len(t, s.claims(), 3, "drifted claim must not be replaced or released") + }) + } +} + +func setEdgeProgramming(lb *networkingv1alpha.NetworkLoadBalancer, status metav1.ConditionStatus) { + for i := range lb.Status.Endpoints { + if lb.Status.Endpoints[i].RequestName != "edge" { + continue + } + for j := range lb.Status.Endpoints[i].Addresses { + apimeta.SetStatusCondition(&lb.Status.Endpoints[i].Addresses[j].Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerProgrammed, Status: status, + Reason: "Observed", Message: "data plane programming status", ObservedGeneration: lb.Generation, + }) + } + } +} + +func setEdgeAnnouncement(lb *networkingv1alpha.NetworkLoadBalancer, status metav1.ConditionStatus) { + for i := range lb.Status.Endpoints { + if lb.Status.Endpoints[i].RequestName != "edge" { + continue + } + for j := range lb.Status.Endpoints[i].Addresses { + address := &lb.Status.Endpoints[i].Addresses[j] + address.Announced = status == metav1.ConditionTrue + apimeta.SetStatusCondition(&address.Conditions, metav1.Condition{ + Type: networkingv1alpha.NetworkLoadBalancerAnnounced, Status: status, + Reason: "Observed", Message: "BGP announcement status", ObservedGeneration: lb.Generation, + }) + } + } +}