diff --git a/api/v1alpha/groupversion_info.go b/api/v1alpha/groupversion_info.go index 4f890ace..c2634bfe 100644 --- a/api/v1alpha/groupversion_info.go +++ b/api/v1alpha/groupversion_info.go @@ -42,6 +42,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &NetworkInterfaceList{}, &NetworkInterfaceClaim{}, &NetworkInterfaceClaimList{}, + &NetworkLoadBalancerBackendHealth{}, + &NetworkLoadBalancerBackendHealthList{}, &NetworkLoadBalancer{}, &NetworkLoadBalancerList{}, &NetworkLoadBalancerRoute{}, diff --git a/api/v1alpha/networkinterface_types.go b/api/v1alpha/networkinterface_types.go index 7319a3a1..89cb9220 100644 --- a/api/v1alpha/networkinterface_types.go +++ b/api/v1alpha/networkinterface_types.go @@ -4,6 +4,7 @@ package v1alpha import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // NetworkInterfaceReclaimPolicy decides what becomes of an interface, and the @@ -262,6 +263,11 @@ type NetworkInterfaceAttachmentRef struct { // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=253 Name string `json:"name"` + + // uid identifies the exact attachment incarnation. Probe consumers require + // this value so a recreated attachment cannot inherit the prior health. + // +kubebuilder:validation:Optional + UID types.UID `json:"uid,omitempty"` } // NetworkInterfaceSpec defines the desired state of NetworkInterface. It is @@ -446,4 +452,8 @@ const ( // published from, so a collector can find the source of a copy it is looking // at. NetworkInterfaceSourceNamespaceLabel = "networking.datumapis.com/source-namespace" + + // NetworkInterfaceSourceUIDLabel identifies the exact cell interface behind + // a published copy. A replacement may reuse the name, but never the UID. + NetworkInterfaceSourceUIDLabel = "networking.datumapis.com/source-uid" ) diff --git a/api/v1alpha/networkloadbalancerbackendhealth_types.go b/api/v1alpha/networkloadbalancerbackendhealth_types.go new file mode 100644 index 00000000..d6db9b01 --- /dev/null +++ b/api/v1alpha/networkloadbalancerbackendhealth_types.go @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// +kubebuilder:validation:Enum=TCP;HTTP;HTTPS +type NetworkLoadBalancerProbeProtocol string + +const ( + NetworkLoadBalancerProbeTCP NetworkLoadBalancerProbeProtocol = "TCP" + NetworkLoadBalancerProbeHTTP NetworkLoadBalancerProbeProtocol = "HTTP" + NetworkLoadBalancerProbeHTTPS NetworkLoadBalancerProbeProtocol = "HTTPS" +) + +type NetworkLoadBalancerBackendHealthRouteRef struct { + // +kubebuilder:validation:MinLength=1 + Namespace string `json:"namespace"` + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + UID types.UID `json:"uid"` + // +kubebuilder:validation:Minimum=1 + Generation int64 `json:"generation"` +} + +type NetworkLoadBalancerBackendHealthParentRef struct { + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + UID types.UID `json:"uid"` + // +kubebuilder:validation:Minimum=1 + Generation int64 `json:"generation"` + // +kubebuilder:validation:MinLength=1 + Listener string `json:"listener"` +} + +// NetworkLoadBalancerProbe describes a resolved active check. Its port is a +// backend port number, not a listener port or a NetworkService port name. +// +kubebuilder:validation:XValidation:message="path is only valid for HTTP and HTTPS",rule="has(self.path) ? self.protocol in ['HTTP', 'HTTPS'] : true" +// +kubebuilder:validation:XValidation:message="serverName is only valid for HTTPS",rule="has(self.serverName) ? self.protocol == 'HTTPS' : true" +type NetworkLoadBalancerProbe struct { + Protocol NetworkLoadBalancerProbeProtocol `json:"protocol"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port int32 `json:"port"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Pattern=`^/.*$` + // +kubebuilder:validation:MaxLength=256 + Path string `json:"path,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxLength=253 + ServerName string `json:"serverName,omitempty"` +} + +// NetworkLoadBalancerBackendHealthTarget identifies one resolved interface +// endpoint and its probe. The source UID prevents a replacement interface +// reusing a name from inheriting a previous endpoint's health. +type NetworkLoadBalancerBackendHealthTarget struct { + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + // +kubebuilder:validation:MinLength=1 + SourceNamespace string `json:"sourceNamespace"` + // +kubebuilder:validation:MinLength=1 + SourceInterface string `json:"sourceInterface"` + SourceUID types.UID `json:"sourceUID"` + // +kubebuilder:validation:MinLength=1 + AttachmentName string `json:"attachmentName"` + AttachmentUID types.UID `json:"attachmentUID"` + // +kubebuilder:validation:MinLength=1 + Address string `json:"address"` + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + ServingPort int32 `json:"servingPort"` + Probe NetworkLoadBalancerProbe `json:"probe"` +} + +// NetworkLoadBalancerBackendHealthSpec carries one route parent and location's +// resolved targets into the cell that can reach them. +type NetworkLoadBalancerBackendHealthSpec struct { + RouteRef NetworkLoadBalancerBackendHealthRouteRef `json:"routeRef"` + ParentRef NetworkLoadBalancerBackendHealthParentRef `json:"parentRef"` + // +kubebuilder:validation:MinLength=1 + Location string `json:"location"` + // +kubebuilder:validation:MinLength=1 + NetworkName string `json:"networkName"` + // +kubebuilder:validation:MinLength=1 + FabricVPCID string `json:"fabricVPCID"` + // +kubebuilder:validation:MaxItems=64 + // +listType=map + // +listMapKey=name + Targets []NetworkLoadBalancerBackendHealthTarget `json:"targets"` +} + +// NetworkLoadBalancerBackendHealthTargetStatus is usable only while its route, +// object generation, target fingerprint, and expiry match current intent. +type NetworkLoadBalancerBackendHealthTargetStatus struct { + Name string `json:"name"` + ObservedGeneration int64 `json:"observedGeneration"` + RouteUID types.UID `json:"routeUID"` + RouteGeneration int64 `json:"routeGeneration"` + TargetFingerprint string `json:"targetFingerprint"` + Healthy bool `json:"healthy"` + ConsecutiveSuccesses int32 `json:"consecutiveSuccesses"` + LastProbeTime metav1.Time `json:"lastProbeTime"` + ValidUntil metav1.Time `json:"validUntil"` + Reason string `json:"reason,omitempty"` +} + +type NetworkLoadBalancerBackendHealthStatus struct { + // ProbesEvaluated remains Unknown until a VPC-capable runner reports results. + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // +kubebuilder:validation:MaxItems=64 + // +listType=map + // +listMapKey=name + Targets []NetworkLoadBalancerBackendHealthTargetStatus `json:"targets,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// NetworkLoadBalancerBackendHealth is an internal, cell-local probe contract. +// A target is eligible for data plane programming only after a fresh success. +type NetworkLoadBalancerBackendHealth struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec NetworkLoadBalancerBackendHealthSpec `json:"spec"` + // +kubebuilder:default={conditions:{{type:"ProbesEvaluated",status:"Unknown",reason:"Pending",message:"Waiting for a VPC-capable probe runner",lastTransitionTime:"1970-01-01T00:00:00Z"}}} + Status NetworkLoadBalancerBackendHealthStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +type NetworkLoadBalancerBackendHealthList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkLoadBalancerBackendHealth `json:"items"` +} diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index f46f7b34..a9a26d0d 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -1948,6 +1948,179 @@ func (in *NetworkLoadBalancerAddressStatus) DeepCopy() *NetworkLoadBalancerAddre return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealth) DeepCopyInto(out *NetworkLoadBalancerBackendHealth) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealth. +func (in *NetworkLoadBalancerBackendHealth) DeepCopy() *NetworkLoadBalancerBackendHealth { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkLoadBalancerBackendHealth) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthList) DeepCopyInto(out *NetworkLoadBalancerBackendHealthList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkLoadBalancerBackendHealth, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthList. +func (in *NetworkLoadBalancerBackendHealthList) DeepCopy() *NetworkLoadBalancerBackendHealthList { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkLoadBalancerBackendHealthList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthParentRef) DeepCopyInto(out *NetworkLoadBalancerBackendHealthParentRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthParentRef. +func (in *NetworkLoadBalancerBackendHealthParentRef) DeepCopy() *NetworkLoadBalancerBackendHealthParentRef { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthParentRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthRouteRef) DeepCopyInto(out *NetworkLoadBalancerBackendHealthRouteRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthRouteRef. +func (in *NetworkLoadBalancerBackendHealthRouteRef) DeepCopy() *NetworkLoadBalancerBackendHealthRouteRef { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthRouteRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthSpec) DeepCopyInto(out *NetworkLoadBalancerBackendHealthSpec) { + *out = *in + out.RouteRef = in.RouteRef + out.ParentRef = in.ParentRef + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]NetworkLoadBalancerBackendHealthTarget, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthSpec. +func (in *NetworkLoadBalancerBackendHealthSpec) DeepCopy() *NetworkLoadBalancerBackendHealthSpec { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthStatus) DeepCopyInto(out *NetworkLoadBalancerBackendHealthStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]NetworkLoadBalancerBackendHealthTargetStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthStatus. +func (in *NetworkLoadBalancerBackendHealthStatus) DeepCopy() *NetworkLoadBalancerBackendHealthStatus { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthTarget) DeepCopyInto(out *NetworkLoadBalancerBackendHealthTarget) { + *out = *in + out.Probe = in.Probe +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthTarget. +func (in *NetworkLoadBalancerBackendHealthTarget) DeepCopy() *NetworkLoadBalancerBackendHealthTarget { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerBackendHealthTargetStatus) DeepCopyInto(out *NetworkLoadBalancerBackendHealthTargetStatus) { + *out = *in + in.LastProbeTime.DeepCopyInto(&out.LastProbeTime) + in.ValidUntil.DeepCopyInto(&out.ValidUntil) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerBackendHealthTargetStatus. +func (in *NetworkLoadBalancerBackendHealthTargetStatus) DeepCopy() *NetworkLoadBalancerBackendHealthTargetStatus { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerBackendHealthTargetStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkLoadBalancerBackendTransport) DeepCopyInto(out *NetworkLoadBalancerBackendTransport) { *out = *in @@ -2154,6 +2327,21 @@ func (in *NetworkLoadBalancerNetworkServiceRef) DeepCopy() *NetworkLoadBalancerN return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerProbe) DeepCopyInto(out *NetworkLoadBalancerProbe) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerProbe. +func (in *NetworkLoadBalancerProbe) DeepCopy() *NetworkLoadBalancerProbe { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerProbe) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkLoadBalancerRoute) DeepCopyInto(out *NetworkLoadBalancerRoute) { *out = *in diff --git a/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml index 4973ac3a..aae8ac3a 100644 --- a/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml +++ b/config/crd/bases/networking.datumapis.com_networkinterfaces.yaml @@ -316,6 +316,11 @@ spec: maxLength: 253 minLength: 1 type: string + uid: + description: |- + uid identifies the exact attachment incarnation. Probe consumers require + this value so a recreated attachment cannot inherit the prior health. + type: string required: - apiGroup - kind diff --git a/config/crd/bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml b/config/crd/bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml new file mode 100644 index 00000000..24aa6c70 --- /dev/null +++ b/config/crd/bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml @@ -0,0 +1,329 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: networkloadbalancerbackendhealths.networking.datumapis.com +spec: + group: networking.datumapis.com + names: + kind: NetworkLoadBalancerBackendHealth + listKind: NetworkLoadBalancerBackendHealthList + plural: networkloadbalancerbackendhealths + singular: networkloadbalancerbackendhealth + scope: Namespaced + versions: + - name: v1alpha + schema: + openAPIV3Schema: + description: |- + NetworkLoadBalancerBackendHealth is an internal, cell-local probe contract. + A target is eligible for data plane programming only after a fresh success. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NetworkLoadBalancerBackendHealthSpec carries one route parent and location's + resolved targets into the cell that can reach them. + properties: + fabricVPCID: + minLength: 1 + type: string + location: + minLength: 1 + type: string + networkName: + minLength: 1 + type: string + parentRef: + properties: + generation: + format: int64 + minimum: 1 + type: integer + listener: + minLength: 1 + type: string + name: + minLength: 1 + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - generation + - listener + - name + - uid + type: object + routeRef: + properties: + generation: + format: int64 + minimum: 1 + type: integer + name: + minLength: 1 + type: string + namespace: + minLength: 1 + type: string + uid: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - generation + - name + - namespace + - uid + type: object + targets: + items: + description: |- + NetworkLoadBalancerBackendHealthTarget identifies one resolved interface + endpoint and its probe. The source UID prevents a replacement interface + reusing a name from inheriting a previous endpoint's health. + properties: + address: + minLength: 1 + type: string + attachmentName: + minLength: 1 + type: string + attachmentUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + name: + maxLength: 63 + minLength: 1 + type: string + probe: + description: |- + NetworkLoadBalancerProbe describes a resolved active check. Its port is a + backend port number, not a listener port or a NetworkService port name. + properties: + path: + maxLength: 256 + pattern: ^/.*$ + type: string + port: + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + enum: + - TCP + - HTTP + - HTTPS + type: string + serverName: + maxLength: 253 + type: string + required: + - port + - protocol + type: object + x-kubernetes-validations: + - message: path is only valid for HTTP and HTTPS + rule: 'has(self.path) ? self.protocol in [''HTTP'', ''HTTPS''] + : true' + - message: serverName is only valid for HTTPS + rule: 'has(self.serverName) ? self.protocol == ''HTTPS'' : + true' + servingPort: + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sourceInterface: + minLength: 1 + type: string + sourceNamespace: + minLength: 1 + type: string + sourceUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + required: + - address + - attachmentName + - attachmentUID + - name + - probe + - servingPort + - sourceInterface + - sourceNamespace + - sourceUID + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - fabricVPCID + - location + - networkName + - parentRef + - routeRef + - targets + type: object + status: + default: + conditions: + - lastTransitionTime: "1970-01-01T00:00:00Z" + message: Waiting for a VPC-capable probe runner + reason: Pending + status: Unknown + type: ProbesEvaluated + properties: + conditions: + description: ProbesEvaluated remains Unknown until a VPC-capable runner + reports results. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + targets: + items: + description: |- + NetworkLoadBalancerBackendHealthTargetStatus is usable only while its route, + object generation, target fingerprint, and expiry match current intent. + properties: + consecutiveSuccesses: + format: int32 + type: integer + healthy: + type: boolean + lastProbeTime: + format: date-time + type: string + name: + type: string + observedGeneration: + format: int64 + type: integer + reason: + type: string + routeGeneration: + format: int64 + type: integer + routeUID: + description: |- + UID is a type that holds unique ID values, including UUIDs. Because we + don't ONLY use UUIDs, this is an alias to string. Being a type captures + intent and helps make sure that UIDs and names do not get conflated. + type: string + targetFingerprint: + type: string + validUntil: + format: date-time + type: string + required: + - consecutiveSuccesses + - healthy + - lastProbeTime + - name + - observedGeneration + - routeGeneration + - routeUID + - targetFingerprint + - validUntil + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/downstream/kustomization.yaml b/config/crd/downstream/kustomization.yaml index 215f6511..123da524 100644 --- a/config/crd/downstream/kustomization.yaml +++ b/config/crd/downstream/kustomization.yaml @@ -46,4 +46,5 @@ resources: # Network interface claim path, served on cells - ../bases/networking.datumapis.com_networkinterfaceclaims.yaml - ../bases/networking.datumapis.com_networkinterfaces.yaml + - ../bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml - ../bases/networking.datumapis.com_subnets.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d556090b..b3dee038 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -362,6 +362,7 @@ rules: - networkcontexts/status - networkinterfaceclaims/status - networkinterfaces/status + - networkloadbalancerbackendhealths/status - networkloadbalancerroutes/status - networkloadbalancers/status - networkpolicies/status @@ -378,6 +379,7 @@ rules: - networking.datumapis.com resources: - connectorclasses + - networkloadbalancerbackendhealths - networkloadbalancerroutes - networkservices verbs: diff --git a/docs/api/networkinterfaces.md b/docs/api/networkinterfaces.md index 84c5ee2b..4ea9d402 100644 --- a/docs/api/networkinterfaces.md +++ b/docs/api/networkinterfaces.md @@ -497,6 +497,14 @@ compute.datumapis.com.
name is the name of the referent.
true + + uid + string + + uid identifies the exact attachment incarnation. Probe consumers require +this value so a recreated attachment cannot inherit the prior health.
+ + false diff --git a/docs/plans/network-load-balancer-cell-contract.md b/docs/plans/network-load-balancer-cell-contract.md new file mode 100644 index 00000000..500e4f86 --- /dev/null +++ b/docs/plans/network-load-balancer-cell-contract.md @@ -0,0 +1,95 @@ +# Network load balancer cell contract + +## Identity available today + +A project's `NetworkInterface` is a copy of a cell interface. The copy has the +cell location, source namespace, and source name. This change also publishes the +source interface UID. These four values identify one source object, including +across deletion and recreation under the same name. The copy deliberately omits +`status.attachmentRef` and `status.vpc`; those values name cell resources and +are not authoritative in a project control plane. + +The route resolver counts only projected endpoints carrying this source UID. +Older copies without the label remain unprogrammable until the cell publishes +them again. The UID on a copy is a hint for the cell to check against its live +object, never authorization by itself. + +Inside a cell, Cloud creates a `VPCAttachment` with +`spec.interfaceRef.name` pointing to the source interface in the same +namespace. The source interface reports the attachment in +`status.attachmentRef` and the fabric VPC identifier in `status.vpc`. Galactic +reads attachment addresses, `status.vpc`, and `status.node`. Matching an address +alone is unsafe because different VPCs can use the same address. + +## Proposed intent + +The project route reconciler should publish a **versioned, location-scoped +intent** through the hub to the target cell. This resource does not exist yet. +The owner reference must identify the route by UID and generation so a deleted +or replaced route cannot keep a VIP active. Its minimum schema is: + +```yaml +apiVersion: networking.datumapis.com/v1alpha +kind: NetworkLoadBalancerCellIntent +spec: + source: + projectCluster: + namespace: + routeName: + routeUID: + routeGeneration: + parentName: + listenerName: + location: + networkName: + vip: + protocol: TCP # or UDP + listenerPort: 443 + backends: + - sourceNamespace: + sourceName: + sourceUID: + address: + port: 8443 + weight: 1 + healthCheck: +``` + +Only a platform controller may write this intent. The cell must treat the +projected source tuple as a lookup hint, not as proof. For every backend it +must read the cell `NetworkInterface` and require the UID, network, bound phase, +holder availability, location, and address to match. It must then read the +same-namespace `VPCAttachment` named by the interface's `status.attachmentRef` +and require its `spec.interfaceRef.name`, controller owner UID, `status.vpc`, +IPv6 address, and programmed generation to match the source. The attachment +must have a node and the source must identify the expected VPC. A missing or +changed field removes the backend from the effective set. The cell must also +verify that the source namespace belongs to the project recorded in the +intent, and that the attachment's `spec.vpc` resolves to a live VPC whose +fabric ID matches both statuses. A source UID change invalidates old intent +even when the name and IP are reused. + +The cell should report resolved attachment identities and probe results per +backend, with the intent generation it observed. A project controller may set +route `Programmed=True` only after the cell has acknowledged that generation +and every announcing gateway node confirms the same healthy backend set. VIP +announcement follows that acknowledgment; an empty healthy set withdraws the +VIP but retains the allocated address. + +## Galactic API dependencies + +The current `NetworkRule` requires a nonempty `backendSelector`. Cloud's +`VPCAttachment` writer does not provide a trusted label that selects all and +only this route's attachments. `backendAddresses` narrows a selector, so it +cannot be used alone today. The Network API should permit an omitted selector +**only when `backendAddresses` is explicitly set**. Galactic should then +select attachments by authoritative `status.vpc == spec.vpcRef` and intersect +their IPv6 addresses with `backendAddresses`. An explicit empty list means no +backends and withdrawal. The writer must verify authorization for `vpcRef`; +the existing Galactic admission contract does not itself prove it. + +Current DSR also cannot safely serve two VIPs sharing a backend +address/port/protocol, and does not implement weighted selection or active +application probes. Those capabilities and an all-gateway programming +acknowledgment are required before this intent can drive the full native API. +Until then, route and load balancer `Programmed` and `Announced` remain false. diff --git a/docs/plans/network-load-balancer-health.md b/docs/plans/network-load-balancer-health.md new file mode 100644 index 00000000..9859c283 --- /dev/null +++ b/docs/plans/network-load-balancer-health.md @@ -0,0 +1,49 @@ +# Cell-local network load balancer health + +The project controller cannot prove that a backend answers on its application +port. It sees a projected interface address, but the cell controller manager's +pod has no established route into each tenant VPC. Host-network connection +failures must not be interpreted as unhealthy applications. + +`NetworkLoadBalancerBackendHealth` is an internal cell-local contract for a +runner on the gateway's VPC-aware socket path. The cell manager registers the +probe reconciler without a dialer today. Its `ProbesEvaluated` condition is +`False/NoDataPlaneDialer`, and no target is eligible from status alone. +Serving requires a dialer that selects the validated fabric VPC's VRF or +network namespace for each outgoing connection. It must never fall back to the +controller pod's ordinary network. + +The cell intent controller must first verify the live source interface, +attachment UID, VPC identity, and project ownership described in the +[cell contract](network-load-balancer-cell-contract.md). It can then create one +health object for a route parent and location. The parent UID and generation +bind the health object to a particular load balancer instance. Each target +records the source interface UID, attachment UID, address, serving port, +and resolved probe. The probe runner also rechecks the live interface +UID, network, location, holder availability, attachment name, VPC ID, and +address before each check. The eventual traffic programmer must revalidate +the attachment UID and VPC association when it consumes healthy targets; probe success is +not an authorization to program or announce a VIP. + +A VPC-aware runner evaluates up to 64 targets per object, with at most eight +concurrent probes per object and 32 across the process. Queueing for a probe +slot does not consume the probe deadline. Each TCP connection or HTTP/HTTPS GET +has a two-second deadline. A full 64-target sweep can take eight waves, or +16 seconds, and the next sweep starts five seconds after completion. Five +seconds is therefore a pause between sweeps, not a per-target interval. +Two consecutive successes are needed to mark a target healthy; a failure +removes it when that sweep is published. Results expire after 39 seconds: +two maximum sweeps, the five-second pause, and one probe timeout of scheduling +slack. A delayed sweep beyond this bound causes health to expire closed. +Consumers use `HealthyTargets`, which rejects a +result when the object generation, route UID or generation, target fingerprint, +probe timestamp, or expiry no longer matches. A removed or replaced source +cannot inherit an earlier success. + +HTTP and HTTPS accept only 2xx responses and do not follow redirects. HTTPS +always validates the server certificate. With the current native route API, +which has no probe hostname, the certificate must have an IP SAN for the +backend IPv6 address. The cell-local contract can carry an explicit server +name once the public route API exposes one; until then a hostname is not +invented by the runner. UDP traffic uses a separate TCP, HTTP, or HTTPS probe +port and is never treated as healthy from a UDP packet alone. diff --git a/internal/agent/catalog_test.go b/internal/agent/catalog_test.go index 9304e31a..2f3c8930 100644 --- a/internal/agent/catalog_test.go +++ b/internal/agent/catalog_test.go @@ -32,22 +32,23 @@ var inScopeAPIFiles = []string{ // rather than ignored so that "scoped" cannot quietly become "stale": a new // file has to be argued into one list or the other. var outOfScopeAPIFiles = map[string]string{ - "edgereachability_types.go": "hub-side record of which addresses an edge should reach; no customer-facing status", - "groupversion_info.go": "scheme registration, no reasons", - "internaldns_types.go": "private resolver access is a network surface, not load balancer status", - "locationbinding_types.go": "placement plumbing behind a location, never named by a load balancer", - "network_types.go": "the network a service sits on; its own product surface", - "networkbinding_types.go": "network attachment plumbing", - "networkcontext_types.go": "network attachment plumbing", - "networkinterface_types.go": "an instance's interface; compute's surface", - "networkinterfaceclaim_types.go": "an instance's interface; compute's surface", - "networkloadbalancer_types.go": "its own Layer 4 product surface, not reached through an application load balancer", - "networkloadbalancerroute_types.go": "its own Layer 4 routing surface, not reached through an application load balancer", - "networkpolicy_types.go": "its own product surface, not reached through a load balancer", - "subnet_types.go": "address management below a network", - "subnetclaim_types.go": "address management below a network", - "trafficprotectionpolicy_types.go": "declares no reasons of its own; the controller writes upstream Gateway API policy reasons", - "zz_generated.deepcopy.go": "generated", + "edgereachability_types.go": "hub-side record of which addresses an edge should reach; no customer-facing status", + "groupversion_info.go": "scheme registration, no reasons", + "internaldns_types.go": "private resolver access is a network surface, not load balancer status", + "locationbinding_types.go": "placement plumbing behind a location, never named by a load balancer", + "network_types.go": "the network a service sits on; its own product surface", + "networkbinding_types.go": "network attachment plumbing", + "networkcontext_types.go": "network attachment plumbing", + "networkinterface_types.go": "an instance's interface; compute's surface", + "networkinterfaceclaim_types.go": "an instance's interface; compute's surface", + "networkloadbalancer_types.go": "its own Layer 4 product surface, not reached through an application load balancer", + "networkloadbalancerbackendhealth_types.go": "cell-local probe state, not an application load balancer customer surface", + "networkloadbalancerroute_types.go": "its own Layer 4 routing surface, not reached through an application load balancer", + "networkpolicy_types.go": "its own product surface, not reached through a load balancer", + "subnet_types.go": "address management below a network", + "subnetclaim_types.go": "address management below a network", + "trafficprotectionpolicy_types.go": "declares no reasons of its own; the controller writes upstream Gateway API policy reasons", + "zz_generated.deepcopy.go": "generated", } // extraReasonConsts are reason constants whose NAME does not contain "Reason", diff --git a/internal/cmd/cell/cell.go b/internal/cmd/cell/cell.go index c862fd21..7ee15531 100644 --- a/internal/cmd/cell/cell.go +++ b/internal/cmd/cell/cell.go @@ -352,6 +352,9 @@ func controllerRegistrations( HubCluster: hubCluster, }).SetupWithManager(mgr) }}) + registrations = append(registrations, namedSetup{"networkloadbalancerbackendhealth", func() (bool, error) { + return true, (&controller.NetworkLoadBalancerBackendHealthReconciler{}).SetupWithManager(mgr.GetLocalManager()) + }}) registrations = append(registrations, namedSetup{"internaldns-cell", func() (bool, error) { if !serverConfig.InternalDNS.Enabled { diff --git a/internal/cmd/manager/manager_test.go b/internal/cmd/manager/manager_test.go index 19f3065f..d97e4fe3 100644 --- a/internal/cmd/manager/manager_test.go +++ b/internal/cmd/manager/manager_test.go @@ -71,6 +71,7 @@ var reconcilerControllerNames = map[string]string{ "NetworkReconciler": "network", "NetworkServiceReconciler": "networkservice", "NetworkLoadBalancerRouteReconciler": "networkloadbalancerroute", + "NetworkLoadBalancerBackendHealthReconciler": "networkloadbalancerbackendhealth", "SubnetClaimReconciler": "subnetclaim", "SubnetReconciler": "subnet", "TrafficProtectionPolicyReconciler": "trafficprotectionpolicy", diff --git a/internal/controller/networkinterface_projection.go b/internal/controller/networkinterface_projection.go index 1cc761c2..8a5119f6 100644 --- a/internal/controller/networkinterface_projection.go +++ b/internal/controller/networkinterface_projection.go @@ -61,6 +61,9 @@ func projectionLabels(source *networkingv1alpha.NetworkInterface, location strin labels[networkingv1alpha.NetworkInterfaceProjectionLabel] = "true" labels[networkingv1alpha.NetworkInterfaceSourceNamespaceLabel] = source.Namespace + if source.UID != "" { + labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel] = string(source.UID) + } if location != "" { labels[networkingv1alpha.NetworkInterfaceLocationLabel] = location diff --git a/internal/controller/networkinterface_projection_test.go b/internal/controller/networkinterface_projection_test.go index 8731e7dd..5d6037e9 100644 --- a/internal/controller/networkinterface_projection_test.go +++ b/internal/controller/networkinterface_projection_test.go @@ -345,7 +345,7 @@ func (v *visibility) projectCopy() (*networkingv1alpha.NetworkInterface, bool) { func TestInterfaceReachesTheProjectControlPlane(t *testing.T) { v := newVisibility(t) - v.interfaceOnCell() + source := v.interfaceOnCell() v.publish() v.handToProject() @@ -365,6 +365,24 @@ func TestInterfaceReachesTheProjectControlPlane(t *testing.T) { require.Equal(t, testLocationName, copied.Labels[networkingv1alpha.NetworkInterfaceLocationLabel]) require.Equal(t, boundInterfaceName, copied.Labels[networkingv1alpha.NetworkInterfaceHolderLabel]) + require.Equal(t, v.cellNamespace, copied.Labels[networkingv1alpha.NetworkInterfaceSourceNamespaceLabel]) + require.Equal(t, string(source.UID), copied.Labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel]) +} + +func TestProjectedInterfaceSourceIdentityChangesWithUID(t *testing.T) { + source := &networkingv1alpha.NetworkInterface{ObjectMeta: metav1.ObjectMeta{ + Namespace: "cell", Name: "nic", UID: "first", + Labels: map[string]string{networkingv1alpha.NetworkInterfaceSourceUIDLabel: "forged"}, + }} + first := projectedInterface(source, "dfw") + require.Equal(t, "first", first.Labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel]) + source.UID = "second" + second := projectedInterface(source, "dfw") + require.Equal(t, "second", second.Labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel]) + require.Equal(t, "cell", second.Labels[networkingv1alpha.NetworkInterfaceSourceNamespaceLabel]) + source.UID = "" + third := projectedInterface(source, "dfw") + require.NotContains(t, third.Labels, networkingv1alpha.NetworkInterfaceSourceUIDLabel) } func TestProjectedInterfaceReportsDNSSettings(t *testing.T) { diff --git a/internal/controller/networkloadbalancerbackendhealth_controller.go b/internal/controller/networkloadbalancerbackendhealth_controller.go new file mode 100644 index 00000000..3ab5e494 --- /dev/null +++ b/internal/controller/networkloadbalancerbackendhealth_controller.go @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync" + "time" + + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/nlbhealth" +) + +const probesPerHealthObject = nlbhealth.ProbeSlotsPerObject + +var cellProbeSlots = make(chan struct{}, 32) + +type NetworkLoadBalancerBackendHealthReconciler struct { + Client client.Client + Prober nlbhealth.Prober + Now func() time.Time +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancerbackendhealths,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancerbackendhealths/status,verbs=get;update;patch + +func (r *NetworkLoadBalancerBackendHealthReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { + var health networkingv1alpha.NetworkLoadBalancerBackendHealth + if err := r.Client.Get(ctx, req.NamespacedName, &health); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !health.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + if len(health.Spec.Targets) > nlbhealth.MaxTargets { + return ctrl.Result{}, fmt.Errorf("health object %s has too many targets", req.NamespacedName) + } + + now := time.Now + if r.Now != nil { + now = r.Now + } + previous := make(map[string]*networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus, len(health.Status.Targets)) + for i := range health.Status.Targets { + previous[health.Status.Targets[i].Name] = &health.Status.Targets[i] + } + statuses := make([]networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus, len(health.Spec.Targets)) + localSlots := make(chan struct{}, probesPerHealthObject) + var workers sync.WaitGroup + for i := range health.Spec.Targets { + workers.Add(1) + go func(i int) { + defer workers.Done() + target := health.Spec.Targets[i] + var probeErr error + if health.Spec.RouteRef.UID == "" || health.Spec.RouteRef.Generation < 1 || + health.Spec.ParentRef.UID == "" || health.Spec.ParentRef.Generation < 1 || + target.SourceUID == "" || target.AttachmentUID == "" { + probeErr = errors.New("source identity is incomplete") + } else { + probeErr = r.validateLiveTarget(ctx, &health, target) + } + if probeErr == nil { + probeErr = takeProbeSlot(ctx, localSlots) + } + if probeErr == nil { + defer func() { <-localSlots }() + if probeErr = takeProbeSlot(ctx, cellProbeSlots); probeErr == nil { + defer func() { <-cellProbeSlots }() + // Probe starts its own timeout after both concurrency slots are held. + probeErr = r.Prober.Probe(ctx, health.Spec.FabricVPCID, target) + } + } + statuses[i] = nlbhealth.NextTargetStatus(&health, target, previous[target.Name], probeErr, now()) + }(i) + } + workers.Wait() + if err := ctx.Err(); err != nil { + return ctrl.Result{}, err + } + + wanted := networkingv1alpha.NetworkLoadBalancerBackendHealthStatus{ + Conditions: slices.Clone(health.Status.Conditions), Targets: statuses, + } + condition := metav1.Condition{ + Type: "ProbesEvaluated", Status: metav1.ConditionTrue, Reason: "Evaluated", + Message: fmt.Sprintf("Evaluated %d targets from the cell network path", len(statuses)), + ObservedGeneration: health.Generation, + } + if r.Prober.DialContext == nil { + condition.Status = metav1.ConditionFalse + condition.Reason = "NoDataPlaneDialer" + condition.Message = "No VPC-aware probe dialer is configured" + } + apimeta.SetStatusCondition(&wanted.Conditions, condition) + if !equality.Semantic.DeepEqual(health.Status, wanted) { + health.Status = wanted + if err := r.Client.Status().Update(ctx, &health); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("update backend health status: %w", err) + } + } + return ctrl.Result{RequeueAfter: nlbhealth.ProbeInterval}, nil +} + +func (r *NetworkLoadBalancerBackendHealthReconciler) validateLiveTarget( + ctx context.Context, + health *networkingv1alpha.NetworkLoadBalancerBackendHealth, + target networkingv1alpha.NetworkLoadBalancerBackendHealthTarget, +) error { + var iface networkingv1alpha.NetworkInterface + if err := r.Client.Get(ctx, client.ObjectKey{Namespace: target.SourceNamespace, Name: target.SourceInterface}, &iface); err != nil { + return fmt.Errorf("read source interface: %w", err) + } + if iface.UID != target.SourceUID || iface.Labels[networkingv1alpha.NetworkInterfaceProjectionLabel] == "true" || + iface.Spec.Network.Name != health.Spec.NetworkName || + iface.Labels[networkingv1alpha.NetworkInterfaceLocationLabel] != health.Spec.Location || + iface.Status.Phase != networkingv1alpha.NetworkInterfacePhaseBound || !isServiceMemberHealthy(&iface) || + iface.Status.VPC != health.Spec.FabricVPCID || iface.Status.AttachmentRef == nil || + iface.Status.AttachmentRef.Name != target.AttachmentName || + iface.Status.AttachmentRef.UID == "" || iface.Status.AttachmentRef.UID != target.AttachmentUID { + return errors.New("source interface no longer matches this target") + } + if !slices.ContainsFunc(iface.Spec.Addresses, func(a networkingv1alpha.NetworkInterfaceAddress) bool { + address, _, _ := strings.Cut(a.Address, "/") + return address == target.Address + }) { + return errors.New("source interface no longer holds this address") + } + return nil +} + +func takeProbeSlot(ctx context.Context, slots chan struct{}) error { + select { + case slots <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r *NetworkLoadBalancerBackendHealthReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Client = mgr.GetClient() + return ctrl.NewControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkLoadBalancerBackendHealth{}). + WithOptions(controller.Options{MaxConcurrentReconciles: 4}). + Named("networkloadbalancerbackendhealth"). + Complete(r) +} diff --git a/internal/controller/networkloadbalancerbackendhealth_controller_test.go b/internal/controller/networkloadbalancerbackendhealth_controller_test.go new file mode 100644 index 00000000..7b1fab2e --- /dev/null +++ b/internal/controller/networkloadbalancerbackendhealth_controller_test.go @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/nlbhealth" +) + +func TestBackendHealthFullProbeSweepPublishesFreshConsecutiveSuccesses(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + iface := &networkingv1alpha.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "cell-project", Name: "nic-1", UID: types.UID("interface-1"), + Labels: map[string]string{networkingv1alpha.NetworkInterfaceLocationLabel: "us-east-1"}, + }, + Spec: networkingv1alpha.NetworkInterfaceSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "default"}, + Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "fd20::1/128"}}, + }, + Status: networkingv1alpha.NetworkInterfaceStatus{ + Phase: networkingv1alpha.NetworkInterfacePhaseBound, VPC: "vpc-1", + AttachmentRef: &networkingv1alpha.NetworkInterfaceAttachmentRef{Name: "attach-1", UID: types.UID("attachment-1")}, + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkInterfaceHolderAvailable, + Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}}, + }, + } + health := &networkingv1alpha.NetworkLoadBalancerBackendHealth{ + ObjectMeta: metav1.ObjectMeta{Namespace: "network-services-operator-system", Name: "health-64", Generation: 1}, + Spec: networkingv1alpha.NetworkLoadBalancerBackendHealthSpec{ + RouteRef: networkingv1alpha.NetworkLoadBalancerBackendHealthRouteRef{ + Namespace: "project", Name: "route-1", UID: types.UID("route-1"), Generation: 1, + }, + ParentRef: networkingv1alpha.NetworkLoadBalancerBackendHealthParentRef{ + Name: "load-balancer", UID: types.UID("load-balancer-1"), Generation: 1, Listener: "game", + }, + Location: "us-east-1", NetworkName: "default", FabricVPCID: "vpc-1", + }, + } + for i := 0; i < 64; i++ { + health.Spec.Targets = append(health.Spec.Targets, networkingv1alpha.NetworkLoadBalancerBackendHealthTarget{ + Name: fmt.Sprintf("origin-%02d", i), SourceNamespace: "cell-project", SourceInterface: "nic-1", SourceUID: iface.UID, + AttachmentName: "attach-1", AttachmentUID: types.UID("attachment-1"), Address: "fd20::1", + ServingPort: 8443, + Probe: networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeTCP, Port: 8443}, + }) + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(iface, health). + WithStatusSubresource(iface, health).Build() + start := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + var clock atomic.Int64 + clock.Store(start.UnixNano()) + var dials, active, maximum atomic.Int32 + waves := make([]chan struct{}, 16) + for i := range waves { + waves[i] = make(chan struct{}) + } + r := &NetworkLoadBalancerBackendHealthReconciler{ + Client: cl, Now: func() time.Time { return time.Unix(0, clock.Load()) }, + Prober: nlbhealth.Prober{DialContext: func(ctx context.Context, _, _, _ string) (net.Conn, error) { + current := active.Add(1) + defer active.Add(-1) + for old := maximum.Load(); current > old; old = maximum.Load() { + if maximum.CompareAndSwap(old, current) { + break + } + } + n := int(dials.Add(1)) + wave := waves[(n-1)/probesPerHealthObject] + if n%probesPerHealthObject == 0 { + clock.Add(int64(nlbhealth.ProbeTimeout)) + close(wave) + } + select { + case <-wave: + case <-ctx.Done(): + return nil, ctx.Err() + } + a, b := net.Pipe() + _ = b.Close() + return a, nil + }}, + } + key := client.ObjectKeyFromObject(health) + for sweep := 0; sweep < 2; sweep++ { + result, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, nlbhealth.ProbeInterval, result.RequeueAfter) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Len(t, health.Status.Targets, 64) + if sweep == 0 { + publishedAt := time.Unix(0, clock.Load()) + for _, status := range health.Status.Targets { + require.True(t, status.ValidUntil.After(publishedAt), "first-sweep result expired before publication: %s", status.Name) + } + require.Empty(t, nlbhealth.HealthyTargets(health, time.Unix(0, clock.Load()))) + } else { + require.Len(t, nlbhealth.HealthyTargets(health, time.Unix(0, clock.Load())), 64) + // The next 64-target sweep consumes eight two-second waves before + // publishing. Every previously healthy target must remain eligible + // immediately before that publication, including the first wave. + beforeNextPublication := time.Unix(0, clock.Load()).Add(nlbhealth.ProbeInterval + nlbhealth.MaxSweepDuration - time.Nanosecond) + require.Len(t, nlbhealth.HealthyTargets(health, beforeNextPublication), 64) + } + clock.Add(int64(nlbhealth.ProbeInterval)) + } + require.EqualValues(t, 128, dials.Load()) + require.LessOrEqual(t, maximum.Load(), int32(probesPerHealthObject)) + // Results are never reusable after the freshness window, even after success. + require.Empty(t, nlbhealth.HealthyTargets(health, time.Unix(0, clock.Load()).Add(nlbhealth.ResultLifetime))) +} + +func TestBackendHealthReconcileRequiresLiveSourceAndVPCDialer(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + iface := &networkingv1alpha.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "cell-project", Name: "nic-1", UID: types.UID("interface-1"), + Labels: map[string]string{networkingv1alpha.NetworkInterfaceLocationLabel: "us-east-1"}, + }, + Spec: networkingv1alpha.NetworkInterfaceSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "default"}, + Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "fd20::1/128"}}, + }, + Status: networkingv1alpha.NetworkInterfaceStatus{ + Phase: networkingv1alpha.NetworkInterfacePhaseBound, VPC: "vpc-1", + AttachmentRef: &networkingv1alpha.NetworkInterfaceAttachmentRef{Name: "attach-1", UID: types.UID("attachment-1")}, + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkInterfaceHolderAvailable, + Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}}, + }, + } + health := &networkingv1alpha.NetworkLoadBalancerBackendHealth{ + ObjectMeta: metav1.ObjectMeta{Namespace: "network-services-operator-system", Name: "route-1-us-east-1", Generation: 4}, + Spec: networkingv1alpha.NetworkLoadBalancerBackendHealthSpec{ + RouteRef: networkingv1alpha.NetworkLoadBalancerBackendHealthRouteRef{Namespace: "project", Name: "route-1", UID: types.UID("route-1"), Generation: 2}, + ParentRef: networkingv1alpha.NetworkLoadBalancerBackendHealthParentRef{Name: "load-balancer", UID: types.UID("load-balancer-1"), Generation: 3, Listener: "game"}, + Location: "us-east-1", NetworkName: "default", FabricVPCID: "vpc-1", + Targets: []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget{{ + Name: "origin-1", SourceNamespace: "cell-project", SourceInterface: "nic-1", SourceUID: iface.UID, + AttachmentName: "attach-1", AttachmentUID: types.UID("attachment-1"), Address: "fd20::1", + ServingPort: 8443, + Probe: networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeTCP, Port: 8443}, + }}, + }, + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(iface, health). + WithStatusSubresource(iface, health).Build() + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + dials := 0 + r := &NetworkLoadBalancerBackendHealthReconciler{ + Client: cl, Now: func() time.Time { return now }, + Prober: nlbhealth.Prober{DialContext: func(_ context.Context, vpc, _, _ string) (net.Conn, error) { + require.Equal(t, "vpc-1", vpc) + dials++ + a, b := net.Pipe() + if err := b.Close(); err != nil { + return nil, err + } + return a, nil + }}, + } + key := client.ObjectKeyFromObject(health) + for i := 0; i < 2; i++ { + result, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, nlbhealth.ProbeInterval, result.RequeueAfter) + now = now.Add(nlbhealth.ProbeInterval) + } + require.Equal(t, 2, dials) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Len(t, nlbhealth.HealthyTargets(health, now), 1) + require.True(t, apimeta.IsStatusConditionTrue(health.Status.Conditions, "ProbesEvaluated")) + + // Replacing an attachment under the same name must not reuse prior health. + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(iface), iface)) + iface.Status.AttachmentRef.UID = types.UID("attachment-2") + require.NoError(t, cl.Status().Update(context.Background(), iface)) + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, 2, dials) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Empty(t, nlbhealth.HealthyTargets(health, now)) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(iface), iface)) + iface.Status.AttachmentRef.UID = "" + require.NoError(t, cl.Status().Update(context.Background(), iface)) + _, err = r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, 2, dials) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Empty(t, nlbhealth.HealthyTargets(health, now)) + + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(iface), iface)) + iface.Status.Phase = networkingv1alpha.NetworkInterfacePhaseAvailable + require.NoError(t, cl.Status().Update(context.Background(), iface)) + _, err = r.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.Equal(t, 2, dials) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Empty(t, nlbhealth.HealthyTargets(health, now)) + + noDialer := &NetworkLoadBalancerBackendHealthReconciler{Client: cl} + _, err = noDialer.Reconcile(context.Background(), ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + require.NoError(t, cl.Get(context.Background(), key, health)) + require.Equal(t, "NoDataPlaneDialer", apimeta.FindStatusCondition(health.Status.Conditions, "ProbesEvaluated").Reason) +} + +func TestBackendHealthProbeDeadlineStartsAfterGlobalSlotAcquisition(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, networkingv1alpha.AddToScheme(scheme)) + iface := &networkingv1alpha.NetworkInterface{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "cell-project", Name: "nic-1", UID: types.UID("interface-1"), + Labels: map[string]string{networkingv1alpha.NetworkInterfaceLocationLabel: "us-east-1"}, + }, + Spec: networkingv1alpha.NetworkInterfaceSpec{ + Network: networkingv1alpha.LocalNetworkRef{Name: "default"}, + Addresses: []networkingv1alpha.NetworkInterfaceAddress{{Address: "fd20::1/128"}}, + }, + Status: networkingv1alpha.NetworkInterfaceStatus{ + Phase: networkingv1alpha.NetworkInterfacePhaseBound, VPC: "vpc-1", + AttachmentRef: &networkingv1alpha.NetworkInterfaceAttachmentRef{Name: "attach-1", UID: types.UID("attachment-1")}, + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkInterfaceHolderAvailable, + Status: metav1.ConditionTrue, Reason: "Ready", LastTransitionTime: metav1.Now()}}, + }, + } + health := &networkingv1alpha.NetworkLoadBalancerBackendHealth{ + ObjectMeta: metav1.ObjectMeta{Namespace: "network-services-operator-system", Name: "health-1", Generation: 1}, + Spec: networkingv1alpha.NetworkLoadBalancerBackendHealthSpec{ + RouteRef: networkingv1alpha.NetworkLoadBalancerBackendHealthRouteRef{ + Namespace: "project", Name: "route-1", UID: types.UID("route-1"), Generation: 1, + }, + ParentRef: networkingv1alpha.NetworkLoadBalancerBackendHealthParentRef{ + Name: "load-balancer", UID: types.UID("load-balancer-1"), Generation: 1, Listener: "game", + }, + Location: "us-east-1", NetworkName: "default", FabricVPCID: "vpc-1", + Targets: []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget{{ + Name: "origin-1", SourceNamespace: "cell-project", SourceInterface: "nic-1", SourceUID: iface.UID, + AttachmentName: "attach-1", AttachmentUID: types.UID("attachment-1"), Address: "fd20::1", + ServingPort: 8443, + Probe: networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeTCP, Port: 8443}, + }}, + }, + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(iface, health). + WithStatusSubresource(iface, health).Build() + probed := make(chan struct{}, 1) + r := &NetworkLoadBalancerBackendHealthReconciler{ + Client: cl, + Prober: nlbhealth.Prober{DialContext: func(ctx context.Context, _, _, _ string) (net.Conn, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + probed <- struct{}{} + a, b := net.Pipe() + _ = b.Close() + return a, nil + }}, + } + for range cap(cellProbeSlots) { + cellProbeSlots <- struct{}{} + } + t.Cleanup(func() { + for len(cellProbeSlots) > 0 { + <-cellProbeSlots + } + }) + finished := make(chan error, 1) + go func() { + _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(health)}) + finished <- err + }() + select { + case <-probed: + t.Fatal("probe ran without acquiring a global slot") + case err := <-finished: + t.Fatalf("reconcile finished before a slot was released: %v", err) + case <-time.After(nlbhealth.ProbeTimeout + 100*time.Millisecond): + } + <-cellProbeSlots + select { + case err := <-finished: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("reconcile did not complete after a slot was released") + } + require.Len(t, probed, 1) + require.NoError(t, cl.Get(context.Background(), client.ObjectKeyFromObject(health), health)) + require.Equal(t, "Warming", health.Status.Targets[0].Reason) +} diff --git a/internal/controller/networkloadbalancerroute_controller.go b/internal/controller/networkloadbalancerroute_controller.go index bc95d517..5bb5f751 100644 --- a/internal/controller/networkloadbalancerroute_controller.go +++ b/internal/controller/networkloadbalancerroute_controller.go @@ -53,11 +53,13 @@ type NetworkLoadBalancerRouteReconciler struct { } type networkLoadBalancerRouteEndpoint struct { - NetworkService string - Interface string - Location string - Address string - Port int32 + NetworkService string + Interface string + SourceNamespace string + SourceUID string + Location string + Address string + Port int32 } type networkLoadBalancerRouteReferenceError struct { @@ -220,7 +222,7 @@ func reconcileRouteParent( fmt.Sprintf("All backend references resolved; %d holder-available IPv6 endpoints are eligible", len(endpoints))) if status.Effective && len(endpoints) == 0 { setRouteParentCondition(&status, route.Generation, networkingv1alpha.NetworkLoadBalancerRouteProgrammed, - metav1.ConditionFalse, nlbRouteReasonNoEligibleEndpoints, "No enabled backend has a holder-available IPv6 endpoint in a selected location") + metav1.ConditionFalse, nlbRouteReasonNoEligibleEndpoints, "No enabled backend has a holder-available IPv6 endpoint with source identity in a selected location") } return status, nil } @@ -493,7 +495,10 @@ func resolveNetworkLoadBalancerRouteBackends( for i := range members { member := &members[i] location := member.Labels[networkingv1alpha.NetworkInterfaceLocationLabel] - if location == "" || !routeLocationSelected(parent, listener, location) || !isServiceMemberHealthy(member) { + sourceNamespace := member.Labels[networkingv1alpha.NetworkInterfaceSourceNamespaceLabel] + sourceUID := member.Labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel] + if !isProjection(member) || sourceNamespace == "" || sourceUID == "" || + location == "" || !routeLocationSelected(parent, listener, location) || !isServiceMemberHealthy(member) { continue } address, ok := addressOfType(interfaceBackhaulAddresses(member), discoveryv1.AddressTypeIPv6) @@ -501,11 +506,13 @@ func resolveNetworkLoadBalancerRouteBackends( continue } endpoints = append(endpoints, networkLoadBalancerRouteEndpoint{ - NetworkService: ref.Name, - Interface: member.Name, - Location: location, - Address: address, - Port: port.Port, + NetworkService: ref.Name, + Interface: member.Name, + SourceNamespace: sourceNamespace, + SourceUID: sourceUID, + Location: location, + Address: address, + Port: port.Port, }) } } diff --git a/internal/controller/networkloadbalancerroute_controller_test.go b/internal/controller/networkloadbalancerroute_controller_test.go index ab16bd8b..de0c8625 100644 --- a/internal/controller/networkloadbalancerroute_controller_test.go +++ b/internal/controller/networkloadbalancerroute_controller_test.go @@ -76,7 +76,13 @@ func testRouteMember(name string, available bool, address string) *networkingv1a return &networkingv1alpha.NetworkInterface{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test", Name: name, - Labels: map[string]string{"app": "storefront", networkingv1alpha.NetworkInterfaceLocationLabel: "dfw"}, + Labels: map[string]string{ + "app": "storefront", + networkingv1alpha.NetworkInterfaceLocationLabel: "dfw", + networkingv1alpha.NetworkInterfaceProjectionLabel: "true", + networkingv1alpha.NetworkInterfaceSourceNamespaceLabel: "cell-test", + networkingv1alpha.NetworkInterfaceSourceUIDLabel: "source-" + name, + }, }, Spec: networkingv1alpha.NetworkInterfaceSpec{ Network: networkingv1alpha.LocalNetworkRef{Name: "default"}, @@ -508,6 +514,8 @@ func TestNetworkLoadBalancerRouteEndpointResolution(t *testing.T) { require.NoError(t, err) require.Len(t, endpoints, 1) assert.Equal(t, "fd00::1", endpoints[0].Address) + assert.Equal(t, "cell-test", endpoints[0].SourceNamespace) + assert.Equal(t, "source-ready-v6", endpoints[0].SourceUID) assert.Equal(t, "dfw", endpoints[0].Location) parent.Spec.Exposure.Addresses[0].Locations = &networkingv1alpha.NetworkLoadBalancerLocationSelection{ @@ -521,6 +529,13 @@ func TestNetworkLoadBalancerRouteEndpointResolution(t *testing.T) { endpoints, err = resolveNetworkLoadBalancerRouteBackends(ctx, cl, route, parent, &parent.Spec.Listeners[0]) require.NoError(t, err) require.Len(t, endpoints, 1, "omitted locations must follow backend membership") + + member := testRouteMember("missing-source-uid", true, "fd00::3/128") + delete(member.Labels, networkingv1alpha.NetworkInterfaceSourceUIDLabel) + cl = fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)).WithObjects(service, member).Build() + endpoints, err = resolveNetworkLoadBalancerRouteBackends(ctx, cl, route, parent, &parent.Spec.Listeners[0]) + require.NoError(t, err) + assert.Empty(t, endpoints) } func TestNetworkLoadBalancerRouteUDPRequiresTCPProbe(t *testing.T) { diff --git a/internal/networkagent/catalog_test.go b/internal/networkagent/catalog_test.go index 26796cb9..1ed7176f 100644 --- a/internal/networkagent/catalog_test.go +++ b/internal/networkagent/catalog_test.go @@ -30,17 +30,18 @@ var inScopeAPIFiles = []string{ } var outOfScopeAPIFiles = map[string]string{ - "domain_types.go": "load balancer hostnames; the ALB agent's", - "edgereachability_types.go": "hub-side record with no customer-facing status", - "groupversion_info.go": "scheme registration, no reasons", - "httpproxy_types.go": "load balancers; the ALB agent's", - "internaldns_types.go": "internal DNS service configuration, no status reasons", - "locationbinding_types.go": "which locations a project is offered; surfaces here only as LocationNotAvailable", - "networkloadbalancer_types.go": "Network Load Balancer has its own product status, separate from networks and network services", - "networkloadbalancerroute_types.go": "Network Load Balancer route status belongs to the load balancer product", - "networkpolicy_types.go": "declares no status reasons", - "trafficprotectionpolicy_types.go": "load balancer protection; the ALB agent's", - "zz_generated.deepcopy.go": "generated", + "domain_types.go": "load balancer hostnames; the ALB agent's", + "edgereachability_types.go": "hub-side record with no customer-facing status", + "groupversion_info.go": "scheme registration, no reasons", + "httpproxy_types.go": "load balancers; the ALB agent's", + "internaldns_types.go": "internal DNS service configuration, no status reasons", + "locationbinding_types.go": "which locations a project is offered; surfaces here only as LocationNotAvailable", + "networkloadbalancer_types.go": "Network Load Balancer has its own product status, separate from networks and network services", + "networkloadbalancerbackendhealth_types.go": "cell-local NLB probe status, not a network-agent customer API", + "networkloadbalancerroute_types.go": "Network Load Balancer route status belongs to the load balancer product", + "networkpolicy_types.go": "declares no status reasons", + "trafficprotectionpolicy_types.go": "load balancer protection; the ALB agent's", + "zz_generated.deepcopy.go": "generated", } var controllerFiles = []string{ diff --git a/internal/nlbhealth/health.go b/internal/nlbhealth/health.go new file mode 100644 index 00000000..05830187 --- /dev/null +++ b/internal/nlbhealth/health.go @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package nlbhealth + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/netip" + "net/url" + "strconv" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +const ( + ProbeTimeout = 2 * time.Second + ProbeInterval = 5 * time.Second + MaxTargets = 64 + ProbeSlotsPerObject = 8 + MaxSweepDuration = (MaxTargets / ProbeSlotsPerObject) * ProbeTimeout + // Status is published after an entire sweep. An early result is already up + // to MaxSweepDuration old at publication, then waits ProbeInterval and up + // to another full sweep for replacement. One timeout of scheduling slack + // preserves eligibility under the bounded 64-target, eight-slot workload; + // a longer delay expires closed rather than extending stale health forever. + ResultLifetime = 2*MaxSweepDuration + ProbeInterval + ProbeTimeout + ConsecutiveSuccessWindow = ResultLifetime + ProbeTimeout + HealthyAfterSuccesses = 2 +) + +type Prober struct { + DialContext func(context.Context, string, string, string) (net.Conn, error) + RootCAs *x509.CertPool +} + +func (p Prober) Probe(ctx context.Context, fabricVPCID string, target networkingv1alpha.NetworkLoadBalancerBackendHealthTarget) error { + if p.DialContext == nil || fabricVPCID == "" { + return errors.New("VPC-aware probe dialer is not configured") + } + address, err := netip.ParseAddr(target.Address) + if err != nil || !address.Is6() || address.Is4In6() || address.IsUnspecified() || address.IsLoopback() || address.IsLinkLocalUnicast() || address.IsMulticast() { + return errors.New("target must be a routable IPv6 unicast address") + } + if target.Probe.Port < 1 || target.Probe.Port > 65535 { + return errors.New("invalid probe port") + } + + ctx, cancel := context.WithTimeout(ctx, ProbeTimeout) + defer cancel() + endpoint := net.JoinHostPort(address.String(), strconv.Itoa(int(target.Probe.Port))) + dial := func(ctx context.Context, network, endpoint string) (net.Conn, error) { + return p.DialContext(ctx, fabricVPCID, network, endpoint) + } + + switch target.Probe.Protocol { + case networkingv1alpha.NetworkLoadBalancerProbeTCP: + conn, err := dial(ctx, "tcp", endpoint) + if err != nil { + return err + } + return conn.Close() + case networkingv1alpha.NetworkLoadBalancerProbeHTTP, networkingv1alpha.NetworkLoadBalancerProbeHTTPS: + path := target.Probe.Path + if path == "" { + path = "/" + } + if path[0] != '/' { + return errors.New("probe path must begin with /") + } + scheme := "http" + if target.Probe.Protocol == networkingv1alpha.NetworkLoadBalancerProbeHTTPS { + scheme = "https" + } + transport := &http.Transport{ + Proxy: nil, + DialContext: dial, + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: p.RootCAs, ServerName: target.Probe.ServerName}, + } + defer transport.CloseIdleConnections() + requestURL := url.URL{Scheme: scheme, Host: endpoint, Path: path} + request, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL.String(), nil) + if err != nil { + return err + } + if target.Probe.ServerName != "" { + request.Host = target.Probe.ServerName + } + response, err := (&http.Client{ + Transport: transport, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }).Do(request) + if err != nil { + return err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode < 200 || response.StatusCode >= 300 { + return fmt.Errorf("probe returned HTTP %d", response.StatusCode) + } + return nil + default: + return errors.New("unsupported probe protocol") + } +} + +func TargetFingerprint(target networkingv1alpha.NetworkLoadBalancerBackendHealthTarget) string { + encoded, _ := json.Marshal(target) + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +func NextTargetStatus( + health *networkingv1alpha.NetworkLoadBalancerBackendHealth, + target networkingv1alpha.NetworkLoadBalancerBackendHealthTarget, + previous *networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus, + probeErr error, + now time.Time, +) networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus { + fingerprint := TargetFingerprint(target) + status := networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{ + Name: target.Name, + ObservedGeneration: health.Generation, + RouteUID: health.Spec.RouteRef.UID, + RouteGeneration: health.Spec.RouteRef.Generation, + TargetFingerprint: fingerprint, + LastProbeTime: metav1.NewTime(now), + ValidUntil: metav1.NewTime(now), + } + if probeErr != nil { + status.Reason = "ProbeFailed" + return status + } + if previous != nil && previous.ObservedGeneration == health.Generation && + previous.RouteUID == health.Spec.RouteRef.UID && previous.RouteGeneration == health.Spec.RouteRef.Generation && + previous.TargetFingerprint == fingerprint && previous.ConsecutiveSuccesses > 0 && + (previous.Reason == "Warming" || previous.Reason == "Healthy") && + previous.ValidUntil.Time.Equal(previous.LastProbeTime.Add(ResultLifetime)) && + !previous.LastProbeTime.After(now) && now.Sub(previous.LastProbeTime.Time) < ConsecutiveSuccessWindow { + status.ConsecutiveSuccesses = previous.ConsecutiveSuccesses + } + if status.ConsecutiveSuccesses < HealthyAfterSuccesses { + status.ConsecutiveSuccesses++ + } + status.Healthy = status.ConsecutiveSuccesses >= HealthyAfterSuccesses + if status.Healthy { + status.Reason = "Healthy" + } else { + status.Reason = "Warming" + } + status.ValidUntil = metav1.NewTime(now.Add(ResultLifetime)) + return status +} + +func HealthyTargets(health *networkingv1alpha.NetworkLoadBalancerBackendHealth, now time.Time) []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget { + condition := apimeta.FindStatusCondition(health.Status.Conditions, "ProbesEvaluated") + if condition == nil || condition.Status != metav1.ConditionTrue || condition.ObservedGeneration != health.Generation { + return nil + } + byName := make(map[string]networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus, len(health.Status.Targets)) + for _, status := range health.Status.Targets { + byName[status.Name] = status + } + var eligible []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget + for _, target := range health.Spec.Targets { + status, found := byName[target.Name] + if !found || !status.Healthy || status.ObservedGeneration != health.Generation || + status.RouteUID != health.Spec.RouteRef.UID || status.RouteGeneration != health.Spec.RouteRef.Generation || + status.TargetFingerprint != TargetFingerprint(target) || status.LastProbeTime.IsZero() || + status.LastProbeTime.After(now) || !status.ValidUntil.After(now) || + status.ValidUntil.Sub(status.LastProbeTime.Time) > ResultLifetime { + continue + } + eligible = append(eligible, target) + } + return eligible +} diff --git a/internal/nlbhealth/health_test.go b/internal/nlbhealth/health_test.go new file mode 100644 index 00000000..51601866 --- /dev/null +++ b/internal/nlbhealth/health_test.go @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package nlbhealth + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "math/big" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +func healthFixture() *networkingv1alpha.NetworkLoadBalancerBackendHealth { + return &networkingv1alpha.NetworkLoadBalancerBackendHealth{ + Spec: networkingv1alpha.NetworkLoadBalancerBackendHealthSpec{ + RouteRef: networkingv1alpha.NetworkLoadBalancerBackendHealthRouteRef{UID: types.UID("route-1"), Generation: 2}, + Targets: []networkingv1alpha.NetworkLoadBalancerBackendHealthTarget{{ + Name: "origin", SourceUID: types.UID("interface-1"), AttachmentUID: types.UID("attachment-1"), + Address: "fd20::1", Probe: networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeTCP, Port: 8080}, + }}, + }, + } +} + +func TestHealthyTargetsRequiresFreshMatchingObservation(t *testing.T) { + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + health := healthFixture() + health.Generation = 4 + target := health.Spec.Targets[0] + first := NextTargetStatus(health, target, nil, nil, now) + require.False(t, first.Healthy) + second := NextTargetStatus(health, target, &first, nil, now.Add(ProbeInterval)) + require.True(t, second.Healthy) + health.Status.Targets = []networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{second} + require.Empty(t, HealthyTargets(health, now.Add(ProbeInterval))) + health.Status.Conditions = []metav1.Condition{{Type: "ProbesEvaluated", Status: metav1.ConditionTrue, Reason: "Evaluated", ObservedGeneration: health.Generation, LastTransitionTime: metav1.NewTime(now)}} + require.Len(t, HealthyTargets(health, now.Add(ProbeInterval)), 1) + + cases := []struct { + name string + change func(*networkingv1alpha.NetworkLoadBalancerBackendHealth) + at time.Time + }{ + {name: "expired", at: now.Add(ProbeInterval + ResultLifetime)}, + {name: "pending evaluation", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Status.Conditions[0].Status = metav1.ConditionUnknown + }}, + {name: "stale evaluation", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Status.Conditions[0].ObservedGeneration-- + }}, + {name: "new object generation", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { h.Generation++ }}, + {name: "new route UID", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { h.Spec.RouteRef.UID = "route-2" }}, + {name: "new route generation", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { h.Spec.RouteRef.Generation++ }}, + {name: "replacement interface", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Spec.Targets[0].SourceUID = "interface-2" + }}, + {name: "replacement attachment", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Spec.Targets[0].AttachmentUID = "attachment-2" + }}, + {name: "probe changed", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { h.Spec.Targets[0].Probe.Port++ }}, + {name: "future probe timestamp", change: func(h *networkingv1alpha.NetworkLoadBalancerBackendHealth) { + h.Status.Targets[0].LastProbeTime = h.Status.Targets[0].ValidUntil + }}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + copy := health.DeepCopy() + if tt.change != nil { + tt.change(copy) + } + at := now.Add(ProbeInterval) + if !tt.at.IsZero() { + at = tt.at + } + require.Empty(t, HealthyTargets(copy, at)) + }) + } + + failed := NextTargetStatus(health, target, &second, errors.New("connection refused"), now.Add(2*ProbeInterval)) + require.False(t, failed.Healthy) + require.Equal(t, int32(0), failed.ConsecutiveSuccesses) + health.Status.Targets = []networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{failed} + require.Empty(t, HealthyTargets(health, now.Add(2*ProbeInterval))) +} + +func TestSuccessContinuityAcrossDifferentProbeWavesDoesNotExtendServingFreshness(t *testing.T) { + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + health := healthFixture() + health.Generation = 4 + health.Status.Conditions = []metav1.Condition{{ + Type: "ProbesEvaluated", Status: metav1.ConditionTrue, Reason: "Evaluated", + ObservedGeneration: health.Generation, LastTransitionTime: metav1.NewTime(now), + }} + target := health.Spec.Targets[0] + warming := NextTargetStatus(health, target, nil, nil, now.Add(-ProbeInterval)) + first := NextTargetStatus(health, target, &warming, nil, now) + require.True(t, first.Healthy) + health.Status.Targets = []networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{first} + require.Len(t, HealthyTargets(health, now.Add(ResultLifetime-time.Nanosecond)), 1) + require.Empty(t, HealthyTargets(health, now.Add(ResultLifetime))) + + // A target may move from an early wave to a late wave of the next sweep. + // Its earlier observation remains unservable, but a fresh success can count. + late := now.Add(ResultLifetime + time.Second) + require.Empty(t, HealthyTargets(health, late)) + second := NextTargetStatus(health, target, &first, nil, late) + require.True(t, second.Healthy) + require.Equal(t, int32(HealthyAfterSuccesses), second.ConsecutiveSuccesses) + health.Status.Targets = []networkingv1alpha.NetworkLoadBalancerBackendHealthTargetStatus{second} + require.Len(t, HealthyTargets(health, late), 1) + + tooLate := NextTargetStatus(health, target, &first, nil, now.Add(ConsecutiveSuccessWindow)) + require.False(t, tooLate.Healthy) + require.Equal(t, int32(1), tooLate.ConsecutiveSuccesses) +} + +func TestProbeRequiresVPCDialerAndRejectsUnsafeTarget(t *testing.T) { + target := healthFixture().Spec.Targets[0] + require.ErrorContains(t, (Prober{}).Probe(context.Background(), "vpc-1", target), "VPC-aware") + prober := Prober{DialContext: func(context.Context, string, string, string) (net.Conn, error) { + a, b := net.Pipe() + if err := b.Close(); err != nil { + return nil, err + } + return a, nil + }} + require.NoError(t, prober.Probe(context.Background(), "vpc-1", target)) + target.Address = "::1" + require.ErrorContains(t, prober.Probe(context.Background(), "vpc-1", target), "routable IPv6") + target.Address = "192.0.2.1" + require.ErrorContains(t, prober.Probe(context.Background(), "vpc-1", target), "routable IPv6") +} + +func TestHTTPAndHTTPSProbeResults(t *testing.T) { + target := healthFixture().Spec.Targets[0] + target.Probe = networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeHTTP, Port: 8080, Path: "/health"} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" { + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + dial := func(ctx context.Context, vpc, network, _ string) (net.Conn, error) { + require.Equal(t, "vpc-1", vpc) + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + } + require.NoError(t, (Prober{DialContext: dial}).Probe(context.Background(), "vpc-1", target)) + target.Probe.Path = "/missing" + require.ErrorContains(t, (Prober{DialContext: dial}).Probe(context.Background(), "vpc-1", target), "404") + + tlsServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })) + defer tlsServer.Close() + cert := tlsServer.Certificate() + serverName := "" + if len(cert.DNSNames) > 0 { + serverName = cert.DNSNames[0] + } else { + serverName = cert.IPAddresses[0].String() + } + target.Probe = networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeHTTPS, Port: 8080, ServerName: serverName} + tlsDial := func(ctx context.Context, _, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, tlsServer.Listener.Addr().String()) + } + require.Error(t, (Prober{DialContext: tlsDial}).Probe(context.Background(), "vpc-1", target)) + roots := x509.NewCertPool() + roots.AddCert(cert) + require.NoError(t, (Prober{DialContext: tlsDial, RootCAs: roots}).Probe(context.Background(), "vpc-1", target)) +} + +func TestProbeHonorsCallerDeadline(t *testing.T) { + target := healthFixture().Spec.Targets[0] + prober := Prober{DialContext: func(ctx context.Context, _, _, _ string) (net.Conn, error) { + <-ctx.Done() + return nil, ctx.Err() + }} + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + require.ErrorIs(t, prober.Probe(ctx, "vpc-1", target), context.DeadlineExceeded) + require.Less(t, time.Since(start), time.Second) +} + +func TestHTTPSDefaultsToVerifiedIPSAN(t *testing.T) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + certificate := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "health"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("fd20::1")}, + } + der, err := x509.CreateCertificate(rand.Reader, certificate, certificate, &key.PublicKey, key) + require.NoError(t, err) + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })) + server.TLS = &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + server.StartTLS() + defer server.Close() + roots := x509.NewCertPool() + parsed, err := x509.ParseCertificate(der) + require.NoError(t, err) + roots.AddCert(parsed) + prober := Prober{ + RootCAs: roots, + DialContext: func(ctx context.Context, _, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, server.Listener.Addr().String()) + }, + } + target := healthFixture().Spec.Targets[0] + target.Probe = networkingv1alpha.NetworkLoadBalancerProbe{Protocol: networkingv1alpha.NetworkLoadBalancerProbeHTTPS, Port: 8080} + require.NoError(t, prober.Probe(context.Background(), "vpc-1", target)) + target.Address = "fd20::2" + require.Error(t, prober.Probe(context.Background(), "vpc-1", target)) +}