diff --git a/api/v1alpha/groupversion_info.go b/api/v1alpha/groupversion_info.go index c2634bfe..c443e4a7 100644 --- a/api/v1alpha/groupversion_info.go +++ b/api/v1alpha/groupversion_info.go @@ -46,6 +46,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &NetworkLoadBalancerBackendHealthList{}, &NetworkLoadBalancer{}, &NetworkLoadBalancerList{}, + &NetworkLoadBalancerCellIntent{}, + &NetworkLoadBalancerCellIntentList{}, &NetworkLoadBalancerRoute{}, &NetworkLoadBalancerRouteList{}, &NetworkPolicy{}, diff --git a/api/v1alpha/networkloadbalancercellintent_types.go b/api/v1alpha/networkloadbalancercellintent_types.go new file mode 100644 index 00000000..b74dfc89 --- /dev/null +++ b/api/v1alpha/networkloadbalancercellintent_types.go @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package v1alpha + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// NetworkLoadBalancerCellIntentState distinguishes desired backend validation +// from an explicit withdrawal. Neither state authorizes traffic programming. +// +kubebuilder:validation:Enum=Validate;Withdrawn +type NetworkLoadBalancerCellIntentState string + +const ( + NetworkLoadBalancerCellIntentValidate NetworkLoadBalancerCellIntentState = "Validate" + NetworkLoadBalancerCellIntentWithdrawn NetworkLoadBalancerCellIntentState = "Withdrawn" +) + +type NetworkLoadBalancerCellIntentSource struct { + // +kubebuilder:validation:MinLength=1 + ProjectCluster string `json:"projectCluster"` + // +kubebuilder:validation:MinLength=1 + Namespace string `json:"namespace"` + // +kubebuilder:validation:MinLength=1 + RouteName string `json:"routeName"` + // +kubebuilder:validation:MinLength=1 + RouteUID string `json:"routeUID"` + RouteGeneration int64 `json:"routeGeneration"` + // +kubebuilder:validation:MinLength=1 + ParentName string `json:"parentName"` + // +kubebuilder:validation:MinLength=1 + ParentUID string `json:"parentUID"` + ParentGeneration int64 `json:"parentGeneration"` + // +kubebuilder:validation:MinLength=1 + ListenerName string `json:"listenerName"` + // +kubebuilder:validation:MinLength=1 + AddressRequest string `json:"addressRequest"` +} + +type NetworkLoadBalancerCellIntentProbe struct { + Protocol string `json:"protocol"` + Port int32 `json:"port"` + Path string `json:"path,omitempty"` +} + +type NetworkLoadBalancerCellIntentBackend struct { + // +kubebuilder:validation:MinLength=1 + SourceNamespace string `json:"sourceNamespace"` + // +kubebuilder:validation:MinLength=1 + SourceName string `json:"sourceName"` + // +kubebuilder:validation:MinLength=1 + SourceUID string `json:"sourceUID"` + // Family identifies the backend transport address independently of the + // public VIP family. IPv4 ingress can use an IPv6 backend through a proxy. + Family IPFamily `json:"family"` + // +kubebuilder:validation:MinLength=1 + Address string `json:"address"` + Port int32 `json:"port"` + Probe NetworkLoadBalancerCellIntentProbe `json:"probe"` +} + +// NetworkLoadBalancerCellIntentSpec is a validation request to one location. +// The cell must independently verify the live interface, attachment, and VPC. +// +kubebuilder:validation:XValidation:message="Validate needs backends; Withdrawn must have none",rule="self.state == 'Validate' ? has(self.backends) && size(self.backends) > 0 : !has(self.backends) || size(self.backends) == 0" +type NetworkLoadBalancerCellIntentSpec struct { + Source NetworkLoadBalancerCellIntentSource `json:"source"` + Location string `json:"location"` + NetworkName string `json:"networkName"` + // VIPFamily selects the ingress packet path. It may differ from a backend's + // family; the serving cell must support that family pair before programming. + VIPFamily IPFamily `json:"vipFamily"` + VIP string `json:"vip"` + Protocol NetworkLoadBalancerProtocol `json:"protocol"` + ListenerPort int32 `json:"listenerPort"` + State NetworkLoadBalancerCellIntentState `json:"state"` + // +kubebuilder:validation:MaxItems=1024 + Backends []NetworkLoadBalancerCellIntentBackend `json:"backends,omitempty"` +} + +// +kubebuilder:object:root=true +// NetworkLoadBalancerCellIntent carries a scoped, non-serving desired state +// from a project control plane through the hub to the serving cell. +type NetworkLoadBalancerCellIntent struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NetworkLoadBalancerCellIntentSpec `json:"spec"` +} + +// +kubebuilder:object:root=true + +type NetworkLoadBalancerCellIntentList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NetworkLoadBalancerCellIntent `json:"items"` +} diff --git a/api/v1alpha/zz_generated.deepcopy.go b/api/v1alpha/zz_generated.deepcopy.go index a9a26d0d..05fa91b1 100644 --- a/api/v1alpha/zz_generated.deepcopy.go +++ b/api/v1alpha/zz_generated.deepcopy.go @@ -2141,6 +2141,131 @@ func (in *NetworkLoadBalancerBackendTransport) DeepCopy() *NetworkLoadBalancerBa return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntent) DeepCopyInto(out *NetworkLoadBalancerCellIntent) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntent. +func (in *NetworkLoadBalancerCellIntent) DeepCopy() *NetworkLoadBalancerCellIntent { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkLoadBalancerCellIntent) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentBackend) DeepCopyInto(out *NetworkLoadBalancerCellIntentBackend) { + *out = *in + out.Probe = in.Probe +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentBackend. +func (in *NetworkLoadBalancerCellIntentBackend) DeepCopy() *NetworkLoadBalancerCellIntentBackend { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentBackend) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentList) DeepCopyInto(out *NetworkLoadBalancerCellIntentList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NetworkLoadBalancerCellIntent, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentList. +func (in *NetworkLoadBalancerCellIntentList) DeepCopy() *NetworkLoadBalancerCellIntentList { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NetworkLoadBalancerCellIntentList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentProbe) DeepCopyInto(out *NetworkLoadBalancerCellIntentProbe) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentProbe. +func (in *NetworkLoadBalancerCellIntentProbe) DeepCopy() *NetworkLoadBalancerCellIntentProbe { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentProbe) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentSource) DeepCopyInto(out *NetworkLoadBalancerCellIntentSource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentSource. +func (in *NetworkLoadBalancerCellIntentSource) DeepCopy() *NetworkLoadBalancerCellIntentSource { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkLoadBalancerCellIntentSpec) DeepCopyInto(out *NetworkLoadBalancerCellIntentSpec) { + *out = *in + out.Source = in.Source + if in.Backends != nil { + in, out := &in.Backends, &out.Backends + *out = make([]NetworkLoadBalancerCellIntentBackend, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkLoadBalancerCellIntentSpec. +func (in *NetworkLoadBalancerCellIntentSpec) DeepCopy() *NetworkLoadBalancerCellIntentSpec { + if in == nil { + return nil + } + out := new(NetworkLoadBalancerCellIntentSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkLoadBalancerEndpointStatus) DeepCopyInto(out *NetworkLoadBalancerEndpointStatus) { *out = *in diff --git a/config/crd/bases/networking.datumapis.com_networkloadbalancercellintents.yaml b/config/crd/bases/networking.datumapis.com_networkloadbalancercellintents.yaml new file mode 100644 index 00000000..6f200c27 --- /dev/null +++ b/config/crd/bases/networking.datumapis.com_networkloadbalancercellintents.yaml @@ -0,0 +1,191 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: networkloadbalancercellintents.networking.datumapis.com +spec: + group: networking.datumapis.com + names: + kind: NetworkLoadBalancerCellIntent + listKind: NetworkLoadBalancerCellIntentList + plural: networkloadbalancercellintents + singular: networkloadbalancercellintent + scope: Namespaced + versions: + - name: v1alpha + schema: + openAPIV3Schema: + description: |- + NetworkLoadBalancerCellIntent carries a scoped, non-serving desired state + from a project control plane through the hub to the serving cell. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NetworkLoadBalancerCellIntentSpec is a validation request to one location. + The cell must independently verify the live interface, attachment, and VPC. + properties: + backends: + items: + properties: + address: + minLength: 1 + type: string + family: + description: |- + Family identifies the backend transport address independently of the + public VIP family. IPv4 ingress can use an IPv6 backend through a proxy. + enum: + - IPv4 + - IPv6 + type: string + port: + format: int32 + type: integer + probe: + properties: + path: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + sourceName: + minLength: 1 + type: string + sourceNamespace: + minLength: 1 + type: string + sourceUID: + minLength: 1 + type: string + required: + - address + - family + - port + - probe + - sourceName + - sourceNamespace + - sourceUID + type: object + maxItems: 1024 + type: array + listenerPort: + format: int32 + type: integer + location: + type: string + networkName: + type: string + protocol: + description: NetworkLoadBalancerProtocol is the transport accepted + by a listener. + enum: + - TCP + - UDP + - TLS + type: string + source: + properties: + addressRequest: + minLength: 1 + type: string + listenerName: + minLength: 1 + type: string + namespace: + minLength: 1 + type: string + parentGeneration: + format: int64 + type: integer + parentName: + minLength: 1 + type: string + parentUID: + minLength: 1 + type: string + projectCluster: + minLength: 1 + type: string + routeGeneration: + format: int64 + type: integer + routeName: + minLength: 1 + type: string + routeUID: + minLength: 1 + type: string + required: + - addressRequest + - listenerName + - namespace + - parentGeneration + - parentName + - parentUID + - projectCluster + - routeGeneration + - routeName + - routeUID + type: object + state: + description: |- + NetworkLoadBalancerCellIntentState distinguishes desired backend validation + from an explicit withdrawal. Neither state authorizes traffic programming. + enum: + - Validate + - Withdrawn + type: string + vip: + type: string + vipFamily: + description: |- + VIPFamily selects the ingress packet path. It may differ from a backend's + family; the serving cell must support that family pair before programming. + enum: + - IPv4 + - IPv6 + type: string + required: + - listenerPort + - location + - networkName + - protocol + - source + - state + - vip + - vipFamily + type: object + x-kubernetes-validations: + - message: Validate needs backends; Withdrawn must have none + rule: 'self.state == ''Validate'' ? has(self.backends) && size(self.backends) + > 0 : !has(self.backends) || size(self.backends) == 0' + required: + - spec + type: object + served: true + storage: true diff --git a/config/crd/downstream/kustomization.yaml b/config/crd/downstream/kustomization.yaml index 123da524..096c7072 100644 --- a/config/crd/downstream/kustomization.yaml +++ b/config/crd/downstream/kustomization.yaml @@ -47,4 +47,5 @@ resources: - ../bases/networking.datumapis.com_networkinterfaceclaims.yaml - ../bases/networking.datumapis.com_networkinterfaces.yaml - ../bases/networking.datumapis.com_networkloadbalancerbackendhealths.yaml + - ../bases/networking.datumapis.com_networkloadbalancercellintents.yaml - ../bases/networking.datumapis.com_subnets.yaml diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index cedb4a3a..ccbd42bf 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -19,6 +19,7 @@ resources: - bases/networking.datumapis.com_networkinterfaceclaims.yaml - bases/networking.datumapis.com_networkservices.yaml - bases/networking.datumapis.com_networkloadbalancers.yaml +- bases/networking.datumapis.com_networkloadbalancercellintents.yaml - bases/networking.datumapis.com_networkloadbalancerroutes.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/config/federation/clusterpropagationpolicy.yaml b/config/federation/clusterpropagationpolicy.yaml index 7cbefd9d..dd88a7c8 100644 --- a/config/federation/clusterpropagationpolicy.yaml +++ b/config/federation/clusterpropagationpolicy.yaml @@ -154,3 +154,9 @@ spec: matchExpressions: - key: meta.datumapis.com/upstream-cluster-name operator: Exists + - apiVersion: networking.datumapis.com/v1alpha + kind: NetworkLoadBalancerCellIntent + labelSelector: + matchExpressions: + - key: meta.datumapis.com/upstream-cluster-name + operator: Exists diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b3dee038..4c9c511f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -343,6 +343,7 @@ rules: - networkcontexts/finalizers - networkinterfaceclaims/finalizers - networkinterfaces/finalizers + - networkloadbalancerroutes/finalizers - networkloadbalancers/finalizers - networkpolicies/finalizers - networks/finalizers @@ -380,7 +381,6 @@ rules: resources: - connectorclasses - networkloadbalancerbackendhealths - - networkloadbalancerroutes - networkservices verbs: - get @@ -389,6 +389,18 @@ rules: - apiGroups: - networking.datumapis.com resources: + - networkloadbalancercellintents + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.datumapis.com + resources: + - networkloadbalancerroutes - networkloadbalancers verbs: - get diff --git a/docs/api/networkloadbalancercellintents.md b/docs/api/networkloadbalancercellintents.md new file mode 100644 index 00000000..30b3b29f --- /dev/null +++ b/docs/api/networkloadbalancercellintents.md @@ -0,0 +1,369 @@ +# API Reference + +Packages: + +- [networking.datumapis.com/v1alpha](#networkingdatumapiscomv1alpha) + +# networking.datumapis.com/v1alpha + +Resource Types: + +- [NetworkLoadBalancerCellIntent](#networkloadbalancercellintent) + + + + +## NetworkLoadBalancerCellIntent +[↩ Parent](#networkingdatumapiscomv1alpha ) + + + + + + +NetworkLoadBalancerCellIntent carries a scoped, non-serving desired state +from a project control plane through the hub to the serving cell. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
apiVersionstringnetworking.datumapis.com/v1alphatrue
kindstringNetworkLoadBalancerCellIntenttrue
metadataobjectRefer to the Kubernetes API documentation for the fields of the `metadata` field.true
specobject + NetworkLoadBalancerCellIntentSpec is a validation request to one location. +The cell must independently verify the live interface, attachment, and VPC.
+
+ Validations:
  • self.state == 'Validate' ? has(self.backends) && size(self.backends) > 0 : !has(self.backends) || size(self.backends) == 0: Validate needs backends; Withdrawn must have none
  • +
    true
    + + +### NetworkLoadBalancerCellIntent.spec +[↩ Parent](#networkloadbalancercellintent) + + + +NetworkLoadBalancerCellIntentSpec is a validation request to one location. +The cell must independently verify the live interface, attachment, and VPC. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    listenerPortinteger +
    +
    + Format: int32
    +
    true
    locationstring +
    +
    true
    networkNamestring +
    +
    true
    protocolenum + NetworkLoadBalancerProtocol is the transport accepted by a listener.
    +
    + Enum: TCP, UDP, TLS
    +
    true
    sourceobject +
    +
    true
    stateenum + NetworkLoadBalancerCellIntentState distinguishes desired backend validation +from an explicit withdrawal. Neither state authorizes traffic programming.
    +
    + Enum: Validate, Withdrawn
    +
    true
    vipstring +
    +
    true
    vipFamilyenum + VIPFamily selects the ingress packet path. It may differ from a backend's +family; the serving cell must support that family pair before programming.
    +
    + Enum: IPv4, IPv6
    +
    true
    backends[]object +
    +
    false
    + + +### NetworkLoadBalancerCellIntent.spec.source +[↩ Parent](#networkloadbalancercellintentspec) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressRequeststring +
    +
    true
    listenerNamestring +
    +
    true
    namespacestring +
    +
    true
    parentGenerationinteger +
    +
    + Format: int64
    +
    true
    parentNamestring +
    +
    true
    parentUIDstring +
    +
    true
    projectClusterstring +
    +
    true
    routeGenerationinteger +
    +
    + Format: int64
    +
    true
    routeNamestring +
    +
    true
    routeUIDstring +
    +
    true
    + + +### NetworkLoadBalancerCellIntent.spec.backends[index] +[↩ Parent](#networkloadbalancercellintentspec) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    addressstring +
    +
    true
    familyenum + Family identifies the backend transport address independently of the +public VIP family. IPv4 ingress can use an IPv6 backend through a proxy.
    +
    + Enum: IPv4, IPv6
    +
    true
    portinteger +
    +
    + Format: int32
    +
    true
    probeobject +
    +
    true
    sourceNamestring +
    +
    true
    sourceNamespacestring +
    +
    true
    sourceUIDstring +
    +
    true
    + + +### NetworkLoadBalancerCellIntent.spec.backends[index].probe +[↩ Parent](#networkloadbalancercellintentspecbackendsindex) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionRequired
    portinteger +
    +
    + Format: int32
    +
    true
    protocolstring +
    +
    true
    pathstring +
    +
    false
    diff --git a/docs/plans/network-load-balancer-cell-contract.md b/docs/plans/network-load-balancer-cell-contract.md index 500e4f86..ec97244c 100644 --- a/docs/plans/network-load-balancer-cell-contract.md +++ b/docs/plans/network-load-balancer-cell-contract.md @@ -21,12 +21,22 @@ namespace. The source interface reports the attachment in reads attachment addresses, `status.vpc`, and `status.node`. Matching an address alone is unsafe because different VPCs can use the same address. -## Proposed intent +## Published intent -The project route reconciler should publish a **versioned, location-scoped -intent** through the hub to the target cell. This resource does not exist yet. -The owner reference must identify the route by UID and generation so a deleted -or replaced route cannot keep a VIP active. Its minimum schema is: +The project reconciler writes a `NetworkLoadBalancerCellIntent` into the mapped +hub namespace. The federation policy carries its spec to gateway-enabled +cells. Only the cell whose configured location equals `spec.location` may +process it. The route's UID and generation and the load balancer's UID and +generation identify the source objects; cross-cluster owner references cannot +provide that guarantee. + +An intent is published only for the oldest route on an existing listener with +resolved backend references, a ready network, and an allocated IPv6 address +whose `Allocated=True` condition observed the current load balancer +generation. Every published address names one location; an empty location or +an address shared by two allocation entries is excluded. +The source interface namespace and UID come from the projected interface; +the cell must verify them against live state. The published shape is: ```yaml apiVersion: networking.datumapis.com/v1alpha @@ -39,22 +49,36 @@ spec: routeUID: routeGeneration: parentName: + parentUID: + parentGeneration: listenerName: + addressRequest:
    location: networkName: vip: protocol: TCP # or UDP listenerPort: 443 + state: Validate backends: - sourceNamespace: sourceName: sourceUID: address: port: 8443 - weight: 1 - healthCheck: + probe: + protocol: TCP + port: 8443 ``` +`Validate` requests identity and health validation only. It never authorizes +`NetworkRule` creation, `Programmed=True`, or VIP announcement. Loss of +eligibility or removal of a route changes the old intent to `Withdrawn` with +an empty backend list. The route holds a finalizer until that withdrawal is +written. Withdrawn objects remain on the hub so a later cell acknowledgment +can prove the old VIP is no longer announced before they are deleted. A new +VIP, listener port, parent UID, or route UID gets a distinct intent name, so +the previous identity is explicitly withdrawn. + Only a platform controller may write this intent. The cell must treat the projected source tuple as a lookup hint, not as proof. For every backend it must read the cell `NetworkInterface` and require the UID, network, bound phase, @@ -70,7 +94,9 @@ fabric ID matches both statuses. A source UID change invalidates old intent even when the name and IP are reused. The cell should report resolved attachment identities and probe results per -backend, with the intent generation it observed. A project controller may set +backend, with the intent generation it observed. Cell status is not propagated +back by Karmada, so a dedicated cell-to-hub acknowledgment is still needed. +A project controller may set route `Programmed=True` only after the cell has acknowledged that generation and every announcing gateway node confirms the same healthy backend set. VIP announcement follows that acknowledgment; an empty healthy set withdraws the @@ -78,10 +104,10 @@ VIP but retains the allocated address. ## Galactic API dependencies -The current `NetworkRule` requires a nonempty `backendSelector`. Cloud's +The deployed `NetworkRule` requires a nonempty `backendSelector`. Cloud's `VPCAttachment` writer does not provide a trusted label that selects all and only this route's attachments. `backendAddresses` narrows a selector, so it -cannot be used alone today. The Network API should permit an omitted selector +cannot be used alone in the deployed API. The Network API follow-up permits an omitted selector **only when `backendAddresses` is explicitly set**. Galactic should then select attachments by authoritative `status.vpc == spec.vpcRef` and intersect their IPv6 addresses with `backendAddresses`. An explicit empty list means no diff --git a/internal/agent/catalog_test.go b/internal/agent/catalog_test.go index 2f3c8930..714136e4 100644 --- a/internal/agent/catalog_test.go +++ b/internal/agent/catalog_test.go @@ -43,6 +43,7 @@ var outOfScopeAPIFiles = map[string]string{ "networkinterfaceclaim_types.go": "an instance's interface; compute's surface", "networkloadbalancer_types.go": "its own Layer 4 product surface, not reached through an application load balancer", "networkloadbalancerbackendhealth_types.go": "cell-local probe state, not an application load balancer customer surface", + "networkloadbalancercellintent_types.go": "internal cell validation request; no customer-facing condition reasons", "networkloadbalancerroute_types.go": "its own Layer 4 routing surface, not reached through an application load balancer", "networkpolicy_types.go": "its own product surface, not reached through a load balancer", "subnet_types.go": "address management below a network", diff --git a/internal/cmd/manager/manager.go b/internal/cmd/manager/manager.go index 09de377b..baa9371d 100644 --- a/internal/cmd/manager/manager.go +++ b/internal/cmd/manager/manager.go @@ -812,6 +812,11 @@ func controllerRegistrations( {"networkloadbalancerroute", true, func() error { return (&controller.NetworkLoadBalancerRouteReconciler{}).SetupWithManager(mgr) }}, + {"networkloadbalancercellintent", true, func() error { + return (&controller.NetworkLoadBalancerCellIntentReconciler{ + HubCluster: deps.downstreamCluster, + }).SetupWithManager(mgr) + }}, // Which workloads are behind a proxy is only answerable in a project // control plane, and only a cell can act on it. The record this writes to // the hub is what carries the answer between them. diff --git a/internal/cmd/manager/manager_test.go b/internal/cmd/manager/manager_test.go index d97e4fe3..6f3185b5 100644 --- a/internal/cmd/manager/manager_test.go +++ b/internal/cmd/manager/manager_test.go @@ -72,6 +72,7 @@ var reconcilerControllerNames = map[string]string{ "NetworkServiceReconciler": "networkservice", "NetworkLoadBalancerRouteReconciler": "networkloadbalancerroute", "NetworkLoadBalancerBackendHealthReconciler": "networkloadbalancerbackendhealth", + "NetworkLoadBalancerCellIntentReconciler": "networkloadbalancercellintent", "SubnetClaimReconciler": "subnetclaim", "SubnetReconciler": "subnet", "TrafficProtectionPolicyReconciler": "trafficprotectionpolicy", diff --git a/internal/controller/networkloadbalancercellintent_controller.go b/internal/controller/networkloadbalancercellintent_controller.go new file mode 100644 index 00000000..e345dac7 --- /dev/null +++ b/internal/controller/networkloadbalancercellintent_controller.go @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/netip" + "slices" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" + mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" + "sigs.k8s.io/multicluster-runtime/pkg/multicluster" + mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + "go.datum.net/network-services-operator/internal/downstreamclient" +) + +const ( + nlbIntentFinalizer = "networking.datumapis.com/nlb-cell-intent-withdrawal" + nlbIntentRouteUIDLabel = "networking.datumapis.com/route-uid" + nlbIntentRetry = 30 * time.Second +) + +type NetworkLoadBalancerCellIntentReconciler struct { + HubCluster cluster.Cluster + mgr mcmanager.Manager +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancerroutes,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancerroutes/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancers;networkservices;networkinterfaces;networks,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkloadbalancercellintents,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch + +func (r *NetworkLoadBalancerCellIntentReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (ctrl.Result, error) { + cl, err := r.mgr.GetCluster(ctx, req.ClusterName) + if err != nil { + return ctrl.Result{}, err + } + return r.reconcileIntent(ctx, string(req.ClusterName), cl.GetClient(), req.NamespacedName) +} + +func (r *NetworkLoadBalancerCellIntentReconciler) reconcileIntent(ctx context.Context, project string, projectClient client.Client, key client.ObjectKey) (ctrl.Result, error) { + var route networkingv1alpha.NetworkLoadBalancerRoute + if err := projectClient.Get(ctx, key, &route); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if route.UID == "" { + return ctrl.Result{}, fmt.Errorf("route %s has no UID", key) + } + + if route.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(&route, nlbIntentFinalizer) { + if err := projectClient.Update(ctx, &route); err != nil { + return ctrl.Result{}, fmt.Errorf("hold route until cell intent withdrawal: %w", err) + } + } + if !route.DeletionTimestamp.IsZero() && !controllerutil.ContainsFinalizer(&route, nlbIntentFinalizer) { + return ctrl.Result{}, nil + } + + strategy := downstreamclient.NewMappedNamespaceResourceStrategy(project, projectClient, r.HubCluster.GetClient()) + hubNamespace, err := strategy.GetDownstreamNamespaceNameForUpstreamNamespace(ctx, route.Namespace) + if apierrors.IsNotFound(err) { + if !route.DeletionTimestamp.IsZero() { + controllerutil.RemoveFinalizer(&route, nlbIntentFinalizer) + return ctrl.Result{}, projectClient.Update(ctx, &route) + } + return ctrl.Result{RequeueAfter: nlbIntentRetry}, nil + } + if err != nil { + return ctrl.Result{}, fmt.Errorf("resolve hub namespace: %w", err) + } + + var desired []networkingv1alpha.NetworkLoadBalancerCellIntent + if route.DeletionTimestamp.IsZero() { + desired, err = buildNetworkLoadBalancerCellIntents(ctx, projectClient, project, hubNamespace, &route) + if err != nil { + return ctrl.Result{}, err + } + } + if err := reconcileHubCellIntents(ctx, r.HubCluster.GetClient(), hubNamespace, project, &route, desired); err != nil { + return ctrl.Result{}, err + } + if !route.DeletionTimestamp.IsZero() { + controllerutil.RemoveFinalizer(&route, nlbIntentFinalizer) + if err := projectClient.Update(ctx, &route); err != nil { + return ctrl.Result{}, fmt.Errorf("release withdrawn route: %w", err) + } + } + return ctrl.Result{}, nil +} + +func buildNetworkLoadBalancerCellIntents(ctx context.Context, cl client.Client, project, hubNamespace string, route *networkingv1alpha.NetworkLoadBalancerRoute) ([]networkingv1alpha.NetworkLoadBalancerCellIntent, error) { + var peers networkingv1alpha.NetworkLoadBalancerRouteList + if err := cl.List(ctx, &peers, client.InNamespace(route.Namespace)); err != nil { + return nil, fmt.Errorf("list route owners: %w", err) + } + var desired []networkingv1alpha.NetworkLoadBalancerCellIntent + claims := newBackendClaimCache() + for _, ref := range route.Spec.ParentRefs { + if ref.Namespace != "" && ref.Namespace != route.Namespace { + continue + } + var parent networkingv1alpha.NetworkLoadBalancer + if err := cl.Get(ctx, client.ObjectKey{Namespace: route.Namespace, Name: ref.Name}, &parent); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, err + } + if !parent.DeletionTimestamp.IsZero() || parent.UID == "" { + continue + } + listenerName, ok := ResolveNetworkLoadBalancerRouteListener(&parent, ref) + if !ok { + continue + } + winner := oldestRouteForListener(peers.Items, &parent, listenerName) + if winner == nil || winner.UID != route.UID { + continue + } + listenerIndex := slices.IndexFunc(parent.Spec.Listeners, func(l networkingv1alpha.NetworkLoadBalancerListener) bool { return l.Name == listenerName }) + if listenerIndex < 0 { + continue + } + listener := &parent.Spec.Listeners[listenerIndex] + if unsupportedRouteListenerCapability(route, &parent, listener) != "" || len(listener.AddressRefs) != 1 { + continue + } + var network networkingv1alpha.Network + if err := cl.Get(ctx, client.ObjectKey{Namespace: route.Namespace, Name: parent.Spec.NetworkRef.Name}, &network); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, err + } + ready := apimeta.FindStatusCondition(network.Status.Conditions, networkingv1alpha.NetworkReady) + if !network.DeletionTimestamp.IsZero() || ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != network.Generation { + continue + } + endpoints, err := resolveNetworkLoadBalancerRouteBackends(ctx, cl, route, &parent, listener) + var refError *networkLoadBalancerRouteReferenceError + if errors.As(err, &refError) { + continue + } + if err != nil { + return nil, err + } + owner, err := olderRouteClaimingBackend(ctx, cl, route, &parent, listener, endpoints, peers.Items, claims) + if err != nil { + return nil, err + } + if owner != "" { + continue + } + sharedVIPs := map[string]bool{} + seenVIPs := map[string]bool{} + for _, endpoint := range parent.Status.Endpoints { + for _, allocation := range endpoint.Addresses { + vip, parseErr := netip.ParseAddr(allocation.Value) + if parseErr != nil { + continue + } + address := vip.String() + if seenVIPs[address] { + sharedVIPs[address] = true + } + seenVIPs[address] = true + } + } + for _, addressRef := range listener.AddressRefs { + requestIndex := slices.IndexFunc(parent.Spec.Exposure.Addresses, func(a networkingv1alpha.NetworkLoadBalancerAddressRequest) bool { return a.Name == addressRef }) + if requestIndex < 0 { + continue + } + request := parent.Spec.Exposure.Addresses[requestIndex] + for _, endpointStatus := range parent.Status.Endpoints { + if endpointStatus.RequestName != addressRef { + continue + } + for _, allocation := range endpointStatus.Addresses { + // The public ingress path currently supports IPv6 only. + if !slices.Contains(request.Families, allocation.Family) || + allocation.Family != networkingv1alpha.IPv6Protocol || allocation.Value == "" || + allocation.ClaimRef == nil || allocation.ClaimRef.Name == "" { + continue + } + allocated := apimeta.FindStatusCondition(allocation.Conditions, networkingv1alpha.NetworkLoadBalancerAllocated) + if allocated == nil || allocated.Status != metav1.ConditionTrue || allocated.ObservedGeneration != parent.Generation { + continue + } + vip, parseErr := netip.ParseAddr(allocation.Value) + if parseErr != nil || !vip.Is6() || vip.Is4In6() || vip.Zone() != "" || !vip.IsGlobalUnicast() || + endpointStatus.Location == "" || sharedVIPs[vip.String()] { + continue + } + location := endpointStatus.Location + if request.Locations != nil && request.Locations.From == networkingv1alpha.NetworkLoadBalancerLocationSourceNames && !slices.Contains(request.Locations.Names, location) { + continue + } + intent := networkingv1alpha.NetworkLoadBalancerCellIntent{ObjectMeta: metav1.ObjectMeta{ + Namespace: hubNamespace, + Labels: map[string]string{ + downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-" + strings.ReplaceAll(project, "/", "_"), + downstreamclient.UpstreamOwnerNamespaceLabel: route.Namespace, + nlbIntentRouteUIDLabel: string(route.UID), + networkingv1alpha.LocationLabel: location, + }, + }, Spec: networkingv1alpha.NetworkLoadBalancerCellIntentSpec{ + Source: networkingv1alpha.NetworkLoadBalancerCellIntentSource{ + ProjectCluster: project, Namespace: route.Namespace, RouteName: route.Name, + RouteUID: string(route.UID), RouteGeneration: route.Generation, + ParentName: parent.Name, ParentUID: string(parent.UID), ParentGeneration: parent.Generation, + ListenerName: listener.Name, AddressRequest: addressRef, + }, + Location: location, NetworkName: parent.Spec.NetworkRef.Name, + VIPFamily: allocation.Family, VIP: vip.String(), Protocol: listener.Protocol, ListenerPort: listener.Port, + State: networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, + }} + for _, endpoint := range endpoints { + if endpoint.Location != location { + continue + } + intent.Spec.Backends = append(intent.Spec.Backends, networkingv1alpha.NetworkLoadBalancerCellIntentBackend{ + SourceNamespace: endpoint.SourceNamespace, SourceName: endpoint.Interface, + SourceUID: endpoint.SourceUID, Family: networkingv1alpha.IPv6Protocol, + Address: endpoint.Address, Port: endpoint.Port, + Probe: endpoint.Probe, + }) + } + // Galactic currently accepts at most 64 explicit backend addresses. + // Keep the consumer and hub APIs open for larger endpoint sets, + // but do not mark such an intent ready for this cell adapter. + if len(intent.Spec.Backends) > 64 { + intent.Spec.Backends = nil + } + if len(intent.Spec.Backends) != 0 { + intent.Spec.State = networkingv1alpha.NetworkLoadBalancerCellIntentValidate + } + intent.Name = networkLoadBalancerCellIntentName(intent.Spec) + desired = append(desired, intent) + } + } + } + } + withdrawConflictingCellIntents(desired) + return desired, nil +} + +// Status can temporarily contain two distinct VIP allocations for one +// address request during replacement. Withdraw all conflicting intents +// until Galactic can reuse backends across VIPs. +func withdrawConflictingCellIntents(desired []networkingv1alpha.NetworkLoadBalancerCellIntent) { + claims := make(map[string]int) + for _, intent := range desired { + if intent.Spec.State != networkingv1alpha.NetworkLoadBalancerCellIntentValidate { + continue + } + for _, backend := range intent.Spec.Backends { + claims[cellIntentBackendClaimKey(&intent, &backend)]++ + } + } + for i := range desired { + intent := &desired[i] + if intent.Spec.State != networkingv1alpha.NetworkLoadBalancerCellIntentValidate { + continue + } + if slices.ContainsFunc(intent.Spec.Backends, func(backend networkingv1alpha.NetworkLoadBalancerCellIntentBackend) bool { + return claims[cellIntentBackendClaimKey(intent, &backend)] > 1 + }) { + intent.Spec.State = networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn + intent.Spec.Backends = nil + } + } +} + +func cellIntentBackendClaimKey(intent *networkingv1alpha.NetworkLoadBalancerCellIntent, + backend *networkingv1alpha.NetworkLoadBalancerCellIntentBackend, +) string { + return fmt.Sprintf("%s/%s/%s/%s/%d", intent.Spec.NetworkName, intent.Spec.Location, + intent.Spec.Protocol, backend.Address, backend.Port) +} + +func networkLoadBalancerCellIntentName(spec networkingv1alpha.NetworkLoadBalancerCellIntentSpec) string { + identity := fmt.Sprintf("%s/%s/%s/%s/%s/%s/%s/%s/%d", spec.Source.ProjectCluster, spec.Source.Namespace, + spec.Source.RouteUID, spec.Source.ParentUID, spec.Source.ListenerName, spec.Source.AddressRequest, + spec.Location, spec.VIP, spec.ListenerPort) + identity += "/" + string(spec.Protocol) + sum := sha256.Sum256([]byte(identity)) + return "nlb-" + hex.EncodeToString(sum[:16]) +} + +func reconcileHubCellIntents(ctx context.Context, hub client.Client, namespace, project string, route *networkingv1alpha.NetworkLoadBalancerRoute, desired []networkingv1alpha.NetworkLoadBalancerCellIntent) error { + labels := client.MatchingLabels{ + downstreamclient.UpstreamOwnerClusterNameLabel: "cluster-" + strings.ReplaceAll(project, "/", "_"), + downstreamclient.UpstreamOwnerNamespaceLabel: route.Namespace, + nlbIntentRouteUIDLabel: string(route.UID), + } + var existing networkingv1alpha.NetworkLoadBalancerCellIntentList + if err := hub.List(ctx, &existing, client.InNamespace(namespace), labels); err != nil { + return fmt.Errorf("list published cell intents: %w", err) + } + wanted := make(map[string]networkingv1alpha.NetworkLoadBalancerCellIntent, len(desired)) + for _, intent := range desired { + wanted[intent.Name] = intent + } + for i := range existing.Items { + intent := &existing.Items[i] + if _, ok := wanted[intent.Name]; ok { + continue + } + if intent.Spec.State == networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn && len(intent.Spec.Backends) == 0 { + continue + } + intent.Spec.State = networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn + intent.Spec.Backends = nil + if err := hub.Update(ctx, intent); err != nil { + return fmt.Errorf("withdraw old cell intent %s: %w", intent.Name, err) + } + } + for _, wantedIntent := range desired { + intent := &networkingv1alpha.NetworkLoadBalancerCellIntent{ObjectMeta: metav1.ObjectMeta{Name: wantedIntent.Name, Namespace: namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, hub, intent, func() error { + if intent.CreationTimestamp.IsZero() && intent.UID == "" || intent.Spec.Source.RouteUID == wantedIntent.Spec.Source.RouteUID { + intent.Labels = wantedIntent.Labels + intent.Spec = wantedIntent.Spec + return nil + } + return fmt.Errorf("cell intent name %s belongs to another route", intent.Name) + }) + if err != nil { + return fmt.Errorf("publish cell intent %s: %w", wantedIntent.Name, err) + } + } + return nil +} + +func (r *NetworkLoadBalancerCellIntentReconciler) SetupWithManager(mgr mcmanager.Manager) error { + if r.HubCluster == nil { + return errors.New("a hub cluster is required") + } + r.mgr = mgr + watch := func(mapper func(context.Context, client.Client, client.Object) []ctrl.Request) func(multicluster.ClusterName, cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return func(clusterName multicluster.ClusterName, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []mcreconcile.Request { + requests := mapper(ctx, cl.GetClient(), obj) + out := make([]mcreconcile.Request, 0, len(requests)) + for _, req := range requests { + out = append(out, mcreconcile.Request{ClusterName: clusterName, Request: req}) + } + return out + }) + } + } + return mcbuilder.ControllerManagedBy(mgr). + For(&networkingv1alpha.NetworkLoadBalancerRoute{}, mcbuilder.WithEngageWithLocalCluster(false)). + Watches(&networkingv1alpha.NetworkLoadBalancerRoute{}, watch(siblingRoutesForRouteEvent)). + Watches(&networkingv1alpha.NetworkLoadBalancer{}, watch(routesForLoadBalancerEvent)). + Watches(&networkingv1alpha.NetworkService{}, watch(routesForNetworkServiceEvent)). + Watches(&networkingv1alpha.NetworkInterface{}, watch(routesForInterfaceEvent)). + Watches(&networkingv1alpha.Network{}, watch(routesForInterfaceEvent)). + Named("networkloadbalancercellintent"). + Complete(r) +} diff --git a/internal/controller/networkloadbalancercellintent_controller_test.go b/internal/controller/networkloadbalancercellintent_controller_test.go new file mode 100644 index 00000000..39913ba5 --- /dev/null +++ b/internal/controller/networkloadbalancercellintent_controller_test.go @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +func allocatedCellIntentAddress(value string, generation int64) networkingv1alpha.NetworkLoadBalancerAddressStatus { + return networkingv1alpha.NetworkLoadBalancerAddressStatus{ + Family: networkingv1alpha.IPv6Protocol, Value: value, + ClaimRef: &networkingv1alpha.NetworkLoadBalancerIPClaimRef{Project: "project", Namespace: "test", Name: "vip-claim"}, + Conditions: []metav1.Condition{{Type: networkingv1alpha.NetworkLoadBalancerAllocated, + Status: metav1.ConditionTrue, Reason: "Allocated", ObservedGeneration: generation, + LastTransitionTime: metav1.Now()}}, + } +} + +func allocatedCellIntentEndpoint(value string, generation int64) networkingv1alpha.NetworkLoadBalancerEndpointStatus { + return networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + RequestName: "regional", Location: "dfw", + Addresses: []networkingv1alpha.NetworkLoadBalancerAddressStatus{allocatedCellIntentAddress(value, generation)}, + } +} + +func TestCellIntentPublishesAndWithdrawsWithoutDeletingHold(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + route.UID = types.UID("route-1") + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", parent.Generation)} + network := readyIntentNetwork() + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test", UID: types.UID("project-ns")}} + project := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(namespace, route, parent, network, testRouteNetworkService(), testRouteMember("member", true, "fd00::1/128")). + WithStatusSubresource(route, parent).Build() + hub := fake.NewClientBuilder().WithScheme(project.Scheme()). + WithObjects(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns-project-ns"}}).Build() + reconciler := &NetworkLoadBalancerCellIntentReconciler{HubCluster: &fakeCluster{cl: hub}} + key := client.ObjectKeyFromObject(route) + + _, err := reconciler.reconcileIntent(ctx, "project", project, key) + require.NoError(t, err) + var first networkingv1alpha.NetworkLoadBalancerCellIntentList + require.NoError(t, hub.List(ctx, &first)) + require.Len(t, first.Items, 1) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, first.Items[0].Spec.State) + require.Equal(t, "2001:db8::1", first.Items[0].Spec.VIP) + require.Equal(t, networkingv1alpha.IPv6Protocol, first.Items[0].Spec.VIPFamily) + require.Equal(t, networkingv1alpha.IPv6Protocol, first.Items[0].Spec.Backends[0].Family) + require.Equal(t, "project", first.Items[0].Spec.Source.ProjectCluster) + require.Equal(t, "route-1", first.Items[0].Spec.Source.RouteUID) + require.Equal(t, "cell-test", first.Items[0].Spec.Backends[0].SourceNamespace) + require.Equal(t, "source-member", first.Items[0].Spec.Backends[0].SourceUID) + require.Equal(t, int32(8080), first.Items[0].Spec.Backends[0].Probe.Port) + + require.NoError(t, project.Get(ctx, client.ObjectKeyFromObject(parent), parent)) + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::2", parent.Generation)} + require.NoError(t, project.Status().Update(ctx, parent)) + _, err = reconciler.reconcileIntent(ctx, "project", project, key) + require.NoError(t, err) + var changed networkingv1alpha.NetworkLoadBalancerCellIntentList + require.NoError(t, hub.List(ctx, &changed)) + require.Len(t, changed.Items, 2) + for _, intent := range changed.Items { + if intent.Spec.VIP == "2001:db8::1" { + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, intent.Spec.State) + require.Empty(t, intent.Spec.Backends) + } else { + require.Equal(t, "2001:db8::2", intent.Spec.VIP) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, intent.Spec.State) + } + } + + require.NoError(t, project.Get(ctx, key, route)) + require.NoError(t, project.Delete(ctx, route)) + _, err = reconciler.reconcileIntent(ctx, "project", project, key) + require.NoError(t, err) + var withdrawn networkingv1alpha.NetworkLoadBalancerCellIntentList + require.NoError(t, hub.List(ctx, &withdrawn)) + require.Len(t, withdrawn.Items, 2) + for _, intent := range withdrawn.Items { + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, intent.Spec.State) + require.Empty(t, intent.Spec.Backends) + } +} + +func TestCellIntentRequiresFreshPerLocationAllocation(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + allocatedCellIntentEndpoint("2001:db8::1", 4), + } + parent.Status.Endpoints[0].Location = "" + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), + testRouteNetworkService(), testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) + + parent.Status.Endpoints[0].Addresses[0].Conditions[0].ObservedGeneration = 5 + require.NoError(t, cl.Update(ctx, parent)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) + + parent.Status.Endpoints[0].Location = "dfw" + require.NoError(t, cl.Update(ctx, parent)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, "dfw", desired[0].Spec.Location) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, desired[0].Spec.State) + + duplicate := allocatedCellIntentEndpoint("2001:db8::1", 5) + duplicate.Location = "ord" + parent.Status.Endpoints = append(parent.Status.Endpoints, duplicate) + require.NoError(t, cl.Update(ctx, parent)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentRejectsUnsupportedVIPFamily(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Spec.Exposure.Addresses[0].Families = []networkingv1alpha.IPFamily{networkingv1alpha.IPv4Protocol} + allocation := allocatedCellIntentAddress("192.0.2.10", parent.Generation) + allocation.Family = networkingv1alpha.IPv4Protocol + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{{RequestName: "regional", Location: "dfw", Addresses: []networkingv1alpha.NetworkLoadBalancerAddressStatus{allocation}}} + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentResolvesPortParentAndEquivalentNamespace(t *testing.T) { + ctx := context.Background() + port := int32(443) + older := testLoadBalancerRoute("older", time.Unix(10, 0)) + older.Spec.ParentRefs[0] = networkingv1alpha.NetworkLoadBalancerRouteParentRef{ + Name: "edge", Port: &port, Namespace: "test", + } + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(older, parent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", older) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, "tcp", desired[0].Spec.Source.ListenerName) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, desired[0].Spec.State) + + newer := testLoadBalancerRoute("newer", time.Unix(20, 0)) + require.NoError(t, cl.Create(ctx, newer)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", newer) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentWithholdsOversizedGalacticBackendSet(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + objects := make([]client.Object, 0, 69) + objects = append(objects, route, parent, readyIntentNetwork(), testRouteNetworkService()) + for i := range 65 { + objects = append(objects, testRouteMember(fmt.Sprintf("member-%d", i), true, + fmt.Sprintf("fd00::%x/128", i+1))) + } + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)).WithObjects(objects...).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, desired[0].Spec.State) + require.Empty(t, desired[0].Spec.Backends) +} + +func readyIntentNetwork() *networkingv1alpha.Network { + return &networkingv1alpha.Network{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "default", Generation: 1}, + Status: networkingv1alpha.NetworkStatus{Conditions: []metav1.Condition{{ + Type: networkingv1alpha.NetworkReady, Status: metav1.ConditionTrue, + Reason: "Ready", ObservedGeneration: 1, LastTransitionTime: metav1.Now(), + }}}, + } +} + +func TestCellIntentFailsClosedOnMissingIdentityAndOlderRoute(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("newer", time.Unix(20, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + member := testRouteMember("member", true, "fd00::1/128") + delete(member.Labels, networkingv1alpha.NetworkInterfaceSourceUIDLabel) + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), member).Build() + + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, desired[0].Spec.State) + require.Empty(t, desired[0].Spec.Backends) + + member.Labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel] = "source-member" + require.NoError(t, cl.Update(ctx, member)) + require.NoError(t, cl.Create(ctx, testLoadBalancerRoute("older", time.Unix(10, 0)))) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentDoesNotValidateConflictingBackendSocket(t *testing.T) { + ctx := context.Background() + older := testLoadBalancerRoute("older", time.Unix(10, 0)) + newer := testLoadBalancerRoute("newer", time.Unix(20, 0)) + newer.Spec.ParentRefs[0].Name = "edge-2" + firstParent := testRouteLoadBalancer() + firstParent.UID = types.UID("first-parent") + firstParent.Generation = 5 + firstParent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + secondParent := firstParent.DeepCopy() + secondParent.Name = "edge-2" + secondParent.UID = types.UID("second-parent") + secondParent.Status.Endpoints[0].Addresses[0].Value = "2001:db8::2" + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(older, newer, firstParent, secondParent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", newer) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentWithholdsMultipleVIPsOnOneListener(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("route", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + second := parent.Spec.Exposure.Addresses[0] + second.Name = "second" + parent.Spec.Exposure.Addresses = append(parent.Spec.Exposure.Addresses, second) + parent.Spec.Listeners[0].AddressRefs = []string{"regional", "second"} + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + allocatedCellIntentEndpoint("2001:db8::1", 5), + allocatedCellIntentEndpoint("2001:db8::2", 5), + } + parent.Status.Endpoints[1].RequestName = "second" + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) +} + +func TestCellIntentWithdrawsTwoAllocatedVIPsForOneRequest(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("route", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + allocatedCellIntentEndpoint("2001:db8::1", 5), + allocatedCellIntentEndpoint("2001:db8::2", 5), + } + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 2) + for _, intent := range desired { + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentWithdrawn, intent.Spec.State) + require.Empty(t, intent.Spec.Backends) + } +} + +func TestCellIntentWithholdsSameRouteSecondParentBackendClaim(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("route", time.Unix(10, 0)) + route.Spec.ParentRefs = append(route.Spec.ParentRefs, + networkingv1alpha.NetworkLoadBalancerRouteParentRef{Name: "edge-2", Listener: "tcp"}) + firstParent := testRouteLoadBalancer() + firstParent.UID = types.UID("first-parent") + firstParent.Generation = 5 + firstParent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{ + allocatedCellIntentEndpoint("2001:db8::1", 5), + } + secondParent := firstParent.DeepCopy() + secondParent.Name = "edge-2" + secondParent.UID = types.UID("second-parent") + secondParent.Status.Endpoints[0].Addresses[0].Value = "2001:db8::2" + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, firstParent, secondParent, readyIntentNetwork(), testRouteNetworkService(), + testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Equal(t, "edge", desired[0].Spec.Source.ParentName) + require.Equal(t, networkingv1alpha.NetworkLoadBalancerCellIntentValidate, desired[0].Spec.State) +} + +func TestCellIntentWithholdsAllocationFromOlderGeneration(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 4)} + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), + testRouteNetworkService(), testRouteMember("member", true, "fd00::1/128")).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Empty(t, desired) + + parent.Status.Endpoints[0].Addresses[0].Conditions[0].ObservedGeneration = 5 + require.NoError(t, cl.Update(ctx, parent)) + desired, err = buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) +} + +func TestCellIntentSkipsBackendsThatAreNotBound(t *testing.T) { + ctx := context.Background() + route := testLoadBalancerRoute("storefront", time.Unix(10, 0)) + parent := testRouteLoadBalancer() + parent.UID = types.UID("parent-1") + parent.Generation = 5 + parent.Status.Endpoints = []networkingv1alpha.NetworkLoadBalancerEndpointStatus{allocatedCellIntentEndpoint("2001:db8::1", 5)} + bound := testRouteMember("bound", true, "fd00::1/128") + released := testRouteMember("released", true, "fd00::2/128") + released.Status.Phase = networkingv1alpha.NetworkInterfacePhaseAvailable + cl := fake.NewClientBuilder().WithScheme(networkServiceTestScheme(t)). + WithObjects(route, parent, readyIntentNetwork(), testRouteNetworkService(), bound, released).Build() + desired, err := buildNetworkLoadBalancerCellIntents(ctx, cl, "project", "hub", route) + require.NoError(t, err) + require.Len(t, desired, 1) + require.Len(t, desired[0].Spec.Backends, 1) + require.Equal(t, "source-bound", desired[0].Spec.Backends[0].SourceUID) +} diff --git a/internal/controller/networkloadbalancerroute_controller.go b/internal/controller/networkloadbalancerroute_controller.go index b0b2780f..58dd9af1 100644 --- a/internal/controller/networkloadbalancerroute_controller.go +++ b/internal/controller/networkloadbalancerroute_controller.go @@ -61,6 +61,7 @@ type networkLoadBalancerRouteEndpoint struct { Location string Address string Port int32 + Probe networkingv1alpha.NetworkLoadBalancerCellIntentProbe } type networkLoadBalancerRouteReferenceError struct { @@ -549,12 +550,21 @@ func resolveNetworkLoadBalancerRouteBackends( fmt.Sprintf("NetworkService %q selects an interface outside network %q", ref.Name, parent.Spec.NetworkRef.Name)} } } + probe := networkingv1alpha.NetworkLoadBalancerCellIntentProbe{Protocol: "TCP", Port: port.Port} + if check := backend.HealthCheck; check != nil { + probe.Protocol = check.Protocol + probe.Path = check.Path + if check.Port != "" { + probeIndex := slices.IndexFunc(service.Spec.Ports, func(p networkingv1alpha.NetworkServicePort) bool { return p.Name == check.Port }) + probe.Port = service.Spec.Ports[probeIndex].Port + } + } for i := range members { member := &members[i] location := member.Labels[networkingv1alpha.NetworkInterfaceLocationLabel] sourceNamespace := member.Labels[networkingv1alpha.NetworkInterfaceSourceNamespaceLabel] sourceUID := member.Labels[networkingv1alpha.NetworkInterfaceSourceUIDLabel] - if !isProjection(member) || sourceNamespace == "" || sourceUID == "" || + if !isProjection(member) || member.Status.Phase != networkingv1alpha.NetworkInterfacePhaseBound || sourceNamespace == "" || sourceUID == "" || location == "" || !routeLocationSelected(parent, listener, location) || !isServiceMemberHealthy(member) { continue } @@ -570,6 +580,7 @@ func resolveNetworkLoadBalancerRouteBackends( Location: location, Address: address, Port: port.Port, + Probe: probe, }) } } diff --git a/internal/networkagent/catalog_test.go b/internal/networkagent/catalog_test.go index 1ed7176f..50cee86b 100644 --- a/internal/networkagent/catalog_test.go +++ b/internal/networkagent/catalog_test.go @@ -38,6 +38,7 @@ var outOfScopeAPIFiles = map[string]string{ "locationbinding_types.go": "which locations a project is offered; surfaces here only as LocationNotAvailable", "networkloadbalancer_types.go": "Network Load Balancer has its own product status, separate from networks and network services", "networkloadbalancerbackendhealth_types.go": "cell-local NLB probe status, not a network-agent customer API", + "networkloadbalancercellintent_types.go": "hub-side NLB serving intent, not a network-agent customer API", "networkloadbalancerroute_types.go": "Network Load Balancer route status belongs to the load balancer product", "networkpolicy_types.go": "declares no status reasons", "trafficprotectionpolicy_types.go": "load balancer protection; the ALB agent's",