diff --git a/api/v1alpha1/internaldns_project_types.go b/api/v1alpha1/internaldns_project_types.go index f7f16461..d20eb0e4 100644 --- a/api/v1alpha1/internaldns_project_types.go +++ b/api/v1alpha1/internaldns_project_types.go @@ -35,10 +35,9 @@ type DNSZoneAssociationSpec struct { } type DNSZoneAssociationStatus struct { - ResolvedDNSZoneRef DNSObjectReference `json:"resolvedDNSZoneRef,omitempty"` - ResolvedResolverContextRef DNSObjectReference `json:"resolvedResolverContextRef,omitempty"` - BindingRef *DNSObjectReference `json:"bindingRef,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` + ResolvedDNSZoneRef DNSNameReference `json:"resolvedDNSZoneRef,omitempty"` + ResolvedResolverContextRef DNSNameReference `json:"resolvedResolverContextRef,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` } // DNSResolverContext is the DNS-owned serving and namespace boundary created by @@ -63,8 +62,8 @@ type DNSResolverContextManagedNamespace struct { } type DNSResolverContextManagedNamespaceStatus struct { - DNSZoneRef DNSObjectReference `json:"dnsZoneRef,omitempty"` - Suffix string `json:"suffix,omitempty"` + DNSZoneRef DNSNameReference `json:"dnsZoneRef,omitempty"` + Suffix string `json:"suffix,omitempty"` } type DNSResolverContextStatus struct { @@ -137,9 +136,8 @@ type DNSResolverAccessBindingSpec struct { } type DNSResolverAccessBindingStatus struct { - ObservedSequence int64 `json:"observedSequence,omitempty"` - BindingRef *DNSObjectReference `json:"bindingRef,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` + ObservedSequence int64 `json:"observedSequence,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` } // +kubebuilder:object:root=true @@ -186,9 +184,9 @@ const ( const DNSRegistrationMaxTTLSeconds int32 = 30 -// +kubebuilder:validation:XValidation:rule="has(self.dnsZoneRef.uid) && self.dnsZoneRef.uid.size() > 0",message="dnsZoneRef must pin the zone UID" +// +kubebuilder:validation:XValidation:rule="self.dnsZoneRef.name.size() > 0",message="dnsZoneRef requires a name" type DNSRegistrationSpec struct { - DNSZoneRef DNSObjectReference `json:"dnsZoneRef"` + DNSZoneRef DNSNameReference `json:"dnsZoneRef"` Name string `json:"name"` RecordTypes []RRType `json:"recordTypes"` PublicationPolicy DNSPublicationPolicy `json:"publicationPolicy"` @@ -358,15 +356,14 @@ func (s *DNSRegistrationSelector) AsLabelSelector() *metav1.LabelSelector { } type DNSNamingPolicyStatus struct { - ResolvedResolverContextRef DNSObjectReference `json:"resolvedResolverContextRef,omitempty"` - ResolvedDNSZoneRef DNSObjectReference `json:"resolvedDNSZoneRef,omitempty"` + ResolvedResolverContextRef DNSNameReference `json:"resolvedResolverContextRef,omitempty"` ResolvedAdditionalNames []DNSResolvedAdditionalNameRule `json:"resolvedAdditionalNames,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty"` } type DNSResolvedAdditionalNameRule struct { Selector *metav1.LabelSelector `json:"selector,omitempty"` - DNSZoneRef DNSObjectReference `json:"dnsZoneRef"` + DNSZoneRef DNSNameReference `json:"dnsZoneRef"` NamePrefix string `json:"namePrefix"` } @@ -388,15 +385,13 @@ type DNSNamingPolicyList struct { // DNSManagedNamespace tracks a resolver context's canonical private namespace. type DNSManagedNamespaceSpec struct { - ProjectUID types.UID `json:"projectUID"` - ResolverContextRef DNSObjectReference `json:"resolverContextRef"` - DomainSuffix string `json:"domainSuffix"` - DNSZoneClassName string `json:"dnsZoneClassName"` + ResolverContextRef DNSNameReference `json:"resolverContextRef"` + DomainSuffix string `json:"domainSuffix"` + DNSZoneClassName string `json:"dnsZoneClassName"` } type DNSManagedNamespaceStatus struct { - DNSZoneRef DNSObjectReference `json:"dnsZoneRef,omitempty"` - AssociationRef DNSObjectReference `json:"associationRef,omitempty"` + DNSZoneRef DNSNameReference `json:"dnsZoneRef,omitempty"` CanonicalSuffix string `json:"canonicalSuffix,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty"` } diff --git a/api/v1alpha1/internaldns_service_types.go b/api/v1alpha1/internaldns_service_types.go index dc57e565..f00fc424 100644 --- a/api/v1alpha1/internaldns_service_types.go +++ b/api/v1alpha1/internaldns_service_types.go @@ -50,6 +50,9 @@ type DNSResolverBindingConfiguration struct { Listeners DNSResolverBindingListeners `json:"listeners"` // +kubebuilder:validation:MaxItems=256 ZoneRefs []DNSObjectReference `json:"zoneRefs"` + // ZoneApices reserves each private namespace before publication is available. + // +kubebuilder:validation:MaxProperties=256 + ZoneApices map[string]string `json:"zoneApices,omitempty"` } // +kubebuilder:validation:XValidation:rule="oldSelf == self",message="listener identity is immutable for an access binding" diff --git a/api/v1alpha1/internaldns_validation_envtest_test.go b/api/v1alpha1/internaldns_validation_envtest_test.go index a4673fa2..c9515064 100644 --- a/api/v1alpha1/internaldns_validation_envtest_test.go +++ b/api/v1alpha1/internaldns_validation_envtest_test.go @@ -127,7 +127,7 @@ func TestPrivateDNSValidationAtAPIServer(t *testing.T) { }) for _, ttl := range []int32{0, 31, 2147483647} { t.Run("ttl", func(t *testing.T) { - a := &DNSRegistration{ObjectMeta: metav1.ObjectMeta{GenerateName: "registration-", Namespace: "validation"}, Spec: DNSRegistrationSpec{DNSZoneRef: DNSObjectReference{Name: "zone", UID: "zone-uid"}, Name: "web", RecordTypes: []RRType{RRTypeAAAA}, PublicationPolicy: DNSPublicationPolicyEligibleContributions, TTLSeconds: ttl}} + a := &DNSRegistration{ObjectMeta: metav1.ObjectMeta{GenerateName: "registration-", Namespace: "validation"}, Spec: DNSRegistrationSpec{DNSZoneRef: DNSNameReference{Name: "zone"}, Name: "web", RecordTypes: []RRType{RRTypeAAAA}, PublicationPolicy: DNSPublicationPolicyEligibleContributions, TTLSeconds: ttl}} requireInvalid(t, cl.Create(ctx, a)) }) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ea2cbae6..c82bb10f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -353,7 +353,6 @@ func (in *DNSManagedNamespaceSpec) DeepCopy() *DNSManagedNamespaceSpec { func (in *DNSManagedNamespaceStatus) DeepCopyInto(out *DNSManagedNamespaceStatus) { *out = *in out.DNSZoneRef = in.DNSZoneRef - out.AssociationRef = in.AssociationRef if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -475,7 +474,6 @@ func (in *DNSNamingPolicySpec) DeepCopy() *DNSNamingPolicySpec { func (in *DNSNamingPolicyStatus) DeepCopyInto(out *DNSNamingPolicyStatus) { *out = *in out.ResolvedResolverContextRef = in.ResolvedResolverContextRef - out.ResolvedDNSZoneRef = in.ResolvedDNSZoneRef if in.ResolvedAdditionalNames != nil { in, out := &in.ResolvedAdditionalNames, &out.ResolvedAdditionalNames *out = make([]DNSResolvedAdditionalNameRule, len(*in)) @@ -1356,11 +1354,6 @@ func (in *DNSResolverAccessBindingSpec) DeepCopy() *DNSResolverAccessBindingSpec // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSResolverAccessBindingStatus) DeepCopyInto(out *DNSResolverAccessBindingStatus) { *out = *in - if in.BindingRef != nil { - in, out := &in.BindingRef, &out.BindingRef - *out = new(DNSObjectReference) - **out = **in - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -1416,6 +1409,13 @@ func (in *DNSResolverBindingConfiguration) DeepCopyInto(out *DNSResolverBindingC *out = make([]DNSObjectReference, len(*in)) copy(*out, *in) } + if in.ZoneApices != nil { + in, out := &in.ZoneApices, &out.ZoneApices + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSResolverBindingConfiguration. @@ -1954,11 +1954,6 @@ func (in *DNSZoneAssociationStatus) DeepCopyInto(out *DNSZoneAssociationStatus) *out = *in out.ResolvedDNSZoneRef = in.ResolvedDNSZoneRef out.ResolvedResolverContextRef = in.ResolvedResolverContextRef - if in.BindingRef != nil { - in, out := &in.BindingRef, &out.BindingRef - *out = new(DNSObjectReference) - **out = **in - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml index c57ef52a..4ff71067 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml @@ -43,64 +43,24 @@ spec: type: string domainSuffix: type: string - projectUID: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string resolverContextRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object required: - dnsZoneClassName - domainSuffix - - projectUID - resolverContextRef type: object status: properties: - associationRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object canonicalSuffix: type: string conditions: @@ -161,23 +121,12 @@ spec: type: array dnsZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml index 1f3312f4..0b5babaf 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml @@ -204,23 +204,12 @@ spec: properties: dnsZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object @@ -280,47 +269,14 @@ spec: - namePrefix type: object type: array - resolvedDNSZoneRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object resolvedResolverContextRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsregistrations.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsregistrations.yaml index 701b0e8a..3fbeb8c3 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsregistrations.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsregistrations.yaml @@ -40,23 +40,12 @@ spec: properties: dnsZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object @@ -105,8 +94,8 @@ spec: - ttlSeconds type: object x-kubernetes-validations: - - message: dnsZoneRef must pin the zone UID - rule: has(self.dnsZoneRef.uid) && self.dnsZoneRef.uid.size() > 0 + - message: dnsZoneRef requires a name + rule: self.dnsZoneRef.name.size() > 0 status: properties: canonicalFQDN: diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml index f743b3d3..c4ac5037 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml @@ -131,28 +131,6 @@ spec: > oldSelf.authorization.sequence status: properties: - bindingRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object conditions: items: description: Condition contains details for one aspect of the current diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml index 4d78b627..81137d8c 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml @@ -124,6 +124,13 @@ spec: format: int64 minimum: 1 type: integer + zoneApices: + additionalProperties: + type: string + description: ZoneApices reserves each private namespace before + publication is available. + maxProperties: 256 + type: object zoneRefs: items: description: |- diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml index 15b51f51..cb730aa9 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml @@ -137,23 +137,12 @@ spec: properties: dnsZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object diff --git a/config/crd/bases/dns.networking.miloapis.com_dnszoneassociations.yaml b/config/crd/bases/dns.networking.miloapis.com_dnszoneassociations.yaml index f2bbd186..f074cd14 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnszoneassociations.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnszoneassociations.yaml @@ -73,28 +73,6 @@ spec: rule: self.resolverContextRef.name.size() > 0 status: properties: - bindingRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object conditions: items: description: Condition contains details for one aspect of the current @@ -153,45 +131,23 @@ spec: type: array resolvedDNSZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object resolvedResolverContextRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 0b3ecd42..3008d094 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -79,15 +79,19 @@ Regional publication proof connects directly to that member's BIND listener, supplies the exact context destination in a protected PROXYv2 header over both UDP and TCP, and verifies the installed SOA serial and fingerprint. Restrict `proxyPeers` to regional dnsdist and the member's local publication agent -network. +network. Local health probes also require the BIND listener address in +`proxyPeers` and loopback in dnsdist ACLs. These peers must remain inside the +protected serving network. ## Transactions, health, and outage recovery -An attached zone that has no active publication gates its context until the -configuration is complete. Pending contexts deny queries, recursion, and cache -answers while retaining the original access authorization. Configuration -completion can restore service without renewing authorization; an actual access -withdrawal still requires a newer source authorization to reactivate. +Each attached private apex has independent readiness. A pending zone returns +SERVFAIL while published zones continue serving. Its names cannot fall through +to public recursion or cached answers. If the private apex is unknown, the +context stays gated until its configuration is complete. Completing publication +restores service without renewing authorization. Resolver access readiness +reports the applied path independently of each zone's publication status. Explicit access withdrawal +requires a newer source authorization to reactivate. The agent writes a complete candidate set into its stage directory. Regional zone files are siblings of `cluster-bind.conf`; validation rewrites file @@ -108,12 +112,11 @@ from persistent checkpoints before waiting for messaging connectivity. New nodes require a complete configuration before accepting queries. Replay cannot regress writer or configuration versions or restore a withdrawn lifetime. -The local watchdog lease is at most five seconds. It is renewed only after a -complete validated transaction and required DNS proofs. -The gate must stop or firewall the local private listener without depending on -Kubernetes or NATS. Validation, reload, readiness wait, and proof must fit -inside the watchdog budget; reduce shard size or improve local operations when -they do not. +The local watchdog lease is at most five seconds. An independent health loop +checks local DNS daemons over UDP and TCP and renews the lease for the activated +configuration. Staging and validation can continue while that configuration +serves. A rejected expansion retains the activated configuration; a withdrawal +that cannot be applied gates serving. The gate runs without Kubernetes or NATS. Mount the transaction stage, active BIND configuration, zone files, checkpoint, and watchdog lease on low-latency storage local to the serving host. Do not put @@ -163,8 +166,28 @@ contribution and eligibility observation use that publisher's current grant. Grants identify the authenticated subject within the project API context. Publication-result updates cannot change a publisher's observation. Resource permissions do not depend on webhook principal lists. -The supported internal envelope version is `v1alpha4` and the serving checkpoint -format is 5. Unsupported versions fail closed. Deploy matching controller and -fleet versions with fresh internal DNS transport state and checkpoints. Preserve -any existing state for recovery. The deployment requires a coordinated rollout; -it does not support mixed versions or automatic state conversion. +## Rollout and recovery + +Project APIs select zones by name and report names and readiness. DNS stores +resolved ownership identities in reference ConfigMaps in the service project. +The collector checks source resources before removing those records. + +The supported envelope version is `v1alpha4` and checkpoint format is 5. Keep +reference records, transport state, and persistent checkpoints across compatible upgrades. Deploy +serving agents before controllers that emit per-zone pending state. Enable the +controller change after every serving member supports that contract; a matching +wire version alone does not establish feature compatibility. + +The checkpoint retains desired version fences and the activated configuration +separately. Restart can restore the activated configuration during a rejected +expansion or broker outage. Explicit withdrawal invalidates that fallback before +installation. Unknown checkpoint or wire formats preserve files and reject +activation. Format conversion requires a tested migration that retains lifetime, +withdrawal, configuration, and publisher fences; never reset those fences to +complete an upgrade. + +Snapshot budgets admit established bindings before new allocations. Overflow +reports `CapacityAvailable=False` for the affected binding while healthy +contexts and withdrawals continue. A pending zone fails only queries beneath +its private apex; its names never fall through to public recursion. Bindings, +publication targets, and context status use the same stable shard assignment. diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml index b5a43141..0c919476 100644 --- a/config/internal-dns/fleet.example.yaml +++ b/config/internal-dns/fleet.example.yaml @@ -11,7 +11,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", - "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64","::1/128"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20","fd70:10::10"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, @@ -28,7 +28,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", - "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, + "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64","::1/128"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}]}, "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, @@ -45,7 +45,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", - "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30","fd70:20::10"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} }, @@ -65,7 +65,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", - "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31","fd70:20::11"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} }, diff --git a/config/internal-dns/rbac.yaml b/config/internal-dns/rbac.yaml index eb4695c1..feec5ab2 100644 --- a/config/internal-dns/rbac.yaml +++ b/config/internal-dns/rbac.yaml @@ -22,7 +22,7 @@ rules: resources: [dnsresolvercontexts, dnsresolveraccessbindings] verbs: [get, list, watch, create, update, patch, delete] - apiGroups: [dns.networking.miloapis.com] - resources: [dnsresolvercontexts/status, dnsresolveraccessbindings/status, dnsmanagednamespaces, dnsmanagednamespaces/status] + resources: [dnsresolvercontexts/status, dnsresolveraccessbindings/status] verbs: [get, list, watch] --- # Project products write declarations and their own freshness observations. @@ -37,7 +37,7 @@ rules: resources: [dnsregistrations, dnsrecordcontributions] verbs: [get, list, watch, create, update, patch, delete] - apiGroups: [dns.networking.miloapis.com] - resources: [dnscontributiongrants, dnsresolvercontexts, dnsmanagednamespaces, dnsnamingpolicies] + resources: [dnscontributiongrants, dnsresolvercontexts, dnsnamingpolicies] verbs: [get, list, watch] - apiGroups: [dns.networking.miloapis.com] resources: [dnsrecordcontributions/status] @@ -54,7 +54,7 @@ rules: resources: [dnszones, dnsrecordsets, dnszoneassociations, dnsnamingpolicies, dnsregistrations] verbs: [get, list, watch, create, update, patch, delete] - apiGroups: [dns.networking.miloapis.com] - resources: [dnsresolvercontexts, dnsmanagednamespaces] + resources: [dnsresolvercontexts] verbs: [get, list, watch] --- # Bind only to a trusted issuer that applies the product's registration policy. @@ -122,7 +122,7 @@ rules: verbs: [get, list, watch, create, update, patch] - apiGroups: [""] resources: [configmaps] - verbs: [get, list, watch, create, update, patch] + verbs: [get, list, watch, create, update, patch, delete] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/internaldns/admission/handler.go b/internal/internaldns/admission/handler.go index 8f062898..0d34a185 100644 --- a/internal/internaldns/admission/handler.go +++ b/internal/internaldns/admission/handler.go @@ -315,16 +315,15 @@ func (h *Handler) registration(ctx context.Context, cl client.Client, req kubead return nil } -// Zone references cannot silently follow a replacement with the same name. -func currentPrivateZone(ctx context.Context, cl client.Client, namespace string, ref dnsv1alpha1.DNSObjectReference) (*dnsv1alpha1.DNSZone, error) { - if ref.Name == "" || ref.UID == "" { - return nil, fmt.Errorf("zone reference must pin name and UID") +func currentPrivateZone(ctx context.Context, cl client.Client, namespace string, ref dnsv1alpha1.DNSNameReference) (*dnsv1alpha1.DNSZone, error) { + if ref.Name == "" { + return nil, fmt.Errorf("zone reference requires a name") } var zone dnsv1alpha1.DNSZone if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ref.Name}, &zone); err != nil { return nil, fmt.Errorf("private zone lookup: %w", err) } - if zone.UID != ref.UID || !zone.DeletionTimestamp.IsZero() || (ref.Generation != 0 && ref.Generation != zone.Generation) { + if !zone.DeletionTimestamp.IsZero() { return nil, fmt.Errorf("zone lifetime is not current") } if zone.Spec.Visibility != dnsv1alpha1.DNSZoneVisibilityPrivate { diff --git a/internal/internaldns/admission/handler_test.go b/internal/internaldns/admission/handler_test.go index 4d498004..928a7a53 100644 --- a/internal/internaldns/admission/handler_test.go +++ b/internal/internaldns/admission/handler_test.go @@ -24,7 +24,7 @@ func TestContributionAuthenticatedWriter(t *testing.T) { producer := "system:serviceaccount:compute-system:dns-publisher" registration := &dnsv1alpha1.DNSRegistration{ ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "project", UID: "reg-uid", Generation: 3}, - Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "private"}, RecordTypes: []dnsv1alpha1.RRType{"AAAA"}}, + Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "private"}, RecordTypes: []dnsv1alpha1.RRType{"AAAA"}}, } grant := &dnsv1alpha1.DNSContributionGrant{ ObjectMeta: metav1.ObjectMeta{Name: "compute", Namespace: "project", UID: "grant-uid"}, @@ -245,7 +245,7 @@ func TestPrivateRegistrationRejectsAuthorityOverrides(t *testing.T) { rt dnsv1alpha1.RRType allowed bool }{{"api", dnsv1alpha1.RRTypeA, true}, {"api", dnsv1alpha1.RRTypeCNAME, true}, {"@", dnsv1alpha1.RRTypeCNAME, false}, {"CORP.INTERNAL.", dnsv1alpha1.RRTypeCNAME, false}, {"@", dnsv1alpha1.RRTypeSOA, false}} { - registration := dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, TTLSeconds: 5, Name: tt.owner, RecordTypes: []dnsv1alpha1.RRType{tt.rt}}} + registration := dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, TTLSeconds: 5, Name: tt.owner, RecordTypes: []dnsv1alpha1.RRType{tt.rt}}} response := h.Handle(context.Background(), request("dnsregistrations", "tenant", registration, nil)) if response.Allowed != tt.allowed { t.Fatalf("owner %s type %s allowed=%v expected=%v", tt.owner, tt.rt, response.Allowed, tt.allowed) @@ -261,15 +261,15 @@ func TestRegistrationRequiresCurrentZone(t *testing.T) { zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", Namespace: "project", UID: "current"}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(zone).Build()} for _, tt := range []struct { - uid types.UID + name string ttl int32 allowed bool }{ - {"current", 1, true}, {"current", 30, true}, {"", 5, false}, {"retired", 5, false}, + {zone.Name, 1, true}, {zone.Name, 30, true}, {"", 5, false}, {"missing", 5, false}, } { - reg := dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: tt.uid}, TTLSeconds: tt.ttl, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}}} + reg := dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: tt.name}, TTLSeconds: tt.ttl, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}}} if got := h.Handle(context.Background(), request("dnsregistrations", "tenant", reg, nil)); got.Allowed != tt.allowed { - t.Errorf("UID=%q TTL=%d: allowed=%v: %v", tt.uid, tt.ttl, got.Allowed, got.Result) + t.Errorf("name=%q TTL=%d: allowed=%v: %v", tt.name, tt.ttl, got.Allowed, got.Result) } } } diff --git a/internal/internaldns/controlplane/access_recovery_envtest_test.go b/internal/internaldns/controlplane/access_recovery_envtest_test.go index 3586d3ab..eacfce64 100644 --- a/internal/internaldns/controlplane/access_recovery_envtest_test.go +++ b/internal/internaldns/controlplane/access_recovery_envtest_test.go @@ -37,7 +37,7 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { } }) scheme := runtime.NewScheme() - requireAccessTestSuccess(t, dnsv1alpha1.AddToScheme(scheme)) + requireAccessTestSuccess(t, addReferenceStateToScheme(scheme)) requireAccessTestSuccess(t, corev1.AddToScheme(scheme)) cl, err := client.New(cfg, client.Options{Scheme: scheme}) if err != nil { diff --git a/internal/internaldns/controlplane/compiler.go b/internal/internaldns/controlplane/compiler.go index 70838bc2..2df38a0d 100644 --- a/internal/internaldns/controlplane/compiler.go +++ b/internal/internaldns/controlplane/compiler.go @@ -53,14 +53,16 @@ type RegistrationFence struct { } type CompileInput struct { - Zone *dnsv1alpha1.DNSZone - Associations []dnsv1alpha1.DNSZoneAssociation - Registrations []dnsv1alpha1.DNSRegistration - Grants []dnsv1alpha1.DNSContributionGrant - Contributions []dnsv1alpha1.DNSRecordContribution - StaticRecords []dnsv1alpha1.DNSRecordSet - Previous *PublicationPlan - Now time.Time + Zone *dnsv1alpha1.DNSZone + Associations []dnsv1alpha1.DNSZoneAssociation + AssociationReferences map[types.UID]associationReferences + RegistrationZoneReferences map[types.UID]dnsv1alpha1.DNSObjectReference + Registrations []dnsv1alpha1.DNSRegistration + Grants []dnsv1alpha1.DNSContributionGrant + Contributions []dnsv1alpha1.DNSRecordContribution + StaticRecords []dnsv1alpha1.DNSRecordSet + Previous *PublicationPlan + Now time.Time } type CompileResult struct { @@ -86,7 +88,7 @@ func Compile(in CompileInput) (CompileResult, error) { result.Plan.ZoneUID = in.Zone.UID result.Plan.ZoneApex = canonicalName(in.Zone.Spec.DomainName) - result.Plan.ContextUIDs = associatedContextUIDs(in.Zone, in.Associations) + result.Plan.ContextUIDs = associatedContextUIDs(in.Zone, in.Associations, in.AssociationReferences) claims := map[ownershipKey]types.UID{} staticClaim := map[ownershipKey]types.UID{} @@ -106,13 +108,17 @@ func Compile(in CompileInput) (CompileResult, error) { result.Reasons[reg.UID] = "InvalidTTL" continue } - if !refMatches(reg.Spec.DNSZoneRef, in.Zone.Name, in.Zone.UID, in.Zone.Generation) { + if reg.Spec.DNSZoneRef.Name != in.Zone.Name { continue } if reg.UID == "" { result.Reasons[reg.UID] = "MissingUID" continue } + if !refMatches(in.RegistrationZoneReferences[reg.UID], in.Zone.Name, in.Zone.UID, in.Zone.Generation) { + result.Reasons[reg.UID] = "InvalidReference" + continue + } name := zoneOwner(reg.Spec.Name, result.Plan.ZoneApex) valid := true for _, rt := range reg.Spec.RecordTypes { @@ -484,12 +490,13 @@ func sortAggregate(records []dnsv1alpha1.RecordEntry, uids []types.UID) { } } -func associatedContextUIDs(zone *dnsv1alpha1.DNSZone, associations []dnsv1alpha1.DNSZoneAssociation) []types.UID { +func associatedContextUIDs(zone *dnsv1alpha1.DNSZone, associations []dnsv1alpha1.DNSZoneAssociation, references map[types.UID]associationReferences) []types.UID { contextsByUID := map[types.UID]struct{}{} for i := range associations { a := &associations[i] - consumerUID := a.Status.ResolvedResolverContextRef.UID - if associationMatchesZone(a, zone) && consumerUID != "" { + refs := references[a.UID] + consumerUID := refs.Context.UID + if associationMatchesZone(a, zone, refs) && consumerUID != "" { contextsByUID[consumerUID] = struct{}{} } } diff --git a/internal/internaldns/controlplane/compiler_test.go b/internal/internaldns/controlplane/compiler_test.go index fe5b6c80..278103d7 100644 --- a/internal/internaldns/controlplane/compiler_test.go +++ b/internal/internaldns/controlplane/compiler_test.go @@ -186,19 +186,35 @@ func TestCompileSRVTargetUsesAbsoluteZoneIdentity(t *testing.T) { func fixture(now time.Time) CompileInput { zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", UID: "zone-a", Generation: 2}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - assoc := func(name string, vpc types.UID) dnsv1alpha1.DNSZoneAssociation { - return dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name)}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: name}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: name, UID: vpc}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} + assoc := func(name string) dnsv1alpha1.DNSZoneAssociation { + return dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name)}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: name}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: name}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} } - reg := dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: "registration-a", Generation: 3}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 30}} + reg := dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: "registration-a", Generation: 3}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 30}} grant := dnsv1alpha1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: "compute", UID: "grant-a", Generation: 2}, Spec: dnsv1alpha1.DNSContributionGrantSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, ProducerID: "compute-east", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}}, Status: dnsv1alpha1.DNSContributionGrantStatus{ActiveWriterEpoch: 7, ObservedGrantGeneration: 2, ObservedRegistrationGeneration: 3}} c := dnsv1alpha1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint-a", UID: "contribution-a", Generation: 4}, Spec: dnsv1alpha1.DNSRecordContributionSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: grant.Name, UID: grant.UID}, RecordSets: []dnsv1alpha1.DNSContributionRecordSet{{RecordType: dnsv1alpha1.RRTypeAAAA, Records: []dnsv1alpha1.RecordEntry{{Name: "api.corp.internal", AAAA: &dnsv1alpha1.AAAARecordSpec{Content: "fd00::1"}}}}}}, Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 4, WriterEpoch: 7, Sequence: 18, Eligible: true}} - return CompileInput{Zone: zone, Associations: []dnsv1alpha1.DNSZoneAssociation{assoc("b", "vpc-b"), assoc("a", "vpc-a")}, Registrations: []dnsv1alpha1.DNSRegistration{reg}, Grants: []dnsv1alpha1.DNSContributionGrant{grant}, Contributions: []dnsv1alpha1.DNSRecordContribution{c}, Now: now} + zoneRef := dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation} + return CompileInput{Zone: zone, Associations: []dnsv1alpha1.DNSZoneAssociation{assoc("b"), assoc("a")}, AssociationReferences: map[types.UID]associationReferences{ + "a": {Zone: zoneRef, Context: dnsv1alpha1.DNSObjectReference{Name: "a", UID: "vpc-a"}}, + "b": {Zone: zoneRef, Context: dnsv1alpha1.DNSObjectReference{Name: "b", UID: "vpc-b"}}, + }, RegistrationZoneReferences: map[types.UID]dnsv1alpha1.DNSObjectReference{reg.UID: zoneRef}, Registrations: []dnsv1alpha1.DNSRegistration{reg}, Grants: []dnsv1alpha1.DNSContributionGrant{grant}, Contributions: []dnsv1alpha1.DNSRecordContribution{c}, Now: now} } func structLocal(name string) corev1.LocalObjectReference { return corev1.LocalObjectReference{Name: name} } +func TestCompileRejectsReplacementBehindANameOnlyReference(t *testing.T) { + in := fixture(time.Now().UTC()) + in.Zone.UID = "replacement-zone" + result, err := Compile(in) + if err != nil { + t.Fatal(err) + } + if result.Accepted[in.Registrations[0].UID] || result.Reasons[in.Registrations[0].UID] != "InvalidReference" || len(result.Plan.RRsets) != 0 { + t.Fatalf("replacement reused a registration's private reference: %#v", result) + } +} + func TestCompileRejectsPrivateAuthorityOverridesWithoutBlockingHealthyRecords(t *testing.T) { for _, tt := range []struct { name string @@ -217,6 +233,7 @@ func TestCompileRejectsPrivateAuthorityOverridesWithoutBlockingHealthyRecords(t bad.Spec.Name = tt.entry.Name bad.Spec.RecordTypes = []dnsv1alpha1.RRType{tt.recordType} in.Registrations = append(in.Registrations, *bad) + in.RegistrationZoneReferences[bad.UID] = in.RegistrationZoneReferences[in.Registrations[0].UID] got, err := Compile(in) if err != nil { t.Fatal(err) diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index eb29962f..744d61b3 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -43,34 +43,43 @@ func pinNamedReference(name string, previous, current dnsv1alpha1.DNSObjectRefer } func (r *Reconciler) reconcileNamingPolicies(ctx context.Context, items []dnsv1alpha1.DNSNamingPolicy, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + associationPins, err := r.associationReferences(ctx, assocs) + if err != nil { + return err + } for i := range items { p := &items[i] base := p.DeepCopy() var resolverContext dnsv1alpha1.DNSResolverContext err := r.Get(ctx, client.ObjectKey{Namespace: p.Namespace, Name: p.Spec.ResolverContextRef.Name}, &resolverContext) - current := dnsv1alpha1.DNSObjectReference{} + if err != nil && !apierrors.IsNotFound(err) { + return err + } + current := dnsv1alpha1.DNSObjectReference{Name: p.Spec.ResolverContextRef.Name} if err == nil && resolverContext.DeletionTimestamp.IsZero() { current = dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID} } - contextPin, contextCurrent := pinNamedReference(p.Spec.ResolverContextRef.Name, p.Status.ResolvedResolverContextRef, current) + selected := map[string]dnsv1alpha1.DNSObjectReference{"context": current} + for _, rule := range p.Spec.AdditionalNames { + ref := dnsv1alpha1.DNSObjectReference{Name: rule.DNSZoneRef.Name} + if zone := zones[rule.DNSZoneRef.Name]; zone != nil && zone.DeletionTimestamp.IsZero() && zone.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate { + ref.UID, ref.Generation = zone.UID, zone.Generation + } + selected["zone/"+rule.DNSZoneRef.Name] = ref + } + pins, contextCurrent, err := r.pinReferences(ctx, p, selected) + if err != nil { + return err + } + contextPin := pins["context"] ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") valid := contextCurrent && p.DeletionTimestamp.IsZero() && ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == resolverContext.Generation - p.Status.ResolvedResolverContextRef = contextPin - - previousZones := map[string]dnsv1alpha1.DNSObjectReference{} - for _, rule := range base.Status.ResolvedAdditionalNames { - previousZones[rule.DNSZoneRef.Name] = rule.DNSZoneRef - } + p.Status.ResolvedResolverContextRef = dnsv1alpha1.DNSNameReference{Name: contextPin.Name} resolved := make([]dnsv1alpha1.DNSResolvedAdditionalNameRule, 0, len(p.Spec.AdditionalNames)) seen := map[string]bool{} for _, rule := range p.Spec.AdditionalNames { - z := zones[rule.DNSZoneRef.Name] - currentZone := dnsv1alpha1.DNSObjectReference{} - if z != nil && z.DeletionTimestamp.IsZero() && z.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate { - currentZone = dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation} - } - zonePin, zoneCurrent := pinNamedReference(rule.DNSZoneRef.Name, previousZones[rule.DNSZoneRef.Name], currentZone) - if !zoneCurrent || !contextCurrent || !associatedWithContext(assocs, zonePin.UID, contextPin.UID) { + zonePin := pins["zone/"+rule.DNSZoneRef.Name] + if !contextCurrent || !associatedWithContext(assocs, associationPins, zonePin.UID, contextPin.UID) { valid = false } selector, selectorErr := namingSelector(rule.Selector.AsLabelSelector()) @@ -85,7 +94,7 @@ func (r *Reconciler) reconcileNamingPolicies(ctx context.Context, items []dnsv1a valid = false } seen[key] = true - resolved = append(resolved, dnsv1alpha1.DNSResolvedAdditionalNameRule{Selector: rule.Selector.AsLabelSelector().DeepCopy(), DNSZoneRef: zonePin, NamePrefix: canonicalName(rule.NamePrefix)}) + resolved = append(resolved, dnsv1alpha1.DNSResolvedAdditionalNameRule{Selector: rule.Selector.AsLabelSelector().DeepCopy(), DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zonePin.Name}, NamePrefix: canonicalName(rule.NamePrefix)}) } p.Status.ResolvedAdditionalNames = resolved status := metav1.ConditionTrue @@ -110,13 +119,14 @@ func associationAccepted(a *dnsv1alpha1.DNSZoneAssociation) bool { return a.DeletionTimestamp.IsZero() && accepted != nil && accepted.Status == metav1.ConditionTrue && accepted.ObservedGeneration == a.Generation } -func associationMatchesZone(a *dnsv1alpha1.DNSZoneAssociation, z *dnsv1alpha1.DNSZone) bool { - return z != nil && associationAccepted(a) && refMatches(a.Status.ResolvedDNSZoneRef, z.Name, z.UID, z.Generation) +func associationMatchesZone(a *dnsv1alpha1.DNSZoneAssociation, z *dnsv1alpha1.DNSZone, refs associationReferences) bool { + return z != nil && associationAccepted(a) && refMatches(refs.Zone, z.Name, z.UID, z.Generation) } -func associatedWithContext(assocs []dnsv1alpha1.DNSZoneAssociation, zoneUID, contextUID types.UID) bool { +func associatedWithContext(assocs []dnsv1alpha1.DNSZoneAssociation, pins map[types.UID]associationReferences, zoneUID, contextUID types.UID) bool { for _, a := range assocs { - if associationAccepted(&a) && a.Status.ResolvedDNSZoneRef.UID == zoneUID && a.Status.ResolvedResolverContextRef.UID == contextUID { + refs := pins[a.UID] + if associationAccepted(&a) && refs.Zone.UID == zoneUID && refs.Context.UID == contextUID { return true } } @@ -131,13 +141,17 @@ func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, } for i := range list.Items { m := &list.Items[i] - if m.Spec.ProjectUID != r.Options.ProjectUID || m.Spec.ResolverContextRef.UID == "" { - if err := r.patchCondition(ctx, m, dnsValueAccepted, metav1.ConditionFalse, "InvalidIdentity", "project and pinned resolver context UID are required", now); err != nil { + var contextObject dnsv1alpha1.DNSResolverContext + if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: m.Spec.ResolverContextRef.Name}, &contextObject); err != nil { + return err + } + if !metav1.IsControlledBy(m, &contextObject) || !contextObject.DeletionTimestamp.IsZero() { + if err := r.patchCondition(ctx, m, dnsValueAccepted, metav1.ConditionFalse, "InvalidIdentity", "managed namespace must be owned by its resolver context", now); err != nil { return err } continue } - lifetimeToken := model.OpaqueToken(m.Labels["internal-dns.miloapis.com/context-uid"]) + lifetimeToken := model.OpaqueToken(string(contextObject.UID)) suffix := canonicalName(m.Spec.DomainSuffix) zoneName := "managed-" + lifetimeToken[:12] @@ -156,10 +170,6 @@ func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, assocName := zoneName + "-context" assocSpec := dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Managed: true} - var contextObject dnsv1alpha1.DNSResolverContext - if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: m.Spec.ResolverContextRef.Name}, &contextObject); err != nil { - return err - } assocSpec.ResolverContextRef = dnsv1alpha1.DNSNameReference{Name: contextObject.Name} assoc := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: assocName, Namespace: ns, Labels: map[string]string{"dns.networking.miloapis.com/managed": "true"}}, Spec: assocSpec} @@ -176,15 +186,17 @@ func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, } base := m.DeepCopy() accepted := apimeta.FindStatusCondition(assoc.Status.Conditions, dnsValueAccepted) - ready := zone.Spec.DomainName == suffix && metav1.IsControlledBy(zone, m) && accepted != nil && accepted.Status == metav1.ConditionTrue && assoc.Status.ResolvedDNSZoneRef.UID == zone.UID + refs, err := r.readReferences(ctx, assoc) + if err != nil { + return err + } + ready := zone.Spec.DomainName == suffix && metav1.IsControlledBy(zone, m) && accepted != nil && accepted.Status == metav1.ConditionTrue && accepted.ObservedGeneration == assoc.Generation && refMatches(refs["zone"], zone.Name, zone.UID, zone.Generation) if ready { - m.Status.DNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID} - m.Status.AssociationRef = dnsv1alpha1.DNSObjectReference{Name: assoc.Name, UID: assoc.UID} + m.Status.DNSZoneRef = dnsv1alpha1.DNSNameReference{Name: zone.Name} m.Status.CanonicalSuffix = suffix setCondition(&m.Status.Conditions, dnsValueAccepted, metav1.ConditionTrue, "Ready", "managed namespace is allocated", m.Generation, now) } else { - m.Status.DNSZoneRef = dnsv1alpha1.DNSObjectReference{} - m.Status.AssociationRef = dnsv1alpha1.DNSObjectReference{} + m.Status.DNSZoneRef = dnsv1alpha1.DNSNameReference{} m.Status.CanonicalSuffix = "" setCondition(&m.Status.Conditions, dnsValueAccepted, metav1.ConditionFalse, "PendingAssociation", "managed namespace association is not accepted", m.Generation, now) } @@ -197,6 +209,10 @@ func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, now time.Time) error { var contexts dnsv1alpha1.DNSResolverContextList + var existing dnsv1alpha1.DNSResolverBindingList + if err := r.List(ctx, &existing, client.InNamespace(r.Options.PlatformNamespace)); err != nil { + return err + } if err := r.List(ctx, &contexts, client.InNamespace(ns)); err != nil { return err } @@ -222,7 +238,7 @@ func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, n } c.Status.ServingTargets = nil if valid { - for _, target := range r.Options.servingRegions() { + for _, target := range r.selectServingTargets(c.UID, existing.Items) { c.Status.ServingTargets = append(c.Status.ServingTargets, dnsv1alpha1.DNSResolverContextServingTarget{Region: target.Region, Shard: target.Shard}) } } @@ -230,7 +246,7 @@ func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, n c.Status.ManagedNamespace = dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{} if c.Spec.ManagedNamespace.Enabled && valid { name := "context-" + model.OpaqueToken(string(c.UID))[:20] - m := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{"internal-dns.miloapis.com/platform-owned": "true", "internal-dns.miloapis.com/context-uid": string(c.UID)}}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: c.Name, UID: c.UID}, DomainSuffix: c.Spec.ManagedNamespace.DomainSuffix, DNSZoneClassName: r.Options.PrivateZoneClassName}} + m := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{"internal-dns.miloapis.com/platform-owned": "true"}}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: c.Name}, DomainSuffix: c.Spec.ManagedNamespace.DomainSuffix, DNSZoneClassName: r.Options.PrivateZoneClassName}} if r.Scheme != nil { if err := controllerutil.SetControllerReference(c, m, r.Scheme); err != nil { return err @@ -242,7 +258,7 @@ func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, n if err := r.Get(ctx, client.ObjectKeyFromObject(m), m); err != nil { return err } - if condition := apimeta.FindStatusCondition(m.Status.Conditions, dnsValueAccepted); condition != nil && condition.Status == metav1.ConditionTrue && condition.ObservedGeneration == m.Generation && m.Status.DNSZoneRef.UID != "" && metav1.IsControlledBy(m, c) && m.Spec.DomainSuffix == c.Spec.ManagedNamespace.DomainSuffix && m.Status.CanonicalSuffix == c.Spec.ManagedNamespace.DomainSuffix { + if condition := apimeta.FindStatusCondition(m.Status.Conditions, dnsValueAccepted); condition != nil && condition.Status == metav1.ConditionTrue && condition.ObservedGeneration == m.Generation && m.Status.DNSZoneRef.Name != "" && metav1.IsControlledBy(m, c) && m.Spec.DomainSuffix == c.Spec.ManagedNamespace.DomainSuffix && m.Status.CanonicalSuffix == c.Spec.ManagedNamespace.DomainSuffix { c.Status.ManagedNamespace.DNSZoneRef = m.Status.DNSZoneRef c.Status.ManagedNamespace.Suffix = m.Status.CanonicalSuffix ready = true @@ -282,23 +298,29 @@ func (r *Reconciler) reconcileAssociations(ctx context.Context, items []dnsv1alp a := &items[i] base := a.DeepCopy() z := zones[a.Spec.DNSZoneRef.Name] - currentZone := dnsv1alpha1.DNSObjectReference{} + currentZone := dnsv1alpha1.DNSObjectReference{Name: a.Spec.DNSZoneRef.Name} if z != nil && z.DeletionTimestamp.IsZero() && z.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate { currentZone = dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation} } - zonePin, zoneCurrent := pinNamedReference(a.Spec.DNSZoneRef.Name, a.Status.ResolvedDNSZoneRef, currentZone) var resolverContext dnsv1alpha1.DNSResolverContext lookupErr := r.Get(ctx, client.ObjectKey{Namespace: a.Namespace, Name: a.Spec.ResolverContextRef.Name}, &resolverContext) - currentContext := dnsv1alpha1.DNSObjectReference{} + if lookupErr != nil && !apierrors.IsNotFound(lookupErr) { + return lookupErr + } + currentContext := dnsv1alpha1.DNSObjectReference{Name: a.Spec.ResolverContextRef.Name} if lookupErr == nil && resolverContext.DeletionTimestamp.IsZero() { currentContext = dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID} } - contextPin, contextCurrent := pinNamedReference(a.Spec.ResolverContextRef.Name, a.Status.ResolvedResolverContextRef, currentContext) - a.Status.ResolvedDNSZoneRef = zonePin - a.Status.ResolvedResolverContextRef = contextPin + pins, referencesCurrent, err := r.pinReferences(ctx, a, map[string]dnsv1alpha1.DNSObjectReference{"zone": currentZone, "context": currentContext}) + if err != nil { + return err + } + zonePin, contextPin := pins["zone"], pins["context"] + a.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSNameReference{Name: zonePin.Name} + a.Status.ResolvedResolverContextRef = dnsv1alpha1.DNSNameReference{Name: contextPin.Name} status := metav1.ConditionTrue reason, msg := dnsValueAccepted, "association accepted" - if !a.DeletionTimestamp.IsZero() || !zoneCurrent || !contextCurrent { + if !a.DeletionTimestamp.IsZero() || !referencesCurrent { status = metav1.ConditionFalse reason = "InvalidReference" msg = "private zone and resolver context must match their resolved lifetimes; recreate the association to reattach replacements" @@ -324,6 +346,10 @@ func (r *Reconciler) reconcileBindings(ctx context.Context, ns string, assocs [] } func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + associationPins, err := r.associationReferences(ctx, assocs) + if err != nil { + return err + } var accesses dnsv1alpha1.DNSResolverAccessBindingList var contexts dnsv1alpha1.DNSResolverContextList var existing dnsv1alpha1.DNSResolverBindingList @@ -350,8 +376,9 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as zonesByContext := map[types.UID][]types.UID{} for _, a := range assocs { z := zones[a.Spec.DNSZoneRef.Name] - if associationMatchesZone(&a, z) { - zonesByContext[a.Status.ResolvedResolverContextRef.UID] = append(zonesByContext[a.Status.ResolvedResolverContextRef.UID], z.UID) + refs := associationPins[a.UID] + if associationMatchesZone(&a, z, refs) { + zonesByContext[refs.Context.UID] = append(zonesByContext[refs.Context.UID], z.UID) } } sort.Slice(accesses.Items, func(i, j int) bool { return accesses.Items[i].Name < accesses.Items[j].Name }) @@ -381,7 +408,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as if owner := destinations[key]; owner != "" && owner != a.UID { valid = false } else if valid { - if r.Options.AddressAllocator == nil || r.Options.AddressAllocator.ClaimDestination(ctx, r.Options.ProjectUID, a.UID, spec.Region, netip.MustParseAddr(spec.QueryIdentity.Value).String(), spec.Port) != nil { + if r.Options.AddressAllocator == nil || r.Options.AddressAllocator.ClaimDestination(ctx, r.Options.ProjectUID, contextObject.UID, spec.Region, netip.MustParseAddr(spec.QueryIdentity.Value).String(), spec.Port) != nil { valid = false } else { destinations[key] = a.UID @@ -389,11 +416,10 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as } status, reason, msg := metav1.ConditionTrue, dnsValueAccepted, "access binding is current" if !valid { - status, reason, msg = metav1.ConditionFalse, "InvalidOrStaleAccess", "context lifetime, epoch, sequence, destination, and bounded deadline must be current" + status, reason, msg = metav1.ConditionFalse, "InvalidOrStaleAccess", "context lifetime, epoch, sequence, and destination must be current" } setCondition(&a.Status.Conditions, dnsValueAccepted, status, reason, msg, a.Generation, now) if !valid { - a.Status.BindingRef = nil setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "AccessRejected", "access binding is not accepted", a.Generation, now) if !jsonEqual(base.Status, a.Status) { if err := r.Status().Patch(ctx, a, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { @@ -433,7 +459,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as return err } } - binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForRegion(spec.Region))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForRegion(spec.Region)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence}}} + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones), ZoneApices: zoneApices(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence}}} if !exists { if err := r.Create(ctx, binding); err != nil { return err @@ -453,10 +479,11 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as binding = &old } a.Status.ObservedSequence = spec.Authorization.Sequence - a.Status.BindingRef = &dnsv1alpha1.DNSObjectReference{Name: binding.Name, UID: binding.UID} setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "PendingReplicaVerification", "serving fleet acknowledgement is pending", a.Generation, now) if c := apimeta.FindStatusCondition(binding.Status.Conditions, "Serving"); c != nil && c.Status == metav1.ConditionTrue && binding.Status.ObservedConfigurationRevision == binding.Spec.Configuration.Revision && bindingAckCurrent(binding, now) { - setCondition(&a.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "required serving replicas applied this access generation", a.Generation, now) + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "serving capacity applied this access generation", a.Generation, now) + } else if c != nil && c.Status == metav1.ConditionUnknown { + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionUnknown, "ObservationUnavailable", "current serving health is unavailable", a.Generation, now) } if !jsonEqual(base.Status, a.Status) { if err := r.Status().Patch(ctx, a, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { @@ -483,7 +510,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as b.Spec.Tombstone = true b.Spec.Configuration.Revision++ // Authorization belongs to the trusted integration. Preserve its fence and - // deadline; withdrawal is ordered independently by configuration revision. + // revision; withdrawal is ordered independently by configuration revision. if err := r.Patch(ctx, b, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { return err } @@ -537,9 +564,13 @@ func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace stri return !valid, nil } -func (r *Reconciler) shardForRegion(region string) string { - t, _ := r.targetForRegion(region) - return t.Shard +func (r *Reconciler) shardForContextRegion(contextUID types.UID, region string, existing []dnsv1alpha1.DNSResolverBinding) string { + for _, target := range r.selectServingTargets(contextUID, existing) { + if target.Region == region { + return target.Shard + } + } + return "" } func (r *Reconciler) targetForRegion(region string) (ServingRegion, bool) { @@ -666,6 +697,20 @@ func bindingEqual(a, b dnsv1alpha1.DNSResolverBindingSpec) bool { return string(xb) == string(yb) } +func zoneApices(uids []types.UID, zones map[string]*dnsv1alpha1.DNSZone) map[string]string { + wanted := map[types.UID]bool{} + for _, uid := range uids { + wanted[uid] = true + } + apices := map[string]string{} + for _, zone := range zones { + if wanted[zone.UID] { + apices[string(zone.UID)] = zone.Spec.DomainName + } + } + return apices +} + func zoneReferences(uids []types.UID, zones map[string]*dnsv1alpha1.DNSZone) []dnsv1alpha1.DNSObjectReference { refs := make([]dnsv1alpha1.DNSObjectReference, 0, len(uids)) for _, uid := range uids { diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index ffe14cf1..a6db2496 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -57,17 +57,18 @@ func TestContextAccessPersistsUntilExplicitWithdrawal(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } contextObject := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}, Status: dnsv1alpha1.DNSResolverContextStatus{AccessWriterEpoch: 4}} setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 1, now) zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", Namespace: "project", UID: "zone-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "assoc", Namespace: "project", UID: "assoc-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "zone"}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: "ctx"}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "zone-uid", Generation: 1}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}}} + association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "assoc", Namespace: "project", UID: "assoc-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "zone"}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: "ctx"}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "zone"}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: "ctx"}}} setCondition(&association.Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 4, Sequence: 12}}} c := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject, zone, association, access).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverBinding{}).Build() r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: testAllocator{}}} + seedReferenceState(t, r, association, map[string]dnsv1alpha1.DNSObjectReference{"zone": {Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, "context": {Name: contextObject.Name, UID: contextObject.UID}}) if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now); err != nil { t.Fatal(err) } @@ -97,7 +98,9 @@ func TestContextAccessPersistsUntilExplicitWithdrawal(t *testing.T) { extraAssociation := association.DeepCopy() extraAssociation.Name = "extra-assoc" extraAssociation.Spec.DNSZoneRef = dnsv1alpha1.DNSNameReference{Name: extraZone.Name} - extraAssociation.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: extraZone.Name, UID: extraZone.UID, Generation: extraZone.Generation} + extraAssociation.UID = "extra-association-uid" + extraAssociation.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSNameReference{Name: extraZone.Name} + seedReferenceState(t, r, extraAssociation, map[string]dnsv1alpha1.DNSObjectReference{"zone": {Name: extraZone.Name, UID: extraZone.UID, Generation: extraZone.Generation}, "context": {Name: contextObject.Name, UID: contextObject.UID}}) if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association, *extraAssociation}, map[string]*dnsv1alpha1.DNSZone{"zone": zone, "extra": extraZone}, now); err != nil { t.Fatal(err) } @@ -224,21 +227,29 @@ func TestContextNamingPolicyResolvesMultipleZonesWithoutNetworking(t *testing.T) ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } resolverContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "ctx-uid", Generation: 2}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}} setCondition(&resolverContext.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 2, now) z1 := dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project", UID: "zone-1", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} z2 := dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "services", Namespace: "project", UID: "zone-2", Generation: 3}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - associations := []dnsv1alpha1.DNSZoneAssociation{{Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: z1.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolverContext.UID}}}, {Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: z2.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolverContext.UID}}}} + associations := []dnsv1alpha1.DNSZoneAssociation{{Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{}}}, {Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{}}}} for i := range associations { + zone := []dnsv1alpha1.DNSZone{z1, z2}[i] + associations[i].Name = zone.Name + "-association" + associations[i].Namespace = "project" + associations[i].UID = types.UID(associations[i].Name) associations[i].Generation = 1 setCondition(&associations[i].Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) } policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "names", Namespace: "project", UID: "policy-uid", Generation: 1}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolverContext.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: z1.Name}, NamePrefix: "instances"}, {DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: z2.Name}, NamePrefix: "services"}}}} c := fake.NewClientBuilder().WithScheme(s).WithObjects(resolverContext, policy).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSNamingPolicy{}).Build() r := &Reconciler{Client: c, Options: ReconcilerOptions{}} + for i := range associations { + zone := []dnsv1alpha1.DNSZone{z1, z2}[i] + seedReferenceState(t, r, &associations[i], map[string]dnsv1alpha1.DNSObjectReference{"zone": {Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, "context": {Name: resolverContext.Name, UID: resolverContext.UID}}) + } if err := r.reconcileNamingPolicies(ctx, []dnsv1alpha1.DNSNamingPolicy{*policy}, associations, map[string]*dnsv1alpha1.DNSZone{z1.Name: &z1, z2.Name: &z2}, now); err != nil { t.Fatal(err) } @@ -249,7 +260,7 @@ func TestContextNamingPolicyResolvesMultipleZonesWithoutNetworking(t *testing.T) if accepted == nil || accepted.Status != metav1.ConditionTrue || len(policy.Status.ResolvedAdditionalNames) != 2 { t.Fatalf("context naming policy not resolved: %#v", policy.Status) } - if policy.Status.ResolvedResolverContextRef.UID != resolverContext.UID { + if policy.Status.ResolvedResolverContextRef.Name != resolverContext.Name { t.Fatalf("identity projection incorrect: %#v", policy.Status) } } @@ -284,3 +295,15 @@ func TestStableOneShardPerRegionAssignment(t *testing.T) { t.Fatalf("adding a shard moved live VPC: before=%s after=%s", central.Shard, after[0].Shard) } } + +func TestAccessPlacementMatchesContextPublicationTargets(t *testing.T) { + r := &Reconciler{Options: ReconcilerOptions{ProjectUID: "project", ServingRegions: []ServingRegion{{Region: "east", Shard: "a"}, {Region: "east", Shard: "b"}, {Region: "west", Shard: "a"}}}} + for i := range 100 { + uid := types.UID(fmt.Sprintf("context-%d", i)) + for _, target := range r.selectServingTargets(uid, nil) { + if shard := r.shardForContextRegion(uid, target.Region, nil); shard != target.Shard { + t.Fatalf("binding shard %s does not match publication target %s", shard, target.Shard) + } + } + } +} diff --git a/internal/internaldns/controlplane/managed_domain_test.go b/internal/internaldns/controlplane/managed_domain_test.go index deed10d8..4f75b12f 100644 --- a/internal/internaldns/controlplane/managed_domain_test.go +++ b/internal/internaldns/controlplane/managed_domain_test.go @@ -18,7 +18,7 @@ import ( func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } domains := map[string]string{"vpc-a": "datum.internal", "vpc-b": "datum.internal", "custom": "private.example.net"} @@ -52,17 +52,12 @@ func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { if err := c.List(ctx, &associations); err != nil { t.Fatal(err) } - for i := range associations.Items { - a := &associations.Items[i] - for _, z := range zones.Items { - if z.Name == a.Spec.DNSZoneRef.Name { - a.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation} - } - } - setCondition(&a.Status.Conditions, "Accepted", metav1.ConditionTrue, "Ready", "", a.Generation, now) - if err := c.Status().Update(ctx, a); err != nil { - t.Fatal(err) - } + zoneByName := map[string]*dnsv1alpha1.DNSZone{} + for i := range zones.Items { + zoneByName[zones.Items[i].Name] = &zones.Items[i] + } + if err := r.reconcileAssociations(ctx, associations.Items, zoneByName, now); err != nil { + t.Fatal(err) } if err := r.reconcileManagedNamespaces(ctx, "project", now); err != nil { t.Fatal(err) @@ -75,7 +70,7 @@ func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { t.Fatal(err) } for _, resolver := range contexts.Items { - if resolver.Status.ManagedNamespace.Suffix != domains[resolver.Name] || resolver.Status.ManagedNamespace.DNSZoneRef.UID == "" { + if resolver.Status.ManagedNamespace.Suffix != domains[resolver.Name] || resolver.Status.ManagedNamespace.DNSZoneRef.Name == "" { t.Fatalf("unallocated context: %#v", resolver.Status) } var zone dnsv1alpha1.DNSZone @@ -86,7 +81,7 @@ func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { t.Fatalf("requested %q, allocated %q", domains[resolver.Name], zone.Spec.DomainName) } } - if contexts.Items[0].Status.ManagedNamespace.DNSZoneRef.UID == contexts.Items[1].Status.ManagedNamespace.DNSZoneRef.UID { + if contexts.Items[0].Status.ManagedNamespace.DNSZoneRef.Name == contexts.Items[1].Status.ManagedNamespace.DNSZoneRef.Name { t.Fatal("contexts share a zone lifetime") } zone := &zones.Items[0] @@ -104,7 +99,7 @@ func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { t.Fatal(err) } for _, resolver := range contexts.Items { - if resolver.Status.ManagedNamespace.DNSZoneRef.UID == zone.UID { + if resolver.Status.ManagedNamespace.DNSZoneRef.Name == zone.Name { t.Fatal("a mismatched zone domain remained allocated") } } @@ -114,7 +109,7 @@ func TestManagedNamespaceRequiresRequestedDomain(t *testing.T) { for _, domain := range []string{"", "bad..internal", "*.internal"} { t.Run(domain, func(t *testing.T) { s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "consumer", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "opaque-consumer", ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespace{Enabled: true, DomainSuffix: domain}}} diff --git a/internal/internaldns/controlplane/named_reference_api_test.go b/internal/internaldns/controlplane/named_reference_api_test.go index bda1cb8f..5a8e3e92 100644 --- a/internal/internaldns/controlplane/named_reference_api_test.go +++ b/internal/internaldns/controlplane/named_reference_api_test.go @@ -27,7 +27,7 @@ func verifyNamedReferencesWithAPI(t *testing.T, cl client.Client) { policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "named-policy", Namespace: "project"}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: &dnsv1alpha1.DNSRegistrationSelector{MatchLabels: map[string]string{"app": "storefront"}}, DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, NamePrefix: "hosts"}}}} requireAccessTestSuccess(t, cl.Create(ctx, association)) requireAccessTestSuccess(t, cl.Create(ctx, policy)) - r := &Reconciler{Client: cl} + r := &Reconciler{Client: cl, Options: Options{PlatformNamespace: "platform"}} reconcile := func(want metav1.ConditionStatus) { t.Helper() requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(association), association)) @@ -56,7 +56,11 @@ func verifyNamedReferencesWithAPI(t *testing.T, cl client.Client) { t.Fatal("replacement did not receive a new API-assigned UID") } reconcile(metav1.ConditionFalse) - if association.Status.ResolvedDNSZoneRef.UID != originalUID || policy.Status.ResolvedAdditionalNames[0].DNSZoneRef.UID != originalUID { + associationPins, err := r.readReferences(ctx, association) + requireAccessTestSuccess(t, err) + policyPins, err := r.readReferences(ctx, policy) + requireAccessTestSuccess(t, err) + if associationPins["zone"].UID != originalUID || policyPins["zone/"+zone.Name].UID != originalUID { t.Fatal("API-backed replacement transferred lifetime pins") } } diff --git a/internal/internaldns/controlplane/named_reference_test.go b/internal/internaldns/controlplane/named_reference_test.go index c030a356..df665a99 100644 --- a/internal/internaldns/controlplane/named_reference_test.go +++ b/internal/internaldns/controlplane/named_reference_test.go @@ -21,16 +21,24 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { ctx := context.Background() now := time.Now().UTC() scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "application", Namespace: "project", UID: "context-original", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer"}} setCondition(&resolver.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", resolver.Generation, now) zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", UID: "zone-original", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "prod.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}}} - policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", Generation: 1}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, NamePrefix: "hosts"}}}} + association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", UID: "source-original", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}}} + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", UID: "source-original", Generation: 1}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, NamePrefix: "hosts"}}}} cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver, zone, association, policy).WithStatusSubresource(association, policy).Build() r := &Reconciler{Client: cl} + references := func(source client.Object) map[string]dnsv1alpha1.DNSObjectReference { + t.Helper() + refs, err := r.readReferences(ctx, source) + if err != nil { + t.Fatal(err) + } + return refs + } zones := map[string]*dnsv1alpha1.DNSZone{zone.Name: zone} reconcile := func() { t.Helper() @@ -65,13 +73,13 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { } reconcile() assertAccepted(true) - if association.Status.ResolvedDNSZoneRef.UID != zone.UID || policy.Status.ResolvedResolverContextRef.UID != resolver.UID { + if references(association)["zone"].UID != zone.UID || references(policy)["context"].UID != resolver.UID { t.Fatal("name-only references did not pin lifetimes") } zone.Generation++ reconcile() assertAccepted(true) - if association.Status.ResolvedDNSZoneRef.Generation != zone.Generation || policy.Status.ResolvedAdditionalNames[0].DNSZoneRef.Generation != zone.Generation { + if references(association)["zone"].Generation != zone.Generation || references(policy)["zone/"+zone.Name].Generation != zone.Generation { t.Fatal("resolved generations were not persisted for the same zone lifetime") } originalZoneUID := zone.UID @@ -88,7 +96,7 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { } reconcile() assertAccepted(false) - if association.Status.ResolvedDNSZoneRef.UID != originalZoneUID || policy.Status.ResolvedResolverContextRef.UID != originalContextUID { + if references(association)["zone"].UID != originalZoneUID || references(policy)["context"].UID != originalContextUID { t.Fatal("deletion cleared lifetime pins") } replaced.SetResourceVersion("") @@ -101,7 +109,7 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { } reconcile() assertAccepted(false) - if association.Status.ResolvedDNSZoneRef.UID != originalZoneUID || policy.Status.ResolvedAdditionalNames[0].DNSZoneRef.UID != originalZoneUID || policy.Status.ResolvedResolverContextRef.UID != originalContextUID { + if references(association)["zone"].UID != originalZoneUID || references(policy)["zone/"+zone.Name].UID != originalZoneUID || references(policy)["context"].UID != originalContextUID { t.Fatal("replacement inherited lifetime pins") } // Recreate both customer resources to explicitly attach the replacement. @@ -109,6 +117,7 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { t.Fatal(err) } association.ResourceVersion = "" + association.UID = "association-recreated" association.Status = dnsv1alpha1.DNSZoneAssociationStatus{} if err := cl.Create(ctx, association); err != nil { t.Fatal(err) @@ -121,6 +130,7 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { t.Fatal(err) } policy.ResourceVersion = "" + policy.UID = "policy-recreated" policy.Status = dnsv1alpha1.DNSNamingPolicyStatus{} if err := cl.Create(ctx, policy); err != nil { t.Fatal(err) diff --git a/internal/internaldns/controlplane/naming.go b/internal/internaldns/controlplane/naming.go index 0c1237ca..72e15a99 100644 --- a/internal/internaldns/controlplane/naming.go +++ b/internal/internaldns/controlplane/naming.go @@ -26,27 +26,24 @@ type ResolvedName struct { // additive policy names after it. It never selects an arbitrary associated // customer zone. // Selectors match DNSRegistration metadata labels supplied by the publisher. -func ResolveNaming(ctx context.Context, c client.Client, namespace string, projectUID, contextUID types.UID, registrationLabels map[string]string, allocatedName string) ([]ResolvedName, error) { +func ResolveNaming(ctx context.Context, c client.Client, namespace string, projectUID types.UID, contextName string, registrationLabels map[string]string, allocatedName string) ([]ResolvedName, error) { label := dnsLabel(allocatedName) if label == "" { return nil, fmt.Errorf("allocated name %q has no DNS-safe label", allocatedName) } - var managed dnsv1alpha1.DNSManagedNamespaceList - if err := c.List(ctx, &managed, client.InNamespace(namespace)); err != nil { + var resolver dnsv1alpha1.DNSResolverContext + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: contextName}, &resolver); err != nil { return nil, err } - var out []ResolvedName - for i := range managed.Items { - m := &managed.Items[i] - accepted := apimeta.FindStatusCondition(m.Status.Conditions, dnsValueAccepted) - if m.Spec.ProjectUID == projectUID && m.Spec.ResolverContextRef.UID == contextUID && accepted != nil && accepted.Status == metav1.ConditionTrue { - out = append(out, ResolvedName{DNSZoneRef: m.Status.DNSZoneRef, Name: label, Canonical: true}) - break - } + if !resolver.DeletionTimestamp.IsZero() || !apimeta.IsStatusConditionTrue(resolver.Status.Conditions, "Ready") { + return nil, fmt.Errorf("managed DNS namespace for context %s is not ready", contextName) } - if len(out) == 0 { - return nil, fmt.Errorf("managed DNS namespace for resolver context %s is not ready", contextUID) + var canonical dnsv1alpha1.DNSZone + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: resolver.Status.ManagedNamespace.DNSZoneRef.Name}, &canonical); err != nil { + return nil, err } + out := []ResolvedName{{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: canonical.Name, UID: canonical.UID}, Name: label, Canonical: true}} + r := &Reconciler{Client: c, Options: Options{ProjectUID: projectUID}} seen := map[string]bool{string(out[0].DNSZoneRef.UID) + "\x00" + out[0].Name: true} var policies dnsv1alpha1.DNSNamingPolicyList if err := c.List(ctx, &policies, client.InNamespace(namespace)); err != nil { @@ -61,7 +58,14 @@ func ResolveNaming(ctx context.Context, c client.Client, namespace string, proje for i := range policies.Items { p := &policies.Items[i] accepted := apimeta.FindStatusCondition(p.Status.Conditions, dnsValueAccepted) - if p.Status.ResolvedResolverContextRef.UID != contextUID || !p.DeletionTimestamp.IsZero() || accepted == nil || accepted.Status != metav1.ConditionTrue || accepted.ObservedGeneration != p.Generation { + if p.Status.ResolvedResolverContextRef.Name != contextName || !p.DeletionTimestamp.IsZero() || accepted == nil || accepted.Status != metav1.ConditionTrue || accepted.ObservedGeneration != p.Generation { + continue + } + pins, err := r.readReferences(ctx, p) + if err != nil { + return nil, err + } + if pins["context"].UID != resolver.UID { continue } for _, rule := range p.Status.ResolvedAdditionalNames { @@ -70,9 +74,17 @@ func ResolveNaming(ctx context.Context, c client.Client, namespace string, proje return nil, fmt.Errorf("naming policy %s has an invalid selector: %w", p.Name, err) } name := label + "." + canonicalName(rule.NamePrefix) - key := string(rule.DNSZoneRef.UID) + "\x00" + name + zoneRef := pins["zone/"+rule.DNSZoneRef.Name] + var zone dnsv1alpha1.DNSZone + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: zoneRef.Name}, &zone); err != nil { + return nil, err + } + if !refMatches(zoneRef, zone.Name, zone.UID, zone.Generation) { + continue + } + key := string(zoneRef.UID) + "\x00" + name if selector.Matches(labels.Set(registrationLabels)) && !seen[key] { - out = append(out, ResolvedName{DNSZoneRef: rule.DNSZoneRef, Name: name}) + out = append(out, ResolvedName{DNSZoneRef: zoneRef, Name: name}) seen[key] = true } } diff --git a/internal/internaldns/controlplane/naming_selector_test.go b/internal/internaldns/controlplane/naming_selector_test.go index 8b7d0117..b8df2b13 100644 --- a/internal/internaldns/controlplane/naming_selector_test.go +++ b/internal/internaldns/controlplane/naming_selector_test.go @@ -18,13 +18,15 @@ import ( func TestNamingSelectorsMatchRegistrationLabelsWithinContext(t *testing.T) { scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } - managed := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project"}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ProjectUID: "project-a", ResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: "consumer-a"}}, Status: dnsv1alpha1.DNSManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "canonical", UID: "canonical-zone"}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} - zone := dnsv1alpha1.DNSObjectReference{Name: "custom", UID: "custom-zone"} + resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "consumer-a", Namespace: "project", UID: "context-a"}, Status: dnsv1alpha1.DNSResolverContextStatus{ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "canonical"}}, Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}}}} + canonical := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "canonical", Namespace: "project", UID: "canonical-zone"}} + custom := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project", UID: "custom-zone"}} + zone := dnsv1alpha1.DNSNameReference{Name: custom.Name} storefront := &metav1.LabelSelector{MatchLabels: map[string]string{"app": "storefront"}} - policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "names", Namespace: "project", Generation: 1}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: "consumer-a"}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue, ObservedGeneration: 1}}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{ + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "names", Namespace: "project", UID: "names-uid", Generation: 1}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue, ObservedGeneration: 1}}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{ {DNSZoneRef: zone, NamePrefix: "all"}, {Selector: &metav1.LabelSelector{}, DNSZoneRef: zone, NamePrefix: "empty"}, {Selector: storefront, DNSZoneRef: zone, NamePrefix: "storefront"}, @@ -33,7 +35,7 @@ func TestNamingSelectorsMatchRegistrationLabelsWithinContext(t *testing.T) { }}} other := policy.DeepCopy() other.Name = "other-context" - other.Status.ResolvedResolverContextRef.UID = "consumer-b" + other.Status.ResolvedResolverContextRef.Name = "consumer-b" other.Status.ResolvedAdditionalNames = []dnsv1alpha1.DNSResolvedAdditionalNameRule{{DNSZoneRef: zone, NamePrefix: "other-context"}} stale := policy.DeepCopy() stale.Name = "stale-policy" @@ -43,7 +45,9 @@ func TestNamingSelectorsMatchRegistrationLabelsWithinContext(t *testing.T) { foreign.Name = "other-namespace" foreign.Namespace = "other-project" foreign.Status.ResolvedAdditionalNames = []dnsv1alpha1.DNSResolvedAdditionalNameRule{{DNSZoneRef: zone, NamePrefix: "foreign"}} - cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(managed, policy, other, stale, foreign).Build() + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver, canonical, custom, policy, other, stale, foreign).Build() + r := &Reconciler{Client: cl, Options: Options{ProjectUID: "project-a"}} + seedReferenceState(t, r, policy, map[string]dnsv1alpha1.DNSObjectReference{"context": {Name: resolver.Name, UID: resolver.UID}, "zone/custom": {Name: custom.Name, UID: custom.UID}}) for _, tc := range []struct { name string labels map[string]string @@ -92,13 +96,13 @@ func TestNamingPolicyValidatesSelectorsAndDuplicateRules(t *testing.T) { ctx := context.Background() now := time.Now() scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer"}} setCondition(&resolver.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 1, now) zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", UID: "zone-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - association := dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Generation: 1}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: zone.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolver.UID}}} + association := dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "association", Namespace: "project", UID: "association-uid", Generation: 1}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}}} setCondition(&association.Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) for _, tc := range []struct { name string @@ -121,6 +125,7 @@ func TestNamingPolicyValidatesSelectorsAndDuplicateRules(t *testing.T) { } cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver, policy).WithStatusSubresource(policy).Build() r := &Reconciler{Client: cl} + seedReferenceState(t, r, &association, map[string]dnsv1alpha1.DNSObjectReference{"context": {Name: resolver.Name, UID: resolver.UID}, "zone": {Name: zone.Name, UID: zone.UID, Generation: zone.Generation}}) if err := r.reconcileNamingPolicies(ctx, []dnsv1alpha1.DNSNamingPolicy{*policy}, []dnsv1alpha1.DNSZoneAssociation{association}, map[string]*dnsv1alpha1.DNSZone{zone.Name: zone}, now); err != nil { t.Fatal(err) } @@ -131,7 +136,8 @@ func TestNamingPolicyValidatesSelectorsAndDuplicateRules(t *testing.T) { if condition == nil || (condition.Status == metav1.ConditionTrue) != tc.valid { t.Fatalf("accepted=%v want %v", condition, tc.valid) } - if policy.Status.ResolvedAdditionalNames[0].DNSZoneRef.UID != zone.UID { + pins, err := r.readReferences(ctx, policy) + if err != nil || pins["zone/"+zone.Name].UID != zone.UID { t.Fatal("selector validation lost the zone lifetime pin") } }) diff --git a/internal/internaldns/controlplane/naming_test.go b/internal/internaldns/controlplane/naming_test.go index 7e21b43a..5cb1aef8 100644 --- a/internal/internaldns/controlplane/naming_test.go +++ b/internal/internaldns/controlplane/naming_test.go @@ -12,13 +12,17 @@ import ( func TestResolveNamingReturnsCanonicalThenAuthorizedAliases(t *testing.T) { s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } - managed := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project"}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ProjectUID: "project-a", ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}}, Status: dnsv1alpha1.DNSManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "canonical", UID: "zone-c"}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} - policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project"}, Spec: dnsv1alpha1.DNSNamingPolicySpec{Priority: 10}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "custom", UID: "zone-x"}, NamePrefix: "instances"}}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} - c := fake.NewClientBuilder().WithScheme(s).WithObjects(managed, policy).Build() - names, err := ResolveNaming(context.Background(), c, "project", "project-a", "vpc-a", nil, "Web 01") + resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "project", UID: "context-a"}, Status: dnsv1alpha1.DNSResolverContextStatus{ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "canonical"}}, Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}}}} + canonical := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "canonical", Namespace: "project", UID: "zone-c"}} + custom := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project", UID: "zone-x"}} + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project", UID: "policy-a"}, Spec: dnsv1alpha1.DNSNamingPolicySpec{Priority: 10}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "custom"}, NamePrefix: "instances"}}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(resolver, canonical, custom, policy).Build() + r := &Reconciler{Client: c, Options: Options{ProjectUID: "project-a"}} + seedReferenceState(t, r, policy, map[string]dnsv1alpha1.DNSObjectReference{"context": {Name: resolver.Name, UID: resolver.UID}, "zone/custom": {Name: custom.Name, UID: custom.UID}}) + names, err := ResolveNaming(context.Background(), c, "project", "project-a", resolver.Name, nil, "Web 01") if err != nil { t.Fatal(err) } diff --git a/internal/internaldns/controlplane/publication_reconciler.go b/internal/internaldns/controlplane/publication_reconciler.go index 9df5ca03..6eac5480 100644 --- a/internal/internaldns/controlplane/publication_reconciler.go +++ b/internal/internaldns/controlplane/publication_reconciler.go @@ -57,6 +57,27 @@ func (r *Reconciler) reconcileProject(ctx context.Context, ns string) (ctrl.Resu if err := r.reconcileNamingPolicies(ctx, policies.Items, assocs.Items, zoneByName, now); err != nil { return ctrl.Result{}, err } + var sources []client.Object + for i := range assocs.Items { + sources = append(sources, &assocs.Items[i]) + } + for i := range policies.Items { + sources = append(sources, &policies.Items[i]) + } + for i := range regs.Items { + registration := ®s.Items[i] + ref := dnsv1alpha1.DNSObjectReference{Name: registration.Spec.DNSZoneRef.Name} + if zone := zoneByName[ref.Name]; zone != nil && zone.DeletionTimestamp.IsZero() && zone.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate { + ref.UID, ref.Generation = zone.UID, zone.Generation + } + if _, _, err := r.pinReferences(ctx, registration, map[string]dnsv1alpha1.DNSObjectReference{"zone": ref}); err != nil { + return ctrl.Result{}, err + } + sources = append(sources, registration) + } + if err := r.collectReferenceStates(ctx, sources); err != nil { + return ctrl.Result{}, err + } if changed, err := r.issueGrants(ctx, grants.Items, regs.Items, now); err != nil || changed { return ctrl.Result{Requeue: changed}, err } @@ -146,7 +167,7 @@ func (r *Reconciler) aggregatePublicationStatus(ctx context.Context, z *dnsv1alp } for i := range regs { reg := ®s[i] - if !refMatches(reg.Spec.DNSZoneRef, z.Name, z.UID, z.Generation) || registrationFences[reg.UID] != reg.Generation { + if reg.Spec.DNSZoneRef.Name != z.Name || registrationFences[reg.UID] != reg.Generation { continue } regUIDs[reg.UID] = true @@ -266,6 +287,18 @@ func (r *Reconciler) compileZone(ctx context.Context, z *dnsv1alpha1.DNSZone, as return false, time.Time{}, err } input := CompileInput{Zone: z, Associations: assocs, Registrations: regs, Grants: grants, Contributions: cs, StaticRecords: records, Previous: previous, Now: now} + input.AssociationReferences, err = r.associationReferences(ctx, assocs) + if err != nil { + return false, time.Time{}, err + } + input.RegistrationZoneReferences = map[types.UID]dnsv1alpha1.DNSObjectReference{} + for i := range regs { + refs, err := r.readReferences(ctx, ®s[i]) + if err != nil { + return false, time.Time{}, err + } + input.RegistrationZoneReferences[regs[i].UID] = refs["zone"] + } tombstone := !z.DeletionTimestamp.IsZero() var payload []byte var ownPlan PublicationPlan @@ -557,7 +590,7 @@ func (r *Reconciler) patchInputStatuses(ctx context.Context, zone *dnsv1alpha1.D relevant := map[types.UID]bool{} for i := range regs { reg := ®s[i] - if !refMatches(reg.Spec.DNSZoneRef, zone.Name, zone.UID, zone.Generation) { + if reg.Spec.DNSZoneRef.Name != zone.Name { continue } relevant[reg.UID] = true @@ -568,6 +601,12 @@ func (r *Reconciler) patchInputStatuses(ctx context.Context, zone *dnsv1alpha1.D } base := reg.DeepCopy() reg.Status.CanonicalFQDN = absoluteOwner(reg.Spec.Name, result.Plan.ZoneApex) + if !accepted { + reg.Status.CanonicalFQDN = "" + reg.Status.PublicationWriterEpoch = 0 + reg.Status.PublicationRevision = 0 + projectstatus.Published(®.Status.Conditions, false, reg.Generation, now) + } reg.Status.ObservedGeneration = reg.Generation setCondition(®.Status.Conditions, "Accepted", boolCondition(accepted), reason, reason, reg.Generation, now) if err := r.Status().Patch(ctx, reg, client.MergeFrom(base)); err != nil && !apierrors.IsNotFound(err) { diff --git a/internal/internaldns/controlplane/publication_reconciler_test.go b/internal/internaldns/controlplane/publication_reconciler_test.go index 25ef84b6..32a81651 100644 --- a/internal/internaldns/controlplane/publication_reconciler_test.go +++ b/internal/internaldns/controlplane/publication_reconciler_test.go @@ -23,7 +23,7 @@ func TestExpiredOwnershipTakeoverFencesResumedWriter(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", Namespace: "project", UID: "zone-a"}} @@ -73,7 +73,7 @@ func TestQuiescentSurvivorRenewsOwnershipBeyondLeaseWindow(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } network := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "project", UID: "vpc-a"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-a"}} @@ -152,7 +152,7 @@ func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } payload, err := json.Marshal(model.PublicationPlan{ZoneUID: "zone-a", Apex: "corp.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contribution-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 19}}, RRSets: []model.RRSet{}}) @@ -209,7 +209,7 @@ func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { func TestLoadPreviousFailsWhenActiveChunkIsMissing(t *testing.T) { ctx := context.Background() s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } manifest := &dnsv1alpha1.DNSPublicationManifest{ @@ -230,7 +230,7 @@ func TestLoadPreviousFailsWhenActiveChunkIsMissing(t *testing.T) { func TestLoadPreviousRejectsHashValidMalformedPayload(t *testing.T) { ctx := context.Background() s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } payload := []byte(`{"zoneUID":`) @@ -249,7 +249,7 @@ func TestLoadPreviousRejectsHashValidMalformedPayload(t *testing.T) { func TestEmptyPublicationUsesJSONArraysForRequiredSlices(t *testing.T) { ctx := context.Background() s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } c := fake.NewClientBuilder().WithScheme(s).Build() @@ -275,7 +275,7 @@ func TestReconcilerPersistsFencedMultiRegionOutbox(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } network := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "project", UID: "vpc-a"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-a"}} diff --git a/internal/internaldns/controlplane/publication_regression_test.go b/internal/internaldns/controlplane/publication_regression_test.go index fc013e93..819f6bc5 100644 --- a/internal/internaldns/controlplane/publication_regression_test.go +++ b/internal/internaldns/controlplane/publication_regression_test.go @@ -57,7 +57,7 @@ func TestCommittedUntargetedTombstoneCompletes(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } cl := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&dnsv1alpha1.DNSPublicationManifest{}).Build() @@ -90,7 +90,7 @@ func TestCommittedUntargetedTombstoneCompletes(t *testing.T) { func TestTerminatingAuthorityDoesNotIssueOrBindWriterEpochs(t *testing.T) { now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } for _, terminating := range []string{"registration", "grant"} { diff --git a/internal/internaldns/controlplane/reconciler.go b/internal/internaldns/controlplane/reconciler.go index 25930e98..a0ac2e2f 100644 --- a/internal/internaldns/controlplane/reconciler.go +++ b/internal/internaldns/controlplane/reconciler.go @@ -48,7 +48,7 @@ type AddressAllocator interface { func (o *ReconcilerOptions) defaults() { if o.PlatformNamespace == "" { - o.PlatformNamespace = "internal-dns-system" + o.PlatformNamespace = defaultPlatformNamespace } if o.Region == "" { o.Region = "default" diff --git a/internal/internaldns/controlplane/reference_state.go b/internal/internaldns/controlplane/reference_state.go new file mode 100644 index 00000000..56a44ba0 --- /dev/null +++ b/internal/internaldns/controlplane/reference_state.go @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const referenceStateLabel = "internal-dns.miloapis.com/reference-state" +const referenceProjectLabel = "internal-dns.miloapis.com/project-uid" +const referenceStateDataKey = "references.json" +const defaultPlatformNamespace = "internal-dns-system" + +type referenceState struct { + SourceNamespace string `json:"sourceNamespace"` + SourceKind string `json:"sourceKind"` + Source dnsv1.DNSObjectReference `json:"source"` + References map[string]dnsv1.DNSObjectReference `json:"references"` +} + +func referenceKind(source client.Object) string { + switch source.(type) { + case *dnsv1.DNSZoneAssociation: + return "DNSZoneAssociation" + case *dnsv1.DNSNamingPolicy: + return "DNSNamingPolicy" + case *dnsv1.DNSRegistration: + return "DNSRegistration" + default: + return "" + } +} + +func (r *Reconciler) referenceStateObject(source client.Object) *corev1.ConfigMap { + namespace := r.Options.PlatformNamespace + if namespace == "" { + namespace = defaultPlatformNamespace + } + identity := fmt.Sprintf("%s/%s/%s/%s/%s", r.Options.ProjectUID, source.GetNamespace(), referenceKind(source), source.GetName(), source.GetUID()) + return &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, Name: "references-" + model.OpaqueToken(identity), + Labels: map[string]string{referenceStateLabel: "true", referenceProjectLabel: string(r.Options.ProjectUID)}, + }} +} + +func (r *Reconciler) readReferences(ctx context.Context, source client.Object) (map[string]dnsv1.DNSObjectReference, error) { + _, state, err := r.loadReferenceState(ctx, source) + return state.References, err +} + +func (r *Reconciler) loadReferenceState(ctx context.Context, source client.Object) (*corev1.ConfigMap, referenceState, error) { + object := r.referenceStateObject(source) + state := referenceState{SourceKind: referenceKind(source), SourceNamespace: source.GetNamespace(), Source: dnsv1.DNSObjectReference{Name: source.GetName(), UID: source.GetUID()}, References: map[string]dnsv1.DNSObjectReference{}} + if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + return object, state, client.IgnoreNotFound(err) + } + if err := json.Unmarshal([]byte(object.Data[referenceStateDataKey]), &state); err != nil { + return object, state, fmt.Errorf("decode DNS reference state: %w", err) + } + if state.SourceKind != referenceKind(source) || state.SourceNamespace != source.GetNamespace() || state.Source.Name != source.GetName() || state.Source.UID != source.GetUID() { + return object, state, fmt.Errorf("DNS reference state source does not match") + } + return object, state, nil +} + +func (r *Reconciler) pinReferences(ctx context.Context, source client.Object, selected map[string]dnsv1.DNSObjectReference) (map[string]dnsv1.DNSObjectReference, bool, error) { + object, state, err := r.loadReferenceState(ctx, source) + if err != nil { + return nil, false, err + } + current := true + resolved := map[string]dnsv1.DNSObjectReference{} + for key, ref := range selected { + pin, valid := pinNamedReference(ref.Name, state.References[key], ref) + resolved[key] = pin + current = current && valid + } + if reflect.DeepEqual(state.References, resolved) { + return resolved, current, nil + } + state.References = resolved + payload, err := json.Marshal(state) + if err != nil { + return nil, false, err + } + if object.ResourceVersion == "" { + object.Data = map[string]string{referenceStateDataKey: string(payload)} + err = r.Create(ctx, object) + } else { + base := object.DeepCopy() + object.Data[referenceStateDataKey] = string(payload) + err = r.Patch(ctx, object, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) + } + return resolved, current, err +} + +func (r *Reconciler) collectReferenceStates(ctx context.Context, sources []client.Object) error { + live := map[string]bool{} + for _, source := range sources { + live[r.referenceStateObject(source).Name] = true + } + var states corev1.ConfigMapList + namespace := r.Options.PlatformNamespace + if namespace == "" { + namespace = defaultPlatformNamespace + } + if err := r.List(ctx, &states, client.InNamespace(namespace), client.MatchingLabels{referenceStateLabel: "true", referenceProjectLabel: string(r.Options.ProjectUID)}); err != nil { + return err + } + for i := range states.Items { + if !live[states.Items[i].Name] { + var state referenceState + if err := json.Unmarshal([]byte(states.Items[i].Data[referenceStateDataKey]), &state); err != nil { + return err + } + var source client.Object + switch state.SourceKind { + case "DNSZoneAssociation": + source = &dnsv1.DNSZoneAssociation{} + case "DNSNamingPolicy": + source = &dnsv1.DNSNamingPolicy{} + case "DNSRegistration": + source = &dnsv1.DNSRegistration{} + default: + return fmt.Errorf("unknown DNS reference source kind %q", state.SourceKind) + } + err := r.Get(ctx, client.ObjectKey{Namespace: state.SourceNamespace, Name: state.Source.Name}, source) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && source.GetUID() == state.Source.UID { + continue + } + if err := r.Delete(ctx, &states.Items[i], client.Preconditions{UID: &states.Items[i].UID}); client.IgnoreNotFound(err) != nil { + return err + } + } + } + return nil +} + +type associationReferences struct { + Zone dnsv1.DNSObjectReference + Context dnsv1.DNSObjectReference +} + +func (r *Reconciler) associationReferences(ctx context.Context, associations []dnsv1.DNSZoneAssociation) (map[types.UID]associationReferences, error) { + result := map[types.UID]associationReferences{} + for i := range associations { + refs, err := r.readReferences(ctx, &associations[i]) + if err != nil { + return nil, err + } + result[associations[i].UID] = associationReferences{Zone: refs["zone"], Context: refs["context"]} + } + return result, nil +} diff --git a/internal/internaldns/controlplane/reference_state_test.go b/internal/internaldns/controlplane/reference_state_test.go new file mode 100644 index 00000000..0709106d --- /dev/null +++ b/internal/internaldns/controlplane/reference_state_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "testing" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func addReferenceStateToScheme(scheme *runtime.Scheme) error { + if err := corev1.AddToScheme(scheme); err != nil { + return err + } + return dnsv1alpha1.AddToScheme(scheme) +} + +func seedReferenceState(t *testing.T, r *Reconciler, source client.Object, selected map[string]dnsv1alpha1.DNSObjectReference) { + t.Helper() + if _, valid, err := r.pinReferences(context.Background(), source, selected); err != nil || !valid { + t.Fatalf("reference state: valid=%t err=%v", valid, err) + } +} + +func TestReferenceStateIsPrivateScopedAndCollected(t *testing.T) { + scheme := runtime.NewScheme() + if err := addReferenceStateToScheme(scheme); err != nil { + t.Fatal(err) + } + ctx := context.Background() + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &Reconciler{Client: cl, Options: Options{ProjectUID: "project-a", PlatformNamespace: "platform"}} + other := &Reconciler{Client: cl, Options: Options{ProjectUID: "project-b", PlatformNamespace: "platform"}} + source := &dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "project", UID: "registration-original"}} + selected := map[string]dnsv1alpha1.DNSObjectReference{"zone": {Name: "private", UID: "zone-original", Generation: 1}} + if _, valid, err := r.pinReferences(ctx, source, selected); err != nil || !valid { + t.Fatalf("initial reference: valid=%t err=%v", valid, err) + } + if _, valid, err := other.pinReferences(ctx, source, selected); err != nil || !valid { + t.Fatalf("other project reference: valid=%t err=%v", valid, err) + } + selected["zone"] = dnsv1alpha1.DNSObjectReference{Name: "private", UID: "zone-replacement", Generation: 1} + pins, valid, err := r.pinReferences(ctx, source, selected) + if err != nil || valid || pins["zone"].UID != "zone-original" { + t.Fatalf("replacement transferred a resolved lifetime: pins=%v valid=%t err=%v", pins, valid, err) + } + var public corev1.ConfigMapList + if err := cl.List(ctx, &public, client.InNamespace(source.Namespace)); err != nil || len(public.Items) != 0 { + t.Fatalf("reference state appeared in the project: count=%d err=%v", len(public.Items), err) + } + if err := r.collectReferenceStates(ctx, nil); err != nil { + t.Fatal(err) + } + var states corev1.ConfigMapList + if err := cl.List(ctx, &states, client.InNamespace("platform")); err != nil || len(states.Items) != 1 || states.Items[0].Labels[referenceProjectLabel] != "project-b" { + t.Fatalf("cleanup removed another project's state: states=%v err=%v", states.Items, err) + } + if err := cl.Create(ctx, source); err != nil { + t.Fatal(err) + } + if err := other.collectReferenceStates(ctx, nil); err != nil { + t.Fatal(err) + } + if err := cl.List(ctx, &states, client.InNamespace("platform")); err != nil || len(states.Items) != 1 { + t.Fatalf("collector removed a live source absent from the list snapshot: count=%d err=%v", len(states.Items), err) + } + if err := cl.Delete(ctx, source); err != nil { + t.Fatal(err) + } + if err := other.collectReferenceStates(ctx, nil); err != nil { + t.Fatal(err) + } + if err := cl.List(ctx, &states, client.InNamespace("platform")); err != nil || len(states.Items) != 0 { + t.Fatalf("deleted source retained private state: count=%d err=%v", len(states.Items), err) + } +} diff --git a/internal/internaldns/model/model.go b/internal/internaldns/model/model.go index 039ac1c9..3dc5b478 100644 --- a/internal/internaldns/model/model.go +++ b/internal/internaldns/model/model.go @@ -89,6 +89,7 @@ type Backend struct { } type ZoneAttachment struct { + Pending bool `json:"pending,omitempty"` ZoneUID string `json:"zoneUID"` Apex string `json:"apex"` RequiredPublicationEpoch uint64 `json:"requiredPublicationEpoch"` diff --git a/internal/internaldns/platform/acks.go b/internal/internaldns/platform/acks.go index 321db0cc..f7fdb945 100644 --- a/internal/internaldns/platform/acks.go +++ b/internal/internaldns/platform/acks.go @@ -20,6 +20,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const dnsValueNode = "node" + // AckSink relies on NATS subject ACLs to bind each authenticated serving member // to its own AckSubject. It also verifies membership, region, resource lifetime, // revision and freshness before changing project-facing publication status. @@ -93,7 +95,7 @@ func (s *AckSink) Apply(ctx context.Context, subject string, env model.Envelope) } return s.publication(ctx, a, row, now) case model.KindServingSnapshot: - if member.Role != "node" && member.Role != dnsValueCluster && member.Role != dnsValueResolver && member.Role != dnsValueRegional { + if member.Role != dnsValueNode && member.Role != dnsValueCluster && member.Role != dnsValueResolver && member.Role != dnsValueRegional { return fmt.Errorf("serving ack is not from a resolver member") } return s.serving(ctx, a, row, now) @@ -169,7 +171,7 @@ func (s *AckSink) serving(ctx context.Context, a model.MemberAck, row dnsv1alpha } else { binding.Status.Phase = dnsValuePending } - condition(&binding.Status.Conditions, dnsValueServing, complete, "MemberVerification", binding.Generation, now) + s.bindingConditions(binding, e, snapshot, now, complete) if err := s.Client.Status().Patch(ctx, binding, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { return err } @@ -263,7 +265,7 @@ func (s *AckSink) projectPublicationForScope(ctx context.Context, project Projec if registrationGenerations[string(r.UID)] != uint64(r.Generation) { continue } - if r.Spec.DNSZoneRef.Name != z.Name || r.Spec.DNSZoneRef.UID != "" && r.Spec.DNSZoneRef.UID != z.UID { + if r.Spec.DNSZoneRef.Name != z.Name { continue } available := false @@ -339,7 +341,7 @@ func (s *AckSink) verified(rows []dnsv1alpha1.DNSApplyAcknowledgement, now time. for _, member := range s.Members { match := member.Role == role if role == dnsValueResolver { - match = member.Role == "node" || member.Role == dnsValueCluster || member.Role == dnsValueResolver || member.Role == dnsValueRegional + match = member.Role == dnsValueNode || member.Role == dnsValueCluster || member.Role == dnsValueResolver || member.Role == dnsValueRegional } if !match { continue @@ -400,7 +402,7 @@ func (s *AckSink) Refresh(ctx context.Context) error { b := &bindings.Items[i] base := b.DeepCopy() complete := s.bindingVerified(b, envelope, snapshot, now) - condition(&b.Status.Conditions, dnsValueServing, complete, "MemberVerification", b.Generation, now) + s.bindingConditions(b, envelope, snapshot, now, complete) if complete { b.Status.Phase = dnsValueServing } else { @@ -524,6 +526,10 @@ func LoadCommittedServingSnapshot(ctx context.Context, c client.Client, namespac } func (s *AckSink) bindingVerified(b *dnsv1alpha1.DNSResolverBinding, envelope *model.Envelope, snapshot *model.ServingSnapshot, now time.Time) bool { + return s.bindingProof(b, envelope, snapshot, now, false) +} + +func (s *AckSink) bindingProof(b *dnsv1alpha1.DNSResolverBinding, envelope *model.Envelope, snapshot *model.ServingSnapshot, now time.Time, all bool) bool { if envelope == nil || snapshot == nil || b.Spec.Tombstone { return false } @@ -543,5 +549,38 @@ func (s *AckSink) bindingVerified(b *dnsv1alpha1.DNSResolverBinding, envelope *m rows = append(rows, row) } } - return s.verified(rows, now, dnsValueResolver, b.Spec.Configuration.Revision, int64(envelope.Epoch), b.Spec.Authorization.WriterEpoch, b.Spec.Authorization.Sequence) + if all { + return s.verified(rows, now, dnsValueResolver, b.Spec.Configuration.Revision, int64(envelope.Epoch), b.Spec.Authorization.WriterEpoch, b.Spec.Authorization.Sequence) + } + tiers := map[string][]model.Member{} + for _, member := range s.Members { + role := member.Role + if role == dnsValueCluster { + role = dnsValueRegional + } + if role == dnsValueNode || role == dnsValueResolver || role == dnsValueRegional { + tiers[role] = append(tiers[role], member) + } + } + for _, members := range tiers { + ready := false + for _, member := range members { + scoped := *s + scoped.Members = []model.Member{member} + ready = ready || scoped.verified(rows, now, dnsValueResolver, b.Spec.Configuration.Revision, int64(envelope.Epoch), b.Spec.Authorization.WriterEpoch, b.Spec.Authorization.Sequence) + } + if !ready { + return false + } + } + return len(tiers) > 0 +} + +func (s *AckSink) bindingConditions(b *dnsv1alpha1.DNSResolverBinding, envelope *model.Envelope, snapshot *model.ServingSnapshot, now time.Time, available bool) { + condition(&b.Status.Conditions, "FullyReplicated", s.bindingProof(b, envelope, snapshot, now, true), "ReplicaVerification", b.Generation, now) + condition(&b.Status.Conditions, "ObservationHealthy", available, "MemberObservation", b.Generation, now) + condition(&b.Status.Conditions, dnsValueServing, available, "MemberVerification", b.Generation, now) + if !available && !b.Spec.Tombstone && b.Status.ObservedConfigurationRevision == b.Spec.Configuration.Revision { + apimeta.SetStatusCondition(&b.Status.Conditions, metav1.Condition{Type: dnsValueServing, Status: metav1.ConditionUnknown, Reason: "ObservationUnavailable", Message: "Current serving health is unavailable", ObservedGeneration: b.Generation, LastTransitionTime: metav1.NewTime(now)}) + } } diff --git a/internal/internaldns/platform/allocator.go b/internal/internaldns/platform/allocator.go index b9ebbaf4..202d5bd8 100644 --- a/internal/internaldns/platform/allocator.go +++ b/internal/internaldns/platform/allocator.go @@ -10,6 +10,7 @@ import ( "net/netip" "time" + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" "go.miloapis.com/dns-operator/internal/internaldns/model" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -46,13 +47,13 @@ func (a *Allocator) Allocate(ctx context.Context, projectUID, contextUID types.U // destination across every project sharing this platform control plane. Claims // are retained after revocation so a stale route cannot become valid for a new // context lifetime. -func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID types.UID, region, address string, port int32) error { +func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, contextUID types.UID, region, address string, port int32) error { parsed, err := netip.ParseAddr(address) - if err != nil || parsed.IsUnspecified() || projectUID == "" || accessUID == "" || region == "" || port != 53 { + if err != nil || parsed.IsUnspecified() || projectUID == "" || contextUID == "" || region == "" || port != 53 { return fmt.Errorf("destination claim identity, region, IP, and DNS port are required") } address = parsed.String() - owner := string(projectUID) + "/" + string(accessUID) + owner := string(projectUID) + "/" + string(contextUID) claim := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dns-destination-claim-" + model.OpaqueToken(region+"/"+address+"/"+fmt.Sprint(port)), Namespace: a.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"owner": owner, "region": region, "address": address, "port": fmt.Sprint(port)}} if err := a.Client.Create(ctx, claim); apierrors.IsAlreadyExists(err) { var existing corev1.ConfigMap @@ -60,6 +61,21 @@ func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID return err } if existing.Data["owner"] != owner || existing.Data["region"] != region || existing.Data["address"] != address || existing.Data["port"] != fmt.Sprint(port) { + var bindings dnsv1alpha1.DNSResolverBindingList + if err := a.Client.List(ctx, &bindings, client.InNamespace(a.Namespace)); err != nil { + return err + } + for _, binding := range bindings.Items { + b := binding.Spec + if b.Source.ProjectUID == projectUID && b.Source.ResolverContextRef.UID == contextUID && + b.Source.AccessBindingRef.UID != "" && existing.Data["owner"] == string(projectUID)+"/"+string(b.Source.AccessBindingRef.UID) && + b.Placement.Region == region && b.Configuration.Listeners.Node.Address == address && b.Configuration.Listeners.Node.Port == port && + existing.Data["region"] == region && existing.Data["address"] == address && existing.Data["port"] == fmt.Sprint(port) { + base := existing.DeepCopy() + existing.Data["owner"] = owner + return a.Client.Patch(ctx, &existing, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) + } + } return fmt.Errorf("service destination is already reserved") } return nil diff --git a/internal/internaldns/platform/allocator_test.go b/internal/internaldns/platform/allocator_test.go index bb0c5e16..abe108d1 100644 --- a/internal/internaldns/platform/allocator_test.go +++ b/internal/internaldns/platform/allocator_test.go @@ -76,3 +76,49 @@ func TestAddressClaimsAreConcurrentStableAndLifetimeScoped(t *testing.T) { t.Fatal("new resource lifetime reused the old address") } } + +func TestDestinationClaimBelongsToContextLifetime(t *testing.T) { + ctx := context.Background() + cl := testClient(t) + allocator := Allocator{Client: cl, Namespace: "platform"} + for range 2 { + if err := allocator.ClaimDestination(ctx, "project", "context", "east", "fd70::53", 53); err != nil { + t.Fatal(err) + } + } + if err := allocator.ClaimDestination(ctx, "project", "replacement-context", "east", "fd70::53", 53); err == nil { + t.Fatal("a replacement context reused a quarantined destination") + } + if err := allocator.ClaimDestination(ctx, "another-project", "context", "east", "fd70::53", 53); err == nil { + t.Fatal("another project reused a destination") + } +} + +func TestDestinationClaimConversionRequiresPinnedBinding(t *testing.T) { + ctx := context.Background() + cl := testClient(t) + allocator := Allocator{Client: cl, Namespace: "platform"} + if err := allocator.ClaimDestination(ctx, "project", "access", "east", "fd70::53", 53); err != nil { + t.Fatal(err) + } + if err := allocator.ClaimDestination(ctx, "project", "context", "east", "fd70::53", 53); err == nil { + t.Fatal("claim changed ownership without lifetime proof") + } + binding := &dnsv1alpha1.DNSResolverBinding{} + binding.Namespace, binding.Name = "platform", "binding" + binding.Spec.Source.ProjectUID = "project" + binding.Spec.Source.ResolverContextRef.UID = "context" + binding.Spec.Source.AccessBindingRef.UID = "access" + binding.Spec.Placement.Region = "east" + binding.Spec.Configuration.Listeners.Node.Address = "fd70::53" + binding.Spec.Configuration.Listeners.Node.Port = 53 + if err := cl.Create(ctx, binding); err != nil { + t.Fatal(err) + } + if err := allocator.ClaimDestination(ctx, "project", "context", "east", "fd70::53", 53); err != nil { + t.Fatal(err) + } + if err := allocator.ClaimDestination(ctx, "project", "access", "east", "fd70::53", 53); err == nil { + t.Fatal("converted claim returned to access lifetime") + } +} diff --git a/internal/internaldns/platform/capacity.go b/internal/internaldns/platform/capacity.go new file mode 100644 index 00000000..632fcc87 --- /dev/null +++ b/internal/internaldns/platform/capacity.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package platform + +import ( + "context" + "encoding/json" + "sort" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func fitSnapshot(snapshot model.ServingSnapshot, prior model.ServingSnapshot, maxBindings, maxBytes int) (model.ServingSnapshot, map[string]bool) { + previous := map[string]model.Binding{} + for _, binding := range prior.Bindings { + previous[binding.BindingUID] = binding + } + candidates := append([]model.Binding(nil), snapshot.Bindings...) + sort.Slice(candidates, func(i, j int) bool { + _, first := previous[candidates[i].BindingUID] + _, second := previous[candidates[j].BindingUID] + if first != second { + return first + } + return candidates[i].BindingUID < candidates[j].BindingUID + }) + snapshot.Bindings = nil + rejected := map[string]bool{} + fits := func(binding model.Binding) bool { + if len(snapshot.Bindings) >= maxBindings { + return false + } + candidate := snapshot + candidate.Bindings = append(append([]model.Binding(nil), snapshot.Bindings...), binding) + envelope, err := model.NewEnvelope(model.KindServingSnapshot, "dns-serving-"+model.OpaqueToken(candidate.Region+"/"+candidate.Shard)+"-e18446744073709551615-r18446744073709551615", candidate.Region, candidate.Shard, "00000000-0000-0000-0000-000000000000", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, time.Now(), candidate) + if err != nil { + return false + } + payload, err := json.Marshal(envelope) + return err == nil && len(payload) <= maxBytes + } + for _, binding := range candidates { + if fits(binding) { + snapshot.Bindings = append(snapshot.Bindings, binding) + continue + } + rejected[binding.BindingUID] = true + if old, ok := previous[binding.BindingUID]; ok { + if safeExpansion(old, binding) && fits(old) { + snapshot.Bindings = append(snapshot.Bindings, old) + continue + } + binding.ConfigurationPending = true + binding.Zones = nil + if fits(binding) { + snapshot.Bindings = append(snapshot.Bindings, binding) + } + } + } + sort.Slice(snapshot.Bindings, func(i, j int) bool { return snapshot.Bindings[i].BindingUID < snapshot.Bindings[j].BindingUID }) + return snapshot, rejected +} + +func safeExpansion(old, candidate model.Binding) bool { + if old.Tombstone || candidate.Tombstone || old.ConfigurationPending || candidate.ConfigurationPending || old.Authorization != candidate.Authorization || old.ProjectUID != candidate.ProjectUID || old.ContextUID != candidate.ContextUID || old.ConsumerAddress != candidate.ConsumerAddress || old.ClusterAddress != candidate.ClusterAddress { + return false + } + for _, zone := range old.Zones { + found := false + for _, next := range candidate.Zones { + if zone.ZoneUID == next.ZoneUID && zone.Apex == next.Apex && !next.Pending { + found = true + } + } + if !found { + return false + } + } + return true +} + +func (p *Planner) reportCapacity(ctx context.Context, bindings []dnsv1alpha1.DNSResolverBinding, rejected map[string]bool, now time.Time) error { + for i := range bindings { + binding := &bindings[i] + if binding.Spec.Placement.Region != p.Config.Region || binding.Spec.Placement.Shard != p.Config.Shard { + continue + } + base := binding.DeepCopy() + condition(&binding.Status.Conditions, "CapacityAvailable", !rejected[string(binding.UID)], "ShardCapacity", binding.Generation, now) + if !jsonEqualStatus(base.Status, binding.Status) { + if err := p.Client.Status().Patch(ctx, binding, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/internaldns/platform/capacity_test.go b/internal/internaldns/platform/capacity_test.go new file mode 100644 index 00000000..1ab8ef18 --- /dev/null +++ b/internal/internaldns/platform/capacity_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package platform + +import ( + "context" + "fmt" + "testing" + "time" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func capacityBinding(index int) model.Binding { + return model.Binding{BindingUID: fmt.Sprintf("binding-%d", index), ProjectUID: "project", ContextUID: fmt.Sprintf("context-%d", index), BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: fmt.Sprintf("fd53::%d", index), ClusterAddress: fmt.Sprintf("fd54::%d", index), Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1}} +} + +func TestCapacityOverflowDoesNotBlockWithdrawal(t *testing.T) { + prior := model.ServingSnapshot{Region: "east", Shard: "a", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: time.Now(), Bindings: []model.Binding{capacityBinding(1), capacityBinding(2)}} + candidate := prior + candidate.ConfigurationRevision++ + candidate.Bindings = []model.Binding{capacityBinding(2), capacityBinding(3), capacityBinding(4)} + bounded, rejected := fitSnapshot(candidate, prior, 2, 512*1024) + if len(bounded.Bindings) != 2 || len(rejected) != 1 { + t.Fatalf("capacity result: %#v, rejected %#v", bounded.Bindings, rejected) + } + for _, binding := range bounded.Bindings { + if binding.BindingUID == "binding-1" { + t.Fatal("overflow prevented withdrawal") + } + } + if bounded.Bindings[0].BindingUID != "binding-2" { + t.Fatal("new allocations displaced an existing context") + } +} + +func TestByteOverflowPreservesExistingContext(t *testing.T) { + prior := model.ServingSnapshot{Region: "east", Shard: "a", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: time.Now(), Bindings: []model.Binding{capacityBinding(1)}} + candidate := prior + binding := capacityBinding(1) + for i := range 256 { + binding.Zones = append(binding.Zones, model.ZoneAttachment{ZoneUID: fmt.Sprintf("zone-%d", i), Apex: fmt.Sprintf("zone-%d.internal", i), RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + } + binding.ConfigurationRevision++ + candidate.Bindings = []model.Binding{binding, capacityBinding(2)} + bounded, rejected := fitSnapshot(candidate, prior, 2, 4096) + if !rejected["binding-1"] || len(bounded.Bindings) != 2 || bounded.Bindings[0].ConfigurationRevision != 1 { + t.Fatalf("overflow interrupted an established context: %#v", bounded) + } +} + +func TestCapacityReportDoesNotOverwriteAnotherShard(t *testing.T) { + ctx := context.Background() + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: "binding", Namespace: "platform", UID: "binding-uid"}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: "west", Shard: "b"}}, Status: dnsv1alpha1.DNSResolverBindingStatus{Conditions: []metav1.Condition{{Type: "CapacityAvailable", Status: metav1.ConditionFalse, Reason: "ShardCapacity", LastTransitionTime: metav1.Now()}}}} + cl := testClient(t, binding) + planner := Planner{Client: cl, Config: PlannerConfig{Region: "east", Shard: "a"}} + if err := planner.reportCapacity(ctx, []dnsv1alpha1.DNSResolverBinding{*binding}, nil, time.Now()); err != nil { + t.Fatal(err) + } + var updated dnsv1alpha1.DNSResolverBinding + if err := cl.Get(ctx, client.ObjectKeyFromObject(binding), &updated); err != nil { + t.Fatal(err) + } + if len(updated.Status.Conditions) != 1 || updated.Status.Conditions[0].Status != metav1.ConditionFalse { + t.Fatal("capacity from another shard was overwritten") + } +} diff --git a/internal/internaldns/platform/planner.go b/internal/internaldns/platform/planner.go index e79cca26..2a8197d5 100644 --- a/internal/internaldns/platform/planner.go +++ b/internal/internaldns/platform/planner.go @@ -136,7 +136,6 @@ func (p *Planner) Step(ctx context.Context) error { for _, b := range prior.Bindings { priorBindings[b.BindingUID] = b } - dependencies := map[string]bool{} for _, obj := range bindings.Items { b := obj.Spec if b.Placement.Region != c.Region || b.Placement.Shard != c.Shard || b.Tombstone || !obj.DeletionTimestamp.IsZero() { @@ -150,17 +149,24 @@ func (p *Planner) Step(ctx context.Context) error { view.Transports = append(view.Transports, model.Transport(t)) } complete := true - bindingDependencies := map[string]bool{} for _, zoneRef := range b.Configuration.ZoneRefs { uid := zoneRef.UID + apex := b.Configuration.ZoneApices[string(uid)] + pending := func() { + if apex == "" { + complete = false + return + } + view.Zones = append(view.Zones, model.ZoneAttachment{Pending: true, ZoneUID: string(uid), Apex: apex, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + } o, ok := byZone[string(uid)] if !ok || o.Spec.ActiveManifestName == "" { - complete = false + pending() continue } var m dnsv1alpha1.DNSPublicationManifest if err := p.Client.Get(ctx, client.ObjectKey{Namespace: c.Namespace, Name: o.Spec.ActiveManifestName}, &m); apierrors.IsNotFound(err) { - complete = false + pending() continue } else if err != nil { return err @@ -173,22 +179,20 @@ func (p *Planner) Step(ctx context.Context) error { } } if m.Spec.ZoneRef.UID != uid || m.Spec.Tombstone || !targeted { - complete = false + pending() continue } attachment := model.ZoneAttachment{ZoneUID: string(uid), Apex: m.Spec.ZoneApex, RequiredPublicationEpoch: uint64(m.Spec.WriterEpoch), RequiredPublicationRevision: uint64(m.Spec.Revision)} // Record updates do not change a view's minimum activation dependency. if old, ok := priorBindings[view.BindingUID]; ok && old.BindingGeneration == view.BindingGeneration { for _, z := range old.Zones { - if z.ZoneUID == attachment.ZoneUID && z.Apex == attachment.Apex { + if !z.Pending && z.ZoneUID == attachment.ZoneUID && z.Apex == attachment.Apex { attachment.RequiredPublicationEpoch = z.RequiredPublicationEpoch attachment.RequiredPublicationRevision = z.RequiredPublicationRevision } } } - activationName := model.PublicationActivationName(m.Name, c.Region, c.Shard) view.Zones = append(view.Zones, attachment) - bindingDependencies[activationName] = true } if !complete { // Missing configuration is not a withdrawal of source authorization. @@ -196,10 +200,6 @@ func (p *Planner) Step(ctx context.Context) error { // pending snapshot can publish without waiting on this tenant's outbox. view.ConfigurationPending = true view.Zones = nil - } else { - for name := range bindingDependencies { - dependencies[name] = true - } } sort.Slice(view.Zones, func(i, j int) bool { return view.Zones[i].ZoneUID < view.Zones[j].ZoneUID }) snapshot.Bindings = append(snapshot.Bindings, view) @@ -210,8 +210,14 @@ func (p *Planner) Step(ctx context.Context) error { if maxBindings <= 0 { maxBindings = 256 } - if len(snapshot.Bindings) > maxBindings { - return fmt.Errorf("shard %s/%s exceeds binding budget: %d > %d; no partial snapshot will be exported", c.Region, c.Shard, len(snapshot.Bindings), maxBindings) + maxBytes := c.MaxSnapshotBytes + if maxBytes <= 0 { + maxBytes = 512 * 1024 + } + var rejected map[string]bool + snapshot, rejected = fitSnapshot(snapshot, prior, maxBindings, maxBytes) + if err := p.reportCapacity(ctx, bindings.Items, rejected, now); err != nil { + return err } if err := snapshot.Validate(); err != nil { return err @@ -251,19 +257,10 @@ func (p *Planner) Step(ctx context.Context) error { return err } payload, _ := json.Marshal(env) - maxBytes := c.MaxSnapshotBytes - if maxBytes <= 0 { - maxBytes = 512 * 1024 - } if len(payload) > maxBytes { return fmt.Errorf("shard serving envelope exceeds configured transport budget: %d > %d", len(payload), maxBytes) } - deps := make([]string, 0, len(dependencies)) - for name := range dependencies { - deps = append(deps, name) - } - sort.Strings(deps) - out := &dnsv1alpha1.DNSTransportOutbox{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: c.Namespace, Labels: map[string]string{shardOwnerLabel: owner.Name, "internal-dns.miloapis.com/region": model.SafeToken(c.Region), "internal-dns.miloapis.com/shard": model.SafeToken(c.Shard)}}, Spec: dnsv1alpha1.DNSTransportOutboxSpec{Subject: model.ServingSubject(c.Region, c.Shard), ResourceUID: owner.UID, WriterEpoch: int64(state.Epoch), Revision: int64(revision), PayloadHash: model.Hash(payload), Payload: payload, Activation: true, DependsOn: deps}} + out := &dnsv1alpha1.DNSTransportOutbox{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: c.Namespace, Labels: map[string]string{shardOwnerLabel: owner.Name, "internal-dns.miloapis.com/region": model.SafeToken(c.Region), "internal-dns.miloapis.com/shard": model.SafeToken(c.Shard)}}, Spec: dnsv1alpha1.DNSTransportOutboxSpec{Subject: model.ServingSubject(c.Region, c.Shard), ResourceUID: owner.UID, WriterEpoch: int64(state.Epoch), Revision: int64(revision), PayloadHash: model.Hash(payload), Payload: payload, Activation: true, DependsOn: nil}} if err := p.Client.Create(ctx, out); apierrors.IsAlreadyExists(err) { var existing dnsv1alpha1.DNSTransportOutbox if err := p.Client.Get(ctx, client.ObjectKeyFromObject(out), &existing); err != nil { diff --git a/internal/internaldns/platform/platform_test.go b/internal/internaldns/platform/platform_test.go index 1adaec72..3471f181 100644 --- a/internal/internaldns/platform/platform_test.go +++ b/internal/internaldns/platform/platform_test.go @@ -52,7 +52,7 @@ func TestProjectPublicationContinuesAfterSourceFailure(t *testing.T) { t.Fatal(err) } zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "healthy", Namespace: "healthy-project", UID: "zone-healthy", Generation: 4}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "healthy.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - registration := &dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: zone.Namespace, UID: "registration-healthy", Generation: 2}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, Name: "api"}} + registration := &dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: zone.Namespace, UID: "registration-healthy", Generation: 2}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api"}} healthy := fake.NewClientBuilder().WithScheme(scheme).WithObjects(zone, registration).WithStatusSubresource(&dnsv1alpha1.DNSZone{}, &dnsv1alpha1.DNSRegistration{}).Build() blocked := &blockingListClient{Client: healthy, started: make(chan struct{})} sink := AckSink{Client: testClient(t, chunk), Namespace: "platform", Projects: []ProjectScope{{Client: blocked, Namespace: "offline-project", RequestTimeout: 200 * time.Millisecond}, {Client: healthy, Namespace: zone.Namespace}}} @@ -162,6 +162,17 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { if pending.Bindings[0].Authorization.Revision != uint64(originalAuthorization.Sequence) { t.Fatal("pending configuration renewed the original authorization") } + changing.Spec.Configuration.ZoneApices = map[string]string{"zone-extra": "apps.internal"} + if err := cl.Update(context.Background(), &changing); err != nil { + t.Fatal(err) + } + if err := p.Step(context.Background()); err != nil { + t.Fatal(err) + } + isolated := latestPlannerSnapshot(t, &p) + if isolated.Bindings[0].ConfigurationPending || len(isolated.Bindings[0].Zones) != 2 || !isolated.Bindings[0].Zones[1].Pending { + t.Fatalf("a known pending apex suppressed the ready zone: %#v", isolated.Bindings[0]) + } for _, obj := range []client.Object{ &dnsv1alpha1.DNSPublicationOwnership{ObjectMeta: metav1.ObjectMeta{Name: "owner-" + model.OpaqueToken("zone-extra")[:20], Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationOwnershipSpec{ZoneUID: "zone-extra", ActiveManifestName: "manifest-extra"}}, &dnsv1alpha1.DNSPublicationManifest{ObjectMeta: metav1.ObjectMeta{Name: "manifest-extra", Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationManifestSpec{ZoneRef: dnsv1alpha1.DNSObjectReference{UID: "zone-extra"}, ZoneApex: "apps.internal", WriterEpoch: 1, Revision: 1, ServingTargets: []dnsv1alpha1.DNSPublicationTarget{{Region: region, Shard: shard}}}}, @@ -189,7 +200,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { t.Fatal(err) } _ = cl.List(context.Background(), &out) - if len(out.Items) != 4 { + if len(out.Items) != 5 { t.Fatal("regional export removal did not change the shared snapshot") } var updatedOwner corev1.ConfigMap @@ -212,7 +223,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { t.Fatal(err) } _ = cl.List(context.Background(), &out) - if len(out.Items) != 4 { + if len(out.Items) != 5 { t.Fatal("another holder published during a current lease") } } @@ -517,3 +528,30 @@ func TestZeroTargetTombstoneNeedsNoReplicaACK(t *testing.T) { t.Fatal("prior-target withdrawal must await its ACKs") } } + +func TestServingAvailabilityDoesNotRequireEveryReplica(t *testing.T) { + now := time.Now().UTC() + until := metav1.NewTime(now.Add(time.Minute)) + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{UID: "binding"}} + binding.Spec.Configuration.Generation = 1 + binding.Spec.Configuration.Revision = 1 + binding.Spec.Authorization = dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1} + for _, member := range []string{"node-a", "regional-a"} { + binding.Status.MemberAcknowledgements = append(binding.Status.MemberAcknowledgements, dnsv1alpha1.DNSApplyAcknowledgement{MemberID: member, WriterEpoch: 1, SnapshotRevision: 1, Revision: 1, AuthorizationIssuerEpoch: 1, AuthorizationRevision: 1, Phase: string(model.AckVerified), ValidUntil: &until}) + } + sink := AckSink{Members: []model.Member{{MemberID: "node-a", Role: "node"}, {MemberID: "node-b", Role: "node"}, {MemberID: "regional-a", Role: "regional"}, {MemberID: "regional-b", Role: "regional"}}} + envelope := &model.Envelope{Epoch: 1, Revision: 1} + snapshot := &model.ServingSnapshot{Bindings: []model.Binding{{BindingUID: "binding", BindingGeneration: 1, ConfigurationRevision: 1, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1}}}} + if !sink.bindingVerified(binding, envelope, snapshot, now) { + t.Fatal("a missing replica report suppressed healthy capacity") + } + sink.bindingConditions(binding, envelope, snapshot, now, true) + if apimeta.IsStatusConditionTrue(binding.Status.Conditions, "FullyReplicated") { + t.Fatal("degraded replication was reported complete") + } + binding.Status.ObservedConfigurationRevision = 1 + sink.bindingConditions(binding, envelope, snapshot, now.Add(2*time.Minute), false) + if c := apimeta.FindStatusCondition(binding.Status.Conditions, "Serving"); c == nil || c.Status != metav1.ConditionUnknown { + t.Fatal("stale observations claimed a serving outage") + } +} diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index 1c1cefc8..9f89ecb7 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -110,8 +110,8 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { {"internal-dns-control-plane", "dnspublicationmanifests", "", "create", true}, {"product", "dnsresolvercontexts", "", "get", true}, {"product", "dnsresolvercontexts", "", "list", true}, - {"product", "dnsmanagednamespaces", "", "get", true}, - {"product", "dnsmanagednamespaces", "", "list", true}, + {"product", "dnsmanagednamespaces", "", "get", false}, + {"product", "dnsmanagednamespaces", "", "list", false}, {"product", "dnsmanagednamespaces", "", "patch", false}, {"product", "dnsmanagednamespaces", "status", "patch", false}, {"product", "dnsnamingpolicies", "", "get", true}, @@ -130,7 +130,7 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { {"customer", "dnszoneassociations", "", "create", true}, {"customer", "dnsnamingpolicies", "", "create", true}, {"customer", "dnsresolvercontexts", "", "get", true}, - {"customer", "dnsmanagednamespaces", "", "get", true}, + {"customer", "dnsmanagednamespaces", "", "get", false}, {"customer", "dnsresolvercontexts", "", "patch", false}, {"customer", "dnscontributiongrants", "", "create", false}, {"customer", "dnscontributiongrants", "status", "patch", false}, @@ -282,9 +282,9 @@ func verifyProjectReferenceSchemas(t *testing.T, ctx context.Context, cl client. object client.Object valid bool }{ - {"registration-valid", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, true}, - {"registration-unpinned", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, - {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, + {"registration-name-only", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, true}, + {"registration-empty-zone", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, + {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, {"association-valid", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}}}, true}, {"association-empty-zone", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}}}, false}, {"naming-valid", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, NamePrefix: "services"}}}}, true}, diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index c61d0d13..2a26054d 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -193,6 +193,8 @@ type Agent struct { tx FileTransaction publicationVerifier PublicationVerifier mu sync.Mutex + healthMu sync.Mutex + healthEnabled bool state checkpoint needsActivation bool lastServingAck map[string]servingAckState @@ -267,7 +269,11 @@ func (a *Agent) Start(ctx context.Context) error { // still serves it, so force one validated apply before renewing the lease. a.mu.Lock() a.needsActivation = true + if err := a.restoreActive(ctx); err != nil { + a.report(err) + } a.mu.Unlock() + go a.runActiveHealth(ctx) for { err := a.bootstrapAndStep(ctx) if err == nil { @@ -320,6 +326,7 @@ func (a *Agent) Start(ctx context.Context) error { } func (a *Agent) failClosed(ctx context.Context) error { + a.suspendActiveHealth() a.needsActivation = true // Invalidate independent watchdog readiness immediately, even if the daemon // gate command itself fails. It must never inherit a prior healthy lease. @@ -476,7 +483,7 @@ func (a *Agent) Handle(ctx context.Context, env model.Envelope) error { case model.KindPublicationManifest: if err := a.handleManifest(ctx, env); err != nil { - _ = a.failClosed(ctx) + a.gateRejectedUpdate(ctx, err) return err } return a.finishEvent(ctx) @@ -525,14 +532,20 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving if err != nil { return err } - if err := a.flushRemovedBindingZones(ctx, snapshot); err != nil { - _ = a.failClosed(ctx) - return err - } + previous := a.state.Snapshot a.state.BindingFences = bindingFences a.state.ServingFence = fence{snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision} a.state.Snapshot = &snapshot + if !a.canRetainActive() { + a.state.Active = nil + a.suspendActiveHealth() + } if err := a.store.Save(a.state); err != nil { + _ = a.failClosed(ctx) + return err + } + if err := a.flushRemovedBindingZones(ctx, previous, snapshot); err != nil { + _ = a.failClosed(ctx) return err } } @@ -545,8 +558,8 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving // A configuration change can detach a zone without changing its publication. // Clear its old cache tree before accepting the mapping, including when the // context is temporarily pending. The cache hook must flush descendants too. -func (a *Agent) flushRemovedBindingZones(ctx context.Context, snapshot model.ServingSnapshot) error { - if a.state.Snapshot == nil { +func (a *Agent) flushRemovedBindingZones(ctx context.Context, previous *model.ServingSnapshot, snapshot model.ServingSnapshot) error { + if previous == nil { return nil } attached := map[string]bool{} @@ -556,7 +569,7 @@ func (a *Agent) flushRemovedBindingZones(ctx context.Context, snapshot model.Ser } } flushed := map[string]bool{} - for _, binding := range a.state.Snapshot.Bindings { + for _, binding := range previous.Bindings { for _, zone := range binding.Zones { if !attached[binding.BindingUID+"/"+zone.ZoneUID+"/"+zone.Apex] && !flushed[zone.Apex] { if err := a.flushName(ctx, zone.Apex); err != nil { @@ -847,13 +860,13 @@ func (a *Agent) Step(ctx context.Context) error { } if a.cfg.Mode == ModeResolver { if err := a.retryPendingPublications(ctx); err != nil { - _ = a.failClosed(ctx) + a.gateRejectedUpdate(ctx, err) return err } } if a.cfg.Mode == ModeCombined { if err := a.retryPendingPublications(ctx); err != nil { - _ = a.failClosed(ctx) + a.gateRejectedUpdate(ctx, err) return err } if a.needsActivation || len(a.installedPublications) == 0 { @@ -893,7 +906,7 @@ func (a *Agent) reconcile(ctx context.Context) error { filtered := *a.state.Snapshot filtered.Bindings = nil for _, b := range a.state.Snapshot.Bindings { - eligible := !b.Tombstone && a.publicationsReady(b) + eligible := !b.Tombstone && !b.ConfigurationPending if eligible { active[b.BindingUID] = true } @@ -903,8 +916,17 @@ func (a *Agent) reconcile(ctx context.Context) error { } renderConfig := a.cfg.Render renderConfig.ReadyMembers = map[string]map[string]bool{} + renderConfig.ReadyZoneMembers = map[string]map[string]map[string]bool{} for _, binding := range filtered.Bindings { - renderConfig.ReadyMembers[binding.BindingUID] = a.eligibleRegionalMembers(binding) + public := binding + public.Zones = nil + renderConfig.ReadyMembers[binding.BindingUID] = a.eligibleRegionalMembers(public) + renderConfig.ReadyZoneMembers[binding.BindingUID] = map[string]map[string]bool{} + for _, zone := range binding.Zones { + private := binding + private.Zones = []model.ZoneAttachment{zone} + renderConfig.ReadyZoneMembers[binding.BindingUID][zone.ZoneUID] = a.eligibleRegionalMembers(private) + } } rendered, err := renderPublications(renderConfig, filtered, active, a.state.Publications) if err != nil { @@ -912,6 +934,13 @@ func (a *Agent) reconcile(ctx context.Context) error { } hash := hashFiles(rendered.Files) if hash != a.state.RenderedHash || a.needsActivation { + if !a.canRetainActive() { + a.suspendActiveHealth() + a.state.Active = nil + if err := a.store.Save(a.state); err != nil { + return err + } + } var err error if a.needsActivation { err = a.tx.ApplyForce(ctx, rendered.Files) @@ -919,7 +948,7 @@ func (a *Agent) reconcile(ctx context.Context) error { err = a.tx.Apply(ctx, rendered.Files) } if err != nil { - _ = a.failClosed(ctx) + a.gateRejectedUpdate(ctx, err) return err } a.state.RenderedHash = hash @@ -959,11 +988,15 @@ func (a *Agent) reconcile(ctx context.Context) error { return err } a.state.RenderedHash = gatedHash + rendered = gated if err := a.store.Save(a.state); err != nil { return err } } } + if err := a.approveConfiguration(rendered.Files); err != nil { + return err + } // Local DNS proof renews the watchdog independently of broker acknowledgements. if a.cfg.Mode == ModeResolver { if err := a.renewSafeWatchdogLease(ctx); err != nil { @@ -1002,6 +1035,7 @@ func (a *Agent) verifyResolverBindings(ctx context.Context, bindings []model.Bin eligible := make([]model.Binding, 0, len(bindings)) for _, binding := range bindings { if active[binding.BindingUID] { + binding.Zones = a.readyZones(binding) eligible = append(eligible, binding) } } @@ -1080,7 +1114,7 @@ func (a *Agent) publicationsReady(b model.Binding) bool { if a.cfg.Mode == ModeCombined { for _, z := range b.Zones { p, ok := a.state.Publications[z.ZoneUID] - if !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + if z.Pending || !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { return false } } @@ -1109,7 +1143,7 @@ func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { ready := true for _, z := range b.Zones { p, ok := a.state.Publications[z.ZoneUID] - if !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + if z.Pending || !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { ready = false break } @@ -1131,6 +1165,18 @@ func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { return out } +func (a *Agent) readyZones(b model.Binding) []model.ZoneAttachment { + var ready []model.ZoneAttachment + for _, zone := range b.Zones { + candidate := b + candidate.Zones = []model.ZoneAttachment{zone} + if len(a.eligibleRegionalMembers(candidate)) > 0 { + ready = append(ready, zone) + } + } + return ready +} + func (a *Agent) verifyPublication(ctx context.Context, p publicationState) error { if p.Manifest.Tombstone { return nil @@ -1162,7 +1208,7 @@ func (a *Agent) publicationViewsAt(p publicationState) []PublicationView { continue } for _, z := range b.Zones { - if z.ZoneUID == p.Plan.ZoneUID && model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) >= 0 { + if !z.Pending && z.ZoneUID == p.Plan.ZoneUID && model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) >= 0 { out = append(out, PublicationView{View: b.ViewName(), DestinationAddress: b.ClusterAddress, DestinationPort: b.Port, Serial: publicationSerial(p), FingerprintMailbox: publicationMailbox(p)}) } } @@ -1450,6 +1496,10 @@ func (a *Agent) renewWatchdogLease(ctx context.Context) error { if a.cfg.WatchdogLeasePath == "" { return nil } + return a.writeWatchdogLease() +} + +func (a *Agent) writeWatchdogLease() error { lease := a.cfg.WatchdogLease if lease <= 0 { lease = 5 * time.Second diff --git a/internal/internaldns/serving/bind_test.go b/internal/internaldns/serving/bind_test.go index d0dfb6b7..4926b1e4 100644 --- a/internal/internaldns/serving/bind_test.go +++ b/internal/internaldns/serving/bind_test.go @@ -151,8 +151,8 @@ func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { if saved.Publications["zone-a"].Verified { t.Fatal("failed publication acknowledged") } - if runner.calls[len(runner.calls)-1].Path != "gate" { - t.Fatal("failed activation did not fail closed") + if runner.calls[len(runner.calls)-1].Path == "gate" || saved.Active == nil { + t.Fatal("rejected expansion discarded the active configuration") } runner.failAt = 0 if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err != nil { diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go new file mode 100644 index 00000000..53478e6d --- /dev/null +++ b/internal/internaldns/serving/continuity.go @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "strconv" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +type activeConfiguration struct { + Snapshot model.ServingSnapshot `json:"snapshot"` + Publications map[string]publicationState `json:"publications"` + Files map[string][]byte `json:"files"` +} + +func (a *Agent) gateRejectedUpdate(ctx context.Context, err error) { + var preserved preservedTransactionError + var uncertain uncertainTransactionError + if !errors.As(err, &uncertain) && a.canRetainActive() && (errors.As(err, &preserved) || !a.needsActivation && a.state.RenderedHash == hashFiles(a.state.Active.Files)) { + return + } + _ = a.failClosed(ctx) +} + +func (a *Agent) canRetainActive() bool { + old := a.state.Active + if old == nil || a.state.Snapshot == nil { + return false + } + next := map[string]model.Binding{} + for _, binding := range a.state.Snapshot.Bindings { + next[binding.BindingUID] = binding + } + for _, binding := range old.Snapshot.Bindings { + candidate, ok := next[binding.BindingUID] + if !ok || candidate.Tombstone || candidate.ConfigurationPending || bindingIdentity(binding) != bindingIdentity(candidate) || binding.Authorization != candidate.Authorization { + return false + } + for _, zone := range binding.Zones { + present := false + for _, attached := range candidate.Zones { + if attached.ZoneUID == zone.ZoneUID && attached.Apex == zone.Apex && !attached.Pending { + present = true + } + } + if !present && !zone.Pending { + return false + } + } + } + for uid, publication := range old.Publications { + if publication.Manifest.Tombstone { + continue + } + candidate, ok := a.state.Publications[uid] + if !ok || candidate.Manifest.Tombstone || candidate.Plan.Apex != publication.Plan.Apex { + return false + } + records := map[string]bool{} + for _, rrset := range candidate.Plan.RRSets { + for _, record := range rrset.Records { + if record.Eligible { + records[rrset.Name+"\x00"+rrset.Type+"\x00"+record.Content] = true + } + } + } + for _, rrset := range publication.Plan.RRSets { + for _, record := range rrset.Records { + if record.Eligible && !records[rrset.Name+"\x00"+rrset.Type+"\x00"+record.Content] { + return false + } + } + } + } + return true +} + +func (a *Agent) approveConfiguration(files map[string][]byte) error { + candidate := activeConfiguration{Snapshot: *a.state.Snapshot, Publications: a.state.Publications, Files: files} + data, err := json.Marshal(candidate) + if err != nil { + return err + } + var active activeConfiguration + if err := json.Unmarshal(data, &active); err != nil { + return err + } + if a.state.Active != nil && equalJSON(*a.state.Active, active) { + a.healthMu.Lock() + a.healthEnabled = true + a.healthMu.Unlock() + return nil + } + a.state.Active = &active + if err := a.store.Save(a.state); err != nil { + return err + } + a.healthMu.Lock() + a.healthEnabled = true + a.healthMu.Unlock() + return nil +} + +func (a *Agent) suspendActiveHealth() { + a.healthMu.Lock() + defer a.healthMu.Unlock() + a.healthEnabled = false + +} + +func (a *Agent) restoreActive(ctx context.Context) error { + if !a.canRetainActive() { + return nil + } + if err := a.tx.ApplyForce(ctx, a.state.Active.Files); err != nil { + return err + } + a.healthMu.Lock() + a.healthEnabled = true + a.healthMu.Unlock() + return nil +} + +func (a *Agent) runActiveHealth(ctx context.Context) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := a.refreshActiveHealth(ctx); err != nil { + a.report(err) + } + } + } +} + +func (a *Agent) refreshActiveHealth(ctx context.Context) error { + a.healthMu.Lock() + enabled := a.healthEnabled + a.healthMu.Unlock() + if !enabled || a.cfg.WatchdogLeasePath == "" { + return nil + } + probeCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + if err := a.probeLocalDaemons(probeCtx); err != nil { + return err + } + a.healthMu.Lock() + defer a.healthMu.Unlock() + if !a.healthEnabled { + return nil + } + return a.writeWatchdogLease() +} + +func (a *Agent) probeLocalDaemons(ctx context.Context) error { + if a.cfg.UnsafeAllowNoDNSVerify { + return nil + } + role := a.cfg.Render.Role + if role == "" { + role = RenderBoth + } + var binds []BINDRenderConfig + var dists []DNSDistRenderConfig + if role == RenderNode || role == RenderBoth { + binds = append(binds, a.cfg.Render.NodeBIND) + dists = append(dists, a.cfg.Render.NodeDNSDist) + } + if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { + binds = append(binds, a.cfg.Render.ClusterBIND) + } + if role == RenderCluster || role == RenderBoth || role == RenderRegionalDNSDist { + dists = append(dists, a.cfg.Render.ClusterDNSDist) + } + for _, bind := range binds { + server := net.JoinHostPort(bind.ListenAddress, strconv.Itoa(int(bind.Port))) + view := PublicationView{DestinationAddress: bind.ListenAddress, DestinationPort: bind.Port} + for _, network := range []string{networkUDP, networkTCP} { + answer, err := exchangePublication(ctx, network, server, ".", view, 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local BIND health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) + } + } + } + for _, dist := range dists { + for _, network := range []string{networkUDP, networkTCP} { + answer, err := exchangeResolverSOA(ctx, network, dist.ListenAddress, ".", 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local dnsdist health %s: %w", dist.ListenAddress, errors.Join(err, errors.New("DNS response required"))) + } + } + } + return nil +} diff --git a/internal/internaldns/serving/continuity_test.go b/internal/internaldns/serving/continuity_test.go new file mode 100644 index 00000000..a99fc60b --- /dev/null +++ b/internal/internaldns/serving/continuity_test.go @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "bytes" + "context" + "encoding/json" + "os" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func TestRejectedExpansionPreservesServingAndRestarts(t *testing.T) { + now := time.Now().UTC() + runner := &recordingRunner{} + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + activeHash := agent.state.RenderedHash + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0].ConfigurationRevision++ + candidate.Bindings[0].Zones = append(candidate.Bindings[0].Zones, model.ZoneAttachment{ZoneUID: "pending-zone", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + runner.failAt = len(runner.calls) + 1 + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)); err == nil { + t.Fatal("invalid expansion was accepted") + } + if agent.state.RenderedHash != activeHash || agent.state.Active == nil || !agent.healthEnabled { + t.Fatal("invalid expansion interrupted the active configuration") + } + now = now.Add(time.Hour) + if err := agent.refreshActiveHealth(context.Background()); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(cfg.WatchdogLeasePath) + if err != nil { + t.Fatal(err) + } + var lease watchdogLease + if err := json.Unmarshal(data, &lease); err != nil || !lease.ValidUntil.After(now) { + t.Fatalf("active health stopped during rejected update: %v", err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil || !restarted.healthEnabled { + t.Fatalf("restart could not recover active configuration: %v", err) + } + withdrawn := candidate + withdrawn.ConfigurationRevision++ + withdrawn.Bindings = nil + runner.failAt = len(runner.calls) + 1 + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", withdrawn.ConfigurationEpoch, withdrawn.ConfigurationRevision, withdrawn)); err == nil { + t.Fatal("failed withdrawal was accepted") + } + if agent.state.Active != nil || agent.healthEnabled { + t.Fatal("failed withdrawal retained revoked access") + } +} + +func TestPendingZoneIsIsolatedFromReadyZone(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + pub := &fakePublisher{} + agent.deps.AckPublisher = pub + now = now.Add(time.Minute) + binding := agent.state.Snapshot.Bindings[0] + binding.Zones = append(binding.Zones, model.ZoneAttachment{ZoneUID: "pending", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + if zones := agent.readyZones(binding); len(zones) != 1 || zones[0].ZoneUID != "zone-a" { + t.Fatalf("pending zone suppressed healthy zone: %#v", zones) + } + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0] = binding + candidate.Bindings[0].ConfigurationRevision++ + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)); err != nil { + t.Fatal(err) + } + if agent.state.Active == nil || agent.publicationsReady(binding) { + t.Fatal("partial serving state was not distinguished from full readiness") + } + if zone := currentZone(t, agent); zone == "" { + t.Fatal("ready zone was not materialized") + } + verified := false + for _, env := range pub.envs { + var ack model.MemberAck + if err := json.Unmarshal(env.Payload, &ack); err != nil { + t.Fatal(err) + } + if ack.Kind == model.KindServingSnapshot && ack.Revision == candidate.ConfigurationRevision && ack.Phase == model.AckVerified { + verified = true + } + } + if !verified { + t.Fatal("pending zone prevented resolver access from becoming ready") + } +} + +type blockedValidationRunner struct { + recordingRunner + blocked bool + started chan struct{} + release chan struct{} +} + +func (r *blockedValidationRunner) Run(ctx context.Context, command Command) ([]byte, error) { + if command.Path == "validate" && r.blocked { + close(r.started) + select { + case <-r.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return r.recordingRunner.Run(ctx, command) +} + +func TestActiveHealthDoesNotWaitForCandidateValidation(t *testing.T) { + now := time.Now().UTC() + runner := &blockedValidationRunner{started: make(chan struct{}), release: make(chan struct{})} + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + runner.blocked = true + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0].ConfigurationRevision++ + candidate.Bindings[0].Zones = append(candidate.Bindings[0].Zones, model.ZoneAttachment{ZoneUID: "pending", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + done <- agent.Handle(ctx, envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)) + }() + select { + case <-runner.started: + case <-ctx.Done(): + t.Fatal("candidate validation did not start") + } + healthDone := make(chan error, 1) + go func() { healthDone <- agent.refreshActiveHealth(ctx) }() + select { + case err := <-healthDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("active health waited for the candidate transaction") + } + close(runner.release) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestWithdrawalTransactionRetainsOnlyExistingWatchdogDeadline(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + before, err := os.ReadFile(cfg.WatchdogLeasePath) + if err != nil { + t.Fatal(err) + } + agent.suspendActiveHealth() + now = now.Add(time.Hour) + if err := agent.refreshActiveHealth(context.Background()); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(cfg.WatchdogLeasePath) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("transaction changed the existing watchdog deadline") + } + var lease watchdogLease + if err := json.Unmarshal(after, &lease); err != nil || lease.ValidUntil.After(now) { + t.Fatal("suspended health extended serving authority") + } + if err := agent.failClosed(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(cfg.WatchdogLeasePath); !os.IsNotExist(err) { + t.Fatal("failed withdrawal retained watchdog readiness") + } +} diff --git a/internal/internaldns/serving/render.go b/internal/internaldns/serving/render.go index df6ddfe6..b74364c7 100644 --- a/internal/internaldns/serving/render.go +++ b/internal/internaldns/serving/render.go @@ -47,12 +47,13 @@ const ( ) type RenderConfig struct { - ReadyMembers map[string]map[string]bool `json:"-"` - Role RenderRole `json:"role"` - NodeDNSDist DNSDistRenderConfig `json:"nodeDNSDist,omitempty"` - ClusterDNSDist DNSDistRenderConfig `json:"clusterDNSDist,omitempty"` - NodeBIND BINDRenderConfig `json:"nodeBIND,omitempty"` - ClusterBIND BINDRenderConfig `json:"clusterBIND,omitempty"` + ReadyZoneMembers map[string]map[string]map[string]bool `json:"-"` + ReadyMembers map[string]map[string]bool `json:"-"` + Role RenderRole `json:"role"` + NodeDNSDist DNSDistRenderConfig `json:"nodeDNSDist,omitempty"` + ClusterDNSDist DNSDistRenderConfig `json:"clusterDNSDist,omitempty"` + NodeBIND BINDRenderConfig `json:"nodeBIND,omitempty"` + ClusterBIND BINDRenderConfig `json:"clusterBIND,omitempty"` } type Rendered struct{ Files map[string][]byte } @@ -90,11 +91,11 @@ func renderPublications(c RenderConfig, snapshot model.ServingSnapshot, active m role = RenderBoth } if role == RenderNode || role == RenderBoth { - files[c.NodeDNSDist.Path] = renderDNSDist(c.NodeDNSDist, nodeBackends, bindings, active, false, nil) + files[c.NodeDNSDist.Path] = renderDNSDist(c.NodeDNSDist, nodeBackends, bindings, active, false, nil, c.ReadyZoneMembers) files[c.NodeBIND.Path] = renderNodeBIND(c.NodeBIND, bindings) } if role == RenderCluster || role == RenderBoth || role == RenderRegionalDNSDist { - files[c.ClusterDNSDist.Path] = renderDNSDist(c.ClusterDNSDist, clusterBackends, bindings, active, true, c.ReadyMembers) + files[c.ClusterDNSDist.Path] = renderDNSDist(c.ClusterDNSDist, clusterBackends, bindings, active, true, c.ReadyMembers, c.ReadyZoneMembers) } if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { cluster, err := renderClusterBIND(c.ClusterBIND, bindings, publications, files) @@ -218,7 +219,7 @@ func collectBackends(bindings []model.Binding, get func(model.Binding) []model.B return out } -func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []model.Binding, active map[string]bool, cluster bool, eligible map[string]map[string]bool) []byte { +func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []model.Binding, active map[string]bool, cluster bool, eligible map[string]map[string]bool, zones map[string]map[string]map[string]bool) []byte { var out bytes.Buffer fmt.Fprintf(&out, "setLocal(%q)\n", c.ListenAddress) fmt.Fprint(&out, "setACL({") @@ -233,6 +234,11 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m fmt.Fprintf(&out, "local server = newServer({address=%q, name=%q, pool=%q, useProxyProtocol=true, checkName=\".\", checkType=\"SOA\", mustResolve=false})\n", net.JoinHostPort(b.Address, strconv.Itoa(int(b.Port))), "b_"+model.OpaqueToken(b.MemberID), c.PoolName) if cluster { for _, binding := range bindings { + for _, zone := range binding.Zones { + if zones != nil && active[binding.BindingUID] && zones[binding.BindingUID][zone.ZoneUID][b.MemberID] { + fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()+"_"+model.OpaqueToken(zone.ZoneUID)) + } + } if !binding.ConfigurationPending && active[binding.BindingUID] && (eligible == nil || eligible[binding.BindingUID][b.MemberID]) { fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()) } @@ -250,6 +256,19 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m fmt.Fprint(&out, "}\n\nfunction classifyContext(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"dns-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") fmt.Fprint(&out, "addAction(AllRule(), LuaAction(classifyContext))\n") fmt.Fprint(&out, "addAction(TagRule(\"context-unready\", \"true\"), RCodeAction(DNSRCode.SERVFAIL))\n") + for _, binding := range bindings { + attached := append([]model.ZoneAttachment(nil), binding.Zones...) + sort.Slice(attached, func(i, j int) bool { return len(attached[i].Apex) > len(attached[j].Apex) }) + for _, zone := range attached { + rule := fmt.Sprintf("AndRule({TagRule(\"dns-context\",%q),QNameSuffixRule(%q)})", binding.ViewName(), model.AbsoluteName(zone.Apex)) + if zone.Pending || zones != nil && len(zones[binding.BindingUID][zone.ZoneUID]) == 0 { + fmt.Fprintf(&out, "addAction(%s,RCodeAction(DNSRCode.SERVFAIL))\n", rule) + } else if cluster && zones != nil { + pool := binding.ViewName() + "_" + model.OpaqueToken(zone.ZoneUID) + fmt.Fprintf(&out, "getPool(%q)\naddAction(AndRule({%s,NotRule(PoolAvailableRule(%q))}),RCodeAction(DNSRCode.SERVFAIL))\naddAction(%s,PoolAction(%q))\n", pool, rule, pool, rule, pool) + } + } + } if cluster { for _, binding := range bindings { fmt.Fprintf(&out, "getPool(%q)\naddAction(AndRule({TagRule(\"dns-context\",%q),NotRule(PoolAvailableRule(%q))}), RCodeAction(DNSRCode.SERVFAIL))\naddAction(TagRule(\"dns-context\",%q),PoolAction(%q))\n", binding.ViewName(), binding.ViewName(), binding.ViewName(), binding.ViewName(), binding.ViewName()) @@ -315,7 +334,7 @@ func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publication renderViewStart(&out, b, b.ClusterAddress, c.DefaultCacheSize, maxNegativeCacheTTL(c)) for _, z := range b.Zones { p, ok := publications[z.ZoneUID] - if !ok || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + if z.Pending || !ok || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { // Never recurse toward the public namespace when a private apex is missing. // The front gate returns immediate SERVFAIL; direct BIND queries fail too. fmt.Fprintf(&out, " zone %q { type forward; forward only; forwarders { 127.0.0.1 port 9; }; };\n", strings.TrimSuffix(model.AbsoluteName(z.Apex), ".")) diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go index bf432057..ead95cfe 100644 --- a/internal/internaldns/serving/state.go +++ b/internal/internaldns/serving/state.go @@ -36,6 +36,7 @@ type replicaAck struct { const checkpointFormatVersion = 5 type checkpoint struct { + Active *activeConfiguration `json:"active,omitempty"` FormatVersion int `json:"formatVersion"` ServingFence fence `json:"servingFence"` Snapshot *model.ServingSnapshot `json:"snapshot,omitempty"` diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index 3043e7e9..467ee010 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -44,6 +44,15 @@ type FileTransaction struct { Config TransactionConfig Runner Runner } + +type preservedTransactionError struct{ error } + +func (e preservedTransactionError) Unwrap() error { return e.error } + +type uncertainTransactionError struct{ error } + +func (e uncertainTransactionError) Unwrap() error { return e.error } + type oldFile struct { data []byte mode os.FileMode @@ -61,7 +70,22 @@ func (t FileTransaction) ApplyForce(ctx context.Context, files map[string][]byte return t.apply(ctx, files, true) } -func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, force bool) error { +func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, force bool) (resultErr error) { + mutating := false + defer func() { + if resultErr == nil { + return + } + var preserved preservedTransactionError + if errors.As(resultErr, &preserved) { + return + } + if mutating { + resultErr = uncertainTransactionError{resultErr} + } else { + resultErr = preservedTransactionError{resultErr} + } + }() if t.Runner == nil { t.Runner = ExecRunner{} } @@ -147,9 +171,10 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for commandChanges = nil } if err := t.commandsForChanges(ctx, t.Config.Validate, map[string]string{"{stageDir}": commandStage}, commandChanges); err != nil { - return fmt.Errorf("validate staged serving configuration: %w", err) + return preservedTransactionError{fmt.Errorf("validate staged serving configuration: %w", err)} } installed := make([]string, 0, len(paths)) + mutating = true for _, path := range paths { if !changed[path] { continue @@ -176,6 +201,9 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for if err := t.commandsForChanges(ctx, t.Config.Reload, nil, commandChanges); err != nil { rollbackErr := restoreFiles(paths, old) reloadErr := t.commandsForChanges(ctx, t.Config.Reload, nil, commandChanges) + if rollbackErr == nil && reloadErr == nil { + return preservedTransactionError{fmt.Errorf("reload serving configuration: %w", err)} + } return errors.Join(fmt.Errorf("reload serving configuration: %w", err), rollbackErr, reloadErr) } return nil diff --git a/test/internaldns/README.md b/test/internaldns/README.md index 637aa9a7..c6d84d64 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -111,5 +111,7 @@ into the separate DNS service VPC. Kind cannot establish physical fabric, microVM attachment, host-loss durability, regional independence, or production capacity. -The suite requires internal wire version `v1alpha4`, fresh format 5 checkpoints, -and matching controller and fleet versions. +The suite uses wire version `v1alpha4` and persistent format 5 checkpoints. +Compatible fleet upgrades retain those checkpoints and transport state. Roll out +serving support before enabling new controller output; see the +[rollout contract](../../config/internal-dns/README.md#rollout-and-recovery). diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go index 1f6a5431..2c2a8951 100644 --- a/test/internaldns/kubernetes/qualify/main.go +++ b/test/internaldns/kubernetes/qualify/main.go @@ -288,7 +288,7 @@ func (s *suite) run(path string) error { } } a, b := &s.env.Projects[0], &s.env.Projects[1] - if a.resolverContext.UID == b.resolverContext.UID || a.zone.UID == b.zone.UID || a.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID == b.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID { + if a.resolverContext.UID == b.resolverContext.UID || a.zone.UID == b.zone.UID || a.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name == b.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name { return errors.New("source projects lost their distinct context, zone, or managed namespace identity") } for _, p := range []*project{a, b} { @@ -335,7 +335,7 @@ func (s *suite) prepare(p *project) error { } if err := s.wait(p.Name+" managed namespace", 120*time.Second, func() (bool, error) { err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&p.resolverContext), &p.resolverContext) - return condition(p.resolverContext.Status.Conditions, "Ready", p.resolverContext.Generation) && p.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID != "", err + return condition(p.resolverContext.Status.Conditions, "Ready", p.resolverContext.Generation) && p.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name != "", err }); err != nil { return err } @@ -394,7 +394,7 @@ func recordSets(name, address, ipv6 string) []dnsv1.DNSContributionRecordSet { return sets } -func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string) (*publication, error) { +func (s *suite) publish(p *project, zone dnsv1.DNSNameReference, prefix, name, address, ipv6 string) (*publication, error) { pub := &publication{registration: dnsv1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-registration", Namespace: p.Namespace}, Spec: dnsv1.DNSRegistrationSpec{DNSZoneRef: zone, Name: name, RecordTypes: []dnsv1.RRType{dnsv1.RRTypeA}, PublicationPolicy: dnsv1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 2}}} if ipv6 != "" { pub.registration.Spec.RecordTypes = append(pub.registration.Spec.RecordTypes, dnsv1.RRTypeAAAA) diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index 39801fd6..538eb80f 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -24,11 +24,11 @@ func (s *suite) scenarios(a, b *project) error { return err } } - pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10") + pa, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "common", "common", "10.42.0.10", "fd42::10") if err != nil { return err } - pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20") + pb, err := s.publish(b, dnsv1.DNSNameReference{Name: b.zone.Name}, "common", "common", "10.42.0.20", "fd42::20") if err != nil { return err } @@ -75,7 +75,7 @@ func (s *suite) scenarios(a, b *project) error { return err } s.pass("an unbound destination cannot resolve either tenant's private records", nil) - if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", ""); err != nil { + if _, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "only-a", "only-a", "10.42.0.20", ""); err != nil { return err } if err := s.expect(a, "only-a.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { @@ -114,7 +114,7 @@ func (s *suite) scenarios(a, b *project) error { if err := s.associate(a, apps, "apps"); err != nil { return err } - if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", ""); err != nil { + if _, err := s.publish(a, dnsv1.DNSNameReference{Name: apps.Name}, "apps", "api", "10.42.0.12", ""); err != nil { return err } if err := s.expect(a, "api.apps.internal", "A", true, dns.RcodeSuccess, "10.42.0.12"); err != nil { @@ -165,7 +165,7 @@ func (s *suite) scenarios(a, b *project) error { if err := s.observe(b, pb, true); err != nil { return err } - deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "") + deleted, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "deleted", "deleted", "10.42.0.14", "") if err != nil { return err } @@ -352,7 +352,7 @@ func (s *suite) offlineServing(a, b *project) (resultErr error) { return err } // The outage spans multiple DNS cache and fleet acknowledgement intervals. - until := time.Now().Add(2 * time.Minute) + until := time.Now().Add(3 * time.Minute) for time.Now().Before(until) { for _, tcp := range []bool{false, true} { for _, tenant := range []struct {