From cbdef6e741a2849e9e4e524f0da76684bed14433 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:01:12 -0500 Subject: [PATCH 01/15] Keep private DNS available during control plane outages --- api/v1alpha1/internaldns_access_test.go | 40 +- api/v1alpha1/internaldns_binding_test.go | 4 +- api/v1alpha1/internaldns_project_types.go | 25 +- api/v1alpha1/internaldns_service_types.go | 12 +- .../internaldns_validation_envtest_test.go | 9 +- api/v1alpha1/zz_generated.deepcopy.go | 16 +- ...ng.miloapis.com_dnscontributiongrants.yaml | 4 - ....miloapis.com_dnspublicationmanifests.yaml | 5 +- ...g.miloapis.com_dnsrecordcontributions.yaml | 3 - ...iloapis.com_dnsresolveraccessbindings.yaml | 4 - ...king.miloapis.com_dnsresolverbindings.yaml | 8 - config/internal-dns/README.md | 48 +-- .../internal-dns/control-plane.example.yaml | 5 +- config/internal-dns/fleet.example.yaml | 8 +- internal/internaldns/admission/handler.go | 64 +--- .../internaldns/admission/handler_test.go | 34 +- .../access_recovery_envtest_test.go | 257 +++---------- internal/internaldns/controlplane/compiler.go | 42 +-- .../internaldns/controlplane/compiler_test.go | 68 ++-- .../controlplane/context_reconciler.go | 22 +- .../controlplane/context_reconciler_test.go | 49 +-- .../controlplane/publication_reconciler.go | 20 +- .../publication_reconciler_test.go | 22 +- .../internaldns/controlplane/reconciler.go | 13 +- internal/internaldns/controlplane/wire.go | 3 +- internal/internaldns/model/model.go | 39 +- internal/internaldns/model/model_test.go | 4 +- internal/internaldns/platform/acks.go | 10 +- internal/internaldns/platform/planner.go | 2 +- .../internaldns/platform/platform_test.go | 8 +- internal/internaldns/runtime/config.go | 38 +- internal/internaldns/runtime/config_test.go | 57 ++- internal/internaldns/runtime/run.go | 117 +++--- internal/internaldns/serving/agent.go | 342 +++++------------- internal/internaldns/serving/agent_test.go | 37 +- internal/internaldns/serving/bind_test.go | 194 ++-------- .../internaldns/serving/binding_fences.go | 3 - .../serving/binding_fences_test.go | 13 +- .../internaldns/serving/publication_hash.go | 5 +- internal/internaldns/serving/render.go | 15 +- internal/internaldns/serving/render_test.go | 9 +- internal/internaldns/serving/state.go | 22 +- internal/internaldns/serving/zone.go | 22 +- .../serving/zone_regression_test.go | 6 +- internal/internaldns/transport/jetstream.go | 2 +- test/internaldns/README.md | 6 +- test/internaldns/kubernetes/qualify/main.go | 15 +- .../kubernetes/qualify/scenarios.go | 187 ++++------ test/internaldns/kubernetes/render/main.go | 5 +- test/internaldns/production-parity.md | 13 +- 50 files changed, 665 insertions(+), 1291 deletions(-) diff --git a/api/v1alpha1/internaldns_access_test.go b/api/v1alpha1/internaldns_access_test.go index d4abda23..e67b646e 100644 --- a/api/v1alpha1/internaldns_access_test.go +++ b/api/v1alpha1/internaldns_access_test.go @@ -1,42 +1,24 @@ -// SPDX-License-Identifier: AGPL-3.0-only - package v1alpha1 import ( "encoding/json" "testing" - "time" ) -func TestAccessAuthorizationPreservesDeadlinePrecision(t *testing.T) { - const raw = `{"writerEpoch":1,"sequence":2,"validUntil":"2026-10-08T23:22:29.022357123Z"}` - var authorization DNSResolverAccessAuthorization - if err := json.Unmarshal([]byte(raw), &authorization); err != nil { +func TestResolverAccessHasNoExpiryOrClusterIdentity(t *testing.T) { + a := DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 2} + b, err := json.Marshal(a) + if err != nil { t.Fatal(err) } - for i := 0; i < 3; i++ { - encoded, err := json.Marshal(authorization) - if err != nil { - t.Fatal(err) - } - if string(encoded) != raw { - t.Fatalf("authorization deadline changed on round trip: %s", encoded) - } - if err := json.Unmarshal(encoded, &authorization); err != nil { - t.Fatal(err) - } - } - if authorization.ValidUntil.Nanosecond() != 22357123 { - t.Fatal("authorization lost nanosecond precision") + if string(b) != `{"writerEpoch":1,"sequence":2}` { + t.Fatalf("unexpected access fence: %s", b) } - // The same instant in another time zone serializes to the same UTC fence. - authorization.ValidUntil.Time = authorization.ValidUntil.In(time.FixedZone("offset", 3600)) - encoded, err := json.Marshal(authorization) - if err != nil || string(encoded) != raw { - t.Fatalf("UTC normalization changed authorization: %s, %v", encoded, err) + b, err = json.Marshal(DNSProducerPrincipal{Subject: "system:serviceaccount:compute:dns-publisher"}) + if err != nil { + t.Fatal(err) } - encoded, err = json.Marshal(DNSResolverAccessAuthorization{}) - if err != nil || string(encoded) != `{"writerEpoch":0,"sequence":0,"validUntil":null}` { - t.Fatalf("zero value encoding changed field names or null deadline: %s, %v", encoded, err) + if string(b) != `{"subject":"system:serviceaccount:compute:dns-publisher"}` { + t.Fatalf("unexpected publisher identity: %s", b) } } diff --git a/api/v1alpha1/internaldns_binding_test.go b/api/v1alpha1/internaldns_binding_test.go index ce16887b..1f48e81d 100644 --- a/api/v1alpha1/internaldns_binding_test.go +++ b/api/v1alpha1/internaldns_binding_test.go @@ -5,8 +5,6 @@ package v1alpha1 import ( "encoding/json" "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestResolverBindingUsesNestedContractAndIndependentCopies(t *testing.T) { @@ -14,7 +12,7 @@ func TestResolverBindingUsesNestedContractAndIndependentCopies(t *testing.T) { Source: DNSResolverBindingSource{ProjectUID: "project", ResolverContextRef: DNSObjectReference{Name: "context", UID: "context-uid"}, AccessBindingRef: DNSObjectReference{Name: "access", UID: "access-uid"}}, Placement: DNSResolverBindingPlacement{Region: "central", Shard: "shared"}, Configuration: DNSResolverBindingConfiguration{Generation: 1, Revision: 2, Listeners: DNSResolverBindingListeners{Node: DNSResolverBindingListener{Address: "fd53::1", Port: 53, Transports: []string{"UDP", "TCP"}}, Regional: DNSResolverBindingListener{Address: "fd54::1", Port: 53, Transports: []string{"UDP", "TCP"}}}, ZoneRefs: []DNSObjectReference{{Name: "private", UID: "zone-uid"}}}, - Authorization: DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 4, ValidUntil: metav1.Now()}, + Authorization: DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 4}, }} copy := original.DeepCopy() copy.Spec.Configuration.ZoneRefs[0].Name = "changed" diff --git a/api/v1alpha1/internaldns_project_types.go b/api/v1alpha1/internaldns_project_types.go index b4114de6..297a1b07 100644 --- a/api/v1alpha1/internaldns_project_types.go +++ b/api/v1alpha1/internaldns_project_types.go @@ -3,9 +3,6 @@ package v1alpha1 import ( - "encoding/json" - "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -117,24 +114,7 @@ type DNSResolverAccessAuthorization struct { // +kubebuilder:validation:Minimum=1 WriterEpoch int64 `json:"writerEpoch"` // +kubebuilder:validation:Minimum=1 - Sequence int64 `json:"sequence"` - ValidUntil metav1.Time `json:"validUntil"` -} - -// MarshalJSON preserves the source authorization deadline exactly. metav1.Time -// accepts fractional seconds on input but its default JSON encoder drops them, -// which would mutate a same-fence deadline when projecting an access binding. -func (a DNSResolverAccessAuthorization) MarshalJSON() ([]byte, error) { - var deadline *time.Time - if !a.ValidUntil.IsZero() { - instant := a.ValidUntil.UTC() - deadline = &instant - } - return json.Marshal(struct { - WriterEpoch int64 `json:"writerEpoch"` - Sequence int64 `json:"sequence"` - ValidUntil *time.Time `json:"validUntil"` - }{WriterEpoch: a.WriterEpoch, Sequence: a.Sequence, ValidUntil: deadline}) + Sequence int64 `json:"sequence"` } // +kubebuilder:validation:XValidation:rule="self.contextRef.name.size() > 0 && has(self.contextRef.uid) && self.contextRef.uid.size() > 0",message="contextRef requires name and UID" @@ -247,8 +227,6 @@ type DNSRegistrationList struct { } type DNSProducerPrincipal struct { - // +kubebuilder:validation:MinLength=1 - ClusterUID string `json:"clusterUID"` // +kubebuilder:validation:MinLength=1 Subject string `json:"subject"` } @@ -306,7 +284,6 @@ type DNSRecordContributionStatus struct { Sequence int64 `json:"sequence,omitempty"` Eligible bool `json:"eligible,omitempty"` Reason string `json:"reason,omitempty"` - ValidUntil *metav1.Time `json:"validUntil,omitempty"` PublishedRevision int64 `json:"publishedRevision,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty"` } diff --git a/api/v1alpha1/internaldns_service_types.go b/api/v1alpha1/internaldns_service_types.go index e97eb874..dc57e565 100644 --- a/api/v1alpha1/internaldns_service_types.go +++ b/api/v1alpha1/internaldns_service_types.go @@ -10,7 +10,6 @@ import ( // DNSResolverBinding is the DNS-service projection of a consumer context and access binding. // +kubebuilder:validation:XValidation:rule="self.configuration.generation >= oldSelf.configuration.generation && self.configuration.revision >= oldSelf.configuration.revision",message="configuration fences must not decrease" // +kubebuilder:validation:XValidation:rule="self.authorization.writerEpoch > oldSelf.authorization.writerEpoch || (self.authorization.writerEpoch == oldSelf.authorization.writerEpoch && self.authorization.sequence >= oldSelf.authorization.sequence)",message="authorization fences must not decrease" -// +kubebuilder:validation:XValidation:rule="self.authorization.writerEpoch != oldSelf.authorization.writerEpoch || self.authorization.sequence != oldSelf.authorization.sequence || self.authorization.validUntil == oldSelf.authorization.validUntil",message="changing an authorization deadline requires a new fence" // +kubebuilder:validation:XValidation:rule="self.configuration.zoneRefs == oldSelf.configuration.zoneRefs || (self.configuration.generation > oldSelf.configuration.generation && self.configuration.revision > oldSelf.configuration.revision)",message="zone membership changes require a new cache generation and configuration revision" // +kubebuilder:validation:XValidation:rule="(has(self.tombstone) && self.tombstone) == (has(oldSelf.tombstone) && oldSelf.tombstone) || self.configuration.revision > oldSelf.configuration.revision",message="changing withdrawal state requires a new configuration revision" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.tombstone) || !oldSelf.tombstone || (has(self.tombstone) && self.tombstone) || self.authorization.writerEpoch > oldSelf.authorization.writerEpoch || (self.authorization.writerEpoch == oldSelf.authorization.writerEpoch && self.authorization.sequence > oldSelf.authorization.sequence)",message="reactivating withdrawn access requires a new source authorization fence" @@ -153,13 +152,12 @@ type DNSRegistrationFence struct { } // DNSContributionFence persists an authenticated high-water sequence even -// while its observation is unhealthy, rejected, or expired. +// while its observation is unhealthy or withdrawn. type DNSContributionFence struct { - UID types.UID `json:"uid"` - GrantUID types.UID `json:"grantUID"` - Epoch int64 `json:"epoch"` - Sequence int64 `json:"sequence"` - ValidUntil metav1.Time `json:"validUntil,omitempty"` + UID types.UID `json:"uid"` + GrantUID types.UID `json:"grantUID"` + Epoch int64 `json:"epoch"` + Sequence int64 `json:"sequence"` } type DNSPublicationManifestStatus struct { diff --git a/api/v1alpha1/internaldns_validation_envtest_test.go b/api/v1alpha1/internaldns_validation_envtest_test.go index 95c8c01a..73b1c0f3 100644 --- a/api/v1alpha1/internaldns_validation_envtest_test.go +++ b/api/v1alpha1/internaldns_validation_envtest_test.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "testing" - "time" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -56,7 +55,7 @@ func TestPrivateDNSValidationAtAPIServer(t *testing.T) { } newAccess := func(name string) *DNSResolverAccessBinding { return &DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "validation"}, Spec: DNSResolverAccessBindingSpec{ - ContextRef: DNSObjectReference{Name: "context", UID: "context-uid"}, Region: "central", QueryIdentity: DNSResolverQueryIdentity{Type: DNSResolverQueryIdentityDestinationAddress, Value: "fd53::1"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1, ValidUntil: metav1.NewTime(time.Now().Add(time.Minute))}, + ContextRef: DNSObjectReference{Name: "context", UID: "context-uid"}, Region: "central", QueryIdentity: DNSResolverQueryIdentity{Type: DNSResolverQueryIdentityDestinationAddress, Value: "fd53::1"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1}, }} } for _, tc := range []struct { @@ -85,9 +84,6 @@ func TestPrivateDNSValidationAtAPIServer(t *testing.T) { {"change-region", func(a *DNSResolverAccessBinding) { a.Spec.Region = "east" }}, {"change-address", func(a *DNSResolverAccessBinding) { a.Spec.QueryIdentity.Value = "fd53::2" }}, {"change-epoch", func(a *DNSResolverAccessBinding) { a.Spec.Authorization.WriterEpoch++ }}, - {"extend-without-sequence", func(a *DNSResolverAccessBinding) { - a.Spec.Authorization.ValidUntil = metav1.NewTime(a.Spec.Authorization.ValidUntil.Add(time.Minute)) - }}, {"decrease-sequence", func(a *DNSResolverAccessBinding) { a.Spec.Authorization.Sequence = 1 }}, } { t.Run(tc.name, func(t *testing.T) { @@ -100,13 +96,12 @@ func TestPrivateDNSValidationAtAPIServer(t *testing.T) { requireInvalid(t, cl.Update(ctx, a)) }) } - t.Run("renewal-and-status", func(t *testing.T) { + t.Run("sequence-and-status", func(t *testing.T) { a := newAccess("renewal") if err := cl.Create(ctx, a); err != nil { t.Fatal(err) } a.Spec.Authorization.Sequence++ - a.Spec.Authorization.ValidUntil = metav1.NewTime(a.Spec.Authorization.ValidUntil.Add(time.Minute)) if err := cl.Update(ctx, a); err != nil { t.Fatal(err) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ce42c420..fba1d9af 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8,7 +8,7 @@ package v1alpha1 import ( "go.datum.net/network-services-operator/api/v1alpha" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) @@ -132,7 +132,6 @@ func (in *DNSApplyAcknowledgement) DeepCopy() *DNSApplyAcknowledgement { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSContributionFence) DeepCopyInto(out *DNSContributionFence) { *out = *in - in.ValidUntil.DeepCopyInto(&out.ValidUntil) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSContributionFence. @@ -705,9 +704,7 @@ func (in *DNSPublicationManifestSpec) DeepCopyInto(out *DNSPublicationManifestSp if in.ContributionFences != nil { in, out := &in.ContributionFences, &out.ContributionFences *out = make([]DNSContributionFence, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + copy(*out, *in) } if in.RegistrationFences != nil { in, out := &in.RegistrationFences, &out.RegistrationFences @@ -939,10 +936,6 @@ func (in *DNSRecordContributionSpec) DeepCopy() *DNSRecordContributionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSRecordContributionStatus) DeepCopyInto(out *DNSRecordContributionStatus) { *out = *in - if in.ValidUntil != nil { - in, out := &in.ValidUntil, &out.ValidUntil - *out = (*in).DeepCopy() - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -1268,7 +1261,6 @@ func (in *DNSResolvedAdditionalNameRule) DeepCopy() *DNSResolvedAdditionalNameRu // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSResolverAccessAuthorization) DeepCopyInto(out *DNSResolverAccessAuthorization) { *out = *in - in.ValidUntil.DeepCopyInto(&out.ValidUntil) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSResolverAccessAuthorization. @@ -1350,7 +1342,7 @@ func (in *DNSResolverAccessBindingSpec) DeepCopyInto(out *DNSResolverAccessBindi *out = make([]string, len(*in)) copy(*out, *in) } - in.Authorization.DeepCopyInto(&out.Authorization) + out.Authorization = in.Authorization } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSResolverAccessBindingSpec. @@ -1545,7 +1537,7 @@ func (in *DNSResolverBindingSpec) DeepCopyInto(out *DNSResolverBindingSpec) { out.Source = in.Source out.Placement = in.Placement in.Configuration.DeepCopyInto(&out.Configuration) - in.Authorization.DeepCopyInto(&out.Authorization) + out.Authorization = in.Authorization } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSResolverBindingSpec. diff --git a/config/crd/bases/dns.networking.miloapis.com_dnscontributiongrants.yaml b/config/crd/bases/dns.networking.miloapis.com_dnscontributiongrants.yaml index 46a238eb..3614543d 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnscontributiongrants.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnscontributiongrants.yaml @@ -44,14 +44,10 @@ spec: type: array principal: properties: - clusterUID: - minLength: 1 - type: string subject: minLength: 1 type: string required: - - clusterUID - subject type: object producerID: diff --git a/config/crd/bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml b/config/crd/bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml index 1a8845d5..018209d8 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml @@ -71,7 +71,7 @@ spec: items: description: |- DNSContributionFence persists an authenticated high-water sequence even - while its observation is unhealthy, rejected, or expired. + while its observation is unhealthy or withdrawn. properties: epoch: format: int64 @@ -91,9 +91,6 @@ spec: don't ONLY use UUIDs, this is an alias to string. Being a type captures intent and helps make sure that UIDs and names do not get conflated. type: string - validUntil: - format: date-time - type: string required: - epoch - grantUID diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml index 163f509d..0f38938b 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml @@ -412,9 +412,6 @@ spec: sequence: format: int64 type: integer - validUntil: - format: date-time - type: string writerEpoch: format: int64 type: integer diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml index 99cd47bf..f743b3d3 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml @@ -44,16 +44,12 @@ spec: format: int64 minimum: 1 type: integer - validUntil: - format: date-time - type: string writerEpoch: format: int64 minimum: 1 type: integer required: - sequence - - validUntil - writerEpoch type: object contextRef: diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml index 4875ae95..4d78b627 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml @@ -45,16 +45,12 @@ spec: format: int64 minimum: 1 type: integer - validUntil: - format: date-time - type: string writerEpoch: format: int64 minimum: 1 type: integer required: - sequence - - validUntil - writerEpoch type: object configuration: @@ -263,10 +259,6 @@ spec: rule: self.authorization.writerEpoch > oldSelf.authorization.writerEpoch || (self.authorization.writerEpoch == oldSelf.authorization.writerEpoch && self.authorization.sequence >= oldSelf.authorization.sequence) - - message: changing an authorization deadline requires a new fence - rule: self.authorization.writerEpoch != oldSelf.authorization.writerEpoch - || self.authorization.sequence != oldSelf.authorization.sequence || - self.authorization.validUntil == oldSelf.authorization.validUntil - message: zone membership changes require a new cache generation and configuration revision rule: self.configuration.zoneRefs == oldSelf.configuration.zoneRefs diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index a1573c32..0b3ecd42 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -35,11 +35,11 @@ projection or publication for healthy sources. DNS always consumes `DNSResolverContext` and `DNSResolverAccessBinding` from the project API. It does not watch a networking API. A trusted integration principal bound to `internal-dns-vpc-integration` creates a context and writes -short-lived regional access intent. RBAC separates integration, grant issuer, +regional access intent. RBAC separates integration, grant issuer, product publisher, and DNS controller permissions. Product publishers cannot issue grants or modify resolver authority. -The context UID, access UID, authorization writer epoch, sequence, and original -deadline are carried into the internal `DNSResolverBinding`. +The context UID, access UID, writer epoch, and sequence are carried into the +internal `DNSResolverBinding`. Access persists until an explicit withdrawal. The control plane allocates a managed namespace when the context requests one. Compute discovers the context status and publishes through @@ -72,7 +72,7 @@ TCP. `verifier.publicName` can override this probe. BIND members need a writable Private packet caching stays disabled in dnsdist. Each BIND view has its own positive and negative cache. `max-ncache-ttl` defaults to five seconds and is bounded at 60 seconds. Public recursion retains DNSSEC validation; each view -adds validation exceptions only for its private apexes. Unknown or expired +adds validation exceptions only for its private apexes. Unknown or withdrawn destinations are rejected before cache lookup. Regional publication proof connects directly to that member's BIND listener, @@ -81,7 +81,7 @@ UDP and TCP, and verifies the installed SOA serial and fingerprint. Restrict `proxyPeers` to regional dnsdist and the member's local publication agent network. -## Transactions, health, and expiry +## Transactions, health, and outage recovery An attached zone that has no active publication gates its context until the configuration is complete. Pending contexts deny queries, recursion, and cache @@ -96,19 +96,20 @@ atomic only after every candidate validates. Reload wrappers must wait for the listener and backend pool before returning so the immediate DNS proof observes the activated state. -Publication ACKs are leases. A regional frontend and node may keep a context -available while at least one assigned regional member has a fresh current ACK; -the project-facing Ready condition can still report degraded membership. If no -member remains, the frontend gates the context. A regional member that cannot -apply or prove a publication fails closed and stops renewing its watchdog. +Publication acknowledgements report verified configuration versions. The fleet +retains accepted proofs during control plane or broker outages and continues +using local DNS probes and backend health checks. Status can report degraded +update delivery without withdrawing an applied resolver binding. -Agents preserve original record and access deadlines in their checkpoints. -They remove expired records, rewrite regional zone files, and flush affected -BIND names locally even while NATS or a source API is unavailable. Replay and -restart cannot extend a deadline or regress writer/configuration fences. +Records and access bindings have no expiry deadline. Product services withdraw +endpoints through explicit health updates or deletion. When delivery stops, DNS +retains the last reported endpoint state. Agents restore applied configuration +from persistent checkpoints before waiting for messaging connectivity. New +nodes require a complete configuration before accepting queries. Replay cannot +regress writer or configuration versions or restore a withdrawn lifetime. -The watchdog lease is at most five seconds. It is renewed only after local -expiry enforcement, a complete validated transaction, and required DNS proofs. +The local watchdog lease is at most five seconds. It is renewed only after a +complete validated transaction and required DNS proofs. The gate must stop or firewall the local private listener without depending on Kubernetes or NATS. Validation, reload, readiness wait, and proof must fit inside the watchdog budget; reduce shard size or improve local operations when @@ -142,8 +143,8 @@ raising those bounds. NATS uses TLS with a verified server name, client certificate, and one scoped identity per process. Control-plane credentials publish serving/publication subjects and consume ACKs. Each serving member can consume its assigned durable -subjects and publish only its own hashed ACK subject. Broker partitions do not -extend record or authorization deadlines. +subjects and publish only its own hashed ACK subject. Broker partitions delay +updates while the fleet continues serving its applied configuration. `fleet.example.yaml` provides node, regional frontend, and regional BIND JSON configurations plus one Deployment template. Instantiate the template for each @@ -157,12 +158,13 @@ The project API server authorizes DNS resource and status operations. Bind the roles in `rbac.yaml` to service principals in each project namespace. Customer access uses the protected resources and DNS roles in `config/iam`. -The webhook validates live references and lease limits. It also verifies that -changes to a publisher's contribution and freshness observation use that -publisher's current grant. Publication-result updates cannot renew a publisher's +The webhook validates live references and verifies that changes to a publisher's +contribution and eligibility observation use that publisher's current grant. +Grants identify the authenticated subject within the project API context. +Publication-result updates cannot change a publisher's observation. Resource permissions do not depend on webhook principal lists. -The supported internal envelope version is `v1alpha3` and the serving checkpoint -format is 4. Unsupported versions fail closed. Deploy matching controller and +The supported internal envelope version is `v1alpha4` and the serving checkpoint +format is 5. Unsupported versions fail closed. Deploy matching controller and fleet versions with fresh internal DNS transport state and checkpoints. Preserve any existing state for recovery. The deployment requires a coordinated rollout; it does not support mixed versions or automatic state conversion. diff --git a/config/internal-dns/control-plane.example.yaml b/config/internal-dns/control-plane.example.yaml index 9f01b945..8d648ea5 100644 --- a/config/internal-dns/control-plane.example.yaml +++ b/config/internal-dns/control-plane.example.yaml @@ -14,7 +14,6 @@ data: { "name": "example-project", "projectUID": "00000000-0000-0000-0000-000000000001", - "sourceClusterUID": "00000000-0000-0000-0000-000000000010", "namespace": "example-project", "kubeAPI": {"qps":20,"burst":40,"requestTimeoutSeconds":10} } @@ -63,9 +62,7 @@ data: "admission": { "enabled": true, "port": 9443, - "certDir": "/var/run/internal-dns/admission", - "maxContributionLeaseSeconds": 90, - "maxAccessLeaseSeconds": 300 + "certDir": "/var/run/internal-dns/admission" } } --- diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml index 66c96aa9..b5a43141 100644 --- a/config/internal-dns/fleet.example.yaml +++ b/config/internal-dns/fleet.example.yaml @@ -13,7 +13,7 @@ data: "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"localServingLease":90000000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]} } @@ -30,7 +30,7 @@ data: "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}]}, - "expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"localServingLease":90000000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 + "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-front"]}]} } @@ -47,7 +47,7 @@ data: "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} } @@ -67,7 +67,7 @@ data: "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, - "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} + "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} } diff --git a/internal/internaldns/admission/handler.go b/internal/internaldns/admission/handler.go index e1fb31f6..575a5e02 100644 --- a/internal/internaldns/admission/handler.go +++ b/internal/internaldns/admission/handler.go @@ -11,7 +11,6 @@ import ( "fmt" "net/http" "reflect" - "time" dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" "go.miloapis.com/dns-operator/internal/internaldns/model" @@ -28,18 +27,13 @@ const contributionResource = "dnsrecordcontributions" const Path = "/validate-internal-dns" -// Handler uses an uncached project client for grant and registration lookups. -// TrustedClusterUID is supplied by the server configuration, never by a record -// publisher. For project-aware deployments, ClientForRequest must resolve the -// authenticated project context and return its immutable source cluster UID. +// Handler uses an uncached client selected by authenticated project context. type Handler struct { - Client client.Client - TrustedClusterUID string - MaxContributionLease time.Duration - MaxAccessLease time.Duration + Client client.Client + ResolverContextsEnabled bool - Now func() time.Time - ClientForRequest func(context.Context, kubeadmission.Request) (client.Client, string, error) + + ClientForRequest func(context.Context, kubeadmission.Request) (client.Client, error) } var _ kubeadmission.Handler = (*Handler)(nil) @@ -57,14 +51,14 @@ func (h *Handler) Handle(ctx context.Context, req kubeadmission.Request) kubeadm } switch req.Resource.Resource { case "dnsregistrations", contributionResource, "dnscontributiongrants", "dnszoneassociations", "dnsnamingpolicies", "dnsresolveraccessbindings": - // These resources require live references or bounded freshness checks. + // These resources require live references. default: return kubeadmission.Allowed("API server authorization and schema validation apply") } - cl, clusterUID := h.Client, h.TrustedClusterUID + cl := h.Client if h.ClientForRequest != nil { var err error - cl, clusterUID, err = h.ClientForRequest(ctx, req) + cl, err = h.ClientForRequest(ctx, req) if err != nil { return kubeadmission.Errored(http.StatusServiceUnavailable, err) } @@ -79,7 +73,7 @@ func (h *Handler) Handle(ctx context.Context, req kubeadmission.Request) kubeadm case "dnsregistrations": err = h.registration(ctx, cl, req) case contributionResource: - err = h.contribution(ctx, cl, clusterUID, req) + err = h.contribution(ctx, cl, req) case "dnscontributiongrants": err = h.grant(ctx, cl, req) case "dnszoneassociations": @@ -100,7 +94,6 @@ func (h *Handler) resolverAccess(ctx context.Context, cl client.Client, req kube if err := decode(req, ¤t); err != nil { return err } - a := current.Spec.Authorization var resolverContext dnsv1alpha1.DNSResolverContext if err := cl.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: current.Spec.ContextRef.Name}, &resolverContext); err != nil { return fmt.Errorf("resolver context lookup: %w", err) @@ -115,24 +108,10 @@ func (h *Handler) resolverAccess(ctx context.Context, cl client.Client, req kube if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != resolverContext.Generation { return fmt.Errorf("resolver context is not ready for its current generation") } - max := h.MaxAccessLease - if max <= 0 { - max = 5 * time.Minute - } - now := time.Now() - if h.Now != nil { - now = h.Now() - } - if !a.ValidUntil.After(now) { - return fmt.Errorf("access deadline is already expired") - } - if a.ValidUntil.After(now.Add(max)) { - return fmt.Errorf("access deadline exceeds platform lease limit") - } return nil } -func (h *Handler) contribution(ctx context.Context, cl client.Client, clusterUID string, req kubeadmission.Request) error { +func (h *Handler) contribution(ctx context.Context, cl client.Client, req kubeadmission.Request) error { var c, old dnsv1alpha1.DNSRecordContribution if err := decode(req, &c); err != nil { return err @@ -150,8 +129,8 @@ func (h *Handler) contribution(ctx context.Context, cl client.Client, clusterUID if reflect.DeepEqual(previous, current) { return nil } - // Binding an issued epoch cannot introduce or renew endpoint eligibility. - if previous.WriterEpoch == 0 && previous.ObservedGeneration == 0 && previous.Sequence == 0 && !previous.Eligible && previous.ValidUntil == nil && current.WriterEpoch > 0 { + // Binding an issued epoch cannot introduce endpoint eligibility. + if previous.WriterEpoch == 0 && previous.ObservedGeneration == 0 && previous.Sequence == 0 && !previous.Eligible && current.WriterEpoch > 0 { current.WriterEpoch = 0 bindEpoch = reflect.DeepEqual(previous, current) } @@ -163,7 +142,7 @@ func (h *Handler) contribution(ctx context.Context, cl client.Client, clusterUID if g.UID != c.Spec.GrantRef.UID || !g.DeletionTimestamp.IsZero() { return fmt.Errorf("grant lifetime is not current") } - if clusterUID == "" || g.Spec.Principal.ClusterUID != clusterUID || (!bindEpoch && g.Spec.Principal.Subject != req.UserInfo.Username) { + if !bindEpoch && g.Spec.Principal.Subject != req.UserInfo.Username { return fmt.Errorf("caller does not match the grant's authenticated producer principal") } if g.Spec.RegistrationRef.UID != c.Spec.RegistrationRef.UID || g.Spec.RegistrationRef.Name != c.Spec.RegistrationRef.Name { @@ -215,23 +194,6 @@ func (h *Handler) contribution(ctx context.Context, cl client.Client, clusterUID if c.Status.ObservedGeneration != c.Generation { return fmt.Errorf("producer observation must match the contribution generation") } - if c.Status.ValidUntil == nil { - return fmt.Errorf("producer observation requires an original freshness deadline") - } - now := time.Now() - if h.Now != nil { - now = h.Now() - } - maxLease := h.MaxContributionLease - if maxLease <= 0 { - maxLease = 90 * time.Second - } - if c.Status.ValidUntil.After(now.Add(maxLease)) { - return fmt.Errorf("freshness deadline exceeds platform lease limit") - } - if c.Status.Eligible && !c.Status.ValidUntil.After(now) { - return fmt.Errorf("eligible observation is already expired") - } } else if req.Operation != admissionv1.Delete && !reflect.DeepEqual(old.Status, c.Status) && req.Operation != admissionv1.Create { return fmt.Errorf("producer observations must use the status subresource") } diff --git a/internal/internaldns/admission/handler_test.go b/internal/internaldns/admission/handler_test.go index 3a840af7..f49d8e79 100644 --- a/internal/internaldns/admission/handler_test.go +++ b/internal/internaldns/admission/handler_test.go @@ -21,7 +21,6 @@ import ( ) func TestContributionAuthenticatedWriter(t *testing.T) { - now := time.Date(2026, 10, 6, 18, 0, 0, 0, time.UTC) producer := "system:serviceaccount:compute-system:dns-publisher" registration := &dnsv1alpha1.DNSRegistration{ ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "project", UID: "reg-uid", Generation: 3}, @@ -31,7 +30,7 @@ func TestContributionAuthenticatedWriter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "compute", Namespace: "project", UID: "grant-uid"}, Spec: dnsv1alpha1.DNSContributionGrantSpec{ RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: "api", UID: "reg-uid"}, - Principal: dnsv1alpha1.DNSProducerPrincipal{ClusterUID: "cluster-uid", Subject: producer}, + Principal: dnsv1alpha1.DNSProducerPrincipal{Subject: producer}, RecordTypes: []dnsv1alpha1.RRType{"AAAA"}, }, Status: dnsv1alpha1.DNSContributionGrantStatus{ActiveWriterEpoch: 8, ObservedRegistrationGeneration: 3}, @@ -42,13 +41,13 @@ func TestContributionAuthenticatedWriter(t *testing.T) { RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: "api", UID: "reg-uid", Generation: 3}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: "compute", UID: "grant-uid"}, }, - Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 2, WriterEpoch: 8, Sequence: 18, Eligible: true, ValidUntil: ptrTime(now.Add(30 * time.Second))}, + Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 2, WriterEpoch: 8, Sequence: 18, Eligible: true}, } scheme := runtime.NewScheme() if err := dnsv1alpha1.AddToScheme(scheme); err != nil { t.Fatal(err) } - h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(registration, grant, &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", Namespace: "project"}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}}).Build(), TrustedClusterUID: "cluster-uid", Now: func() time.Time { return now }} + h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(registration, grant, &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", Namespace: "project"}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}}).Build()} cases := []struct { name string user string @@ -70,8 +69,6 @@ func TestContributionAuthenticatedWriter(t *testing.T) { {"forged grant UID", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Spec.GrantRef.UID = "other-grant" }, false}, {"retired registration UID", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Spec.RegistrationRef.UID = "old-reg" }, false}, {"publication fields", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Status.PublishedRevision = 1000 }, false}, - {"overlong lease", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Status.ValidUntil = ptrTime(now.Add(time.Hour)) }, false}, - {"expired eligible lease", producer, func(c *dnsv1alpha1.DNSRecordContribution) { c.Status.ValidUntil = ptrTime(now.Add(-time.Second)) }, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -117,8 +114,6 @@ func TestUnavailableProjectContextFailsClosed(t *testing.T) { } } -func ptrTime(t time.Time) *metav1.Time { v := metav1.NewTime(t); return &v } - func request(resource, username string, obj, old any) kubeadmission.Request { raw, _ := json.Marshal(obj) var oldRaw []byte @@ -160,7 +155,7 @@ func TestRetiredProductIntentCanBeDeletedWithoutLiveGrant(t *testing.T) { } } -func TestResolverAccessRequiresCurrentContextAndBoundedLease(t *testing.T) { +func TestResolverAccessRequiresCurrentContext(t *testing.T) { now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) scheme := runtime.NewScheme() if err := dnsv1alpha1.AddToScheme(scheme); err != nil { @@ -168,20 +163,13 @@ func TestResolverAccessRequiresCurrentContextAndBoundedLease(t *testing.T) { } resolverContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "ctx-uid", Generation: 1}, Status: dnsv1alpha1.DNSResolverContextStatus{AccessWriterEpoch: 3}} setAdmissionCondition(&resolverContext.Status.Conditions, "Ready", metav1.ConditionTrue, 1, now) - h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolverContext).Build(), MaxAccessLease: 5 * time.Minute, Now: func() time.Time { return now }} - old := &dnsv1alpha1.DNSResolverAccessBinding{Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "ctx-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 9, ValidUntil: metav1.NewTime(now.Add(2 * time.Minute))}}} + h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolverContext).Build()} + old := &dnsv1alpha1.DNSResolverAccessBinding{Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "ctx-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 9}}} current := old.DeepCopy() current.Spec.Authorization.Sequence = 10 - current.Spec.Authorization.ValidUntil = metav1.NewTime(now.Add(3 * time.Minute)) req := request("dnsresolveraccessbindings", "system:serviceaccount:network-system:integration", current, old) if res := h.Handle(context.Background(), req); !res.Allowed { - t.Fatalf("fresh integration renewal denied: %v", res.Result) - } - overlong := current.DeepCopy() - overlong.Spec.Authorization.ValidUntil = metav1.NewTime(now.Add(6 * time.Minute)) - req = request("dnsresolveraccessbindings", "system:serviceaccount:network-system:integration", overlong, old) - if h.Handle(context.Background(), req).Allowed { - t.Fatal("overlong access lease accepted") + t.Fatalf("current integration update denied: %v", res.Result) } forged := current.DeepCopy() forged.Spec.ContextRef.UID = "replacement" @@ -300,9 +288,9 @@ func TestRegistrationRequiresCurrentZone(t *testing.T) { } } -func TestPublicationStatusDoesNotRenewPublisherObservation(t *testing.T) { +func TestPublicationStatusDoesNotChangePublisherObservation(t *testing.T) { h := Handler{Client: fake.NewClientBuilder().WithScheme(runtime.NewScheme()).Build()} - old := &dnsv1alpha1.DNSRecordContribution{Status: dnsv1alpha1.DNSRecordContributionStatus{WriterEpoch: 4, Sequence: 10, Eligible: true, ValidUntil: ptrTime(time.Now().Add(-time.Minute))}} + old := &dnsv1alpha1.DNSRecordContribution{Status: dnsv1alpha1.DNSRecordContributionStatus{WriterEpoch: 4, Sequence: 10, Eligible: true}} current := old.DeepCopy() current.Status.PublishedRevision = 12 current.Status.Conditions = []metav1.Condition{{Type: "Published", Status: metav1.ConditionTrue}} @@ -311,10 +299,10 @@ func TestPublicationStatusDoesNotRenewPublisherObservation(t *testing.T) { if result := h.Handle(context.Background(), req); !result.Allowed { t.Fatalf("publication result needs no producer grant: %v", result.Result) } - current.Status.ValidUntil = ptrTime(time.Now().Add(time.Minute)) + current.Status.Sequence++ req = request("dnsrecordcontributions", "rbac-authorized-controller", current, old) req.SubResource = "status" if h.Handle(context.Background(), req).Allowed { - t.Fatal("publication update renewed producer freshness without a grant") + t.Fatal("publication update changed the producer observation without a grant") } } diff --git a/internal/internaldns/controlplane/access_recovery_envtest_test.go b/internal/internaldns/controlplane/access_recovery_envtest_test.go index 4fd1edc2..3586d3ab 100644 --- a/internal/internaldns/controlplane/access_recovery_envtest_test.go +++ b/internal/internaldns/controlplane/access_recovery_envtest_test.go @@ -2,7 +2,6 @@ package controlplane import ( "context" - "encoding/json" "os" "path/filepath" "testing" @@ -85,22 +84,9 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { t.Run("name-only customer references", func(t *testing.T) { verifyNamedReferencesWithAPI(t, cl) }) - deadline := metav1.NewTime(now.Add(time.Minute + 22357123*time.Nanosecond)) - access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 2, ValidUntil: deadline}}} - access.APIVersion = dnsv1alpha1.GroupVersion.String() - access.Kind = "DNSResolverAccessBinding" - raw, err := json.Marshal(access) - if err != nil { - t.Fatal(err) - } - // Submit unstructured JSON, as kubectl does, so the API initially stores a - // fractional deadline independently of the typed client encoder. - rawAccess := &unstructured.Unstructured{} - requireAccessTestSuccess(t, json.Unmarshal(raw, &rawAccess.Object)) - rawAccess.Object["spec"].(map[string]interface{})["authorization"].(map[string]interface{})["validUntil"] = deadline.UTC().Format(time.RFC3339Nano) - requireAccessTestSuccess(t, cl.Create(ctx, rawAccess)) - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - r := &Reconciler{Client: cl, Scheme: scheme, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", MaxAccessLease: 5 * time.Minute, AddressAllocator: &platform.Allocator{Client: cl, Namespace: "platform"}}} + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 2}}} + requireAccessTestSuccess(t, cl.Create(ctx, access)) + r := &Reconciler{Client: cl, Scheme: scheme, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: &platform.Allocator{Client: cl, Namespace: "platform"}}} reconcile := func(at time.Time) { t.Helper() requireAccessTestSuccess(t, r.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, at)) @@ -112,213 +98,60 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { t.Fatalf("got %d bindings", len(bindings.Items)) } binding := bindings.Items[0] - if !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { - t.Fatalf("binding projection changed fractional deadline: %v != %v", binding.Spec.Authorization.ValidUntil, deadline) - } - bindingKey := client.ObjectKeyFromObject(&binding) - readBinding := func() { - t.Helper() - requireAccessTestSuccess(t, cl.Get(ctx, bindingKey, &binding)) - } - reconcile(now) - readBinding() - if binding.Spec.Tombstone || !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { - t.Fatal("unchanged fractional authorization was withdrawn on second reconcile") + key := client.ObjectKeyFromObject(&binding) + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 2 || binding.Spec.Configuration.Revision != 1 { + t.Fatalf("elapsed time changed access: %#v", binding.Spec) } requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - if accepted := apimeta.FindStatusCondition(access.Status.Conditions, "Accepted"); accepted == nil || accepted.Status != metav1.ConditionTrue { - t.Fatal("fractional source authorization lost Accepted status") - } - subsecondMutation := binding.DeepCopy() - subsecondMutation.Spec.Authorization.ValidUntil = metav1.NewTime(deadline.Add(time.Nanosecond)) - if err := cl.Update(ctx, subsecondMutation); !apierrors.IsInvalid(err) { - t.Fatalf("same-fence nanosecond deadline mutation accepted: %v", err) - } - // False is omitted from the wire. An ordinary active update must still - // evaluate the optional tombstone transition rules without a CEL error. - active := binding.DeepCopy() - active.Labels["validation"] = "active-update" - if err := cl.Update(ctx, active); err != nil { - t.Fatalf("active update with omitted false tombstone failed: %v", err) - } - readBinding() - for _, change := range []struct { - name string - mutate func(*dnsv1alpha1.DNSResolverBinding) - }{ - {"context UID", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Source.ResolverContextRef.UID = "replacement-context" }}, - {"access UID", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Source.AccessBindingRef.UID = "replacement-access" }}, - {"placement", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Placement.Shard = "replacement-shard" }}, - {"listener", func(b *dnsv1alpha1.DNSResolverBinding) { b.Spec.Configuration.Listeners.Node.Address = "fd70:100::11" }}, - } { - mutated := binding.DeepCopy() - change.mutate(mutated) - mutated.Spec.Configuration.Revision++ - if err := cl.Update(ctx, mutated); !apierrors.IsInvalid(err) { - t.Fatalf("immutable %s change accepted: %v", change.name, err) + // Schema validation preserves listener identity and ordered source revisions. + for _, mutate := range []func(*dnsv1alpha1.DNSResolverAccessBinding){func(a *dnsv1alpha1.DNSResolverAccessBinding) { a.Spec.QueryIdentity.Value = "fd70:100::20" }, func(a *dnsv1alpha1.DNSResolverAccessBinding) { a.Spec.Authorization.Sequence = 1 }, func(a *dnsv1alpha1.DNSResolverAccessBinding) { a.Spec.ContextRef.UID = "another-context" }} { + candidate := access.DeepCopy() + mutate(candidate) + if err := cl.Update(ctx, candidate); !apierrors.IsInvalid(err) { + t.Fatalf("identity or revision mutation accepted: %v", err) } } - // Expiry advances the DNS-owned configuration fence, preserving the exact - // integration-owned authorization. Source sequence 3 remains available. - expiredAt := now.Add(61 * time.Second) - reconcile(expiredAt) - readBinding() - if !binding.Spec.Tombstone || binding.Spec.Configuration.Revision != 2 || binding.Spec.Authorization.Sequence != 2 || !binding.Spec.Authorization.ValidUntil.Equal(&deadline) { - t.Fatalf("withdrawal consumed source authority: %#v", binding.Spec) - } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - staleExpiredAccess := *access.DeepCopy() - // A direct same-fence replay cannot reopen the retired configuration. - replay := binding.DeepCopy() - replay.Spec.Tombstone = false - replay.Spec.Configuration.Revision++ - if err := cl.Update(ctx, replay); !apierrors.IsInvalid(err) { - t.Fatalf("same-fence reactivation accepted: %v", err) - } - mutated := binding.DeepCopy() - mutated.Spec.Authorization.ValidUntil = metav1.NewTime(expiredAt.Add(time.Minute + 987654321*time.Nanosecond)) - if err := cl.Update(ctx, mutated); !apierrors.IsInvalid(err) { - t.Fatalf("same-fence deadline mutation accepted: %v", err) - } - // Source CEL rejects deadline changes without an increasing sequence. - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - baseAccess := access.DeepCopy() - renewedDeadline := metav1.NewTime(expiredAt.Add(time.Minute + 987654321*time.Nanosecond)) - access.Spec.Authorization.ValidUntil = renewedDeadline - if err := cl.Patch(ctx, access, client.MergeFromWithOptions(baseAccess, client.MergeFromWithOptimisticLock{})); !apierrors.IsInvalid(err) { - t.Fatalf("same-sequence source deadline mutation accepted: %v", err) - } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - if !access.Spec.Authorization.ValidUntil.Equal(&deadline) { - t.Fatal("rejected source patch changed its retained deadline") - } - reconcile(expiredAt) - readBinding() - if !binding.Spec.Tombstone { - t.Fatal("same source fence reopened withdrawn access") - } - // Ordinary next-sequence renewal is independent of private DNS counters. - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - access.Spec.Authorization.Sequence = 3 - access.Spec.Authorization.ValidUntil = renewedDeadline - requireAccessTestSuccess(t, cl.Update(ctx, access)) - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(access), access)) - if !access.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { - t.Fatal("source renewal lost deadline precision") - } - reconcile(expiredAt) - readBinding() - if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || binding.Spec.Configuration.Revision != 3 || !binding.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) { - t.Fatalf("next source sequence did not recover access: %#v", binding.Spec) + // An incomplete cached List cannot revoke a live access binding. + stale := &Reconciler{Client: &accessListSnapshotClient{Client: cl, accesses: nil}, Scheme: scheme, Options: r.Options} + requireAccessTestSuccess(t, stale.reconcileContextBindings(ctx, "project", nil, nil, now.Add(365*24*time.Hour))) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if binding.Spec.Tombstone { + t.Fatal("incomplete List withdrew live access") } - // Another controller can List expired source sequence 2 before this renewed - // platform binding is visible. Cleanup must freshly read sequence 3 instead - // of converting an unchanged stale rejection status into revocation. - staleController := &Reconciler{Client: &accessListSnapshotClient{Client: cl, accesses: []dnsv1alpha1.DNSResolverAccessBinding{staleExpiredAccess}}, Scheme: scheme, Options: r.Options} - requireAccessTestSuccess(t, staleController.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, expiredAt)) - readBinding() - if binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || binding.Spec.Configuration.Revision != 3 { - t.Fatalf("stale source List withdrew renewed binding: %#v", binding.Spec) - } - staleController.Client = &accessListSnapshotClient{Client: cl} - requireAccessTestSuccess(t, staleController.reconcileContextBindings(ctx, "project", nil, map[string]*dnsv1alpha1.DNSZone{}, expiredAt)) - readBinding() - if binding.Spec.Tombstone || binding.Spec.Configuration.Revision != 3 { - t.Fatal("omitted source List withdrew renewed binding") - } - // Deleting the context revokes access before its source deadline. Recreating - // the same name cannot satisfy the old UID-pinned integration reference. - oldContextUID := resolverContext.UID requireAccessTestSuccess(t, cl.Delete(ctx, resolverContext)) - reconcile(expiredAt) - readBinding() - if !binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 3 || !binding.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) { - t.Fatalf("context withdrawal rewrote source authority: %#v", binding.Spec) - } - freshContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "project"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "opaque-consumer"}} - readyContext(freshContext) - if freshContext.UID == oldContextUID { - t.Fatal("context UID did not rotate") - } - reconcile(expiredAt) - readBinding() + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) + if !binding.Spec.Tombstone || binding.Spec.Authorization.Sequence != 2 { + t.Fatal("explicit context deletion did not withdraw access") + } + replacement := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: resolverContext.Name, Namespace: "project"}, Spec: resolverContext.Spec} + readyContext(replacement) + reconcile(now.Add(365 * 24 * time.Hour)) + requireAccessTestSuccess(t, cl.Get(ctx, key, &binding)) if !binding.Spec.Tombstone { - t.Fatal("recreated context name restored old access") + t.Fatal("replacement context inherited retired access") } - // A new access UID cannot reuse a quarantined destination. A fresh protected - // destination creates a separate binding and leaves the retired one closed. requireAccessTestSuccess(t, cl.Delete(ctx, access)) - freshAccess := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: access.Spec} - freshAccess.Spec.ContextRef = dnsv1alpha1.DNSObjectReference{Name: freshContext.Name, UID: freshContext.UID} - freshAccess.Spec.Authorization.Sequence = 1 - requireAccessTestSuccess(t, cl.Create(ctx, freshAccess)) - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) - if !freshAccess.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { - t.Fatal("typed create lost source deadline precision") - } - reconcile(expiredAt) + access = &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "replacement", Namespace: "project"}, Spec: access.Spec} + access.Spec.ContextRef.UID = replacement.UID + access.Spec.QueryIdentity.Value = "fd70:100::20" + access.Spec.Authorization.Sequence = 1 + requireAccessTestSuccess(t, cl.Create(ctx, access)) + reconcile(now.Add(365 * 24 * time.Hour)) requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) - if len(bindings.Items) != 1 { - t.Fatal("new access lifetime reused a protected destination") - } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) - freshAccess.Spec.QueryIdentity.Value = "fd70:100::11" - freshAccess.Spec.Authorization.Sequence = 2 - if err := cl.Update(ctx, freshAccess); !apierrors.IsInvalid(err) { - t.Fatalf("access identity changed in place: %v", err) - } - freshSpec := freshAccess.Spec - requireAccessTestSuccess(t, cl.Delete(ctx, freshAccess)) - freshAccess = &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project"}, Spec: freshSpec} - requireAccessTestSuccess(t, cl.Create(ctx, freshAccess)) - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) - reconcile(expiredAt) - requireAccessTestSuccess(t, cl.List(ctx, &bindings, client.InNamespace("platform"))) - if len(bindings.Items) != 2 { - t.Fatalf("fresh protected identity created %d bindings", len(bindings.Items)) - } - readBinding() - if !binding.Spec.Tombstone { - t.Fatal("old binding reopened after access UID recreation") - } - // A renewal committed after the source confirmation but before destructive - // platform Patch must win through the platform resourceVersion fence. - var freshBinding dnsv1alpha1.DNSResolverBinding - for _, candidate := range bindings.Items { - if candidate.Spec.Source.AccessBindingRef.UID == freshAccess.UID { - freshBinding = candidate - } - } - if freshBinding.UID == "" { - t.Fatal("fresh access binding missing") - } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess)) - freshAccess.Spec.Authorization.Sequence = 3 - freshAccess.Spec.Authorization.ValidUntil = metav1.NewTime(expiredAt.Add(-time.Second)) - requireAccessTestSuccess(t, cl.Update(ctx, freshAccess)) - raceClient := &bindingPatchRaceClient{Client: cl, beforeWithdrawal: func(ctx context.Context) error { - if err := cl.Get(ctx, client.ObjectKeyFromObject(freshAccess), freshAccess); err != nil { - return err - } - freshAccess.Spec.Authorization.Sequence = 4 - freshAccess.Spec.Authorization.ValidUntil = renewedDeadline - if err := cl.Update(ctx, freshAccess); err != nil { - return err - } - if err := cl.Get(ctx, client.ObjectKeyFromObject(&freshBinding), &freshBinding); err != nil { - return err + active := 0 + for _, b := range bindings.Items { + if !b.Spec.Tombstone { + active++ + if b.Spec.Source.ResolverContextRef.UID != replacement.UID { + t.Fatal("new binding uses retired context") + } } - freshBinding.Spec.Authorization = freshAccess.Spec.Authorization - freshBinding.Spec.Configuration.Revision++ - return cl.Update(ctx, &freshBinding) - }} - raceController := &Reconciler{Client: &accessListSnapshotClient{Client: raceClient}, Scheme: scheme, Options: r.Options} - if err := raceController.reconcileContextBindings(ctx, "project", nil, nil, expiredAt); !apierrors.IsConflict(err) { - t.Fatalf("concurrent platform renewal did not fence withdrawal: %v", err) } - requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(&freshBinding), &freshBinding)) - if freshBinding.Spec.Tombstone || freshBinding.Spec.Authorization.Sequence != 4 || !freshBinding.Spec.Authorization.ValidUntil.Equal(&renewedDeadline) { - t.Fatal("destructive cleanup overwrote concurrent platform renewal") + if active != 1 { + t.Fatalf("active bindings=%d", active) } } diff --git a/internal/internaldns/controlplane/compiler.go b/internal/internaldns/controlplane/compiler.go index 1ed1c5e9..30d95e7d 100644 --- a/internal/internaldns/controlplane/compiler.go +++ b/internal/internaldns/controlplane/compiler.go @@ -14,9 +14,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// PublicationPlan is serialized into DNSPublicationChunk payloads. It carries -// original observation deadlines so a disconnected serving agent can expire an -// endpoint without receiving a central deletion. +// PublicationPlan is the canonical record configuration serialized into publication chunks. type PublicationPlan struct { ZoneUID types.UID `json:"zoneUID"` ZoneApex string `json:"zoneApex"` @@ -40,15 +38,13 @@ type PublicationRRSet struct { TTLSeconds int32 `json:"ttlSeconds"` Records []dnsv1alpha1.RecordEntry `json:"records"` ContributionUIDs []types.UID `json:"contributionUIDs,omitempty"` - ValidUntil *time.Time `json:"validUntil,omitempty"` } type ContributionFence struct { - UID types.UID `json:"uid"` - GrantUID types.UID `json:"grantUID"` - Epoch int64 `json:"epoch"` - Sequence int64 `json:"sequence"` - ValidUntil time.Time `json:"validUntil"` + UID types.UID `json:"uid"` + GrantUID types.UID `json:"grantUID"` + Epoch int64 `json:"epoch"` + Sequence int64 `json:"sequence"` } type RegistrationFence struct { @@ -86,10 +82,6 @@ func Compile(in CompileInput) (CompileResult, error) { return CompileResult{}, err } - now := in.Now - if now.IsZero() { - now = time.Now().UTC() - } result := CompileResult{Accepted: map[types.UID]bool{}, Reasons: map[types.UID]string{}, Sequences: map[types.UID]int64{}} result.Plan.ZoneUID = in.Zone.UID result.Plan.ZoneApex = canonicalName(in.Zone.Spec.DomainName) @@ -193,7 +185,6 @@ func Compile(in CompileInput) (CompileResult, error) { previousSeq := map[types.UID]ContributionFence{} if in.Previous != nil { for _, f := range in.Previous.Contributions { - f.ValidUntil = f.ValidUntil.UTC() previousSeq[f.UID] = f } } @@ -208,7 +199,6 @@ func Compile(in CompileInput) (CompileResult, error) { ttl int32 records []dnsv1alpha1.RecordEntry uids []types.UID - until *time.Time } aggregates := map[ownershipKey]*aggregate{} for i := range in.Contributions { @@ -220,11 +210,7 @@ func Compile(in CompileInput) (CompileResult, error) { result.Reasons[c.UID] = reason continue } - until := time.Time{} - if c.Status.ValidUntil != nil { - until = c.Status.ValidUntil.UTC() - } - currentFence := ContributionFence{UID: c.UID, GrantUID: grant.UID, Epoch: c.Status.WriterEpoch, Sequence: c.Status.Sequence, ValidUntil: until} + currentFence := ContributionFence{UID: c.UID, GrantUID: grant.UID, Epoch: c.Status.WriterEpoch, Sequence: c.Status.Sequence} previousFence := previousSeq[c.UID] if previousFence.UID == "" || currentFence.Epoch > previousFence.Epoch || currentFence.Sequence > previousFence.Sequence { publicationFences[c.UID] = currentFence @@ -238,10 +224,6 @@ func Compile(in CompileInput) (CompileResult, error) { result.Reasons[c.UID] = "Ineligible" continue } - if c.Status.ValidUntil == nil || !c.Status.ValidUntil.After(now) { - result.Reasons[c.UID] = "Expired" - continue - } validBundle := true for _, set := range c.Spec.RecordSets { if len(set.Records) == 0 || !containsType(reg.Spec.RecordTypes, set.RecordType) || !containsType(grant.Spec.RecordTypes, set.RecordType) { @@ -279,10 +261,6 @@ func Compile(in CompileInput) (CompileResult, error) { record.Name = key.name a.records = append(a.records, record) a.uids = append(a.uids, c.UID) - if a.until == nil || until.Before(*a.until) { - u := until - a.until = &u - } } } } @@ -309,7 +287,7 @@ func Compile(in CompileInput) (CompileResult, error) { } for key, a := range aggregates { sortAggregate(a.records, a.uids) - result.Plan.RRsets = append(result.Plan.RRsets, PublicationRRSet{Name: key.name, RecordType: key.rrtype, TTLSeconds: a.ttl, Records: a.records, ContributionUIDs: a.uids, ValidUntil: a.until}) + result.Plan.RRsets = append(result.Plan.RRsets, PublicationRRSet{Name: key.name, RecordType: key.rrtype, TTLSeconds: a.ttl, Records: a.records, ContributionUIDs: a.uids}) } sort.Slice(result.Plan.RRsets, func(i, j int) bool { if result.Plan.RRsets[i].Name == result.Plan.RRsets[j].Name { @@ -340,9 +318,6 @@ func validateContribution(c *dnsv1alpha1.DNSRecordContribution, reg *dnsv1alpha1 if c.Status.Sequence <= 0 { return "InvalidSequence" } - if c.Status.ValidUntil == nil || c.Status.ValidUntil.IsZero() { - return "MissingValidityDeadline" - } if previous.UID != "" { if previous.GrantUID != "" && previous.GrantUID != grant.UID { return "FenceIdentityMismatch" @@ -354,9 +329,6 @@ func validateContribution(c *dnsv1alpha1.DNSRecordContribution, reg *dnsv1alpha1 if c.Status.Sequence < previous.Sequence { return dnsValueNonIncreasingSequence } - if c.Status.Sequence == previous.Sequence && !c.Status.ValidUntil.Time.Equal(previous.ValidUntil) { - return "FenceDeadlineMismatch" - } } } return "" diff --git a/internal/internaldns/controlplane/compiler_test.go b/internal/internaldns/controlplane/compiler_test.go index 545debea..fe5b6c80 100644 --- a/internal/internaldns/controlplane/compiler_test.go +++ b/internal/internaldns/controlplane/compiler_test.go @@ -76,25 +76,23 @@ func TestCompileRejectsStaleGenerationAndWholeInvalidBundle(t *testing.T) { }) } -func TestCompileExpiryStillAdvancesHighWaterAndRejectsRollback(t *testing.T) { +func TestCompileWithdrawalAdvancesHighWaterAndRejectsRollback(t *testing.T) { now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) in := fixture(now) previous := in.Contributions[0] - in.Previous = &PublicationPlan{Contributions: []ContributionFence{{UID: previous.UID, GrantUID: previous.Spec.GrantRef.UID, Epoch: 7, Sequence: 18, ValidUntil: now}}} + in.Previous = &PublicationPlan{Contributions: []ContributionFence{{UID: previous.UID, GrantUID: previous.Spec.GrantRef.UID, Epoch: 7, Sequence: 18}}} in.Contributions[0].Status.Sequence = 19 - expired := metav1.NewTime(now.Add(-time.Second)) - in.Contributions[0].Status.ValidUntil = &expired + in.Contributions[0].Status.Eligible = false got, err := Compile(in) if err != nil { t.Fatal(err) } - if got.Reasons[previous.UID] != "Expired" || len(got.Plan.RRsets) != 0 || got.Plan.Contributions[0].Sequence != 19 { - t.Fatalf("expiry fence incorrect: %#v", got) + if got.Reasons[previous.UID] != "Ineligible" || len(got.Plan.RRsets) != 0 || got.Plan.Contributions[0].Sequence != 19 { + t.Fatalf("withdrawal fence incorrect: %#v", got) } in.Previous = &got.Plan in.Contributions[0].Status.Sequence = 18 - fresh := metav1.NewTime(now.Add(time.Minute)) - in.Contributions[0].Status.ValidUntil = &fresh + in.Contributions[0].Status.Eligible = true rolled, err := Compile(in) if err != nil { t.Fatal(err) @@ -141,28 +139,6 @@ func TestCompileRetainsHighWaterAcrossRejectedPublications(t *testing.T) { } } -func TestCompileRejectsEqualSequenceDeadlineMutation(t *testing.T) { - now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - in := fixture(now) - first, err := Compile(in) - if err != nil { - t.Fatal(err) - } - in.Previous = &first.Plan - changed := metav1.NewTime(in.Contributions[0].Status.ValidUntil.Add(time.Minute)) - in.Contributions[0].Status.ValidUntil = &changed - got, err := Compile(in) - if err != nil { - t.Fatal(err) - } - if got.Reasons["contribution-a"] != "FenceDeadlineMismatch" || len(got.Plan.RRsets) != 0 { - t.Fatalf("equal sequence changed its immutable deadline: %#v", got) - } - if len(got.Plan.Contributions) != 1 || !got.Plan.Contributions[0].ValidUntil.Equal(first.Plan.Contributions[0].ValidUntil) { - t.Fatalf("deadline mutation replaced durable fence: %#v", got.Plan.Contributions) - } -} - func TestCompileNormalizesFQDNAndEnforcesCNAMEExclusivity(t *testing.T) { in := fixture(time.Now().UTC()) in.StaticRecords = []dnsv1alpha1.DNSRecordSet{{ @@ -215,8 +191,7 @@ func fixture(now time.Time) CompileInput { } reg := dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: "registration-a", Generation: 3}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 30}} grant := dnsv1alpha1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: "compute", UID: "grant-a", Generation: 2}, Spec: dnsv1alpha1.DNSContributionGrantSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, ProducerID: "compute-east", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}}, Status: dnsv1alpha1.DNSContributionGrantStatus{ActiveWriterEpoch: 7, ObservedGrantGeneration: 2, ObservedRegistrationGeneration: 3}} - until := metav1.NewTime(now.Add(time.Minute)) - c := dnsv1alpha1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint-a", UID: "contribution-a", Generation: 4}, Spec: dnsv1alpha1.DNSRecordContributionSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: grant.Name, UID: grant.UID}, RecordSets: []dnsv1alpha1.DNSContributionRecordSet{{RecordType: dnsv1alpha1.RRTypeAAAA, Records: []dnsv1alpha1.RecordEntry{{Name: "api.corp.internal", AAAA: &dnsv1alpha1.AAAARecordSpec{Content: "fd00::1"}}}}}}, Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 4, WriterEpoch: 7, Sequence: 18, Eligible: true, ValidUntil: &until}} + c := dnsv1alpha1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint-a", UID: "contribution-a", Generation: 4}, Spec: dnsv1alpha1.DNSRecordContributionSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: grant.Name, UID: grant.UID}, RecordSets: []dnsv1alpha1.DNSContributionRecordSet{{RecordType: dnsv1alpha1.RRTypeAAAA, Records: []dnsv1alpha1.RecordEntry{{Name: "api.corp.internal", AAAA: &dnsv1alpha1.AAAARecordSpec{Content: "fd00::1"}}}}}}, Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 4, WriterEpoch: 7, Sequence: 18, Eligible: true}} return CompileInput{Zone: zone, Associations: []dnsv1alpha1.DNSZoneAssociation{assoc("b", "vpc-b"), assoc("a", "vpc-a")}, Registrations: []dnsv1alpha1.DNSRegistration{reg}, Grants: []dnsv1alpha1.DNSContributionGrant{grant}, Contributions: []dnsv1alpha1.DNSRecordContribution{c}, Now: now} } @@ -316,3 +291,32 @@ func TestCompileRejectsUnboundedDynamicTTL(t *testing.T) { t.Fatal("unbounded TTL published") } } + +func TestCompileRetainsHealthyRecordsUntilExplicitWithdrawal(t *testing.T) { + now := time.Now().UTC() + input := fixture(now) + first, err := Compile(input) + if err != nil { + t.Fatal(err) + } + input.Now = now.Add(365 * 24 * time.Hour) + input.Previous = &first.Plan + after, err := Compile(input) + if err != nil { + t.Fatal(err) + } + beforeJSON, _ := json.Marshal(first.Plan) + afterJSON, _ := json.Marshal(after.Plan) + if string(beforeJSON) != string(afterJSON) { + t.Fatal("elapsed time changed the last reported endpoint state") + } + input.Contributions[0].Status.Eligible = false + input.Contributions[0].Status.Sequence++ + withdrawn, err := Compile(input) + if err != nil { + t.Fatal(err) + } + if len(withdrawn.Plan.RRsets) != 0 { + t.Fatal("explicit unhealthy observation did not withdraw the endpoint") + } +} diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index f554999f..54384359 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -367,11 +367,11 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as spec := a.Spec contextObject := contextByUID[spec.ContextRef.UID] contextReady := resolverContextReady(contextObject) - valid := a.DeletionTimestamp.IsZero() && contextObject != nil && contextObject.DeletionTimestamp.IsZero() && contextObject.Name == spec.ContextRef.Name && contextReady && contextObject.Status.AccessWriterEpoch > 0 && spec.Authorization.WriterEpoch == contextObject.Status.AccessWriterEpoch && spec.Authorization.Sequence >= a.Status.ObservedSequence && spec.Authorization.ValidUntil.After(now) && !spec.Authorization.ValidUntil.After(now.Add(r.Options.MaxAccessLease)) + valid := a.DeletionTimestamp.IsZero() && contextObject != nil && contextObject.DeletionTimestamp.IsZero() && contextObject.Name == spec.ContextRef.Name && contextReady && contextObject.Status.AccessWriterEpoch > 0 && spec.Authorization.WriterEpoch == contextObject.Status.AccessWriterEpoch && spec.Authorization.Sequence >= a.Status.ObservedSequence bindingName := "binding-" + model.OpaqueToken(string(r.Options.ProjectUID) + "/" + string(a.UID))[:20] if prior := bindingsByName[bindingName]; prior != nil { compare := model.Compare(uint64(spec.Authorization.WriterEpoch), uint64(spec.Authorization.Sequence), uint64(prior.Spec.Authorization.WriterEpoch), uint64(prior.Spec.Authorization.Sequence)) - if compare < 0 || (compare == 0 && (!spec.Authorization.ValidUntil.Equal(&prior.Spec.Authorization.ValidUntil) || prior.Spec.Tombstone)) { + if compare < 0 || (compare == 0 && prior.Spec.Tombstone) { valid = false } } @@ -437,7 +437,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as return err } } - binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForRegion(spec.Region))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForRegion(spec.Region)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence, ValidUntil: spec.Authorization.ValidUntil}}} + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForRegion(spec.Region))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForRegion(spec.Region)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence}}} if !exists { if err := r.Create(ctx, binding); err != nil { return err @@ -476,7 +476,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as if b.Labels["internal-dns.miloapis.com/context-uid"] == "" { continue } - withdraw, err := r.confirmAccessWithdrawal(ctx, ns, b, now) + withdraw, err := r.confirmAccessWithdrawal(ctx, ns, b) if err != nil { return err } @@ -497,8 +497,8 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as // confirmAccessWithdrawal prevents a stale or incomplete source List from // withdrawing a binding renewed by another controller. Read uncertainty leaves -// the existing authorization intact; serving agents still enforce its deadline. -func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace string, binding *dnsv1alpha1.DNSResolverBinding, now time.Time) (bool, error) { +// the existing authorization intact; serving agents retain the applied configuration. +func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace string, binding *dnsv1alpha1.DNSResolverBinding) (bool, error) { var access dnsv1alpha1.DNSResolverAccessBinding if err := r.Get(ctx, client.ObjectKey{Namespace: namespace, Name: binding.Spec.Source.AccessBindingRef.Name}, &access); err != nil { if apierrors.IsNotFound(err) { @@ -512,7 +512,7 @@ func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace stri authorization := access.Spec.Authorization prior := binding.Spec.Authorization compare := model.Compare(uint64(authorization.WriterEpoch), uint64(authorization.Sequence), uint64(prior.WriterEpoch), uint64(prior.Sequence)) - if compare < 0 || (compare == 0 && !authorization.ValidUntil.Equal(&prior.ValidUntil)) { + if compare < 0 { return false, nil } if access.Spec.ContextRef.Name != binding.Spec.Source.ResolverContextRef.Name || access.Spec.ContextRef.UID != binding.Spec.Source.ResolverContextRef.UID { @@ -529,11 +529,11 @@ func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace stri return true, nil } ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") - if !authorization.ValidUntil.After(now) || resolverContext.Status.AccessWriterEpoch > authorization.WriterEpoch { + if resolverContext.Status.AccessWriterEpoch > authorization.WriterEpoch { return true, nil } // Readiness or epoch propagation can lag a healthy authorization. Such - // uncertainty is not retirement evidence and must not consume its lease. + // uncertainty is not retirement evidence and must not withdraw its binding. if ready == nil || ready.Status != metav1.ConditionTrue || ready.ObservedGeneration != resolverContext.Generation || resolverContext.Status.AccessWriterEpoch < authorization.WriterEpoch { return false, nil } @@ -576,7 +576,7 @@ func bindingAckCurrent(binding *dnsv1alpha1.DNSResolverBinding, now time.Time) b // past the exact configuration/authorization generation or its local ACK // lease. An ACK lease may be shorter than the access deadline, but never // longer because AckSink caps it before persisting the row. - if ack.Phase == string(model.AckVerified) && ack.Revision == binding.Spec.Configuration.Revision && ack.AuthorizationIssuerEpoch == binding.Spec.Authorization.WriterEpoch && ack.AuthorizationRevision == binding.Spec.Authorization.Sequence && ack.ValidUntil != nil && ack.ValidUntil.After(now) && !ack.ValidUntil.After(binding.Spec.Authorization.ValidUntil.Time) { + if ack.Phase == string(model.AckVerified) && ack.Revision == binding.Spec.Configuration.Revision && ack.AuthorizationIssuerEpoch == binding.Spec.Authorization.WriterEpoch && ack.AuthorizationRevision == binding.Spec.Authorization.Sequence && ack.ValidUntil != nil && ack.ValidUntil.After(now) { return true } } @@ -665,8 +665,6 @@ func BindingAddressCandidates(uid types.UID) (string, string) { func bindingEqual(a, b dnsv1alpha1.DNSResolverBindingSpec) bool { x := a y := b - x.Authorization.ValidUntil = metav1.Time{} - y.Authorization.ValidUntil = metav1.Time{} xb, _ := json.Marshal(x) yb, _ := json.Marshal(y) return string(xb) == string(yb) diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index 8da2e5b6..ba6eed14 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -28,22 +28,6 @@ type accessListSnapshotClient struct { getErrorFor string } -type bindingPatchRaceClient struct { - client.Client - beforeWithdrawal func(context.Context) error -} - -func (c *bindingPatchRaceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - if binding, ok := obj.(*dnsv1alpha1.DNSResolverBinding); ok && binding.Spec.Tombstone && c.beforeWithdrawal != nil { - before := c.beforeWithdrawal - c.beforeWithdrawal = nil - if err := before(ctx); err != nil { - return err - } - } - return c.Client.Patch(ctx, obj, patch, opts...) -} - func (c *accessListSnapshotClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { _, access := obj.(*dnsv1alpha1.DNSResolverAccessBinding) _, resolverContext := obj.(*dnsv1alpha1.DNSResolverContext) @@ -69,7 +53,7 @@ func (testAllocator) ClaimDestination(context.Context, types.UID, types.UID, str return nil } -func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { +func TestContextAccessPersistsUntilExplicitWithdrawal(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() @@ -81,10 +65,9 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", Namespace: "project", UID: "zone-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "assoc", Namespace: "project", UID: "assoc-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "zone"}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: "ctx"}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "zone-uid", Generation: 1}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}}} setCondition(&association.Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) - until := metav1.NewTime(now.Add(2 * time.Minute)) - access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 4, Sequence: 12, ValidUntil: until}}} + access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 4, Sequence: 12}}} c := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject, zone, association, access).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverBinding{}).Build() - r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", MaxAccessLease: 5 * time.Minute, AddressAllocator: testAllocator{}}} + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: testAllocator{}}} if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now); err != nil { t.Fatal(err) } @@ -96,7 +79,7 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { t.Fatalf("got %d bindings", len(bindings.Items)) } b := bindings.Items[0] - if b.Spec.Configuration.Listeners.Node.Address != "fd70:100::10" || b.Spec.Source.ResolverContextRef.UID != "context-uid" || b.Spec.Authorization.WriterEpoch != 4 || b.Spec.Authorization.Sequence != 12 || !b.Spec.Authorization.ValidUntil.Equal(&until) { + if b.Spec.Configuration.Listeners.Node.Address != "fd70:100::10" || b.Spec.Source.ResolverContextRef.UID != "context-uid" || b.Spec.Authorization.WriterEpoch != 4 || b.Spec.Authorization.Sequence != 12 { t.Fatalf("access projection drifted: %#v", b.Spec) } if b.Spec.Source.ProjectUID != "project-uid" || b.Spec.Source.ResolverContextRef.Name != "ctx" || b.Spec.Source.AccessBindingRef.Name != "access" || b.Spec.Source.AccessBindingRef.UID != "access-uid" || b.Spec.Configuration.ZoneRefs[0].Name != "zone" || b.Spec.Configuration.ZoneRefs[0].UID != "zone-uid" || b.Spec.Placement.Region != "central" { @@ -125,13 +108,17 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { t.Fatalf("zone membership change reused configuration fences: %#v", b.Spec.Configuration) } + contextObject.Status.AccessWriterEpoch++ + if err := c.Status().Update(ctx, contextObject); err != nil { + t.Fatal(err) + } if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now.Add(3*time.Minute)); err != nil { t.Fatal(err) } if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { t.Fatal(err) } - if !b.Spec.Tombstone || b.Spec.Authorization.Sequence != 12 || !b.Spec.Authorization.ValidUntil.Equal(&until) || b.Spec.Configuration.Revision != 3 { + if !b.Spec.Tombstone || b.Spec.Authorization.Sequence != 12 || b.Spec.Configuration.Revision != 3 { t.Fatalf("withdrawal changed integration authority or failed to advance configuration: %#v", b.Spec) } if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { @@ -141,8 +128,7 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { if ready == nil || ready.Status != metav1.ConditionFalse { t.Fatalf("invalid access retained Ready: %#v", access.Status.Conditions) } - staleExpiredAccess := *access.DeepCopy() - access.Spec.Authorization.ValidUntil = metav1.NewTime(now.Add(4 * time.Minute)) + staleAccess := *access.DeepCopy() if err := c.Update(ctx, access); err != nil { t.Fatal(err) } @@ -153,11 +139,12 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { t.Fatal(err) } if !b.Spec.Tombstone { - t.Fatal("same-fence source deadline replay reopened access") + t.Fatal("same-fence replay reopened access") } if err := c.Get(ctx, client.ObjectKeyFromObject(access), access); err != nil { t.Fatal(err) } + access.Spec.Authorization.WriterEpoch = contextObject.Status.AccessWriterEpoch access.Spec.Authorization.Sequence++ if err := c.Update(ctx, access); err != nil { t.Fatal(err) @@ -168,10 +155,10 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { if err := c.Get(ctx, client.ObjectKeyFromObject(&b), &b); err != nil { t.Fatal(err) } - if b.Spec.Tombstone || b.Spec.Authorization.Sequence != 13 || !b.Spec.Authorization.ValidUntil.Equal(&access.Spec.Authorization.ValidUntil) || b.Spec.Configuration.Revision != 4 { + if b.Spec.Tombstone || b.Spec.Authorization.Sequence != 13 || b.Spec.Configuration.Revision != 4 { t.Fatalf("ordinary next source sequence failed to renew access: %#v", b.Spec) } - for _, snapshot := range [][]dnsv1alpha1.DNSResolverAccessBinding{{staleExpiredAccess}, nil} { + for _, snapshot := range [][]dnsv1alpha1.DNSResolverAccessBinding{{staleAccess}, nil} { staleController := &Reconciler{Client: &accessListSnapshotClient{Client: c, accesses: snapshot}, Scheme: s, Options: r.Options} if err := staleController.reconcileContextBindings(ctx, "project", nil, nil, now.Add(3*time.Minute)); err != nil { t.Fatal(err) @@ -216,9 +203,8 @@ func TestContextAccessProjectsExactAuthorizationAndExpiresClosed(t *testing.T) { func TestBindingAckCurrentRequiresVerifiedExactAuthorization(t *testing.T) { now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) - deadline := metav1.NewTime(now.Add(time.Minute)) ackDeadline := metav1.NewTime(now.Add(30 * time.Second)) - b := &dnsv1alpha1.DNSResolverBinding{Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 11, ValidUntil: deadline}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{Phase: string(model.AckVerified), Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 11, ValidUntil: &ackDeadline}}}} + b := &dnsv1alpha1.DNSResolverBinding{Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 11}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{Phase: string(model.AckVerified), Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 11, ValidUntil: &ackDeadline}}}} if !bindingAckCurrent(b, now) { t.Fatal("exact verified ACK was rejected") } @@ -232,11 +218,6 @@ func TestBindingAckCurrentRequiresVerifiedExactAuthorization(t *testing.T) { t.Fatal("stale authorization ACK was accepted") } b.Status.MemberAcknowledgements[0].AuthorizationRevision = 11 - overlong := metav1.NewTime(now.Add(2 * time.Minute)) - b.Status.MemberAcknowledgements[0].ValidUntil = &overlong - if bindingAckCurrent(b, now) { - t.Fatal("ACK beyond access deadline was accepted") - } } func TestContextNamingPolicyResolvesMultipleZonesWithoutNetworking(t *testing.T) { diff --git a/internal/internaldns/controlplane/publication_reconciler.go b/internal/internaldns/controlplane/publication_reconciler.go index 367e1923..7214859c 100644 --- a/internal/internaldns/controlplane/publication_reconciler.go +++ b/internal/internaldns/controlplane/publication_reconciler.go @@ -101,7 +101,7 @@ func (r *Reconciler) reconcileProject(ctx context.Context, ns string) (ctrl.Resu return ctrl.Result{RequeueAfter: nextProjectRequeue(now, earliest, r.Options.LeaseDuration)}, nil } -// nextProjectRequeue never lets a later contribution or authorization expiry +// nextProjectRequeue never lets a later scheduled reconciliation // postpone ownership renewal. A standby polls more frequently while another // holder owns a zone, but the active holder must still run within half a lease // even when the compiled publication is otherwise quiescent. @@ -311,7 +311,7 @@ func (r *Reconciler) compileZone(ctx context.Context, z *dnsv1alpha1.DNSZone, as return false, time.Time{}, err } } - return r.tombstoneAcknowledged(ctx, own), nextExpiry(ownPlan, now), nil + return r.tombstoneAcknowledged(ctx, own), time.Time{}, nil } previousRevision, err := r.activeRevision(ctx, own) if err != nil { @@ -342,7 +342,7 @@ func (r *Reconciler) compileZone(ctx context.Context, z *dnsv1alpha1.DNSZone, as if err := r.Patch(ctx, own, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { return false, time.Time{}, err } - return false, nextExpiry(ownPlan, now), nil + return false, time.Time{}, nil } func (r *Reconciler) activeRevision(ctx context.Context, own *dnsv1alpha1.DNSPublicationOwnership) (int64, error) { @@ -430,7 +430,7 @@ func (r *Reconciler) persistPublication(ctx context.Context, z *dnsv1alpha1.DNSZ } apiFences := make([]dnsv1alpha1.DNSContributionFence, 0, len(fences)) for _, f := range fences { - apiFences = append(apiFences, dnsv1alpha1.DNSContributionFence{UID: f.UID, GrantUID: f.GrantUID, Epoch: f.Epoch, Sequence: f.Sequence, ValidUntil: metav1.NewTime(f.ValidUntil)}) + apiFences = append(apiFences, dnsv1alpha1.DNSContributionFence{UID: f.UID, GrantUID: f.GrantUID, Epoch: f.Epoch, Sequence: f.Sequence}) } apiRegistrations := make([]dnsv1alpha1.DNSRegistrationFence, 0, len(registrations)) for _, f := range registrations { @@ -548,7 +548,7 @@ func (r *Reconciler) loadPrevious(ctx context.Context, o *dnsv1alpha1.DNSPublica // fields may lose fractional seconds during API round trips, so rebuilding // from manifest metadata can make an unchanged observation appear mutated. for _, f := range wire.ObservationFences { - p.Contributions = append(p.Contributions, ContributionFence{UID: types.UID(f.ContributionUID), GrantUID: types.UID(f.GrantUID), Epoch: int64(f.WriterEpoch), Sequence: int64(f.Sequence), ValidUntil: f.ValidUntil.UTC()}) + p.Contributions = append(p.Contributions, ContributionFence{UID: types.UID(f.ContributionUID), GrantUID: types.UID(f.GrantUID), Epoch: int64(f.WriterEpoch), Sequence: int64(f.Sequence)}) } return p, nil } @@ -672,13 +672,3 @@ func chunk(b []byte, size int) [][]byte { } return out } - -func nextExpiry(p PublicationPlan, now time.Time) time.Time { - var next time.Time - for _, f := range p.Contributions { - if f.ValidUntil.After(now) && (next.IsZero() || f.ValidUntil.Before(next)) { - next = f.ValidUntil - } - } - return next -} diff --git a/internal/internaldns/controlplane/publication_reconciler_test.go b/internal/internaldns/controlplane/publication_reconciler_test.go index 77f0f649..596da265 100644 --- a/internal/internaldns/controlplane/publication_reconciler_test.go +++ b/internal/internaldns/controlplane/publication_reconciler_test.go @@ -58,14 +58,14 @@ func TestExpiredOwnershipTakeoverFencesResumedWriter(t *testing.T) { } } -func TestContributionDeadlineCannotDelayOwnershipRenewal(t *testing.T) { +func TestScheduledReconciliationCannotDelayOwnershipRenewal(t *testing.T) { now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) lease := 10 * time.Second if got := nextProjectRequeue(now, now.Add(90*time.Second), lease); got != 5*time.Second { - t.Fatalf("long-lived contribution delayed lease renewal: got %s want 5s", got) + t.Fatalf("later reconciliation delayed lease renewal: got %s want 5s", got) } if got := nextProjectRequeue(now, now.Add(2*time.Second), lease); got != 2*time.Second { - t.Fatalf("near contribution expiry was not honored: got %s want 2s", got) + t.Fatalf("earlier reconciliation was not honored: got %s want 2s", got) } } @@ -96,12 +96,10 @@ func TestQuiescentSurvivorRenewsOwnershipBeyondLeaseWindow(t *testing.T) { contribution.Namespace = "project" contribution.Status.WriterEpoch = 0 contribution.Status.Sequence = 1 - until := metav1.NewTime(now.Add(90 * time.Second)) - contribution.Status.ValidUntil = &until c := fake.NewClientBuilder().WithScheme(s).WithObjects(network, zone, assoc, reg, grant, contribution). WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSZoneAssociation{}, &dnsv1alpha1.DNSRegistration{}, &dnsv1alpha1.DNSContributionGrant{}, &dnsv1alpha1.DNSRecordContribution{}, &dnsv1alpha1.DNSPublicationManifest{}).Build() options := ReconcilerOptions{ - ProjectUID: "project-a", SourceClusterUID: "cluster-a", ProjectNamespace: "project", PlatformNamespace: "platform", + ProjectUID: "project-a", ProjectNamespace: "project", PlatformNamespace: "platform", Identity: "writer-a", LeaseDuration: 10 * time.Second, ServingRegions: []ServingRegion{{Region: "east", Shard: "shared"}}, AddressAllocator: testAllocator{}, Now: func() time.Time { return now }, } @@ -154,12 +152,11 @@ func TestQuiescentSurvivorRenewsOwnershipBeyondLeaseWindow(t *testing.T) { func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - deadline := now.Add(-time.Minute).Add(101582 * time.Microsecond) s := runtime.NewScheme() if err := dnsv1alpha1.AddToScheme(s); err != nil { t.Fatal(err) } - payload, err := json.Marshal(model.PublicationPlan{ZoneUID: "zone-a", Apex: "corp.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contribution-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 19, ValidUntil: deadline}}, RRSets: []model.RRSet{}}) + payload, err := json.Marshal(model.PublicationPlan{ZoneUID: "zone-a", Apex: "corp.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contribution-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 19}}, RRSets: []model.RRSet{}}) if err != nil { t.Fatal(err) } @@ -175,7 +172,7 @@ func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { ContentHash: model.Hash(payload), // The metadata deliberately represents the precision an API server may // retain. Recovery must use the exact immutable wire observation. - ContributionFences: []dnsv1alpha1.DNSContributionFence{{UID: "contribution-a", GrantUID: "grant-a", Epoch: 7, Sequence: 19, ValidUntil: metav1.NewTime(deadline.Truncate(time.Second))}}, + ContributionFences: []dnsv1alpha1.DNSContributionFence{{UID: "contribution-a", GrantUID: "grant-a", Epoch: 7, Sequence: 19}}, }, } owner := &dnsv1alpha1.DNSPublicationOwnership{ @@ -198,9 +195,6 @@ func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { if len(previous.Contributions) != 1 || previous.Contributions[0].Sequence != 19 { t.Fatalf("old epoch high-water fence was lost: %#v", previous) } - if !previous.Contributions[0].ValidUntil.Equal(deadline) { - t.Fatalf("wire deadline precision was lost: got %s want %s", previous.Contributions[0].ValidUntil, deadline) - } in := fixture(now) in.Previous = previous in.Contributions[0].Status.Sequence = 18 @@ -304,14 +298,14 @@ func TestReconcilerPersistsFencedMultiRegionOutbox(t *testing.T) { contrib.Namespace = "project" contrib.Status.WriterEpoch = 0 contrib.Status.Sequence = 1 - accessEast := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "prod-east", Namespace: "project", UID: "access-east"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}, Region: "east", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd53::11"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1, ValidUntil: metav1.NewTime(now.Add(3 * time.Minute))}}} + accessEast := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "prod-east", Namespace: "project", UID: "access-east"}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}, Region: "east", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd53::11"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1}}} accessWest := accessEast.DeepCopy() accessWest.Name = "prod-west" accessWest.UID = "access-west" accessWest.Spec.Region = "west" accessWest.Spec.QueryIdentity.Value = "fd53::12" c := fake.NewClientBuilder().WithScheme(s).WithObjects(network, zone, assoc, reg, grant, contrib, accessEast, accessWest).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSZoneAssociation{}, &dnsv1alpha1.DNSRegistration{}, &dnsv1alpha1.DNSContributionGrant{}, &dnsv1alpha1.DNSRecordContribution{}, &dnsv1alpha1.DNSPublicationManifest{}).Build() - r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-a", SourceClusterUID: "cluster-a", ProjectNamespace: "project", PlatformNamespace: "internal-dns-system", Identity: "compiler-a", ServingRegions: []ServingRegion{{Region: "east", Shard: "s1"}, {Region: "west", Shard: "s2"}}, AddressAllocator: testAllocator{}, Now: func() time.Time { return now }}} + r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-a", ProjectNamespace: "project", PlatformNamespace: "internal-dns-system", Identity: "compiler-a", ServingRegions: []ServingRegion{{Region: "east", Shard: "s1"}, {Region: "west", Shard: "s2"}}, AddressAllocator: testAllocator{}, Now: func() time.Time { return now }}} for i := 0; i < 8; i++ { if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "project", Name: "project"}}); err != nil { t.Fatalf("reconcile %d: %v", i, err) diff --git a/internal/internaldns/controlplane/reconciler.go b/internal/internaldns/controlplane/reconciler.go index 05755851..02cfdc39 100644 --- a/internal/internaldns/controlplane/reconciler.go +++ b/internal/internaldns/controlplane/reconciler.go @@ -20,7 +20,6 @@ import ( type ReconcilerOptions struct { ProjectUID types.UID - SourceClusterUID string ProjectNamespace string PlatformNamespace string Region string @@ -32,7 +31,6 @@ type ReconcilerOptions struct { AddressAllocator AddressAllocator PrivateZoneClassName string ServingRegions []ServingRegion - MaxAccessLease time.Duration } type ServingRegion struct { @@ -68,9 +66,6 @@ func (o *ReconcilerOptions) defaults() { o.LeaseDuration = 30 * time.Second } - if o.MaxAccessLease <= 0 { - o.MaxAccessLease = 5 * time.Minute - } if o.Now == nil { o.Now = func() time.Time { return time.Now().UTC() } } @@ -85,9 +80,7 @@ func (o ReconcilerOptions) servingRegions() []ServingRegion { type Options = ReconcilerOptions -// Reconciler compiles one project's desired DNS state. ProjectUID and -// SourceClusterUID are trusted configuration supplied by the replication -// bridge; Kubernetes namespace identity is never treated as project identity. +// Reconciler compiles records within a configured project scope. type Reconciler struct { client.Client Scheme *runtime.Scheme @@ -96,8 +89,8 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Options.defaults() - if r.Options.ProjectUID == "" || r.Options.SourceClusterUID == "" { - return ctrl.Result{}, errors.New("projectUID and sourceClusterUID are required") + if r.Options.ProjectUID == "" { + return ctrl.Result{}, errors.New("projectUID is required") } ns := r.Options.ProjectNamespace if ns == "" { diff --git a/internal/internaldns/controlplane/wire.go b/internal/internaldns/controlplane/wire.go index bab5827a..c7a9e38c 100644 --- a/internal/internaldns/controlplane/wire.go +++ b/internal/internaldns/controlplane/wire.go @@ -22,7 +22,7 @@ func WirePlan(plan PublicationPlan) (model.PublicationPlan, error) { for _, contribution := range plan.Contributions { out.ObservationFences = append(out.ObservationFences, model.ObservationFence{ ContributionUID: string(contribution.UID), GrantUID: string(contribution.GrantUID), - WriterEpoch: uint64(contribution.Epoch), Sequence: uint64(contribution.Sequence), ValidUntil: contribution.ValidUntil, + WriterEpoch: uint64(contribution.Epoch), Sequence: uint64(contribution.Sequence), }) } sort.Slice(out.ObservationFences, func(i, j int) bool { @@ -56,7 +56,6 @@ func WirePlan(plan PublicationPlan) (model.PublicationPlan, error) { if f.UID == uid { record.WriterEpoch = uint64(f.Epoch) record.Sequence = uint64(f.Sequence) - record.ValidUntil = f.ValidUntil break } } diff --git a/internal/internaldns/model/model.go b/internal/internaldns/model/model.go index aef57230..039ac1c9 100644 --- a/internal/internaldns/model/model.go +++ b/internal/internaldns/model/model.go @@ -17,7 +17,7 @@ import ( "github.com/miekg/dns" ) -const APIVersion = "internal.dns.networking.miloapis.com/v1alpha3" +const APIVersion = "internal.dns.networking.miloapis.com/v1alpha4" type Kind string @@ -78,9 +78,8 @@ const ( ) type Authorization struct { - IssuerEpoch uint64 `json:"issuerEpoch"` - Revision uint64 `json:"revision"` - ValidUntil time.Time `json:"validUntil"` + IssuerEpoch uint64 `json:"issuerEpoch"` + Revision uint64 `json:"revision"` } type Backend struct { @@ -192,7 +191,7 @@ func (s ServingSnapshot) Validate() error { } seenVIP[raw] = b.BindingUID } - if b.Port == 0 || len(b.Transports) == 0 || b.Authorization.IssuerEpoch == 0 || b.Authorization.Revision == 0 || b.Authorization.ValidUntil.IsZero() { + if b.Port == 0 || len(b.Transports) == 0 || b.Authorization.IssuerEpoch == 0 || b.Authorization.Revision == 0 { return fmt.Errorf("binding %q has incomplete transport or authorization", b.BindingUID) } if b.ConfigurationPending && len(b.Zones) != 0 { @@ -323,12 +322,11 @@ func (c PublicationChunk) Validate() error { } type RRRecord struct { - Content string `json:"content"` - ContributionUID string `json:"contributionUID,omitempty"` - WriterEpoch uint64 `json:"writerEpoch,omitempty"` - Sequence uint64 `json:"sequence,omitempty"` - Eligible bool `json:"eligible"` - ValidUntil time.Time `json:"validUntil,omitempty"` + Content string `json:"content"` + ContributionUID string `json:"contributionUID,omitempty"` + WriterEpoch uint64 `json:"writerEpoch,omitempty"` + Sequence uint64 `json:"sequence,omitempty"` + Eligible bool `json:"eligible"` } type RRSet struct { @@ -352,15 +350,12 @@ type RegistrationFence struct { Generation uint64 `json:"generation"` } -// ObservationFence carries the accepted high-water mark even when an -// unhealthy or withdrawn observation produces no records. ValidUntil is the -// original contributor deadline and is part of the immutable fence identity. +// ObservationFence carries the accepted high-water mark, including withdrawn observations. type ObservationFence struct { - ContributionUID string `json:"contributionUID"` - GrantUID string `json:"grantUID"` - WriterEpoch uint64 `json:"writerEpoch"` - Sequence uint64 `json:"sequence"` - ValidUntil time.Time `json:"validUntil"` + ContributionUID string `json:"contributionUID"` + GrantUID string `json:"grantUID"` + WriterEpoch uint64 `json:"writerEpoch"` + Sequence uint64 `json:"sequence"` } func (p PublicationPlan) Validate() error { @@ -384,7 +379,7 @@ func (p PublicationPlan) Validate() error { observationByUID := make(map[string]ObservationFence, len(p.ObservationFences)) previousContribution := "" for _, observation := range p.ObservationFences { - if observation.ContributionUID == "" || observation.GrantUID == "" || observation.WriterEpoch == 0 || observation.Sequence == 0 || observation.ValidUntil.IsZero() { + if observation.ContributionUID == "" || observation.GrantUID == "" || observation.WriterEpoch == 0 || observation.Sequence == 0 { return errors.New("publication observation fence is incomplete") } if previousContribution != "" && observation.ContributionUID <= previousContribution { @@ -433,12 +428,12 @@ func (p PublicationPlan) Validate() error { return fmt.Errorf("RRset %q contains an invalid record", key) } if rec.ContributionUID == "" { - if rec.WriterEpoch != 0 || rec.Sequence != 0 || !rec.ValidUntil.IsZero() { + if rec.WriterEpoch != 0 || rec.Sequence != 0 { return fmt.Errorf("RRset %q static record contains contribution fence data", key) } } else { observation, ok := observationByUID[rec.ContributionUID] - if !ok || rec.WriterEpoch != observation.WriterEpoch || rec.Sequence != observation.Sequence || !rec.ValidUntil.Equal(observation.ValidUntil) { + if !ok || rec.WriterEpoch != observation.WriterEpoch || rec.Sequence != observation.Sequence { return fmt.Errorf("RRset %q record fence does not match contribution observation %q", key, rec.ContributionUID) } } diff --git a/internal/internaldns/model/model_test.go b/internal/internaldns/model/model_test.go index 8cdef63c..5365bf09 100644 --- a/internal/internaldns/model/model_test.go +++ b/internal/internaldns/model/model_test.go @@ -42,7 +42,7 @@ func TestServingSnapshotRejectsSharedDestination(t *testing.T) { t.Parallel() now := time.Now() b := func(uid, vpc string) Binding { - return Binding{BindingUID: uid, ProjectUID: "project", ContextUID: vpc, BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "2001:db8::1", ClusterAddress: "2001:db8::2", Port: 53, Transports: []Transport{TransportUDP, TransportTCP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1, ValidUntil: now.Add(time.Minute)}} + return Binding{BindingUID: uid, ProjectUID: "project", ContextUID: vpc, BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "2001:db8::1", ClusterAddress: "2001:db8::2", Port: 53, Transports: []Transport{TransportUDP, TransportTCP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1}} } s := ServingSnapshot{Region: "east", Shard: "s1", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []Binding{b("one", "vpc-a"), b("two", "vpc-b")}} if err := s.Validate(); err == nil { @@ -94,7 +94,7 @@ func TestEnvelopeRejectsObsoleteWireVersion(t *testing.T) { func TestPendingConfigurationCannotExposePartialZoneAttachments(t *testing.T) { now := time.Now().UTC() - binding := Binding{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::a", ClusterAddress: "fd54::a", Port: 53, Transports: []Transport{TransportUDP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1, ValidUntil: now.Add(time.Minute)}, ConfigurationPending: true, Zones: []ZoneAttachment{{ZoneUID: "zone", Apex: "private.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}}} + binding := Binding{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::a", ClusterAddress: "fd54::a", Port: 53, Transports: []Transport{TransportUDP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1}, ConfigurationPending: true, Zones: []ZoneAttachment{{ZoneUID: "zone", Apex: "private.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}}} snapshot := ServingSnapshot{Region: "east", Shard: "shared", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []Binding{binding}} if err := snapshot.Validate(); err == nil { t.Fatal("partial pending context accepted") diff --git a/internal/internaldns/platform/acks.go b/internal/internaldns/platform/acks.go index 28b6fd1c..321db0cc 100644 --- a/internal/internaldns/platform/acks.go +++ b/internal/internaldns/platform/acks.go @@ -161,10 +161,6 @@ func (s *AckSink) serving(ctx context.Context, a model.MemberAck, row dnsv1alpha bindingRow.SnapshotRevision = int64(a.Revision) bindingRow.AuthorizationIssuerEpoch = binding.Spec.Authorization.WriterEpoch bindingRow.AuthorizationRevision = binding.Spec.Authorization.Sequence - if bindingRow.ValidUntil != nil && bindingRow.ValidUntil.After(b.Authorization.ValidUntil) { - u := metav1.NewTime(b.Authorization.ValidUntil) - bindingRow.ValidUntil = &u - } binding.Status.MemberAcknowledgements = upsertAck(binding.Status.MemberAcknowledgements, bindingRow) complete := s.bindingVerified(binding, e, snapshot, now) if complete { @@ -283,7 +279,7 @@ func (s *AckSink) projectPublicationForScope(ctx context.Context, project Projec continue } for _, record := range rrset.Records { - if record.Eligible && (record.ContributionUID == "" || record.ValidUntil.After(now)) { + if record.Eligible { available = true } } @@ -385,7 +381,7 @@ func hasApplied(rows []dnsv1alpha1.DNSApplyAcknowledgement, epoch, revision int6 return false } -// Refresh withdraws status readiness when a member lease or authorization expires, +// Refresh withdraws status readiness when member acknowledgment freshness expires, // even if its process or event stream has stopped producing acknowledgements. func (s *AckSink) Refresh(ctx context.Context) error { now := time.Now().UTC() @@ -528,7 +524,7 @@ func LoadCommittedServingSnapshot(ctx context.Context, c client.Client, namespac } func (s *AckSink) bindingVerified(b *dnsv1alpha1.DNSResolverBinding, envelope *model.Envelope, snapshot *model.ServingSnapshot, now time.Time) bool { - if envelope == nil || snapshot == nil || b.Spec.Tombstone || !b.Spec.Authorization.ValidUntil.After(now) { + if envelope == nil || snapshot == nil || b.Spec.Tombstone { return false } present := false diff --git a/internal/internaldns/platform/planner.go b/internal/internaldns/platform/planner.go index e5c974b6..e79cca26 100644 --- a/internal/internaldns/platform/planner.go +++ b/internal/internaldns/platform/planner.go @@ -145,7 +145,7 @@ func (p *Planner) Step(ctx context.Context) error { if b.Configuration.Listeners.Node.Port != 53 || b.Configuration.Listeners.Regional.Port != 53 || !reflect.DeepEqual(b.Configuration.Listeners.Node.Transports, b.Configuration.Listeners.Regional.Transports) { return fmt.Errorf("binding %s requires matching DNS listener ports and transports", obj.Name) } - view := model.Binding{BindingUID: string(obj.UID), ProjectUID: string(b.Source.ProjectUID), ContextUID: string(b.Source.ResolverContextRef.UID), BindingGeneration: uint64(b.Configuration.Generation), ConfigurationRevision: uint64(b.Configuration.Revision), ConsumerAddress: b.Configuration.Listeners.Node.Address, ClusterAddress: b.Configuration.Listeners.Regional.Address, Port: uint16(b.Configuration.Listeners.Node.Port), NodeBackends: c.NodeBackends, ClusterBackends: c.ClusterBackends, Authorization: model.Authorization{IssuerEpoch: uint64(b.Authorization.WriterEpoch), Revision: uint64(b.Authorization.Sequence), ValidUntil: b.Authorization.ValidUntil.Time}} + view := model.Binding{BindingUID: string(obj.UID), ProjectUID: string(b.Source.ProjectUID), ContextUID: string(b.Source.ResolverContextRef.UID), BindingGeneration: uint64(b.Configuration.Generation), ConfigurationRevision: uint64(b.Configuration.Revision), ConsumerAddress: b.Configuration.Listeners.Node.Address, ClusterAddress: b.Configuration.Listeners.Regional.Address, Port: uint16(b.Configuration.Listeners.Node.Port), NodeBackends: c.NodeBackends, ClusterBackends: c.ClusterBackends, Authorization: model.Authorization{IssuerEpoch: uint64(b.Authorization.WriterEpoch), Revision: uint64(b.Authorization.Sequence)}} for _, t := range b.Configuration.Listeners.Node.Transports { view.Transports = append(view.Transports, model.Transport(t)) } diff --git a/internal/internaldns/platform/platform_test.go b/internal/internaldns/platform/platform_test.go index b14b1767..1adaec72 100644 --- a/internal/internaldns/platform/platform_test.go +++ b/internal/internaldns/platform/platform_test.go @@ -114,7 +114,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { mname := fmt.Sprintf("manifest-%d", i) objects = append(objects, &dnsv1alpha1.DNSPublicationOwnership{ObjectMeta: metav1.ObjectMeta{Name: "owner-" + model.OpaqueToken(string(zuid))[:20], Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationOwnershipSpec{ZoneUID: zuid, WriterEpoch: 1, ActiveManifestName: mname}}) objects = append(objects, &dnsv1alpha1.DNSPublicationManifest{ObjectMeta: metav1.ObjectMeta{Name: mname, Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationManifestSpec{ZoneRef: dnsv1alpha1.DNSObjectReference{UID: zuid}, ZoneApex: "prod.internal", WriterEpoch: 1, Revision: 1, ServingTargets: []dnsv1alpha1.DNSPublicationTarget{{Region: region, Shard: shard}}}}) - objects = append(objects, &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("binding-%d", i), Namespace: ns, UID: types.UID(fmt.Sprintf("binding-uid-%d", i)), Labels: map[string]string{"internal-dns.miloapis.com/region": region, "internal-dns.miloapis.com/shard": shard}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: types.UID(fmt.Sprintf("project-%d", i)), ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: string(vuid), UID: vuid}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: region, Shard: shard}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: 1, Revision: 1, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: fmt.Sprintf("fd53::%d", i), Port: 53, Transports: []string{"UDP", "TCP"}}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: fmt.Sprintf("fd54::%d", i), Port: 53, Transports: []string{"UDP", "TCP"}}}, ZoneRefs: zoneReferences([]types.UID{zuid})}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1, ValidUntil: metav1.NewTime(now.Add(time.Minute))}}}) + objects = append(objects, &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("binding-%d", i), Namespace: ns, UID: types.UID(fmt.Sprintf("binding-uid-%d", i)), Labels: map[string]string{"internal-dns.miloapis.com/region": region, "internal-dns.miloapis.com/shard": shard}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: types.UID(fmt.Sprintf("project-%d", i)), ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: string(vuid), UID: vuid}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: region, Shard: shard}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: 1, Revision: 1, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: fmt.Sprintf("fd53::%d", i), Port: 53, Transports: []string{"UDP", "TCP"}}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: fmt.Sprintf("fd54::%d", i), Port: 53, Transports: []string{"UDP", "TCP"}}}, ZoneRefs: zoneReferences([]types.UID{zuid})}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1}}}) } cl := testClient(t, objects...) p := Planner{Client: cl, Allocator: &Allocator{Client: cl, Namespace: ns}, Config: PlannerConfig{Namespace: ns, Region: region, Shard: shard, Identity: "planner", ClusterBackends: []model.Backend{{MemberID: "cluster-1", Address: "fd00::20", Port: 5300}}}, Now: func() time.Time { return now }} @@ -159,7 +159,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { if len(pending.Bindings) != 2 || !pending.Bindings[0].ConfigurationPending || pending.Bindings[0].Tombstone || len(pending.Bindings[0].Zones) != 0 || pending.Bindings[1].ConfigurationPending { t.Fatal("incomplete zone addition withdrew authorization, exposed partial zones, or gated the healthy project") } - if pending.Bindings[0].Authorization.Revision != uint64(originalAuthorization.Sequence) || !pending.Bindings[0].Authorization.ValidUntil.Equal(originalAuthorization.ValidUntil.Time) { + if pending.Bindings[0].Authorization.Revision != uint64(originalAuthorization.Sequence) { t.Fatal("pending configuration renewed the original authorization") } for _, obj := range []client.Object{ @@ -381,7 +381,7 @@ func TestReadinessCannotMixAuthorizationRenewals(t *testing.T) { func TestReadinessExpiresWithoutAnotherAck(t *testing.T) { now := time.Now().UTC() until := metav1.NewTime(now.Add(-time.Second)) - b := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: "binding", Namespace: "platform", Labels: map[string]string{"internal-dns.miloapis.com/region": "east", "internal-dns.miloapis.com/shard": "shared"}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 1}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1, ValidUntil: metav1.NewTime(now.Add(time.Minute))}}, Status: dnsv1alpha1.DNSResolverBindingStatus{Phase: "Serving", MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{MemberID: "resolver", WriterEpoch: 1, Revision: 1, AuthorizationIssuerEpoch: 1, AuthorizationRevision: 1, Phase: string(model.AckVerified), ValidUntil: &until}}}} + b := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: "binding", Namespace: "platform", Labels: map[string]string{"internal-dns.miloapis.com/region": "east", "internal-dns.miloapis.com/shard": "shared"}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Revision: 1}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1}}, Status: dnsv1alpha1.DNSResolverBindingStatus{Phase: "Serving", MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{{MemberID: "resolver", WriterEpoch: 1, Revision: 1, AuthorizationIssuerEpoch: 1, AuthorizationRevision: 1, Phase: string(model.AckVerified), ValidUntil: &until}}}} cl := testClient(t, b) s := AckSink{Client: cl, Namespace: "platform", Region: "east", Shard: "shared", Members: []model.Member{{MemberID: "resolver", Role: "resolver"}}, Now: func() time.Time { return now }} if err := s.Refresh(context.Background()); err != nil { @@ -455,7 +455,7 @@ func TestOutOfOrderAckDoesNotReplaceWithdrawalInSameSecond(t *testing.T) { func TestBindingReadinessRequiresCurrentCommittedSnapshot(t *testing.T) { now := time.Now().UTC() until := metav1.NewTime(now.Add(time.Minute)) - b := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{UID: "binding-uid"}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: 1, Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 5, ValidUntil: until}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{ + b := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{UID: "binding-uid"}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: 1, Revision: 7}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 3, Sequence: 5}}, Status: dnsv1alpha1.DNSResolverBindingStatus{MemberAcknowledgements: []dnsv1alpha1.DNSApplyAcknowledgement{ {MemberID: "node", WriterEpoch: 2, SnapshotRevision: 8, Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 5, Phase: string(model.AckVerified), ValidUntil: &until}, {MemberID: "cluster", WriterEpoch: 2, SnapshotRevision: 8, Revision: 7, AuthorizationIssuerEpoch: 3, AuthorizationRevision: 5, Phase: string(model.AckVerified), ValidUntil: &until}, }}} diff --git a/internal/internaldns/runtime/config.go b/internal/internaldns/runtime/config.go index cc561b6a..ff31fa49 100644 --- a/internal/internaldns/runtime/config.go +++ b/internal/internaldns/runtime/config.go @@ -27,12 +27,12 @@ const ( ) type ProjectConfig struct { - Name string `json:"name,omitempty"` - ProjectUID types.UID `json:"projectUID"` - SourceClusterUID string `json:"sourceClusterUID"` - Namespace string `json:"namespace"` - Kubeconfig string `json:"kubeconfig,omitempty"` - KubeAPI KubeAPIClientConfig `json:"kubeAPI,omitempty"` + Name string `json:"name,omitempty"` + ProjectUID types.UID `json:"projectUID"` + + Namespace string `json:"namespace"` + Kubeconfig string `json:"kubeconfig,omitempty"` + KubeAPI KubeAPIClientConfig `json:"kubeAPI,omitempty"` } type KubeAPIClientConfig struct { @@ -42,11 +42,9 @@ type KubeAPIClientConfig struct { } type AdmissionConfig struct { - Enabled bool `json:"enabled"` - Port int `json:"port,omitempty"` - CertDir string `json:"certDir,omitempty"` - MaxContributionLeaseSeconds int64 `json:"maxContributionLeaseSeconds,omitempty"` - MaxAccessLeaseSeconds int64 `json:"maxAccessLeaseSeconds,omitempty"` + Enabled bool `json:"enabled"` + Port int `json:"port,omitempty"` + CertDir string `json:"certDir,omitempty"` } type RegionalTransportConfig struct { @@ -116,13 +114,6 @@ func (c *Config) defaults() { if c.Admission.Port == 0 { c.Admission.Port = 9443 } - if c.Admission.MaxContributionLeaseSeconds <= 0 { - c.Admission.MaxContributionLeaseSeconds = 90 - } - if c.Admission.MaxAccessLeaseSeconds <= 0 { - c.Admission.MaxAccessLeaseSeconds = 300 - } - if c.NATS.Name == "" { c.NATS.Name = "internal-dns" } @@ -210,8 +201,8 @@ func (c *Config) Validate(role string) error { uids := map[types.UID]bool{} namespaces := map[string]bool{} for i, p := range c.Projects { - if p.ProjectUID == "" || p.SourceClusterUID == "" || p.Namespace == "" { - return fmt.Errorf("project %d requires projectUID, sourceClusterUID, and namespace", i) + if p.ProjectUID == "" || p.Namespace == "" { + return fmt.Errorf("project %d requires projectUID and namespace", i) } if p.Namespace == c.PlatformNamespace { return fmt.Errorf("project %d namespace collides with platformNamespace", i) @@ -275,10 +266,6 @@ func (c *Config) Validate(role string) error { return nil } -func (c AdmissionConfig) maxAccessLease() time.Duration { - return time.Duration(c.MaxAccessLeaseSeconds) * time.Second -} - func validateTransportConfig(nats transport.Config, stream transport.StreamConfig, ensure bool) error { if nats.URL == "" || stream.Name == "" { return errors.New("nats.url and stream.name are required") @@ -342,6 +329,3 @@ func (c KubeAPIClientConfig) requestTimeout() time.Duration { func (c Config) ownershipLease() time.Duration { return time.Duration(c.OwnershipLeaseSeconds) * time.Second } -func (c AdmissionConfig) maxLease() time.Duration { - return time.Duration(c.MaxContributionLeaseSeconds) * time.Second -} diff --git a/internal/internaldns/runtime/config_test.go b/internal/internaldns/runtime/config_test.go index 629e668e..a3a1c433 100644 --- a/internal/internaldns/runtime/config_test.go +++ b/internal/internaldns/runtime/config_test.go @@ -41,7 +41,7 @@ func TestKubeAPIRateLimitDefaultsOverridesAndValidation(t *testing.T) { Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, - Projects: []ProjectConfig{{Name: "project", ProjectUID: "p1", SourceClusterUID: "c1", Namespace: "project", + Projects: []ProjectConfig{{Name: "project", ProjectUID: "p1", Namespace: "project", KubeAPI: KubeAPIClientConfig{QPS: 30, Burst: 60}}}, Members: []model.Member{{MemberID: "regional-1", Role: "regional"}}, } @@ -102,7 +102,7 @@ func TestControlPlaneRejectsAmbiguousProjectsAndMemberIDs(t *testing.T) { base := Config{ NATS: transport.Config{URL: "nats://fixture:4222", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, - Projects: []ProjectConfig{{ProjectUID: "p1", SourceClusterUID: "c1", Namespace: "n1"}, {ProjectUID: "p2", SourceClusterUID: "c2", Namespace: "n2"}}, + Projects: []ProjectConfig{{ProjectUID: "p1", Namespace: "n1"}, {ProjectUID: "p2", Namespace: "n2"}}, } if err := base.Validate("control-plane"); err == nil { t.Fatal("unnamed multi-project configuration was accepted") @@ -152,3 +152,56 @@ func TestConfigLoaderRejectsRemovedPrivateServingFields(t *testing.T) { t.Fatal("multiple config objects accepted") } } + +func TestAgentRestoresCheckpointWhileBrokerUnavailable(t *testing.T) { + dir := t.TempDir() + cfg := Config{Region: "central", Shard: "one", NATS: transport.Config{URL: "nats://127.0.0.1:1", AllowInsecure: true}, Stream: transport.StreamConfig{Name: "DNS", SnapshotsBucket: "DNS_BOOTSTRAP"}, Agent: serving.Config{MemberID: "node-0", StateDir: filepath.Join(dir, "state"), Mode: serving.ModeResolver, Render: serving.RenderConfig{Role: serving.RenderNode, NodeDNSDist: serving.DNSDistRenderConfig{Path: filepath.Join(dir, "dnsdist.conf"), ListenAddress: "[::1]:53", PoolName: "node"}, NodeBIND: serving.BINDRenderConfig{Path: filepath.Join(dir, "named.conf"), ListenAddress: "::1", Port: 5353}}, Transaction: serving.TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, WatchdogLeasePath: filepath.Join(dir, "lease.json"), ReconcileInterval: 10 * time.Millisecond}} + if err := cfg.Validate(roleAgent); err != nil { + t.Fatal(err) + } + seed, err := serving.NewAgent(cfg.Agent, serving.Dependencies{}) + if err != nil { + t.Fatal(err) + } + snapshot := model.ServingSnapshot{Region: "central", Shard: "one", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: time.Now(), Bindings: []model.Binding{{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::1", ClusterAddress: "fd53::2", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1}}}} + env, err := model.NewEnvelope(model.KindServingSnapshot, "initial", "central", "one", "shard", 1, 1, time.Now(), snapshot) + if err != nil { + t.Fatal(err) + } + if err := seed.Handle(context.Background(), env); err != nil { + t.Fatal(err) + } + if err := os.Remove(cfg.Agent.WatchdogLeasePath); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- runAgent(ctx, cfg) }() + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Stat(cfg.Agent.WatchdogLeasePath); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("checkpoint activation waited for broker connectivity") + } + time.Sleep(10 * time.Millisecond) + } + data, err := os.ReadFile(cfg.Agent.Render.NodeDNSDist.Path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "fd53::1") || !strings.Contains(string(data), "ready=true") { + t.Fatalf("persisted resolver mapping was not restored: %s", data) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("agent did not stop") + } +} diff --git a/internal/internaldns/runtime/run.go b/internal/internaldns/runtime/run.go index ae58f04c..01811722 100644 --- a/internal/internaldns/runtime/run.go +++ b/internal/internaldns/runtime/run.go @@ -117,13 +117,12 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { // contexts, access bindings, associations, registrations, or producer observations. routed := &RoutingClient{Client: platformClient, Source: direct, PlatformNamespace: cfg.PlatformNamespace} reconciler := &controlplane.Reconciler{Client: routed, Scheme: scheme, Options: controlplane.ReconcilerOptions{ - ProjectUID: project.ProjectUID, SourceClusterUID: project.SourceClusterUID, + ProjectUID: project.ProjectUID, ProjectNamespace: project.Namespace, PlatformNamespace: cfg.PlatformNamespace, Region: cfg.Region, Shard: cfg.Shard, Identity: identity, LeaseDuration: cfg.ownershipLease(), AddressAllocator: allocator, PrivateZoneClassName: cfg.PrivateZoneClassName, ServingRegions: regions, - MaxAccessLease: cfg.Admission.maxAccessLease(), }} if err := reconciler.SetupWithManager(mgr); err != nil { return fmt.Errorf("setup reconciler: %w", err) @@ -146,8 +145,6 @@ func runControlPlane(ctx context.Context, cfg Config, kubeconfig string) error { return fmt.Errorf("create admission manager: %w", err) } handler := &dnsadmission.Handler{ - MaxContributionLease: cfg.Admission.maxLease(), - MaxAccessLease: cfg.Admission.maxAccessLease(), ResolverContextsEnabled: true, ClientForRequest: projectSelector(projects), } @@ -219,7 +216,7 @@ func runProjectSource(ctx context.Context, project projectRuntime) { err = errors.New("source manager stopped unexpectedly") } log.Error(err, "source manager unavailable; rebuilding and retrying") - if !waitForRetry(ctx, time.Second) { + if !waitForRetry(ctx) { return } } @@ -247,7 +244,7 @@ func runControlPlaneRegion(ctx context.Context, cfg Config, region string, plann return nil } log.Error(err, "regional transport setup failed; retrying") - if !waitForRetry(ctx, time.Second) { + if !waitForRetry(ctx) { return nil } continue @@ -275,7 +272,7 @@ func runControlPlaneRegion(ctx context.Context, cfg Config, region string, plann } if setupFailed { regional.Close() - if !waitForRetry(ctx, time.Second) { + if !waitForRetry(ctx) { return nil } continue @@ -295,8 +292,8 @@ func runControlPlaneRegion(ctx context.Context, cfg Config, region string, plann return nil } -func waitForRetry(ctx context.Context, delay time.Duration) bool { - timer := time.NewTimer(delay) +func waitForRetry(ctx context.Context) bool { + timer := time.NewTimer(time.Second) defer timer.Stop() select { case <-ctx.Done(): @@ -307,45 +304,79 @@ func waitForRetry(ctx context.Context, delay time.Duration) bool { } func runAgent(ctx context.Context, cfg Config) error { - regional, err := connectRegional(ctx, cfg, []string{cfg.Agent.Region}) + log := ctrl.LoggerFrom(ctx).WithName("internal-dns-agent") + publisher := &agentPublisher{} + agent, err := serving.NewAgent(cfg.Agent, serving.Dependencies{AckPublisher: publisher, OnError: func(err error) { log.Error(err, "serving agent operation failed; retrying") }}) if err != nil { return err } - defer regional.Close() - route, _ := regional.route(cfg.Agent.Region) - consumer := func(suffix string, filters ...string) (*transport.Consumer, error) { - return route.bus.Consumer(ctx, transport.ConsumerConfig{Stream: route.stream.Name, - Durable: durable(cfg.Agent.MemberID, suffix), FilterSubjects: filters, - AckWait: 30 * time.Second, MaxAckPending: 64}) + workerCtx, cancel := context.WithCancel(ctx) + defer cancel() + done := make(chan error, 1) + go func() { done <- agent.Start(workerCtx) }() + go func() { + for workerCtx.Err() == nil { + err := runAgentTransport(workerCtx, cfg, agent, publisher) + if workerCtx.Err() != nil { + return + } + if err != nil { + log.Error(err, "DNS state delivery unavailable; retrying") + } + if !waitForRetry(workerCtx) { + return + } + } + }() + select { + case <-ctx.Done(): + return <-done + case err := <-done: + return err } - servingSource, err := consumer("serving", model.ServingSubject(cfg.Agent.Region, cfg.Agent.Shard)) +} + +// agentPublisher does not block local activation while the transport connects. +type agentPublisher struct { + mu sync.RWMutex + delegate transport.ExportPublisher +} + +func (p *agentPublisher) set(delegate transport.ExportPublisher) { + p.mu.Lock() + defer p.mu.Unlock() + p.delegate = delegate +} +func (p *agentPublisher) Publish(ctx context.Context, subject string, env model.Envelope) (transport.PublishAck, error) { + p.mu.RLock() + delegate := p.delegate + p.mu.RUnlock() + if delegate == nil { + return transport.PublishAck{}, errors.New("DNS state transport unavailable") + } + return delegate.Publish(ctx, subject, env) +} +func runAgentTransport(ctx context.Context, cfg Config, agent *serving.Agent, publisher *agentPublisher) error { + regional, err := connectRegional(ctx, cfg, []string{cfg.Agent.Region}) if err != nil { return err } - var recordSource, ackSource *transport.Consumer - if cfg.Agent.Mode == serving.ModeResolver || cfg.Agent.Mode == serving.ModeCombined { - recordSource, err = consumer("records", model.RecordConsumerFilter(cfg.Agent.Region, cfg.Agent.Shard)) - if err != nil { - return err - } + defer regional.Close() + route, _ := regional.route(cfg.Agent.Region) + consumer := func(suffix string, filters ...string) (*transport.Consumer, error) { + return route.bus.Consumer(ctx, transport.ConsumerConfig{Stream: route.stream.Name, Durable: durable(cfg.Agent.MemberID, suffix), FilterSubjects: filters, AckWait: 30 * time.Second, MaxAckPending: 64}) } - if cfg.Agent.Mode == serving.ModeResolver || cfg.Agent.Mode == serving.ModeCombined { - ackSource, err = consumer("acks", ackFilter(cfg.Agent.Region, cfg.Agent.Shard)) + var sources []serving.EventSource + for _, feed := range []struct{ suffix, filter string }{{"serving", model.ServingSubject(cfg.Agent.Region, cfg.Agent.Shard)}, {"records", model.RecordConsumerFilter(cfg.Agent.Region, cfg.Agent.Shard)}, {"acks", ackFilter(cfg.Agent.Region, cfg.Agent.Shard)}} { + source, err := consumer(feed.suffix, feed.filter) if err != nil { return err } + sources = append(sources, source) } - log := ctrl.LoggerFrom(ctx).WithName("internal-dns-agent") - agent, err := serving.NewAgent(cfg.Agent, serving.Dependencies{ - ServingSource: servingSource, RecordSource: recordSource, AckSource: ackSource, - Bootstrap: serving.SnapshotBootstrap{Store: route.store, Region: cfg.Agent.Region, Shard: cfg.Agent.Shard}, - AckPublisher: route.bus, - OnError: func(err error) { log.Error(err, "serving agent operation failed; retrying") }, - }) - if err != nil { - return err - } - return agent.Start(ctx) + publisher.set(route.bus) + defer publisher.set(nil) + return agent.RunTransport(ctx, serving.SnapshotBootstrap{Store: route.store, Region: cfg.Agent.Region, Shard: cfg.Agent.Shard}, sources...) } func newScheme() (*runtime.Scheme, error) { @@ -438,25 +469,25 @@ func consumeAcks(ctx context.Context, source *transport.Consumer, sink *platform } } -func projectSelector(projects []projectRuntime) func(context.Context, kubeadmission.Request) (client.Client, string, error) { - return func(_ context.Context, req kubeadmission.Request) (client.Client, string, error) { +func projectSelector(projects []projectRuntime) func(context.Context, kubeadmission.Request) (client.Client, error) { + return func(_ context.Context, req kubeadmission.Request) (client.Client, error) { parents := req.UserInfo.Extra[parentNameExtra] if len(parents) == 0 && len(projects) == 1 { p := projects[0] if req.Namespace != p.config.Namespace { - return nil, "", fmt.Errorf("request namespace is outside the configured project") + return nil, fmt.Errorf("request namespace is outside the configured project") } - return p.direct, p.config.SourceClusterUID, nil + return p.direct, nil } if len(parents) != 1 { - return nil, "", fmt.Errorf("authenticated project parent is missing or ambiguous") + return nil, fmt.Errorf("authenticated project parent is missing or ambiguous") } for _, p := range projects { if p.config.Name == parents[0] && req.Namespace == p.config.Namespace { - return p.direct, p.config.SourceClusterUID, nil + return p.direct, nil } } - return nil, "", fmt.Errorf("authenticated project parent is not configured for this namespace") + return nil, fmt.Errorf("authenticated project parent is not configured for this namespace") } } diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index dbd40ed7..182773bb 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -37,12 +37,11 @@ type Config struct { Render RenderConfig `json:"render"` Transaction TransactionConfig `json:"transaction"` CacheFlush []Command `json:"cacheFlush,omitempty"` - ExpiryInterval time.Duration `json:"expiryInterval,omitempty"` + ReconcileInterval time.Duration `json:"reconcileInterval,omitempty"` RetryDelay time.Duration `json:"retryDelay,omitempty"` AckInterval time.Duration `json:"ackInterval,omitempty"` AckLease time.Duration `json:"ackLease,omitempty"` AckPublishTimeout time.Duration `json:"ackPublishTimeout,omitempty"` - LocalServingLease time.Duration `json:"localServingLease,omitempty"` MaxBindings int `json:"maxBindings,omitempty"` WatchdogLeasePath string `json:"watchdogLeasePath,omitempty"` WatchdogLease time.Duration `json:"watchdogLease,omitempty"` @@ -201,8 +200,7 @@ type Agent struct { installedPublications map[string]installedPublication } -// This identity describes the exact files successfully activated, not a -// fingerprint recomputed later when an observation deadline may have passed. +// installedPublication identifies the exact files activated by a validated transaction. type installedPublication struct { Identity string EffectiveHash string @@ -255,13 +253,12 @@ func NewAgent(c Config, d Dependencies) (*Agent, error) { } func (a *Agent) Start(ctx context.Context) error { - defer func() { _ = a.tx.FailClosed(context.Background()) }() - // A restarted regional materialization must enforce deadlines persisted in its local - // checkpoint before waiting on any remote service. The watchdog lease is not - // renewed here, so an inconsistent or unavailable bootstrap stays gated. - if err := a.expirePersisted(ctx); err != nil { - a.report(fmt.Errorf("enforce persisted expiry at startup: %w", err)) - } + defer func() { + a.mu.Lock() + defer a.mu.Unlock() + _ = a.failClosed(context.Background()) + }() + // Verify the persisted configuration locally before serving after restart. if !a.gateUntilClosed(ctx) { return nil } @@ -280,9 +277,6 @@ func (a *Agent) Start(ctx context.Context) error { return nil } a.report(err) - if !a.gateUntilClosed(ctx) { - return nil - } retry := a.cfg.RetryDelay if retry <= 0 { retry = time.Second @@ -303,7 +297,7 @@ func (a *Agent) Start(ctx context.Context) error { if a.cfg.Mode == ModeResolver || a.cfg.Mode == ModeCombined { start(a.deps.AckSource) } - interval := a.cfg.ExpiryInterval + interval := a.cfg.ReconcileInterval if interval <= 0 { interval = time.Second } @@ -341,7 +335,10 @@ func (a *Agent) gateUntilClosed(ctx context.Context) bool { retry = time.Second } for { - if err := a.failClosed(ctx); err == nil { + a.mu.Lock() + err := a.failClosed(ctx) + a.mu.Unlock() + if err == nil { return true } else { a.report(fmt.Errorf("gate serving during startup: %w", err)) @@ -352,40 +349,10 @@ func (a *Agent) gateUntilClosed(ctx context.Context) bool { } } -func (a *Agent) expirePersisted(ctx context.Context) error { - if a.cfg.Mode != ModeCombined { - return nil - } - a.mu.Lock() - defer a.mu.Unlock() - return a.expireRecords(ctx) -} - func (a *Agent) bootstrapAndStep(ctx context.Context) error { if a.deps.Bootstrap != nil { - envs, err := a.deps.Bootstrap.Load(ctx) - if err != nil { - return fmt.Errorf("load bootstrap snapshot: %w", err) - } - sort.SliceStable(envs, func(i, j int) bool { - order := func(k model.Kind) int { - switch k { - case model.KindPublicationChunk: - return 0 - case model.KindServingSnapshot: - // Context destination mappings are required to query-proof a - // publication before its manifest can be acknowledged. - return 1 - default: - return 2 - } - } - return order(envs[i].Kind) < order(envs[j].Kind) - }) - for _, env := range envs { - if err := a.Handle(ctx, env); err != nil { - return fmt.Errorf("apply bootstrap %s: %w", env.Kind, err) - } + if err := a.loadBootstrap(ctx, a.deps.Bootstrap); err != nil { + return err } } if err := a.Step(ctx); err != nil { @@ -394,6 +361,49 @@ func (a *Agent) bootstrapAndStep(ctx context.Context) error { return nil } +func (a *Agent) loadBootstrap(ctx context.Context, loader BootstrapSource) error { + envs, err := loader.Load(ctx) + if err != nil { + return fmt.Errorf("load bootstrap snapshot: %w", err) + } + order := func(k model.Kind) int { + switch k { + case model.KindPublicationChunk: + return 0 + case model.KindServingSnapshot: + return 1 + default: + return 2 + } + } + sort.SliceStable(envs, func(i, j int) bool { return order(envs[i].Kind) < order(envs[j].Kind) }) + for _, env := range envs { + if err := a.Handle(ctx, env); err != nil { + return fmt.Errorf("apply bootstrap %s: %w", env.Kind, err) + } + } + return nil +} + +// RunTransport synchronizes desired state independently of local DNS serving. +func (a *Agent) RunTransport(ctx context.Context, loader BootstrapSource, sources ...EventSource) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + if err := a.loadBootstrap(ctx, loader); err != nil { + return err + } + results := make(chan error, len(sources)) + for _, source := range sources { + go func() { results <- a.consume(ctx, source) }() + } + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-results: + return err + } +} + func (a *Agent) consume(ctx context.Context, source EventSource) error { retry := a.cfg.RetryDelay if retry <= 0 { @@ -454,6 +464,9 @@ func (a *Agent) Handle(ctx context.Context, env model.Envelope) error { } a.mu.Lock() defer a.mu.Unlock() + if err := ctx.Err(); err != nil { + return err + } switch env.Kind { case model.KindServingSnapshot: return a.handleServing(ctx, env) @@ -609,7 +622,7 @@ func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { return errors.New("conflicting publication manifest at the same epoch and revision") } if err := a.verifyPublication(ctx, old); err != nil { - _ = a.emitAck(ctx, model.KindPublicationManifest, m.ZoneUID, m.WriterEpoch, m.Revision, model.AckRejected, time.Time{}, err.Error()) + _ = a.emitAck(ctx, model.KindPublicationManifest, m.ZoneUID, m.WriterEpoch, m.Revision, model.AckRejected, err.Error()) return err } for _, name := range publicationNames(old.Plan) { @@ -660,7 +673,7 @@ func (a *Agent) handleManifest(ctx context.Context, env model.Envelope) error { if err := a.store.Save(a.state); err != nil { return err } - next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: plan, EffectiveHash: effectivePlanHash(plan, a.deps.Clock())} + next := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Manifest: m, Plan: plan, EffectiveHash: effectivePlanHash(plan)} if a.cfg.Mode == ModeResolver { return a.recordResolverPublication(ctx, m.ZoneUID, next, old) } @@ -781,7 +794,7 @@ func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { if cmp < 0 || (cmp == 0 && !ack.ObservedAt.After(old.ObservedAt)) { return nil } - a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ObservedAt: ack.ObservedAt} + a.state.Acks[ack.ResourceUID][key] = replicaAck{Fence: fence{ack.Epoch, ack.Revision}, ObservedAt: ack.ObservedAt, Rejected: true} if err := a.store.Save(a.state); err != nil { return err } @@ -829,25 +842,20 @@ func (a *Agent) handleAck(ctx context.Context, env model.Envelope) error { func (a *Agent) Step(ctx context.Context) error { a.mu.Lock() defer a.mu.Unlock() + if a.state.Snapshot == nil { + return errors.New("no verified serving configuration is available") + } if a.cfg.Mode == ModeResolver { if err := a.retryPendingPublications(ctx); err != nil { _ = a.failClosed(ctx) return err } - if err := a.expireResolverCache(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } } if a.cfg.Mode == ModeCombined { if err := a.retryPendingPublications(ctx); err != nil { _ = a.failClosed(ctx) return err } - if err := a.expireRecords(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } if a.needsActivation || len(a.installedPublications) == 0 { if err := a.reconcile(ctx); err != nil { return err @@ -860,12 +868,6 @@ func (a *Agent) Step(ctx context.Context) error { if err := a.reconcile(ctx); err != nil { return err } - if a.cfg.Mode == ModeCombined { - if err := a.expireRecords(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } - } return a.renewSafeWatchdogLease(ctx) } @@ -874,17 +876,7 @@ func (a *Agent) Step(ctx context.Context) error { // lease. This matters when a sustained JetStream backlog repeatedly acquires // the agent lock and delays the ticker. func (a *Agent) finishEvent(ctx context.Context) error { - if a.cfg.Mode == ModeResolver { - if err := a.expireResolverCache(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } - } if a.cfg.Mode == ModeCombined { - if err := a.expireRecords(ctx); err != nil { - _ = a.failClosed(ctx) - return err - } if err := a.heartbeatPublications(ctx); err != nil { return err } @@ -897,29 +889,15 @@ func (a *Agent) reconcile(ctx context.Context) error { return nil } - now := a.deps.Clock() active := map[string]bool{} filtered := *a.state.Snapshot filtered.Bindings = nil for _, b := range a.state.Snapshot.Bindings { - eligible := !b.Tombstone && now.Before(b.Authorization.ValidUntil) && a.publicationsReady(b) + eligible := !b.Tombstone && a.publicationsReady(b) if eligible { active[b.BindingUID] = true } - if !b.Tombstone && now.Before(b.Authorization.ValidUntil) { - lease := a.cfg.LocalServingLease - if lease <= 0 { - lease = 90 * time.Second - } - horizon := now.Add(lease) - quantum := a.cfg.AckInterval - if quantum <= 0 { - quantum = 30 * time.Second - } - horizon = horizon.Truncate(quantum).Add(quantum) - if horizon.Before(b.Authorization.ValidUntil) { - b.Authorization.ValidUntil = horizon - } + if !b.Tombstone { filtered.Bindings = append(filtered.Bindings, b) } } @@ -928,7 +906,7 @@ func (a *Agent) reconcile(ctx context.Context) error { for _, binding := range filtered.Bindings { renderConfig.ReadyMembers[binding.BindingUID] = a.eligibleRegionalMembers(binding) } - rendered, err := renderPublications(renderConfig, filtered, active, a.state.Publications, now) + rendered, err := renderPublications(renderConfig, filtered, active, a.state.Publications) if err != nil { return err } @@ -953,7 +931,7 @@ func (a *Agent) reconcile(ctx context.Context) error { if a.cfg.Mode == ModeCombined { installed := map[string]installedPublication{} for uid, p := range a.state.Publications { - installed[uid] = installedPublication{Identity: publicationIdentity(p), EffectiveHash: effectivePlanHash(p.Plan, now), Views: a.publicationViewsAt(p, now)} + installed[uid] = installedPublication{Identity: publicationIdentity(p), EffectiveHash: effectivePlanHash(p.Plan), Views: a.publicationViewsAt(p)} } a.installedPublications = installed for _, p := range a.state.Publications { @@ -970,7 +948,7 @@ func (a *Agent) reconcile(ctx context.Context) error { // Any binding that fails either UDP or TCP is immediately removed again. probeFailed := a.verifyResolverBindings(ctx, filtered.Bindings, active) if probeFailed { - gated, err := renderPublications(renderConfig, filtered, active, a.state.Publications, now) + gated, err := renderPublications(renderConfig, filtered, active, a.state.Publications) if err != nil { return err } @@ -986,13 +964,7 @@ func (a *Agent) reconcile(ctx context.Context) error { } } } - // Resolver health is a local safety decision. Once every eligible binding - // has passed its end-to-end proof (and every failed binding has been - // durably re-rendered gated), renew the independent watchdog before any - // best-effort broker ACK. A NATS partition must expire remote readiness - // leases, but it must not consume the local five-second watchdog window for - // a dataplane that has just been proved healthy. Combined mode renews later, - // after its regional materialization expiry and health checks also complete. + // Local DNS proof renews the watchdog independently of broker acknowledgements. if a.cfg.Mode == ModeResolver { if err := a.renewSafeWatchdogLease(ctx); err != nil { return err @@ -1005,9 +977,6 @@ func (a *Agent) reconcile(ctx context.Context) error { p := model.AckVerified if !active[b.BindingUID] { p = model.AckApplied - if !now.Before(b.Authorization.ValidUntil) { - p = model.AckExpired - } } if err := a.emitServingAck(ctx, b, p); err != nil { a.report(fmt.Errorf("publish serving ACK %s: %w", b.BindingUID, err)) @@ -1150,7 +1119,7 @@ func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { } } proof := a.state.Acks[z.ZoneUID][member.MemberID+"/"+member.ReplicaID] - if !a.deps.Clock().Before(proof.ValidUntil) || proof.Fence != p.Fence { + if proof.Rejected || proof.ObservedAt.IsZero() || proof.Fence != p.Fence { ready = false break } @@ -1180,21 +1149,21 @@ func (a *Agent) verifyPublication(ctx context.Context, p publicationState) error } func (a *Agent) publicationViews(p publicationState) []PublicationView { - return a.publicationViewsAt(p, a.deps.Clock()) + return a.publicationViewsAt(p) } -func (a *Agent) publicationViewsAt(p publicationState, now time.Time) []PublicationView { +func (a *Agent) publicationViewsAt(p publicationState) []PublicationView { if a.state.Snapshot == nil { return nil } var out []PublicationView for _, b := range a.state.Snapshot.Bindings { - if b.Tombstone || b.ConfigurationPending || !now.Before(b.Authorization.ValidUntil) { + if b.Tombstone || b.ConfigurationPending { continue } for _, z := range b.Zones { if z.ZoneUID == p.Plan.ZoneUID && model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) >= 0 { - out = append(out, PublicationView{View: b.ViewName(), DestinationAddress: b.ClusterAddress, DestinationPort: b.Port, Serial: publicationSerial(p, now), FingerprintMailbox: publicationMailbox(p, now)}) + out = append(out, PublicationView{View: b.ViewName(), DestinationAddress: b.ClusterAddress, DestinationPort: b.Port, Serial: publicationSerial(p), FingerprintMailbox: publicationMailbox(p)}) } } } @@ -1227,84 +1196,11 @@ func (a *Agent) retryPendingPublications(ctx context.Context) error { } } p.Verified = true - p.EffectiveHash = effectivePlanHash(p.Plan, a.deps.Clock()) - a.state.Publications[uid] = p - if err := a.store.Save(a.state); err != nil { - return err - } - } - return nil -} - -// A node retains the publication deadlines so a broker outage cannot leave -// healthy records in its positive cache after the regional member withdraws. -func (a *Agent) expireResolverCache(ctx context.Context) error { - now := a.deps.Clock() - for uid, p := range a.state.Publications { - if p.Manifest.Tombstone { - continue - } - hash := effectivePlanHash(p.Plan, now) - if hash == p.EffectiveHash { - continue - } - for _, name := range publicationNames(p.Plan) { - if err := a.flushName(ctx, name); err != nil { - return err - } - } - p.EffectiveHash = hash - p.LocalRevision++ - a.state.Publications[uid] = p - if err := a.store.Save(a.state); err != nil { - return err - } - } - return nil -} - -func (a *Agent) expireRecords(ctx context.Context) error { - now := a.deps.Clock() - for uid, p := range a.state.Publications { - if !p.Verified || p.Manifest.Tombstone { - continue - } - hash := effectivePlanHash(p.Plan, now) - if hash == p.EffectiveHash { - continue - } - p.LocalRevision++ - p.EffectiveHash = hash - p.Verified = false - a.state.Publications[uid] = p - if err := a.store.Save(a.state); err != nil { - return err - } - if err := a.reconcile(ctx); err != nil { - return err - } - if len(a.publicationViews(p)) == 0 { - continue - } - if err := a.verifyPublication(ctx, p); err != nil { - return err - } - for _, name := range publicationNames(p.Plan) { - if err := a.flushName(ctx, name); err != nil { - return err - } - } - p.Verified = true + p.EffectiveHash = effectivePlanHash(p.Plan) a.state.Publications[uid] = p if err := a.store.Save(a.state); err != nil { return err } - if err := a.reconcile(ctx); err != nil { - return err - } - if err := a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckExpired, time.Time{}, ""); err != nil { - a.report(err) - } } return nil } @@ -1341,7 +1237,7 @@ func (a *Agent) flushPublicationZone(ctx context.Context, zoneUID string) error return nil } -func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, revision uint64, phase model.AckPhase, validUntil time.Time, message string) error { +func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, revision uint64, phase model.AckPhase, message string) error { if a.deps.AckPublisher == nil { return nil } @@ -1351,9 +1247,6 @@ func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, lease = 90 * time.Second } leaseUntil := now.Add(lease) - if !validUntil.IsZero() && validUntil.Before(leaseUntil) { - leaseUntil = validUntil - } ack := model.MemberAck{MemberID: a.cfg.MemberID, ReplicaID: a.cfg.ReplicaID, ResourceUID: uid, Kind: kind, Epoch: epoch, Revision: revision, Phase: phase, ObservedAt: now, ValidUntil: leaseUntil, Error: message} eventID := fmt.Sprintf("ack-%s-%s-%d-%d-%s-%d", model.OpaqueToken(a.cfg.MemberID+"/"+a.cfg.ReplicaID), model.OpaqueToken(uid), epoch, revision, phase, now.UnixNano()) env, err := model.NewEnvelope(model.KindMemberAck, eventID, a.cfg.Region, a.cfg.Shard, uid, epoch, revision, a.deps.Clock(), ack) @@ -1371,13 +1264,11 @@ func (a *Agent) emitAck(ctx context.Context, kind model.Kind, uid string, epoch, } func (a *Agent) emitVerifiedPublicationAck(ctx context.Context, uid string, epoch, revision uint64) error { - // The file transaction, proof and original-deadline enforcement are local - // safety gates. Broker delivery is independent: renew only a safe installed - // state first, then let remote readiness expire if ACK transport is down. + // Renew local readiness before best-effort acknowledgement delivery. if err := a.renewSafeWatchdogLease(ctx); err != nil { return err } - if err := a.emitAck(ctx, model.KindPublicationManifest, uid, epoch, revision, model.AckVerified, time.Time{}, ""); err != nil { + if err := a.emitAck(ctx, model.KindPublicationManifest, uid, epoch, revision, model.AckVerified, ""); err != nil { a.report(fmt.Errorf("publish verified publication ACK %s: %w", uid, err)) return nil } @@ -1398,7 +1289,7 @@ func (a *Agent) emitServingAck(ctx context.Context, b model.Binding, phase model if last.phase == phase && now.Sub(last.at) < interval { return nil } - if err := a.emitAck(ctx, model.KindServingSnapshot, b.BindingUID, a.state.ServingFence.Epoch, a.state.ServingFence.Revision, phase, b.Authorization.ValidUntil, ""); err != nil { + if err := a.emitAck(ctx, model.KindServingSnapshot, b.BindingUID, a.state.ServingFence.Epoch, a.state.ServingFence.Revision, phase, ""); err != nil { return err } a.lastServingAck[b.BindingUID] = servingAckState{phase: phase, at: now} @@ -1424,18 +1315,15 @@ func (a *Agent) heartbeatPublications(ctx context.Context) error { continue } if err := a.verifyPublication(ctx, p); err != nil { - _ = a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckRejected, time.Time{}, err.Error()) + _ = a.emitAck(ctx, model.KindPublicationManifest, uid, p.Fence.Epoch, p.Fence.Revision, model.AckRejected, err.Error()) return fmt.Errorf("reverify publication %s before ACK renewal: %w", uid, err) } due = append(due, duePublication{uid: uid, p: p}) } - // Local watchdog eligibility depends on the complete local verification - // above, not broker availability. Stop after the first stalled publish so a - // partition consumes at most one bounded publish timeout per Step. Missing - // remote ACK leases still expire independently and gate resolver eligibility. + // Publish acknowledgements within a bounded budget after local DNS verification. for _, publication := range due { p := publication.p - if err := a.emitAck(ctx, model.KindPublicationManifest, publication.uid, p.Fence.Epoch, p.Fence.Revision, model.AckVerified, time.Time{}, ""); err != nil { + if err := a.emitAck(ctx, model.KindPublicationManifest, publication.uid, p.Fence.Epoch, p.Fence.Revision, model.AckVerified, ""); err != nil { a.report(fmt.Errorf("publish verified publication ACK %s: %w", publication.uid, err)) break } @@ -1506,9 +1394,6 @@ func (a *Agent) validateContributionFences(plan model.PublicationPlan) error { return fmt.Errorf("contribution %q regressed from epoch %d sequence %d", observation.ContributionUID, current.Epoch, current.Sequence) } if comparison == 0 { - if current.ValidUntil.IsZero() || !current.ValidUntil.Equal(observation.ValidUntil) { - return fmt.Errorf("contribution %q changed the deadline of an existing fence", observation.ContributionUID) - } if current.GrantUID != "" && current.GrantUID != observation.GrantUID { return fmt.Errorf("contribution %q changed the grant of an existing fence", observation.ContributionUID) } @@ -1519,10 +1404,10 @@ func (a *Agent) validateContributionFences(plan model.PublicationPlan) error { func (a *Agent) commitContributionFences(plan model.PublicationPlan) { for _, observation := range plan.ObservationFences { - next := contributionFence{GrantUID: observation.GrantUID, Epoch: observation.WriterEpoch, Sequence: observation.Sequence, ValidUntil: observation.ValidUntil} + next := contributionFence{GrantUID: observation.GrantUID, Epoch: observation.WriterEpoch, Sequence: observation.Sequence} current := a.state.ContributionFences[observation.ContributionUID] comparison := model.Compare(next.Epoch, next.Sequence, current.Epoch, current.Sequence) - if comparison > 0 || (comparison == 0 && current.GrantUID == "" && !current.ValidUntil.IsZero() && current.ValidUntil.Equal(next.ValidUntil)) { + if comparison > 0 || (comparison == 0 && current.GrantUID == "") { a.state.ContributionFences[observation.ContributionUID] = next } } @@ -1535,61 +1420,26 @@ type watchdogLease struct { } func (a *Agent) watchdogPublicationError() error { - now := a.deps.Clock() for uid, p := range a.state.Publications { if p.Manifest.Tombstone { continue } - current := effectivePlanHash(p.Plan, now) + current := effectivePlanHash(p.Plan) if a.cfg.Mode == ModeCombined { - if len(a.publicationViewsAt(p, now)) == 0 { + if len(a.publicationViewsAt(p)) == 0 { continue } installed, ok := a.installedPublications[uid] if !ok || installed.Identity != publicationIdentity(p) || installed.EffectiveHash != current { - return fmt.Errorf("publication %s deadline changed after activation; refusing watchdog renewal", uid) + return fmt.Errorf("publication %s configuration differs from activation; refusing watchdog renewal", uid) } } else if p.EffectiveHash != current { - return fmt.Errorf("publication %s deadline changed after cache withdrawal; refusing watchdog renewal", uid) + return fmt.Errorf("publication %s configuration differs from the applied checkpoint; refusing watchdog renewal", uid) } } return nil } -// A deadline can cross during reload or DNS proof. Settle that transition -// locally before renewing readiness, under one budget across every retry. -func (a *Agent) renewSafeWatchdogLease(ctx context.Context) error { - timeout := a.cfg.PublicationDNSProbe.Timeout - if a.cfg.Mode == ModeResolver { - timeout = a.cfg.ResolverDNSProbe.Timeout - } - if timeout <= 0 { - timeout = 3 * time.Second - } - settleCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - for { - if err := settleCtx.Err(); err != nil { - return errors.Join(fmt.Errorf("settle local publication deadlines: %w", err), a.failClosed(ctx)) - } - var err error - if a.cfg.Mode == ModeCombined { - err = a.expireRecords(settleCtx) - } else { - err = a.expireResolverCache(settleCtx) - } - if err != nil { - return errors.Join(err, a.failClosed(ctx)) - } - if a.watchdogPublicationError() == nil { - return a.renewWatchdogLease(settleCtx) - } - if err := a.reconcile(settleCtx); err != nil { - return errors.Join(err, a.failClosed(ctx)) - } - } -} - func (a *Agent) renewWatchdogLease(ctx context.Context) error { if err := ctx.Err(); err != nil { return errors.Join(err, a.failClosed(ctx)) @@ -1610,3 +1460,11 @@ func (a *Agent) renewWatchdogLease(ctx context.Context) error { } return writeAtomicLease(a.cfg.WatchdogLeasePath, b, 0o600) } + +// Renew the local watchdog only when the activated publication matches its checkpoint. +func (a *Agent) renewSafeWatchdogLease(ctx context.Context) error { + if err := a.watchdogPublicationError(); err != nil { + return errors.Join(err, a.failClosed(ctx)) + } + return a.renewWatchdogLease(ctx) +} diff --git a/internal/internaldns/serving/agent_test.go b/internal/internaldns/serving/agent_test.go index 9db5bd59..a892d334 100644 --- a/internal/internaldns/serving/agent_test.go +++ b/internal/internaldns/serving/agent_test.go @@ -160,14 +160,13 @@ func envelope(t *testing.T, kind model.Kind, uid string, epoch, revision uint64, } func testSnapshot(now time.Time) model.ServingSnapshot { - b := model.Binding{BindingUID: "binding-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 2, RequiredPublicationRevision: 4}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1, ValidUntil: now.Add(10 * time.Minute)}} + b := model.Binding{BindingUID: "binding-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 2, RequiredPublicationRevision: 4}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1}} return model.ServingSnapshot{Region: "east", Shard: "s1", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []model.Binding{b}, Members: []model.Member{{MemberID: "regional-0", ReplicaID: "regional-0", Role: "regional"}}} } func testPublication(t *testing.T, now time.Time) (model.PublicationChunk, model.PublicationManifest) { t.Helper() - validUntil := now.Add(time.Minute) - plan := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 1, Sequence: 1, ValidUntil: validUntil}}, Owners: []string{"api.prod.internal"}, RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 1, Sequence: 1, Eligible: true, ValidUntil: validUntil}}}}} + plan := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 1, Sequence: 1}}, Owners: []string{"api.prod.internal"}, RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 1, Sequence: 1, Eligible: true}}}}} payload, err := json.Marshal(plan) if err != nil { t.Fatal(err) @@ -179,23 +178,21 @@ func testPublication(t *testing.T, now time.Time) (model.PublicationChunk, model func TestWithdrawnObservationFencesOlderHealthyRecord(t *testing.T) { t.Parallel() - now := time.Now().UTC() agent := &Agent{state: newCheckpoint()} withdrawn := model.PublicationPlan{ ZoneUID: "zone-a", Apex: "prod.internal", - ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 33, ValidUntil: now.Add(-time.Minute)}}, + ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 33}}, RRSets: []model.RRSet{}, } if err := withdrawn.Validate(); err != nil { t.Fatal(err) } agent.commitContributionFences(withdrawn) - olderDeadline := now.Add(time.Minute) olderHealthy := model.PublicationPlan{ ZoneUID: "zone-a", Apex: "prod.internal", - ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 32, ValidUntil: olderDeadline}}, + ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 32}}, RRSets: []model.RRSet{{Name: "api.prod.internal", Type: "A", TTL: 30, Records: []model.RRRecord{{ - Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 7, Sequence: 32, Eligible: true, ValidUntil: olderDeadline, + Content: "192.0.2.10", ContributionUID: "contrib-a", WriterEpoch: 7, Sequence: 32, Eligible: true, }}}}, } if err := olderHealthy.Validate(); err != nil { @@ -206,17 +203,16 @@ func TestWithdrawnObservationFencesOlderHealthyRecord(t *testing.T) { } } -func TestObservationFenceReplayCannotExtendDeadline(t *testing.T) { +func TestObservationFenceReplayCannotChangeGrant(t *testing.T) { t.Parallel() - now := time.Now().UTC() agent := &Agent{state: newCheckpoint()} - original := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 2, Sequence: 4, ValidUntil: now}}, RRSets: []model.RRSet{}} + original := model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contrib-a", GrantUID: "grant-a", WriterEpoch: 2, Sequence: 4}}, RRSets: []model.RRSet{}} agent.commitContributionFences(original) replay := original replay.ObservationFences = append([]model.ObservationFence(nil), original.ObservationFences...) - replay.ObservationFences[0].ValidUntil = now.Add(time.Hour) + replay.ObservationFences[0].GrantUID = "another-grant" if err := agent.validateContributionFences(replay); err == nil { - t.Fatal("equal observation fence extended its original deadline") + t.Fatal("equal observation fence changed its grant") } } @@ -277,7 +273,7 @@ func TestStartForceActivatesPersistedUnchangedConfigBeforeLease(t *testing.T) { config := Config{ Region: "east", Shard: "s1", MemberID: "node-0", StateDir: stateDir, Mode: ModeResolver, Render: renderConfig, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, - WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, ExpiryInterval: time.Hour, + WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, ReconcileInterval: time.Hour, Transaction: TransactionConfig{ StageDir: filepath.Join(dir, "stage"), FailClosed: []Command{{Path: "gate"}}, @@ -347,13 +343,19 @@ func TestStepBoundsStalledAckPublishAndRenewsLocalLease(t *testing.T) { Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, WatchdogLease: 5 * time.Second, AckPublishTimeout: 30 * time.Millisecond, - }, Dependencies{AckPublisher: stalledPublisher{}, Runner: &recordingRunner{}, OnError: func(err error) { reported = append(reported, err) }}) + }, Dependencies{Runner: &recordingRunner{}, OnError: func(err error) { reported = append(reported, err) }}) if err != nil { t.Fatal(err) } agent.state.Snapshot = &snapshot agent.state.BindingFences, _ = advanceBindingFences(nil, snapshot) agent.state.ServingFence = fence{Epoch: 1, Revision: 1} + // Complete durable activation before measuring the ACK publication bound. + if err := agent.Step(context.Background()); err != nil { + t.Fatal(err) + } + agent.deps.AckPublisher = stalledPublisher{} + agent.lastServingAck = make(map[string]servingAckState) started := time.Now() if err := agent.Step(context.Background()); err != nil { t.Fatal(err) @@ -512,7 +514,6 @@ func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing. renderConfig.ClusterDNSDist = DNSDistRenderConfig{} renderConfig.ClusterBIND = BINDRenderConfig{} snapshot.Bindings[0].Zones = nil - snapshot.Bindings[0].Authorization.ValidUntil = time.Now().Add(10 * time.Second) leasePath := filepath.Join(dir, "lease.json") publishStarted := make(chan struct{}, 1) publishRelease := make(chan struct{}) @@ -522,8 +523,8 @@ func TestStartContinuouslyRenewsResolverLeaseWhileBrokerDisconnected(t *testing. Render: renderConfig, Transaction: TransactionConfig{StageDir: filepath.Join(dir, "stage")}, UnsafeAllowNoFailClosed: true, UnsafeAllowNoDNSVerify: true, ExternalWatchdog: true, WatchdogLeasePath: leasePath, WatchdogLease: 500 * time.Millisecond, - ExpiryInterval: 20 * time.Millisecond, AckInterval: 50 * time.Millisecond, - AckPublishTimeout: 10 * time.Millisecond, LocalServingLease: time.Second, + ReconcileInterval: 20 * time.Millisecond, AckInterval: 50 * time.Millisecond, + AckPublishTimeout: 10 * time.Millisecond, }, Dependencies{AckPublisher: publisher, Runner: &recordingRunner{}}) if err != nil { t.Fatal(err) diff --git a/internal/internaldns/serving/bind_test.go b/internal/internaldns/serving/bind_test.go index a12b1fc7..d0dfb6b7 100644 --- a/internal/internaldns/serving/bind_test.go +++ b/internal/internaldns/serving/bind_test.go @@ -47,7 +47,7 @@ func installTestPublication(t *testing.T, a *Agent, now time.Time) { } func currentZone(t *testing.T, a *Agent) string { t.Helper() - files, err := renderPublications(a.cfg.Render, *a.state.Snapshot, map[string]bool{}, a.state.Publications, a.deps.Clock()) + files, err := renderPublications(a.cfg.Render, *a.state.Snapshot, map[string]bool{}, a.state.Publications) if err != nil { t.Fatal(err) } @@ -70,7 +70,7 @@ func TestBINDPublicationActivatesLocalPrivateAuthority(t *testing.T) { } installTestPublication(t, agent, now) data := currentZone(t, agent) - if !strings.Contains(data, "192.0.2.10") || !strings.Contains(data, publicationMailbox(agent.state.Publications["zone-a"], now)) { + if !strings.Contains(data, "192.0.2.10") || !strings.Contains(data, publicationMailbox(agent.state.Publications["zone-a"])) { t.Fatalf("bad zone: %s", data) } config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) @@ -84,7 +84,7 @@ func TestBINDPublicationActivatesLocalPrivateAuthority(t *testing.T) { t.Fatal("publication not verified") } } -func TestBINDExpirySurvivesBrokerOutageAndRestart(t *testing.T) { +func TestBINDPublicationSurvivesBrokerOutageAndRestart(t *testing.T) { t.Parallel() now := time.Now().UTC() clock := now @@ -98,12 +98,12 @@ func TestBINDExpirySurvivesBrokerOutageAndRestart(t *testing.T) { installTestPublication(t, a, now) a.deps.AckPublisher = newBoundedAckPublisher(stalledPublisher{}) a.cfg.AckPublishTimeout = time.Millisecond - clock = now.Add(61 * time.Second) + clock = now.Add(24 * time.Hour) if err := a.Step(context.Background()); err != nil { t.Fatal(err) } - if data := currentZone(t, a); strings.Contains(data, "192.0.2.10") || !strings.Contains(data, "api.prod.internal. 5 IN "+model.OwnershipMarkerType) { - t.Fatalf("expiry did not preserve NODATA ownership: %s", data) + if data := currentZone(t, a); !strings.Contains(data, "192.0.2.10") { + t.Fatalf("outage removed the published record: %s", data) } local := a.state.Publications["zone-a"].LocalRevision restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) @@ -114,10 +114,10 @@ func TestBINDExpirySurvivesBrokerOutageAndRestart(t *testing.T) { t.Fatal(err) } if restarted.state.Publications["zone-a"].LocalRevision != local { - t.Fatal("restart renewed expired observation") + t.Fatal("restart changed the applied publication") } - if data := currentZone(t, restarted); strings.Contains(data, "192.0.2.10") { - t.Fatal("restart resurrected record") + if data := currentZone(t, restarted); !strings.Contains(data, "192.0.2.10") { + t.Fatal("restart lost the published record") } } func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { @@ -145,7 +145,7 @@ func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { if err != nil { t.Fatal(err) } - if f := saved.ContributionFences["contrib-a"]; f.Epoch != 1 || f.Sequence != 1 || !f.ValidUntil.Equal(now.Add(time.Minute)) { + if f := saved.ContributionFences["contrib-a"]; f.Epoch != 1 || f.Sequence != 1 { t.Fatalf("observation fence lost: %#v", f) } if saved.Publications["zone-a"].Verified { @@ -159,36 +159,6 @@ func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { t.Fatalf("equal fenced retry failed: %v", err) } } -func TestBINDFailedExpiryDoesNotRenewWatchdog(t *testing.T) { - t.Parallel() - now := time.Now().UTC() - clock := now - cfg := regionalTestConfig(t) - runner := &recordingRunner{} - a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, a, now) - if err := os.Remove(cfg.WatchdogLeasePath); err != nil { - t.Fatal(err) - } - clock = now.Add(61 * time.Second) - runner.failAt = len(runner.calls) + 1 - if err := a.Step(context.Background()); err == nil { - t.Fatal("expiry validation failure accepted") - } - if _, err := os.Stat(cfg.WatchdogLeasePath); !errors.Is(err, os.ErrNotExist) { - t.Fatal("failed expiry renewed watchdog") - } - runner.failAt = 0 - if err := a.Step(context.Background()); err != nil { - t.Fatalf("pending expiry retry failed: %v", err) - } - if strings.Contains(currentZone(t, a), "192.0.2.10") { - t.Fatal("retry resurrected expired record") - } -} func TestBINDTombstoneFencesReplayAcrossRestart(t *testing.T) { t.Parallel() now := time.Now().UTC() @@ -214,7 +184,7 @@ func TestBINDTombstoneFencesReplayAcrossRestart(t *testing.T) { t.Fatal("old snapshot defeated tombstone") } } -func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { +func TestNodeRequiresPublicationAndRetainsCacheDuringOutage(t *testing.T) { t.Parallel() now := time.Now().UTC() clock := now @@ -234,9 +204,9 @@ func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { snapshot = testSnapshot(now) a.state.Snapshot = &snapshot a.state.BindingFences, _ = advanceBindingFences(nil, snapshot) - a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ValidUntil: now.Add(time.Minute)}} + a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)}} if a.publicationsReady(snapshot.Bindings[0]) { - t.Fatal("node enabled from ACK without deadline plan") + t.Fatal("node enabled from ACK without publication plan") } chunk, m := testPublication(t, now) if err := a.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { @@ -249,7 +219,7 @@ func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { t.Fatal("complete plan + current proof not eligible") } runner.calls = nil - clock = now.Add(61 * time.Second) + clock = now.Add(24 * time.Hour) if err := a.Step(context.Background()); err != nil { t.Fatal(err) } @@ -259,15 +229,15 @@ func TestNodeRequiresDeadlinePlanAndExpiresCacheLocally(t *testing.T) { flushed = true } } - if !flushed { - t.Fatal("node did not flush original-deadline record without broker") + if flushed { + t.Fatal("node flushed a valid record during the outage") } restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) if err != nil { t.Fatal(err) } - if restarted.state.Publications["zone-a"].Plan.ObservationFences[0].ValidUntil != now.Add(time.Minute) { - t.Fatal("restart changed original deadline") + if !restarted.publicationsReady(snapshot.Bindings[0]) { + t.Fatal("restart lost the installed publication proof") } } func TestBINDPublicationVersionProofIncludesFullContentFingerprint(t *testing.T) { @@ -279,9 +249,9 @@ func TestBINDPublicationVersionProofIncludesFullContentFingerprint(t *testing.T) t.Fatal(err) } p := publicationState{Fence: fence{m.WriterEpoch, m.Revision}, Plan: plan} - first := publicationMailbox(p, now) + first := publicationMailbox(p) p.Plan.RRSets[0].Records[0].Content = "192.0.2.11" - if first == publicationMailbox(p, now) { + if first == publicationMailbox(p) { t.Fatal("different installed data shares fingerprint") } if len(first) > 255 { @@ -323,7 +293,7 @@ func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { a.state.Publications[z.ZoneUID] = publicationState{Fence: fence{2, 4}, Verified: true} a.state.Acks[z.ZoneUID] = map[string]replicaAck{} } - proof := replicaAck{Fence: fence{2, 4}, ValidUntil: now.Add(time.Minute)} + proof := replicaAck{Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)} a.state.Acks["zone-a"]["regional-0/"] = proof a.state.Acks["zone-b"]["regional-1/"] = proof if a.publicationsReady(b) { @@ -346,11 +316,11 @@ func TestRegionalPoolRequiresOneMemberWithEveryCurrentZone(t *testing.T) { if strings.Count(conf, "server:addPool(") != 1 { t.Fatalf("stale live member remains in context pool:\n%s", conf) } - // A newer ACK without the matching deadline plan gates even the formerly - // current member; a missed record stream cannot serve data with old leases. - a.state.Acks["zone-a"]["regional-1/"] = replicaAck{Fence: fence{2, 5}, ValidUntil: now.Add(time.Minute)} + // A newer ACK without the matching publication plan gates even the formerly + // current member; a resolver must hold the publication version confirmed by the member. + a.state.Acks["zone-a"]["regional-1/"] = replicaAck{Fence: fence{2, 5}, ObservedAt: now, ValidUntil: now.Add(time.Minute)} if a.publicationsReady(b) { - t.Fatal("ACK ahead of local deadline plan activated context") + t.Fatal("ACK ahead of local publication plan activated context") } } func TestMissingOrStalePrivatePublicationHasNoPublicFallback(t *testing.T) { @@ -358,7 +328,7 @@ func TestMissingOrStalePrivatePublicationHasNoPublicFallback(t *testing.T) { c, snapshot := renderFixture(t) c.Role = RenderRegionalBIND publications := map[string]publicationState{"zone-a": {Fence: fence{1, 1}, Plan: model.PublicationPlan{ZoneUID: "zone-a", Apex: "prod.internal"}}} - rendered, err := renderPublications(c, snapshot, map[string]bool{}, publications, time.Now()) + rendered, err := renderPublications(c, snapshot, map[string]bool{}, publications) if err != nil { t.Fatal(err) } @@ -393,7 +363,7 @@ func TestRejectedMemberProofRevokesRoutingAndIgnoresOlderReplay(t *testing.T) { if err := json.Unmarshal(chunk.Payload, &plan); err != nil { t.Fatal(err) } - a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan, now)} + a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan)} a.state.Acks["zone-a"] = map[string]replicaAck{"regional-0/regional-0": {Fence: fence{2, 4}, ObservedAt: now, ValidUntil: now.Add(time.Minute)}} ack := model.MemberAck{MemberID: "regional-0", ReplicaID: "regional-0", ResourceUID: "zone-a", Kind: model.KindPublicationManifest, Epoch: 2, Revision: 4, Phase: model.AckRejected, ObservedAt: now.Add(-time.Second), ValidUntil: now.Add(time.Minute)} if err := a.Handle(context.Background(), envelope(t, model.KindMemberAck, "zone-a", 2, 4, ack)); err != nil { @@ -506,47 +476,6 @@ func installedFileVerifier(t *testing.T, configPath string) PublicationVerifier }) } -func TestBINDProofPinsActivatedFilesWhenDeadlineCrossesReload(t *testing.T) { - t.Parallel() - now := time.Now().UTC() - clock := now - cfg := regionalTestConfig(t) - runner := &afterReloadRunner{} - a, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return clock }, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, a, now) - clock = now.Add(59 * time.Second) - a.needsActivation = true - runner.after = func() { clock = now.Add(61 * time.Second) } - if err := a.Step(context.Background()); err != nil { - t.Fatal(err) - } - if a.state.Publications["zone-a"].LocalRevision == 0 || strings.Contains(currentZone(t, a), "192.0.2.10") { - t.Fatal("expiry crossing reload was not materialized before watchdog renewal") - } - if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { - t.Fatalf("safe expired authority did not renew watchdog: %v", err) - } - // Restart must reconstruct file proofs through activation, never trust the - // earlier process's in-memory proof or its persisted rendered hash alone. - restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }, PublicationVerifier: installedFileVerifier(t, cfg.Render.ClusterBIND.Path)}) - if err != nil { - t.Fatal(err) - } - if len(restarted.installedPublications) != 0 { - t.Fatal("restart inherited unverified process file proofs") - } - restarted.needsActivation = true - if err := restarted.Step(context.Background()); err != nil { - t.Fatal(err) - } - if len(restarted.installedPublications["zone-a"].Views) == 0 { - t.Fatal("restart did not reconstruct exact activated zone proof") - } -} - func TestFailedBINDActivationRetainsPriorInstalledProof(t *testing.T) { t.Parallel() now := time.Now().UTC() @@ -592,7 +521,7 @@ func TestBINDDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { if err := json.Unmarshal(chunk.Payload, &plan); err != nil { t.Fatal(err) } - a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Manifest: manifest, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan, now)} + a.state.Publications["zone-a"] = publicationState{Fence: fence{2, 4}, Manifest: manifest, Plan: plan, Verified: true, EffectiveHash: effectivePlanHash(plan)} if err := a.reconcile(context.Background()); err != nil { t.Fatal(err) } @@ -605,58 +534,6 @@ func TestBINDDoesNotProbeAuthorityBelowRequiredPublicationFence(t *testing.T) { } } -func TestBINDWatchdogCannotRenewExpiredInstalledSnapshot(t *testing.T) { - t.Parallel() - now := time.Now().UTC() - clock := now - cfg := regionalTestConfig(t) - a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, a, now) - clock = now.Add(61 * time.Second) - if err := a.renewWatchdogLease(context.Background()); err == nil { - t.Fatal("expired installed snapshot renewed watchdog") - } - if _, err := os.Stat(cfg.WatchdogLeasePath); !errors.Is(err, os.ErrNotExist) || !a.needsActivation { - t.Fatalf("expired snapshot was not gated: %v", err) - } -} - -func TestBINDExpiryCrossingProofSettlesBeforeWatchdogRenewal(t *testing.T) { - t.Parallel() - now := time.Now().UTC() - clock := now - cfg := regionalTestConfig(t) - files := installedFileVerifier(t, cfg.Render.ClusterBIND.Path) - advance := false - verifier := publicationVerifierFunc(func(ctx context.Context, plan model.PublicationPlan, views []PublicationView) error { - err := files.VerifyPublication(ctx, plan, views) - if advance { - advance = false - clock = now.Add(61 * time.Second) - } - return err - }) - a, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return clock }, PublicationVerifier: verifier}) - if err != nil { - t.Fatal(err) - } - installTestPublication(t, a, now) - clock = now.Add(59 * time.Second) - advance = true - if err := a.Step(context.Background()); err != nil { - t.Fatal(err) - } - if strings.Contains(currentZone(t, a), "192.0.2.10") || a.state.Publications["zone-a"].LocalRevision == 0 { - t.Fatal("deadline crossing proof renewed stale authority") - } - if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil { - t.Fatalf("settled expiry unnecessarily gated process: %v", err) - } -} - type partitioningAckPublisher struct { disconnected atomic.Bool verified atomic.Int64 @@ -676,7 +553,7 @@ func (p *partitioningAckPublisher) Publish(_ context.Context, _ string, env mode return transport.PublishAck{}, nil } -func TestBINDActivationRacingBrokerDisconnectKeepsSafeLeaseAndOriginalDeadline(t *testing.T) { +func TestBINDActivationRacingBrokerDisconnectKeepsServing(t *testing.T) { t.Parallel() now := time.Now().UTC() clock := now @@ -724,7 +601,7 @@ func TestBINDActivationRacingBrokerDisconnectKeepsSafeLeaseAndOriginalDeadline(t t.Fatalf("ACK partition removed safe local lease: %v", err) } // Identical manifest redelivery and snapshot delivery must share the same - // best-effort ACK boundary without extending observation deadlines. + // best-effort ACK boundary without changing the observation. if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 5, manifest)); err != nil { t.Fatal(err) } @@ -734,18 +611,15 @@ func TestBINDActivationRacingBrokerDisconnectKeepsSafeLeaseAndOriginalDeadline(t if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, snapshot)); err != nil { t.Fatal(err) } - clock = now.Add(61 * time.Second) + clock = now.Add(24 * time.Hour) if err := a.Step(context.Background()); err != nil { t.Fatal(err) } - if strings.Contains(currentZone(t, a), "192.0.2.11") { - t.Fatal("ACK partition left records past their original deadline") - } - if !a.state.Publications["zone-a"].Plan.ObservationFences[0].ValidUntil.Equal(now.Add(time.Minute)) { - t.Fatal("activation or ACK retry renewed the observation deadline") + if !strings.Contains(currentZone(t, a), "192.0.2.11") { + t.Fatal("ACK partition removed an applied record") } if _, err := os.Stat(cfg.WatchdogLeasePath); err != nil || a.needsActivation { - t.Fatalf("safe local expiry was gated by ACK outage: %v", err) + t.Fatalf("healthy local serving was gated by ACK outage: %v", err) } before := publisher.verified.Load() publisher.disconnected.Store(false) diff --git a/internal/internaldns/serving/binding_fences.go b/internal/internaldns/serving/binding_fences.go index 18bb83d6..ebb2867c 100644 --- a/internal/internaldns/serving/binding_fences.go +++ b/internal/internaldns/serving/binding_fences.go @@ -53,9 +53,6 @@ func advanceBindingFences(prior map[string]bindingFence, snapshot model.ServingS if cmp < 0 { return nil, fmt.Errorf("binding %s source authorization regressed", b.BindingUID) } - if cmp == 0 && !candidate.Authorization.ValidUntil.Equal(old.Authorization.ValidUntil) { - return nil, fmt.Errorf("binding %s changed the original deadline at the same source fence", b.BindingUID) - } if old.Tombstone && !candidate.Tombstone && cmp <= 0 { return nil, fmt.Errorf("binding %s reactivation requires a newer source authorization", b.BindingUID) } diff --git a/internal/internaldns/serving/binding_fences_test.go b/internal/internaldns/serving/binding_fences_test.go index bc065561..21a35e5d 100644 --- a/internal/internaldns/serving/binding_fences_test.go +++ b/internal/internaldns/serving/binding_fences_test.go @@ -23,7 +23,6 @@ func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testi } active := testSnapshot(now) active.Bindings[0].Authorization.Revision = 2 - active.Bindings[0].Authorization.ValidUntil = now.Add(10 * time.Second) if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { t.Fatal(err) } @@ -43,18 +42,17 @@ func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testi if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, withdrawn)); err != nil { t.Fatal(err) } - if f := a.state.BindingFences["binding-a"]; !f.Tombstone || f.Authorization.Revision != 2 || !f.Authorization.ValidUntil.Equal(active.Bindings[0].Authorization.ValidUntil) { + if f := a.state.BindingFences["binding-a"]; !f.Tombstone || f.Authorization.Revision != 2 { t.Fatal("withdrawal fabricated source authorization") } // Source2 -> Source3 is the next legitimate renewal; DNS must not have - // consumed sequence3 itself while withdrawing the expired authorization. + // consumed sequence3 itself while withdrawing the withdrawn authorization. renewed := withdrawn renewed.Bindings = append([]model.Binding(nil), withdrawn.Bindings...) renewed.ConfigurationRevision = 3 renewed.Bindings[0].ConfigurationRevision = 3 renewed.Bindings[0].Tombstone = false renewed.Bindings[0].Authorization.Revision = 3 - renewed.Bindings[0].Authorization.ValidUntil = now.Add(2 * time.Minute) if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, renewed)); err != nil { t.Fatalf("normal source2 -> source3 renewal rejected: %v", err) } @@ -66,9 +64,9 @@ func TestBindingWithdrawalPreservesSourceSequenceAndAllowsNormalRenewal(t *testi t.Fatal(err) } if a.state.Snapshot.Bindings[0].Tombstone || a.state.ServingFence.Revision != 3 { - t.Fatal("stale expiry replay withdrew current renewal") + t.Fatal("stale withdrawal replay removed current access") } - // A higher shard revision cannot legitimize an older source lease. + // A higher shard revision cannot legitimize an older source revision. stale := withdrawn stale.ConfigurationRevision = 4 if err := a.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 4, stale)); err == nil { @@ -112,9 +110,8 @@ func TestBindingOmissionFenceSurvivesRestartAndRequiresNewSourceAuthority(t *tes if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err == nil { t.Fatal("same source sequence resurrected omitted binding") } - candidate.Bindings[0].Authorization.ValidUntil = now.Add(20 * time.Minute) if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err == nil { - t.Fatal("same source sequence changed its original deadline") + t.Fatal("same source sequence changed its source revision") } candidate.Bindings[0].Authorization.Revision = 3 if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, candidate)); err != nil { diff --git a/internal/internaldns/serving/publication_hash.go b/internal/internaldns/serving/publication_hash.go index 73b74d95..11b72acf 100644 --- a/internal/internaldns/serving/publication_hash.go +++ b/internal/internaldns/serving/publication_hash.go @@ -5,16 +5,15 @@ package serving import ( "sort" "strings" - "time" "go.miloapis.com/dns-operator/internal/internaldns/model" ) -func effectivePlanHash(plan model.PublicationPlan, now time.Time) string { +func effectivePlanHash(plan model.PublicationPlan) string { var values []string for _, rr := range plan.RRSets { for _, r := range rr.Records { - if !r.Eligible || (r.ContributionUID != "" && !now.Before(r.ValidUntil)) { + if !r.Eligible { continue } values = append(values, model.AbsoluteName(rr.Name)+"\x00"+strings.ToUpper(rr.Type)+"\x00"+r.Content) diff --git a/internal/internaldns/serving/render.go b/internal/internaldns/serving/render.go index 05d0a25a..972934d0 100644 --- a/internal/internaldns/serving/render.go +++ b/internal/internaldns/serving/render.go @@ -10,7 +10,6 @@ import ( "sort" "strconv" "strings" - "time" "go.miloapis.com/dns-operator/internal/internaldns/model" ) @@ -59,10 +58,10 @@ type RenderConfig struct { type Rendered struct{ Files map[string][]byte } func Render(c RenderConfig, snapshot model.ServingSnapshot, active map[string]bool) (Rendered, error) { - return renderPublications(c, snapshot, active, nil, time.Now()) + return renderPublications(c, snapshot, active, nil) } -func renderPublications(c RenderConfig, snapshot model.ServingSnapshot, active map[string]bool, publications map[string]publicationState, now time.Time) (Rendered, error) { +func renderPublications(c RenderConfig, snapshot model.ServingSnapshot, active map[string]bool, publications map[string]publicationState) (Rendered, error) { if err := snapshot.Validate(); err != nil { return Rendered{}, err } @@ -98,7 +97,7 @@ func renderPublications(c RenderConfig, snapshot model.ServingSnapshot, active m files[c.ClusterDNSDist.Path] = renderDNSDist(c.ClusterDNSDist, clusterBackends, bindings, active, true, c.ReadyMembers) } if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { - cluster, err := renderClusterBIND(c.ClusterBIND, bindings, publications, now, files) + cluster, err := renderClusterBIND(c.ClusterBIND, bindings, publications, files) if err != nil { return Rendered{}, err } @@ -246,9 +245,9 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m if cluster { addr = b.ClusterAddress } - fmt.Fprintf(&out, " [%q] = {context=%q, validUntil=%d, ready=%t},\n", addr, b.ViewName(), b.Authorization.ValidUntil.Unix(), !b.ConfigurationPending && active[b.BindingUID]) + fmt.Fprintf(&out, " [%q] = {context=%q, ready=%t},\n", addr, b.ViewName(), !b.ConfigurationPending && active[b.BindingUID]) } - fmt.Fprint(&out, "}\n\nfunction classifyVPC(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil or os.time() >= binding.validUntil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"vpc-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") + fmt.Fprint(&out, "}\n\nfunction classifyVPC(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"vpc-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") fmt.Fprint(&out, "addAction(AllRule(), LuaAction(classifyVPC))\n") fmt.Fprint(&out, "addAction(TagRule(\"context-unready\", \"true\"), RCodeAction(DNSRCode.SERVFAIL))\n") if cluster { @@ -309,7 +308,7 @@ func renderNodeBIND(c BINDRenderConfig, bindings []model.Binding) []byte { return out.Bytes() } -func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publications map[string]publicationState, now time.Time, files map[string][]byte) ([]byte, error) { +func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publications map[string]publicationState, files map[string][]byte) ([]byte, error) { var out bytes.Buffer renderOptions(&out, c) for _, b := range bindings { @@ -322,7 +321,7 @@ func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publication fmt.Fprintf(&out, " zone %q { type forward; forward only; forwarders { 127.0.0.1 port 9; }; };\n", strings.TrimSuffix(model.AbsoluteName(z.Apex), ".")) continue } - data, err := renderZone(p, now) + data, err := renderZone(p) if err != nil { return nil, err } diff --git a/internal/internaldns/serving/render_test.go b/internal/internaldns/serving/render_test.go index 0b2914ec..d252c513 100644 --- a/internal/internaldns/serving/render_test.go +++ b/internal/internaldns/serving/render_test.go @@ -11,7 +11,7 @@ import ( func renderFixture(t *testing.T) (RenderConfig, model.ServingSnapshot) { t.Helper() now := time.Now() - b := model.Binding{BindingUID: "bind-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 3, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 2}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 2, ValidUntil: now.Add(time.Minute)}} + b := model.Binding{BindingUID: "bind-a", ProjectUID: "project-a", ContextUID: "vpc-a", BindingGeneration: 1, ConfigurationRevision: 3, ConsumerAddress: "10.253.0.51", ClusterAddress: "10.253.0.41", Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Zones: []model.ZoneAttachment{{ZoneUID: "zone-a", Apex: "prod.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 2}}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 2}} c := RenderConfig{NodeDNSDist: DNSDistRenderConfig{Path: "/tmp/node-dnsdist.conf", ListenAddress: "0.0.0.0:53", ACLs: []string{"10.253.0.0/24"}, PoolName: "node-shared", Backends: []model.Backend{{MemberID: "node-bind-0", Address: "10.253.0.30", Port: 5300}}}, ClusterDNSDist: DNSDistRenderConfig{Path: "/tmp/cluster-dnsdist.conf", ListenAddress: "0.0.0.0:53", ACLs: []string{"10.253.0.0/24"}, PoolName: "cluster-shared", Backends: []model.Backend{{MemberID: "cluster-bind-0", Address: "10.253.0.20", Port: 5300}}}, NodeBIND: BINDRenderConfig{Path: "/tmp/node-bind.conf", ListenAddress: "10.253.0.30", Port: 5300, ProxyPeers: []string{"10.253.0.50"}, DefaultCacheSize: "16M"}, ClusterBIND: BINDRenderConfig{Path: "/tmp/cluster-bind.conf", ListenAddress: "10.253.0.20", Port: 5300, ProxyPeers: []string{"10.253.0.40"}, DefaultCacheSize: "16M"}} s := model.ServingSnapshot{Region: "east", Shard: "s1", ConfigurationEpoch: 1, ConfigurationRevision: 3, GeneratedAt: now, Bindings: []model.Binding{b}} return c, s @@ -134,7 +134,7 @@ func TestRenderRoleOwnsOnlyItsTier(t *testing.T) { } } -func TestDNSDistIncludesDeadlineAndBackendHealthCheck(t *testing.T) { +func TestDNSDistUsesPersistentIdentityAndBackendHealthCheck(t *testing.T) { t.Parallel() c, s := renderFixture(t) r, err := Render(c, s, map[string]bool{"bind-a": true}) @@ -142,7 +142,10 @@ func TestDNSDistIncludesDeadlineAndBackendHealthCheck(t *testing.T) { t.Fatal(err) } conf := string(r.Files[c.NodeDNSDist.Path]) - for _, required := range []string{"validUntil=", "os.time() >= binding.validUntil", "checkName=\".\"", "useProxyProtocol=true"} { + if strings.Contains(conf, "validUntil") || strings.Contains(conf, "os.time()") { + t.Fatal("resolver access depends on a deadline") + } + for _, required := range []string{"checkName=\".\"", "useProxyProtocol=true"} { if !strings.Contains(conf, required) { t.Fatalf("dnsdist config missing %q:\n%s", required, conf) } diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go index d2386829..bf432057 100644 --- a/internal/internaldns/serving/state.go +++ b/internal/internaldns/serving/state.go @@ -19,20 +19,21 @@ const legacyObservationGrantUID = "legacy-checkpoint" // contributionFence persists the complete immutable observation identity. // The optional revision field supplies the sequence when the decoded fence omits it. type contributionFence struct { - GrantUID string `json:"grantUID,omitempty"` - Epoch uint64 `json:"epoch"` - Sequence uint64 `json:"sequence,omitempty"` - ValidUntil time.Time `json:"validUntil,omitempty"` - LegacyRevision uint64 `json:"revision,omitempty"` + GrantUID string `json:"grantUID,omitempty"` + Epoch uint64 `json:"epoch"` + Sequence uint64 `json:"sequence,omitempty"` + + LegacyRevision uint64 `json:"revision,omitempty"` } type replicaAck struct { + Rejected bool `json:"rejected,omitempty"` ObservedAt time.Time `json:"observedAt"` Fence fence `json:"fence"` ValidUntil time.Time `json:"validUntil"` } -const checkpointFormatVersion = 4 +const checkpointFormatVersion = 5 type checkpoint struct { FormatVersion int `json:"formatVersion"` @@ -74,7 +75,7 @@ func (s stateStore) Load() (checkpoint, error) { return state, errors.New("checkpoint lacks required binding authority history") } for uid, f := range state.BindingFences { - if uid == "" || f.Identity == "" || f.ConfigurationGeneration == 0 || f.ConfigurationRevision == 0 || f.Authorization.IssuerEpoch == 0 || f.Authorization.Revision == 0 || f.Authorization.ValidUntil.IsZero() { + if uid == "" || f.Identity == "" || f.ConfigurationGeneration == 0 || f.ConfigurationRevision == 0 || f.Authorization.IssuerEpoch == 0 || f.Authorization.Revision == 0 { return state, fmt.Errorf("checkpoint binding %s has incomplete authority history", uid) } } @@ -132,8 +133,8 @@ func migrateLegacyObservationFences(plan model.PublicationPlan) (model.Publicati if record.ContributionUID == "" { continue } - next := model.ObservationFence{ContributionUID: record.ContributionUID, GrantUID: legacyObservationGrantUID, WriterEpoch: record.WriterEpoch, Sequence: record.Sequence, ValidUntil: record.ValidUntil} - if old, ok := byUID[record.ContributionUID]; ok && (old.WriterEpoch != next.WriterEpoch || old.Sequence != next.Sequence || !old.ValidUntil.Equal(next.ValidUntil)) { + next := model.ObservationFence{ContributionUID: record.ContributionUID, GrantUID: legacyObservationGrantUID, WriterEpoch: record.WriterEpoch, Sequence: record.Sequence} + if old, ok := byUID[record.ContributionUID]; ok && (old.WriterEpoch != next.WriterEpoch || old.Sequence != next.Sequence) { return plan, fmt.Errorf("legacy contribution %q has inconsistent record fences", record.ContributionUID) } byUID[record.ContributionUID] = next @@ -164,9 +165,6 @@ func migrateContributionHighWater(state *checkpoint) error { } current.LegacyRevision = 0 if proof, ok := proofs[uid]; ok && proof.WriterEpoch == current.Epoch && proof.Sequence == current.Sequence { - if current.ValidUntil.IsZero() { - current.ValidUntil = proof.ValidUntil - } if current.GrantUID == "" && proof.GrantUID != legacyObservationGrantUID { current.GrantUID = proof.GrantUID } diff --git a/internal/internaldns/serving/zone.go b/internal/internaldns/serving/zone.go index b995bfab..7a00f1a4 100644 --- a/internal/internaldns/serving/zone.go +++ b/internal/internaldns/serving/zone.go @@ -6,16 +6,14 @@ import ( "fmt" "strconv" "strings" - "time" "github.com/miekg/dns" "go.miloapis.com/dns-operator/internal/internaldns/model" ) -// publicationSerial proves the exact publication and locally enforced expiry -// content. It is unrelated to a wall clock and cannot renew contributor leases. -func publicationSerial(p publicationState, now time.Time) uint32 { - hash := publicationFingerprint(p, now) +// publicationSerial proves the exact publication and eligible record content. +func publicationSerial(p publicationState) uint32 { + hash := publicationFingerprint(p) serial, _ := strconv.ParseUint(hash[:8], 16, 32) if serial == 0 { serial = 1 @@ -23,28 +21,28 @@ func publicationSerial(p publicationState, now time.Time) uint32 { return uint32(serial) } -func publicationFingerprint(p publicationState, now time.Time) string { +func publicationFingerprint(p publicationState) string { data, _ := json.Marshal(p.Plan) - return model.Hash([]byte(fmt.Sprintf("%d/%d/%d/%s/%s", p.Fence.Epoch, p.Fence.Revision, p.LocalRevision, model.Hash(data), effectivePlanHash(p.Plan, now)))) + return model.Hash([]byte(fmt.Sprintf("%d/%d/%d/%s/%s", p.Fence.Epoch, p.Fence.Revision, p.LocalRevision, model.Hash(data), effectivePlanHash(p.Plan)))) } -func publicationMailbox(p publicationState, now time.Time) string { - hash := publicationFingerprint(p, now) +func publicationMailbox(p publicationState) string { + hash := publicationFingerprint(p) return hash[:32] + "." + hash[32:] + ".publication.internal." } -func renderZone(p publicationState, now time.Time) ([]byte, error) { +func renderZone(p publicationState) ([]byte, error) { if err := p.Plan.Validate(); err != nil { return nil, fmt.Errorf("invalid publication: %w", err) } var out bytes.Buffer apex := model.AbsoluteName(p.Plan.Apex) - fmt.Fprintf(&out, "$ORIGIN %s\n$TTL 5\n@ IN SOA ns1.internal-dns.invalid. %s %d 60 30 3600 5\n@ IN NS ns1.internal-dns.invalid.\n", apex, publicationMailbox(p, now), publicationSerial(p, now)) + fmt.Fprintf(&out, "$ORIGIN %s\n$TTL 5\n@ IN SOA ns1.internal-dns.invalid. %s %d 60 30 3600 5\n@ IN NS ns1.internal-dns.invalid.\n", apex, publicationMailbox(p), publicationSerial(p)) ownerHasRecord := map[string]bool{} for _, rrset := range p.Plan.RRSets { if strings.EqualFold(rrset.Type, "SOA") { return nil, fmt.Errorf("publication SOA is reserved for version proof") } for _, record := range rrset.Records { - if !record.Eligible || (record.ContributionUID != "" && !now.Before(record.ValidUntil)) { + if !record.Eligible { continue } line := fmt.Sprintf("%s %d IN %s %s", model.AbsoluteName(rrset.Name), rrset.TTL, strings.ToUpper(rrset.Type), record.Content) diff --git a/internal/internaldns/serving/zone_regression_test.go b/internal/internaldns/serving/zone_regression_test.go index d8ab91d0..b2abacdf 100644 --- a/internal/internaldns/serving/zone_regression_test.go +++ b/internal/internaldns/serving/zone_regression_test.go @@ -3,7 +3,6 @@ package serving import ( "strings" "testing" - "time" "github.com/miekg/dns" "go.miloapis.com/dns-operator/internal/internaldns/model" @@ -23,10 +22,9 @@ func parseRenderedZone(t *testing.T, zone []byte) []dns.RR { } func TestNODATAOwnershipDoesNotGrowNamesOrCollideWithTenantCNAME(t *testing.T) { - now := time.Now() owner := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." + strings.Repeat("c", 63) + "." + strings.Repeat("d", 45) + ".x.internal." plan := model.PublicationPlan{ZoneUID: "zone", Apex: "x.internal", Owners: []string{owner, "api.x.internal"}, RRSets: []model.RRSet{{Name: "_dns-ownership.api.x.internal", Type: "CNAME", TTL: 5, Records: []model.RRRecord{{Content: "target.example.", Eligible: true}}}}} - data, err := renderZone(publicationState{Plan: plan}, now) + data, err := renderZone(publicationState{Plan: plan}) if err != nil { t.Fatal(err) } @@ -41,7 +39,7 @@ func TestNODATAOwnershipDoesNotGrowNamesOrCollideWithTenantCNAME(t *testing.T) { t.Fatalf("missing ownership metadata: %v", markers) } plan.RRSets = append(plan.RRSets, model.RRSet{Name: "api.x.internal", Type: "CNAME", TTL: 5, Records: []model.RRRecord{{Content: "healthy.example.", Eligible: true}}}) - data, err = renderZone(publicationState{Plan: plan}, now) + data, err = renderZone(publicationState{Plan: plan}) if err != nil { t.Fatal(err) } diff --git a/internal/internaldns/transport/jetstream.go b/internal/internaldns/transport/jetstream.go index 7aa00976..8f300b8a 100644 --- a/internal/internaldns/transport/jetstream.go +++ b/internal/internaldns/transport/jetstream.go @@ -92,7 +92,7 @@ func Connect(c Config) (*JetStream, error) { nats.Name(c.Name), nats.Timeout(defaultDuration(c.ConnectTimeout, 10*time.Second)), // A regional partition must not permanently close a long-running agent. - // Callers keep retrying while local expiry and the watchdog gate serving. + // Callers retry delivery while local health checks govern serving. nats.MaxReconnects(-1), } if c.CredentialsFile != "" { diff --git a/test/internaldns/README.md b/test/internaldns/README.md index 93e75daa..637aa9a7 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -83,8 +83,8 @@ The Go qualifier covers overlapping tenant names over UDP and TCP, A/AAAA and negative-cache isolation, multiple zones, managed name discovery, scoped publication and admission denial, record updates and deletion, health withdrawal and recovery, fixed fleet size, regional publication proof, -controller takeover, local record/access expiry while controllers and the -broker are stopped, and publication after broker/controller restoration using +controller takeover, continued tenant-isolated UDP/TCP resolution while controllers +and the broker are stopped, and publication after broker/controller restoration using the retained broker PVCs. Each assertion records its result; skipped scenarios do not count as passes. @@ -111,5 +111,5 @@ into the separate DNS service VPC. Kind cannot establish physical fabric, microVM attachment, host-loss durability, regional independence, or production capacity. -The suite requires internal wire version `v1alpha3`, fresh format 4 checkpoints, +The suite requires internal wire version `v1alpha4`, fresh format 5 checkpoints, and matching controller and fleet versions. diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go index e84f7c9d..1f6a5431 100644 --- a/test/internaldns/kubernetes/qualify/main.go +++ b/test/internaldns/kubernetes/qualify/main.go @@ -34,7 +34,6 @@ import ( type project struct { Name string `json:"name"` ProjectUID string `json:"projectUID"` - SourceClusterUID string `json:"sourceClusterUID"` Namespace string `json:"namespace"` CustomerKubeconfig string `json:"customerKubeconfig"` ProductKubeconfig string `json:"productKubeconfig"` @@ -268,7 +267,7 @@ func (s *suite) run(path string) error { return fmt.Errorf("%s client: %w", p.Name, err) } } - if net.ParseIP(p.ConsumerAddress) == nil || p.ProjectUID == "" || p.SourceClusterUID == "" || p.PublisherSubject == "" { + if net.ParseIP(p.ConsumerAddress) == nil || p.ProjectUID == "" || p.PublisherSubject == "" { return fmt.Errorf("%s has incomplete identity or resolver address", p.Name) } } @@ -351,7 +350,7 @@ func (s *suite) prepare(p *project) error { p.access = dnsv1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "application-network-access", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverAccessBindingSpec{ ContextRef: objectRef(&p.resolverContext), Region: s.env.Region, QueryIdentity: dnsv1.DNSResolverQueryIdentity{Type: dnsv1.DNSResolverQueryIdentityDestinationAddress, Value: p.ConsumerAddress}, Port: 53, Transports: []string{"UDP", "TCP"}, - Authorization: dnsv1.DNSResolverAccessAuthorization{WriterEpoch: p.resolverContext.Status.AccessWriterEpoch, Sequence: 1, ValidUntil: metav1.NewTime(time.Now().Add(5 * time.Minute).UTC())}, + Authorization: dnsv1.DNSResolverAccessAuthorization{WriterEpoch: p.resolverContext.Status.AccessWriterEpoch, Sequence: 1}, }} if err := p.integration.Create(s.ctx, &p.access); err != nil { return err @@ -395,7 +394,7 @@ func recordSets(name, address, ipv6 string) []dnsv1.DNSContributionRecordSet { return sets } -func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string, lifetime time.Duration) (*publication, error) { +func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string) (*publication, error) { pub := &publication{registration: dnsv1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-registration", Namespace: p.Namespace}, Spec: dnsv1.DNSRegistrationSpec{DNSZoneRef: zone, Name: name, RecordTypes: []dnsv1.RRType{dnsv1.RRTypeA}, PublicationPolicy: dnsv1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 2}}} if ipv6 != "" { pub.registration.Spec.RecordTypes = append(pub.registration.Spec.RecordTypes, dnsv1.RRTypeAAAA) @@ -403,7 +402,7 @@ func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, if err := p.product.Create(s.ctx, &pub.registration); err != nil { return nil, err } - pub.grant = dnsv1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-compute", Namespace: p.Namespace}, Spec: dnsv1.DNSContributionGrantSpec{RegistrationRef: objectRef(&pub.registration), ProducerID: "compute", Principal: dnsv1.DNSProducerPrincipal{ClusterUID: p.SourceClusterUID, Subject: p.PublisherSubject}, RecordTypes: pub.registration.Spec.RecordTypes}} + pub.grant = dnsv1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-compute", Namespace: p.Namespace}, Spec: dnsv1.DNSContributionGrantSpec{RegistrationRef: objectRef(&pub.registration), ProducerID: "compute", Principal: dnsv1.DNSProducerPrincipal{Subject: p.PublisherSubject}, RecordTypes: pub.registration.Spec.RecordTypes}} if err := p.issuer.Create(s.ctx, &pub.grant); err != nil { return nil, err } @@ -423,13 +422,13 @@ func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, }); err != nil { return nil, err } - if err := s.observe(p, pub, true, lifetime); err != nil { + if err := s.observe(p, pub, true); err != nil { return nil, err } return pub, nil } -func (s *suite) observe(p *project, pub *publication, eligible bool, lifetime time.Duration) error { +func (s *suite) observe(p *project, pub *publication, eligible bool) error { if err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&pub.contribution), &pub.contribution); err != nil { return err } @@ -442,8 +441,6 @@ func (s *suite) observe(p *project, pub *publication, eligible bool, lifetime ti if !eligible { pub.contribution.Status.Reason = "EndpointNotReady" } - deadline := metav1.NewTime(time.Now().Add(lifetime).UTC()) - pub.contribution.Status.ValidUntil = &deadline if err := p.product.Status().Patch(s.ctx, &pub.contribution, client.MergeFrom(before)); err != nil { return err } diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index fa7e3cd8..afb1ed07 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -5,6 +5,7 @@ package main import ( "errors" "fmt" + "os/exec" "strings" "time" @@ -23,11 +24,11 @@ func (s *suite) scenarios(a, b *project) error { return err } } - pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10", 90*time.Second) + pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10") if err != nil { return err } - pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20", 90*time.Second) + pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20") if err != nil { return err } @@ -74,7 +75,7 @@ func (s *suite) scenarios(a, b *project) error { return err } s.pass("an unbound destination cannot resolve either tenant's private records", nil) - if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", "", 90*time.Second); err != nil { + if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", ""); err != nil { return err } if err := s.expect(a, "only-a.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { @@ -85,7 +86,7 @@ func (s *suite) scenarios(a, b *project) error { } s.pass("negative caches and overlapping endpoint address spaces stay isolated", nil) // Products discover their managed zone through context status. - if _, err := s.publish(a, a.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.11", "", 90*time.Second); err != nil { + if _, err := s.publish(a, a.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.11", ""); err != nil { return err } managedName := "web." + a.resolverContext.Status.ManagedNamespace.Suffix @@ -94,7 +95,7 @@ func (s *suite) scenarios(a, b *project) error { return err } } - if _, err := s.publish(b, b.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.21", "", 90*time.Second); err != nil { + if _, err := s.publish(b, b.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web", "10.42.0.21", ""); err != nil { return err } for _, tcp := range []bool{false, true} { @@ -113,14 +114,14 @@ func (s *suite) scenarios(a, b *project) error { if err := s.associate(a, apps, "apps"); err != nil { return err } - if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", "", 90*time.Second); err != nil { + if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", ""); err != nil { return err } if err := s.expect(a, "api.apps.internal", "A", true, dns.RcodeSuccess, "10.42.0.12"); err != nil { return err } s.pass("one resolver context serves its managed namespace and multiple custom zones", nil) - if err := s.observe(b, pb, true, 90*time.Second); err != nil { + if err := s.observe(b, pb, true); err != nil { return err } if err := s.update(a, pa, "10.42.0.30", "fd42::30"); err != nil { @@ -132,7 +133,7 @@ func (s *suite) scenarios(a, b *project) error { if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { return err } - if err := s.observe(a, pa, false, 90*time.Second); err != nil { + if err := s.observe(a, pa, false); err != nil { return err } for _, tcp := range []bool{false, true} { @@ -141,7 +142,7 @@ func (s *suite) scenarios(a, b *project) error { } } s.pass("an unhealthy endpoint stops producing DNS answers", nil) - if err := s.observe(a, pa, true, 90*time.Second); err != nil { + if err := s.observe(a, pa, true); err != nil { return err } if err := s.expect(a, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.30"); err != nil { @@ -161,10 +162,10 @@ func (s *suite) scenarios(a, b *project) error { if err := s.takeover(a, pa); err != nil { return err } - if err := s.observe(b, pb, true, 90*time.Second); err != nil { + if err := s.observe(b, pb, true); err != nil { return err } - deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "", 90*time.Second) + deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "") if err != nil { return err } @@ -178,10 +179,10 @@ func (s *suite) scenarios(a, b *project) error { return err } s.pass("deleting a contribution withdraws its answer and preserves registration ownership", nil) - if err := s.offlineExpiry(a, b, pa, pb); err != nil { + if err := s.offlineServing(a, b); err != nil { return err } - if err := s.observe(b, pb, true, 90*time.Second); err != nil { + if err := s.observe(b, pb, true); err != nil { return err } if err := s.expect(b, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.20"); err != nil { @@ -200,7 +201,7 @@ func (s *suite) update(p *project, pub *publication, address, ipv6 string) error if err := p.product.Patch(s.ctx, &pub.contribution, client.MergeFrom(before)); err != nil { return err } - return s.observe(p, pub, true, 90*time.Second) + return s.observe(p, pub, true) } func (s *suite) owner(zoneUID string) (dnsv1.DNSPublicationOwnership, error) { @@ -284,7 +285,7 @@ func (s *suite) takeover(p *project, pub *publication) (resultErr error) { }); err != nil { return err } - if err := s.observe(p, pub, true, 90*time.Second); err != nil { + if err := s.observe(p, pub, true); err != nil { return err } if err := s.expect(p, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.30"); err != nil { @@ -322,49 +323,18 @@ func (s *suite) scaleBroker(replicas int32) error { }) } -func (s *suite) offlineExpiry(a, b *project, pa, pb *publication) (resultErr error) { - // Keep the healthy tenant's original record and access lease long enough to - // cover the outage. Only the expired tenant receives the short access lease. - // Finish access readiness before starting the record leases: serving ACKs - // follow the production 30-second interval and can consume that setup time. - if err := s.renewAccess(b, 5*time.Minute); err != nil { - return err - } - if err := s.observe(a, pa, true, 90*time.Second); err != nil { - return err - } - if err := s.observe(b, pb, true, 90*time.Second); err != nil { - return err - } - expiring, err := s.publish(a, objectRef(&a.zone), "expires", "expires", "10.42.0.15", "", 45*time.Second) - if err != nil { - return err - } - if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.15"); err != nil { - return err - } - if err := a.integration.Get(s.ctx, client.ObjectKeyFromObject(&a.access), &a.access); err != nil { - return err - } - before := a.access.DeepCopy() - a.access.Spec.Authorization.Sequence++ - a.access.Spec.Authorization.ValidUntil = metav1.NewTime(time.Now().Add(50 * time.Second).UTC()) - if err := a.integration.Patch(s.ctx, &a.access, client.MergeFrom(before)); err != nil { - return err - } - if err := s.waitAccess(a); err != nil { - return err - } +// offlineServing checks uncached queries while control-plane delivery is unavailable. +func (s *suite) offlineServing(a, b *project) (resultErr error) { var broker appsv1.StatefulSet if err := s.platform.Get(s.ctx, client.ObjectKey{Namespace: s.env.Namespace, Name: s.env.BrokerStatefulSet}, &broker); err != nil { return err } - brokerReplicas := int32(1) + replicas := int32(3) if broker.Spec.Replicas != nil { - brokerReplicas = *broker.Spec.Replicas + replicas = *broker.Spec.Replicas } defer func() { - if err := s.scaleBroker(brokerReplicas); err != nil { + if err := s.scaleBroker(replicas); err != nil { resultErr = errors.Join(resultErr, fmt.Errorf("broker restoration: %w", err)) } for _, name := range s.env.ControllerDeployments { @@ -381,83 +351,74 @@ func (s *suite) offlineExpiry(a, b *project, pa, pb *publication) (resultErr err if err := s.scaleBroker(0); err != nil { return err } - if expiring.contribution.Status.ValidUntil == nil || !expiring.contribution.Status.ValidUntil.After(time.Now().Add(3*time.Second)) { - return fmt.Errorf("record expired before the outage; no offline expiry proof") - } - if !a.access.Spec.Authorization.ValidUntil.After(time.Now().Add(10 * time.Second)) { - return fmt.Errorf("access expired before the outage; no offline authorization proof") - } - if !a.access.Spec.Authorization.ValidUntil.After(expiring.contribution.Status.ValidUntil.Add(5 * time.Second)) { - return fmt.Errorf("record and access deadlines overlap; no independent expiry proof") - } - if pb.contribution.Status.ValidUntil == nil || - !pb.contribution.Status.ValidUntil.After(a.access.Spec.Authorization.ValidUntil.Add(15*time.Second)) { - return fmt.Errorf("healthy record lease cannot cover access expiry; no outage positive control") - } - if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.15"); err != nil { - return err - } - s.pass("Kubernetes controllers and broker stop without stopping the serving pods", nil) - if deadline := expiring.contribution.Status.ValidUntil; deadline != nil { - if remaining := time.Until(deadline.Add(3 * time.Second)); remaining > 0 { - time.Sleep(remaining) + // The outage spans multiple DNS cache and fleet acknowledgement intervals. + until := time.Now().Add(2 * time.Minute) + for time.Now().Before(until) { + for _, tcp := range []bool{false, true} { + for _, tenant := range []struct { + project *project + address string + }{{a, "10.42.0.30"}, {b, "10.42.0.20"}} { + answer, err := s.query(tenant.project, "common.prod.internal", "A", tcp) + if err != nil { + return err + } + if answer.Error != "" || answer.Rcode != dns.RcodeSuccess || len(answer.Answers) != 1 || answer.Answers[0] != tenant.address { + return fmt.Errorf("outage query for %s TCP=%t: %#v", tenant.project.Name, tcp, answer) + } + } + } + select { + case <-s.ctx.Done(): + return s.ctx.Err() + case <-time.After(time.Second): } } - if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess); err != nil { - return err - } - if err := s.expect(b, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.20"); err != nil { + s.pass("both tenants keep resolving over UDP and TCP with controllers and messaging stopped", nil) + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), client.MatchingLabels{"internal-dns-role": "node"}); err != nil { return err } - s.pass("local record expiry preserves the original deadline during a broker and controller outage", expiring.contribution.Status.ValidUntil) - if remaining := time.Until(a.access.Spec.Authorization.ValidUntil.Add(-3 * time.Second)); remaining > 0 { - time.Sleep(remaining) + if len(pods.Items) != 1 { + return fmt.Errorf("expected one node resolver, got %d", len(pods.Items)) } - answer, err := s.query(a, "expires.prod.internal", "A", false) - if err != nil { - return err + pod := pods.Items[0] + var restarts int32 + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "agent" { + restarts = status.RestartCount + } } - if answer.Error != "" || answer.Rcode != dns.RcodeSuccess || len(answer.Answers) != 0 || - !a.access.Spec.Authorization.ValidUntil.After(time.Now()) { - return fmt.Errorf("access was not usable just before its original deadline: %+v", answer) + cmd := exec.CommandContext(s.ctx, "kubectl", "--kubeconfig", s.env.PlatformKubeconfig, "-n", s.env.Namespace, "exec", pod.Name, "-c", "agent", "--", "pkill", "-TERM", "-f", "^/internal-dns --role=agent") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("restart resolver agent: %w: %s", err, output) } - s.pass("unexpired access remains usable after record expiry", map[string]any{ - "validUntil": a.access.Spec.Authorization.ValidUntil, "answer": answer, - }) - if remaining := time.Until(a.access.Spec.Authorization.ValidUntil.Add(3 * time.Second)); remaining > 0 { - time.Sleep(remaining) + if err := s.wait("resolver agent restart without messaging", time.Minute, func() (bool, error) { + var current corev1.Pod + if err := s.platform.Get(s.ctx, client.ObjectKeyFromObject(&pod), ¤t); err != nil { + return false, err + } + for _, status := range current.Status.ContainerStatuses { + if status.Name == "agent" && status.RestartCount > restarts && status.State.Running != nil { + return true, nil + } + } + return false, nil + }); err != nil { + return err } for _, tcp := range []bool{false, true} { - if err := s.wait("expired access rejected", 15*time.Second, func() (bool, error) { - answer, err := s.query(a, "common.prod.internal", "A", tcp) - if err != nil { - return false, err - } - return answer.Error != "" || answer.Rcode == dns.RcodeRefused || answer.Rcode == dns.RcodeServerFailure, nil - }); err != nil { + if err := s.expect(a, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.20"); err != nil { return err } } - if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { - return err - } - s.pass("expired access fails closed while another tenant keeps resolving", a.access.Spec.Authorization.ValidUntil) + s.pass("resolver agent restores both tenant contexts from persistent state without messaging", nil) return nil } -func (s *suite) renewAccess(p *project, lifetime time.Duration) error { - if err := p.integration.Get(s.ctx, client.ObjectKeyFromObject(&p.access), &p.access); err != nil { - return err - } - before := p.access.DeepCopy() - p.access.Spec.Authorization.Sequence++ - p.access.Spec.Authorization.ValidUntil = metav1.NewTime(time.Now().Add(lifetime).UTC()) - if err := p.integration.Patch(s.ctx, &p.access, client.MergeFrom(before)); err != nil { - return err - } - return s.waitAccess(p) -} - func readyPods(pods []corev1.Pod, replicas int32) bool { if len(pods) != int(replicas) { return false diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index daa64a3d..67f7bdfb 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -384,7 +384,7 @@ func configure() { tokenConfig(admin, "", project, "dns-publisher", "compute-system", "product") tokenConfig(admin, "", project, "dns-grant-issuer", "compute-system", "issuer") tokenConfig(admin, "", project, "dns-vpc-integration", "network-system", "integration") - projectRows = append(projectRows, object{"name": project, "projectUID": project + "-uid", "sourceClusterUID": []string{"source-cluster-a-uid", "source-cluster-b-uid"}[i], "namespace": projectNS, "customerKubeconfig": filepath.Join(out, project+"-customer.kubeconfig"), "productKubeconfig": filepath.Join(out, project+"-product.kubeconfig"), "issuerKubeconfig": filepath.Join(out, project+"-issuer.kubeconfig"), "integrationKubeconfig": filepath.Join(out, project+"-integration.kubeconfig"), "publisherSubject": "system:serviceaccount:compute-system:dns-publisher", "consumerAddress": []string{"fd53::a", "fd53::b"}[i]}) + projectRows = append(projectRows, object{"name": project, "projectUID": project + "-uid", "namespace": projectNS, "customerKubeconfig": filepath.Join(out, project+"-customer.kubeconfig"), "productKubeconfig": filepath.Join(out, project+"-product.kubeconfig"), "issuerKubeconfig": filepath.Join(out, project+"-issuer.kubeconfig"), "integrationKubeconfig": filepath.Join(out, project+"-integration.kubeconfig"), "publisherSubject": "system:serviceaccount:compute-system:dns-publisher", "consumerAddress": []string{"fd53::a", "fd53::b"}[i]}) } sources["stringData"] = sourceData items := make([]object, 0, 8) @@ -393,10 +393,9 @@ func configure() { cfg.ConsumerPrefix = "fd53::/64" cfg.ClusterPrefix = "fd54::/64" cfg.OwnershipLeaseSeconds = 10 - cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", SourceClusterUID: "source-cluster-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", SourceClusterUID: "source-cluster-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} + cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} cfg.NodeBackends = []model.Backend{{MemberID: "node-bind-0", Address: "127.0.0.1", Port: 5300}} cfg.ClusterBackends = []model.Backend{{MemberID: memberNames[2], Address: bindIP[0], Port: 5300}, {MemberID: memberNames[3], Address: bindIP[1], Port: 5300}} - cfg.Admission.MaxAccessLeaseSeconds = 600 for _, name := range []string{"control-a", "control-b"} { c := cfg c.NATS.Name = name diff --git a/test/internaldns/production-parity.md b/test/internaldns/production-parity.md index 84dfb42b..7126f71f 100644 --- a/test/internaldns/production-parity.md +++ b/test/internaldns/production-parity.md @@ -33,7 +33,7 @@ cluster health or that the proposed private DNS fleet is deployed. - [NATS infrastructure](https://github.com/datum-cloud/infra/blob/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/infrastructure/databases/nats/base/nats-hr.yaml) declares three broker replicas, persistent JetStream storage, and client and route TLS. The Kubernetes lab uses that three-replica storage and TLS pattern. - Its full broker outage test proves local expiry and replica restart using + Its full broker outage test proves local serving continuity and replica restart using retained PVCs. Quorum loss, disk/host-loss recovery, and regional transport still need dedicated scenarios. - [Public edge DNS](https://github.com/datum-cloud/infra/tree/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/apps/dns-operator/downstream/edge) @@ -137,7 +137,7 @@ intended for staging. Do not mount the Docker socket into those workloads. 2. **Kubernetes deployment (implemented).** Reuse pinned test-infra, serving images and command helpers, TLS admission, scoped worker/product credentials, two controllers, three TLS brokers, and persistent serving state. Run - publication, isolation, takeover, and expiry checks against real pods. ACK + publication, isolation, takeover, and outage checks against real pods. ACK timing is 30/90 seconds; ownership lease and termination grace are shortened to 10 and 2 seconds. CI runs the public regression chain in the same job. Publication resumes after broker/controller restoration using retained @@ -172,11 +172,14 @@ Required deployment and consumer-path scenarios: service route cannot return another VPC's records. Product credentials cannot issue grants or authorize network access. - Endpoint health withdrawal and recovery, deletion, source API loss, and - stale-observation replay obey the original deadlines and configured TTLs. + stale-observation replay preserve current source fences and configured TTLs. + Records remain available during control-plane outages until an explicit update + withdraws them. - Either controller can take ownership without stale publication; a healthy project continues publishing while another project's API is unavailable. -- Broker leader loss retains publication with quorum; quorum loss, agent - restart, member failure, and node failure preserve expiry and fail closed. +- Broker leader loss retains publication with quorum. Agent restart during + quorum loss restores verified checkpoints without broker access. Member and node + failures stop unsafe local serving through independent health checks. - Certificate rotation, webhook unavailability, rolling member replacement, and PVC reuse do not bypass admission or regress checkpoint authority. - Adding contexts leaves the workload count fixed, and public authoritative From 023b793c5c97eb9c331b42e614407d82157cbc9d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:51:56 -0500 Subject: [PATCH 02/15] fix: Preserve internal DNS availability through incomplete updates --- api/v1alpha1/internaldns_service_types.go | 3 + api/v1alpha1/zz_generated.deepcopy.go | 9 +- ...king.miloapis.com_dnsresolverbindings.yaml | 7 + config/internal-dns/README.md | 37 +++- .../controlplane/context_reconciler.go | 42 +++- .../controlplane/context_reconciler_test.go | 12 ++ internal/internaldns/model/model.go | 1 + internal/internaldns/platform/acks.go | 43 +++- internal/internaldns/platform/allocator.go | 22 +- .../internaldns/platform/allocator_test.go | 46 ++++ internal/internaldns/platform/capacity.go | 80 +++++++ .../internaldns/platform/capacity_test.go | 49 +++++ internal/internaldns/platform/planner.go | 52 ++--- .../internaldns/platform/platform_test.go | 40 +++- internal/internaldns/serving/agent.go | 64 +++++- internal/internaldns/serving/continuity.go | 204 ++++++++++++++++++ .../internaldns/serving/continuity_test.go | 93 ++++++++ internal/internaldns/serving/render.go | 39 +++- internal/internaldns/serving/state.go | 1 + internal/internaldns/serving/transaction.go | 10 +- test/internaldns/README.md | 6 +- 21 files changed, 785 insertions(+), 75 deletions(-) create mode 100644 internal/internaldns/platform/capacity.go create mode 100644 internal/internaldns/platform/capacity_test.go create mode 100644 internal/internaldns/serving/continuity.go create mode 100644 internal/internaldns/serving/continuity_test.go diff --git a/api/v1alpha1/internaldns_service_types.go b/api/v1alpha1/internaldns_service_types.go index dc57e565..f00fc424 100644 --- a/api/v1alpha1/internaldns_service_types.go +++ b/api/v1alpha1/internaldns_service_types.go @@ -50,6 +50,9 @@ type DNSResolverBindingConfiguration struct { Listeners DNSResolverBindingListeners `json:"listeners"` // +kubebuilder:validation:MaxItems=256 ZoneRefs []DNSObjectReference `json:"zoneRefs"` + // ZoneApices reserves each private namespace before publication is available. + // +kubebuilder:validation:MaxProperties=256 + ZoneApices map[string]string `json:"zoneApices,omitempty"` } // +kubebuilder:validation:XValidation:rule="oldSelf == self",message="listener identity is immutable for an access binding" diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index fba1d9af..55998ec0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8,7 +8,7 @@ package v1alpha1 import ( "go.datum.net/network-services-operator/api/v1alpha" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) @@ -1418,6 +1418,13 @@ func (in *DNSResolverBindingConfiguration) DeepCopyInto(out *DNSResolverBindingC *out = make([]DNSObjectReference, len(*in)) copy(*out, *in) } + if in.ZoneApices != nil { + in, out := &in.ZoneApices, &out.ZoneApices + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSResolverBindingConfiguration. diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml index 4d78b627..81137d8c 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolverbindings.yaml @@ -124,6 +124,13 @@ spec: format: int64 minimum: 1 type: integer + zoneApices: + additionalProperties: + type: string + description: ZoneApices reserves each private namespace before + publication is available. + maxProperties: 256 + type: object zoneRefs: items: description: |- diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 0b3ecd42..b2ba8aa5 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -108,12 +108,11 @@ from persistent checkpoints before waiting for messaging connectivity. New nodes require a complete configuration before accepting queries. Replay cannot regress writer or configuration versions or restore a withdrawn lifetime. -The local watchdog lease is at most five seconds. It is renewed only after a -complete validated transaction and required DNS proofs. -The gate must stop or firewall the local private listener without depending on -Kubernetes or NATS. Validation, reload, readiness wait, and proof must fit -inside the watchdog budget; reduce shard size or improve local operations when -they do not. +The local watchdog lease is at most five seconds. An independent health loop +checks local DNS daemons over UDP and TCP and renews the lease for the activated +configuration. Staging and validation can continue while that configuration +serves. A rejected expansion retains the activated configuration; a withdrawal +that cannot be applied gates serving. The gate runs without Kubernetes or NATS. Mount the transaction stage, active BIND configuration, zone files, checkpoint, and watchdog lease on low-latency storage local to the serving host. Do not put @@ -163,8 +162,24 @@ contribution and eligibility observation use that publisher's current grant. Grants identify the authenticated subject within the project API context. Publication-result updates cannot change a publisher's observation. Resource permissions do not depend on webhook principal lists. -The supported internal envelope version is `v1alpha4` and the serving checkpoint -format is 5. Unsupported versions fail closed. Deploy matching controller and -fleet versions with fresh internal DNS transport state and checkpoints. Preserve -any existing state for recovery. The deployment requires a coordinated rollout; -it does not support mixed versions or automatic state conversion. +## Rollout and recovery + +The supported envelope version is `v1alpha4` and checkpoint format is 5. Keep +transport state and persistent checkpoints across compatible upgrades. Deploy +serving agents before controllers that emit per-zone pending state. Enable the +controller change after every serving member supports that contract; a matching +wire version alone does not establish feature compatibility. + +The checkpoint retains desired version fences and the activated configuration +separately. Restart can restore the activated configuration during a rejected +expansion or broker outage. Explicit withdrawal invalidates that fallback before +installation. Unknown checkpoint or wire formats preserve files and reject +activation. Format conversion requires a tested migration that retains lifetime, +withdrawal, configuration, and publisher fences; never reset those fences to +complete an upgrade. + +Snapshot budgets admit established bindings before new allocations. Overflow +reports `CapacityAvailable=False` for the affected binding while healthy +contexts and withdrawals continue. A pending zone fails only queries beneath +its private apex; its names never fall through to public recursion. Bindings, +publication targets, and context status use the same stable shard assignment. diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index 54384359..4c7f27f9 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -200,6 +200,10 @@ func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, now time.Time) error { var contexts dnsv1alpha1.DNSResolverContextList + var existing dnsv1alpha1.DNSResolverBindingList + if err := r.List(ctx, &existing, client.InNamespace(r.Options.PlatformNamespace)); err != nil { + return err + } if err := r.List(ctx, &contexts, client.InNamespace(ns)); err != nil { return err } @@ -225,7 +229,7 @@ func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, n } c.Status.ServingTargets = nil if valid { - for _, target := range r.Options.servingRegions() { + for _, target := range r.selectServingTargets(c.UID, existing.Items) { c.Status.ServingTargets = append(c.Status.ServingTargets, dnsv1alpha1.DNSResolverContextServingTarget{Region: target.Region, Shard: target.Shard}) } } @@ -385,7 +389,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as if owner := destinations[key]; owner != "" && owner != a.UID { valid = false } else if valid { - if r.Options.AddressAllocator == nil || r.Options.AddressAllocator.ClaimDestination(ctx, r.Options.ProjectUID, a.UID, spec.Region, netip.MustParseAddr(spec.QueryIdentity.Value).String(), spec.Port) != nil { + if r.Options.AddressAllocator == nil || r.Options.AddressAllocator.ClaimDestination(ctx, r.Options.ProjectUID, contextObject.UID, spec.Region, netip.MustParseAddr(spec.QueryIdentity.Value).String(), spec.Port) != nil { valid = false } else { destinations[key] = a.UID @@ -393,7 +397,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as } status, reason, msg := metav1.ConditionTrue, dnsValueAccepted, "access binding is current" if !valid { - status, reason, msg = metav1.ConditionFalse, "InvalidOrStaleAccess", "context lifetime, epoch, sequence, destination, and bounded deadline must be current" + status, reason, msg = metav1.ConditionFalse, "InvalidOrStaleAccess", "context lifetime, epoch, sequence, and destination must be current" } setCondition(&a.Status.Conditions, dnsValueAccepted, status, reason, msg, a.Generation, now) if !valid { @@ -437,7 +441,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as return err } } - binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForRegion(spec.Region))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForRegion(spec.Region)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence}}} + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: r.Options.PlatformNamespace, Labels: map[string]string{"internal-dns.miloapis.com/context-uid": string(contextObject.UID), "internal-dns.miloapis.com/access-uid": string(a.UID), "internal-dns.miloapis.com/region": model.SafeToken(spec.Region), "internal-dns.miloapis.com/shard": model.SafeToken(r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items))}}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Source: dnsv1alpha1.DNSResolverBindingSource{ProjectUID: r.Options.ProjectUID, ResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: contextObject.Name, UID: contextObject.UID}, AccessBindingRef: dnsv1alpha1.DNSObjectReference{Name: a.Name, UID: a.UID}}, Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: spec.Region, Shard: r.shardForContextRegion(contextObject.UID, spec.Region, existing.Items)}, Configuration: dnsv1alpha1.DNSResolverBindingConfiguration{Generation: generation, Revision: revision, Listeners: dnsv1alpha1.DNSResolverBindingListeners{Node: dnsv1alpha1.DNSResolverBindingListener{Address: spec.QueryIdentity.Value, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}, Regional: dnsv1alpha1.DNSResolverBindingListener{Address: cluster, Port: spec.Port, Transports: append([]string(nil), spec.Transports...)}}, ZoneRefs: zoneReferences(zids, zones), ZoneApices: zoneApices(zids, zones)}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: spec.Authorization.WriterEpoch, Sequence: spec.Authorization.Sequence}}} if !exists { if err := r.Create(ctx, binding); err != nil { return err @@ -460,7 +464,9 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as a.Status.BindingRef = &dnsv1alpha1.DNSObjectReference{Name: binding.Name, UID: binding.UID} setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "PendingReplicaVerification", "serving fleet acknowledgement is pending", a.Generation, now) if c := apimeta.FindStatusCondition(binding.Status.Conditions, "Serving"); c != nil && c.Status == metav1.ConditionTrue && binding.Status.ObservedConfigurationRevision == binding.Spec.Configuration.Revision && bindingAckCurrent(binding, now) { - setCondition(&a.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "required serving replicas applied this access generation", a.Generation, now) + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "serving capacity applied this access generation", a.Generation, now) + } else if c != nil && c.Status == metav1.ConditionUnknown { + setCondition(&a.Status.Conditions, "Ready", metav1.ConditionUnknown, "ObservationUnavailable", "current serving health is unavailable", a.Generation, now) } if !jsonEqual(base.Status, a.Status) { if err := r.Status().Patch(ctx, a, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { @@ -487,7 +493,7 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as b.Spec.Tombstone = true b.Spec.Configuration.Revision++ // Authorization belongs to the trusted integration. Preserve its fence and - // deadline; withdrawal is ordered independently by configuration revision. + // revision; withdrawal is ordered independently by configuration revision. if err := r.Patch(ctx, b, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { return err } @@ -541,9 +547,13 @@ func (r *Reconciler) confirmAccessWithdrawal(ctx context.Context, namespace stri return !valid, nil } -func (r *Reconciler) shardForRegion(region string) string { - t, _ := r.targetForRegion(region) - return t.Shard +func (r *Reconciler) shardForContextRegion(contextUID types.UID, region string, existing []dnsv1alpha1.DNSResolverBinding) string { + for _, target := range r.selectServingTargets(contextUID, existing) { + if target.Region == region { + return target.Shard + } + } + return "" } func (r *Reconciler) targetForRegion(region string) (ServingRegion, bool) { @@ -670,6 +680,20 @@ func bindingEqual(a, b dnsv1alpha1.DNSResolverBindingSpec) bool { return string(xb) == string(yb) } +func zoneApices(uids []types.UID, zones map[string]*dnsv1alpha1.DNSZone) map[string]string { + wanted := map[types.UID]bool{} + for _, uid := range uids { + wanted[uid] = true + } + apices := map[string]string{} + for _, zone := range zones { + if wanted[zone.UID] { + apices[string(zone.UID)] = zone.Spec.DomainName + } + } + return apices +} + func zoneReferences(uids []types.UID, zones map[string]*dnsv1alpha1.DNSZone) []dnsv1alpha1.DNSObjectReference { refs := make([]dnsv1alpha1.DNSObjectReference, 0, len(uids)) for _, uid := range uids { diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index ba6eed14..c019ab4a 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -284,3 +284,15 @@ func TestStableOneShardPerRegionAssignment(t *testing.T) { t.Fatalf("adding a shard moved live VPC: before=%s after=%s", central.Shard, after[0].Shard) } } + +func TestAccessPlacementMatchesContextPublicationTargets(t *testing.T) { + r := &Reconciler{Options: ReconcilerOptions{ProjectUID: "project", ServingRegions: []ServingRegion{{Region: "east", Shard: "a"}, {Region: "east", Shard: "b"}, {Region: "west", Shard: "a"}}}} + for i := range 100 { + uid := types.UID(fmt.Sprintf("context-%d", i)) + for _, target := range r.selectServingTargets(uid, nil) { + if shard := r.shardForContextRegion(uid, target.Region, nil); shard != target.Shard { + t.Fatalf("binding shard %s does not match publication target %s", shard, target.Shard) + } + } + } +} diff --git a/internal/internaldns/model/model.go b/internal/internaldns/model/model.go index 039ac1c9..3dc5b478 100644 --- a/internal/internaldns/model/model.go +++ b/internal/internaldns/model/model.go @@ -89,6 +89,7 @@ type Backend struct { } type ZoneAttachment struct { + Pending bool `json:"pending,omitempty"` ZoneUID string `json:"zoneUID"` Apex string `json:"apex"` RequiredPublicationEpoch uint64 `json:"requiredPublicationEpoch"` diff --git a/internal/internaldns/platform/acks.go b/internal/internaldns/platform/acks.go index 321db0cc..53acd839 100644 --- a/internal/internaldns/platform/acks.go +++ b/internal/internaldns/platform/acks.go @@ -169,7 +169,7 @@ func (s *AckSink) serving(ctx context.Context, a model.MemberAck, row dnsv1alpha } else { binding.Status.Phase = dnsValuePending } - condition(&binding.Status.Conditions, dnsValueServing, complete, "MemberVerification", binding.Generation, now) + s.bindingConditions(binding, e, snapshot, now, complete) if err := s.Client.Status().Patch(ctx, binding, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { return err } @@ -400,7 +400,7 @@ func (s *AckSink) Refresh(ctx context.Context) error { b := &bindings.Items[i] base := b.DeepCopy() complete := s.bindingVerified(b, envelope, snapshot, now) - condition(&b.Status.Conditions, dnsValueServing, complete, "MemberVerification", b.Generation, now) + s.bindingConditions(b, envelope, snapshot, now, complete) if complete { b.Status.Phase = dnsValueServing } else { @@ -524,6 +524,10 @@ func LoadCommittedServingSnapshot(ctx context.Context, c client.Client, namespac } func (s *AckSink) bindingVerified(b *dnsv1alpha1.DNSResolverBinding, envelope *model.Envelope, snapshot *model.ServingSnapshot, now time.Time) bool { + return s.bindingProof(b, envelope, snapshot, now, false) +} + +func (s *AckSink) bindingProof(b *dnsv1alpha1.DNSResolverBinding, envelope *model.Envelope, snapshot *model.ServingSnapshot, now time.Time, all bool) bool { if envelope == nil || snapshot == nil || b.Spec.Tombstone { return false } @@ -543,5 +547,38 @@ func (s *AckSink) bindingVerified(b *dnsv1alpha1.DNSResolverBinding, envelope *m rows = append(rows, row) } } - return s.verified(rows, now, dnsValueResolver, b.Spec.Configuration.Revision, int64(envelope.Epoch), b.Spec.Authorization.WriterEpoch, b.Spec.Authorization.Sequence) + if all { + return s.verified(rows, now, dnsValueResolver, b.Spec.Configuration.Revision, int64(envelope.Epoch), b.Spec.Authorization.WriterEpoch, b.Spec.Authorization.Sequence) + } + tiers := map[string][]model.Member{} + for _, member := range s.Members { + role := member.Role + if role == dnsValueCluster { + role = dnsValueRegional + } + if role == "node" || role == dnsValueResolver || role == dnsValueRegional { + tiers[role] = append(tiers[role], member) + } + } + for _, members := range tiers { + ready := false + for _, member := range members { + scoped := *s + scoped.Members = []model.Member{member} + ready = ready || scoped.verified(rows, now, dnsValueResolver, b.Spec.Configuration.Revision, int64(envelope.Epoch), b.Spec.Authorization.WriterEpoch, b.Spec.Authorization.Sequence) + } + if !ready { + return false + } + } + return len(tiers) > 0 +} + +func (s *AckSink) bindingConditions(b *dnsv1alpha1.DNSResolverBinding, envelope *model.Envelope, snapshot *model.ServingSnapshot, now time.Time, available bool) { + condition(&b.Status.Conditions, "FullyReplicated", s.bindingProof(b, envelope, snapshot, now, true), "ReplicaVerification", b.Generation, now) + condition(&b.Status.Conditions, "ObservationHealthy", available, "MemberObservation", b.Generation, now) + condition(&b.Status.Conditions, dnsValueServing, available, "MemberVerification", b.Generation, now) + if !available && !b.Spec.Tombstone && b.Status.ObservedConfigurationRevision == b.Spec.Configuration.Revision { + apimeta.SetStatusCondition(&b.Status.Conditions, metav1.Condition{Type: dnsValueServing, Status: metav1.ConditionUnknown, Reason: "ObservationUnavailable", Message: "Current serving health is unavailable", ObservedGeneration: b.Generation, LastTransitionTime: metav1.NewTime(now)}) + } } diff --git a/internal/internaldns/platform/allocator.go b/internal/internaldns/platform/allocator.go index 3a169eb5..80a339be 100644 --- a/internal/internaldns/platform/allocator.go +++ b/internal/internaldns/platform/allocator.go @@ -10,6 +10,7 @@ import ( "net/netip" "time" + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" "go.miloapis.com/dns-operator/internal/internaldns/model" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -48,13 +49,13 @@ func (a *Allocator) Allocate(ctx context.Context, projectUID, vpcUID types.UID) // destination across every project sharing this platform control plane. Claims // are retained after revocation so a stale route cannot become valid for a new // context lifetime. -func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID types.UID, region, address string, port int32) error { +func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, contextUID types.UID, region, address string, port int32) error { parsed, err := netip.ParseAddr(address) - if err != nil || parsed.IsUnspecified() || projectUID == "" || accessUID == "" || region == "" || port != 53 { + if err != nil || parsed.IsUnspecified() || projectUID == "" || contextUID == "" || region == "" || port != 53 { return fmt.Errorf("destination claim identity, region, IP, and DNS port are required") } address = parsed.String() - owner := string(projectUID) + "/" + string(accessUID) + owner := string(projectUID) + "/" + string(contextUID) claim := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "dns-destination-claim-" + model.OpaqueToken(region+"/"+address+"/"+fmt.Sprint(port)), Namespace: a.Namespace, Labels: map[string]string{leaseLabel: "true"}}, Data: map[string]string{"owner": owner, "region": region, "address": address, "port": fmt.Sprint(port)}} if err := a.Client.Create(ctx, claim); apierrors.IsAlreadyExists(err) { var existing corev1.ConfigMap @@ -62,6 +63,21 @@ func (a *Allocator) ClaimDestination(ctx context.Context, projectUID, accessUID return err } if existing.Data["owner"] != owner || existing.Data["region"] != region || existing.Data["address"] != address || existing.Data["port"] != fmt.Sprint(port) { + var bindings dnsv1alpha1.DNSResolverBindingList + if err := a.Client.List(ctx, &bindings, client.InNamespace(a.Namespace)); err != nil { + return err + } + for _, binding := range bindings.Items { + b := binding.Spec + if b.Source.ProjectUID == projectUID && b.Source.ResolverContextRef.UID == contextUID && + b.Source.AccessBindingRef.UID != "" && existing.Data["owner"] == string(projectUID)+"/"+string(b.Source.AccessBindingRef.UID) && + b.Placement.Region == region && b.Configuration.Listeners.Node.Address == address && b.Configuration.Listeners.Node.Port == port && + existing.Data["region"] == region && existing.Data["address"] == address && existing.Data["port"] == fmt.Sprint(port) { + base := existing.DeepCopy() + existing.Data["owner"] = owner + return a.Client.Patch(ctx, &existing, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) + } + } return fmt.Errorf("service destination is already reserved") } return nil diff --git a/internal/internaldns/platform/allocator_test.go b/internal/internaldns/platform/allocator_test.go index 5ff77432..35215490 100644 --- a/internal/internaldns/platform/allocator_test.go +++ b/internal/internaldns/platform/allocator_test.go @@ -76,3 +76,49 @@ func TestAddressClaimsAreConcurrentStableAndLifetimeScoped(t *testing.T) { t.Fatal("new resource lifetime reused the old address") } } + +func TestDestinationClaimBelongsToContextLifetime(t *testing.T) { + ctx := context.Background() + cl := testClient(t) + allocator := Allocator{Client: cl, Namespace: "platform"} + for range 2 { + if err := allocator.ClaimDestination(ctx, "project", "context", "east", "fd70::53", 53); err != nil { + t.Fatal(err) + } + } + if err := allocator.ClaimDestination(ctx, "project", "replacement-context", "east", "fd70::53", 53); err == nil { + t.Fatal("a replacement context reused a quarantined destination") + } + if err := allocator.ClaimDestination(ctx, "another-project", "context", "east", "fd70::53", 53); err == nil { + t.Fatal("another project reused a destination") + } +} + +func TestDestinationClaimConversionRequiresPinnedBinding(t *testing.T) { + ctx := context.Background() + cl := testClient(t) + allocator := Allocator{Client: cl, Namespace: "platform"} + if err := allocator.ClaimDestination(ctx, "project", "access", "east", "fd70::53", 53); err != nil { + t.Fatal(err) + } + if err := allocator.ClaimDestination(ctx, "project", "context", "east", "fd70::53", 53); err == nil { + t.Fatal("claim changed ownership without lifetime proof") + } + binding := &dnsv1alpha1.DNSResolverBinding{} + binding.Namespace, binding.Name = "platform", "binding" + binding.Spec.Source.ProjectUID = "project" + binding.Spec.Source.ResolverContextRef.UID = "context" + binding.Spec.Source.AccessBindingRef.UID = "access" + binding.Spec.Placement.Region = "east" + binding.Spec.Configuration.Listeners.Node.Address = "fd70::53" + binding.Spec.Configuration.Listeners.Node.Port = 53 + if err := cl.Create(ctx, binding); err != nil { + t.Fatal(err) + } + if err := allocator.ClaimDestination(ctx, "project", "context", "east", "fd70::53", 53); err != nil { + t.Fatal(err) + } + if err := allocator.ClaimDestination(ctx, "project", "access", "east", "fd70::53", 53); err == nil { + t.Fatal("converted claim returned to access lifetime") + } +} diff --git a/internal/internaldns/platform/capacity.go b/internal/internaldns/platform/capacity.go new file mode 100644 index 00000000..6dc5e7fc --- /dev/null +++ b/internal/internaldns/platform/capacity.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package platform + +import ( + "encoding/json" + "sort" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func fitSnapshot(snapshot model.ServingSnapshot, prior model.ServingSnapshot, maxBindings, maxBytes int) (model.ServingSnapshot, map[string]bool) { + previous := map[string]model.Binding{} + for _, binding := range prior.Bindings { + previous[binding.BindingUID] = binding + } + candidates := append([]model.Binding(nil), snapshot.Bindings...) + sort.Slice(candidates, func(i, j int) bool { + _, first := previous[candidates[i].BindingUID] + _, second := previous[candidates[j].BindingUID] + if first != second { + return first + } + return candidates[i].BindingUID < candidates[j].BindingUID + }) + snapshot.Bindings = nil + rejected := map[string]bool{} + fits := func(binding model.Binding) bool { + if len(snapshot.Bindings) >= maxBindings { + return false + } + candidate := snapshot + candidate.Bindings = append(append([]model.Binding(nil), snapshot.Bindings...), binding) + envelope, err := model.NewEnvelope(model.KindServingSnapshot, "dns-serving-"+model.OpaqueToken(candidate.Region+"/"+candidate.Shard)+"-e18446744073709551615-r18446744073709551615", candidate.Region, candidate.Shard, "00000000-0000-0000-0000-000000000000", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, time.Now(), candidate) + if err != nil { + return false + } + payload, err := json.Marshal(envelope) + return err == nil && len(payload) <= maxBytes + } + for _, binding := range candidates { + if fits(binding) { + snapshot.Bindings = append(snapshot.Bindings, binding) + continue + } + rejected[binding.BindingUID] = true + if old, ok := previous[binding.BindingUID]; ok { + if safeExpansion(old, binding) && fits(old) { + snapshot.Bindings = append(snapshot.Bindings, old) + continue + } + binding.ConfigurationPending = true + binding.Zones = nil + if fits(binding) { + snapshot.Bindings = append(snapshot.Bindings, binding) + } + } + } + sort.Slice(snapshot.Bindings, func(i, j int) bool { return snapshot.Bindings[i].BindingUID < snapshot.Bindings[j].BindingUID }) + return snapshot, rejected +} + +func safeExpansion(old, candidate model.Binding) bool { + if old.Tombstone || candidate.Tombstone || old.ConfigurationPending || candidate.ConfigurationPending || old.Authorization != candidate.Authorization || old.ProjectUID != candidate.ProjectUID || old.ContextUID != candidate.ContextUID || old.ConsumerAddress != candidate.ConsumerAddress || old.ClusterAddress != candidate.ClusterAddress { + return false + } + for _, zone := range old.Zones { + found := false + for _, next := range candidate.Zones { + if zone.ZoneUID == next.ZoneUID && zone.Apex == next.Apex && !next.Pending { + found = true + } + } + if !found { + return false + } + } + return true +} diff --git a/internal/internaldns/platform/capacity_test.go b/internal/internaldns/platform/capacity_test.go new file mode 100644 index 00000000..bb59bb75 --- /dev/null +++ b/internal/internaldns/platform/capacity_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package platform + +import ( + "fmt" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func capacityBinding(index int) model.Binding { + return model.Binding{BindingUID: fmt.Sprintf("binding-%d", index), ProjectUID: "project", ContextUID: fmt.Sprintf("context-%d", index), BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: fmt.Sprintf("fd53::%d", index), ClusterAddress: fmt.Sprintf("fd54::%d", index), Port: 53, Transports: []model.Transport{model.TransportUDP, model.TransportTCP}, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1}} +} + +func TestCapacityOverflowDoesNotBlockWithdrawal(t *testing.T) { + prior := model.ServingSnapshot{Region: "east", Shard: "a", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: time.Now(), Bindings: []model.Binding{capacityBinding(1), capacityBinding(2)}} + candidate := prior + candidate.ConfigurationRevision++ + candidate.Bindings = []model.Binding{capacityBinding(2), capacityBinding(3), capacityBinding(4)} + bounded, rejected := fitSnapshot(candidate, prior, 2, 512*1024) + if len(bounded.Bindings) != 2 || len(rejected) != 1 { + t.Fatalf("capacity result: %#v, rejected %#v", bounded.Bindings, rejected) + } + for _, binding := range bounded.Bindings { + if binding.BindingUID == "binding-1" { + t.Fatal("overflow prevented withdrawal") + } + } + if bounded.Bindings[0].BindingUID != "binding-2" { + t.Fatal("new allocations displaced an existing context") + } +} + +func TestByteOverflowPreservesExistingContext(t *testing.T) { + prior := model.ServingSnapshot{Region: "east", Shard: "a", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: time.Now(), Bindings: []model.Binding{capacityBinding(1)}} + candidate := prior + binding := capacityBinding(1) + for i := range 256 { + binding.Zones = append(binding.Zones, model.ZoneAttachment{ZoneUID: fmt.Sprintf("zone-%d", i), Apex: fmt.Sprintf("zone-%d.internal", i), RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + } + binding.ConfigurationRevision++ + candidate.Bindings = []model.Binding{binding, capacityBinding(2)} + bounded, rejected := fitSnapshot(candidate, prior, 2, 4096) + if !rejected["binding-1"] || len(bounded.Bindings) != 2 || bounded.Bindings[0].ConfigurationRevision != 1 { + t.Fatalf("overflow interrupted an established context: %#v", bounded) + } +} diff --git a/internal/internaldns/platform/planner.go b/internal/internaldns/platform/planner.go index e79cca26..666d3075 100644 --- a/internal/internaldns/platform/planner.go +++ b/internal/internaldns/platform/planner.go @@ -136,7 +136,6 @@ func (p *Planner) Step(ctx context.Context) error { for _, b := range prior.Bindings { priorBindings[b.BindingUID] = b } - dependencies := map[string]bool{} for _, obj := range bindings.Items { b := obj.Spec if b.Placement.Region != c.Region || b.Placement.Shard != c.Shard || b.Tombstone || !obj.DeletionTimestamp.IsZero() { @@ -150,17 +149,24 @@ func (p *Planner) Step(ctx context.Context) error { view.Transports = append(view.Transports, model.Transport(t)) } complete := true - bindingDependencies := map[string]bool{} for _, zoneRef := range b.Configuration.ZoneRefs { uid := zoneRef.UID + apex := b.Configuration.ZoneApices[string(uid)] + pending := func() { + if apex == "" { + complete = false + return + } + view.Zones = append(view.Zones, model.ZoneAttachment{Pending: true, ZoneUID: string(uid), Apex: apex, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + } o, ok := byZone[string(uid)] if !ok || o.Spec.ActiveManifestName == "" { - complete = false + pending() continue } var m dnsv1alpha1.DNSPublicationManifest if err := p.Client.Get(ctx, client.ObjectKey{Namespace: c.Namespace, Name: o.Spec.ActiveManifestName}, &m); apierrors.IsNotFound(err) { - complete = false + pending() continue } else if err != nil { return err @@ -173,22 +179,20 @@ func (p *Planner) Step(ctx context.Context) error { } } if m.Spec.ZoneRef.UID != uid || m.Spec.Tombstone || !targeted { - complete = false + pending() continue } attachment := model.ZoneAttachment{ZoneUID: string(uid), Apex: m.Spec.ZoneApex, RequiredPublicationEpoch: uint64(m.Spec.WriterEpoch), RequiredPublicationRevision: uint64(m.Spec.Revision)} // Record updates do not change a view's minimum activation dependency. if old, ok := priorBindings[view.BindingUID]; ok && old.BindingGeneration == view.BindingGeneration { for _, z := range old.Zones { - if z.ZoneUID == attachment.ZoneUID && z.Apex == attachment.Apex { + if !z.Pending && z.ZoneUID == attachment.ZoneUID && z.Apex == attachment.Apex { attachment.RequiredPublicationEpoch = z.RequiredPublicationEpoch attachment.RequiredPublicationRevision = z.RequiredPublicationRevision } } } - activationName := model.PublicationActivationName(m.Name, c.Region, c.Shard) view.Zones = append(view.Zones, attachment) - bindingDependencies[activationName] = true } if !complete { // Missing configuration is not a withdrawal of source authorization. @@ -196,10 +200,6 @@ func (p *Planner) Step(ctx context.Context) error { // pending snapshot can publish without waiting on this tenant's outbox. view.ConfigurationPending = true view.Zones = nil - } else { - for name := range bindingDependencies { - dependencies[name] = true - } } sort.Slice(view.Zones, func(i, j int) bool { return view.Zones[i].ZoneUID < view.Zones[j].ZoneUID }) snapshot.Bindings = append(snapshot.Bindings, view) @@ -210,8 +210,21 @@ func (p *Planner) Step(ctx context.Context) error { if maxBindings <= 0 { maxBindings = 256 } - if len(snapshot.Bindings) > maxBindings { - return fmt.Errorf("shard %s/%s exceeds binding budget: %d > %d; no partial snapshot will be exported", c.Region, c.Shard, len(snapshot.Bindings), maxBindings) + maxBytes := c.MaxSnapshotBytes + if maxBytes <= 0 { + maxBytes = 512 * 1024 + } + var rejected map[string]bool + snapshot, rejected = fitSnapshot(snapshot, prior, maxBindings, maxBytes) + for i := range bindings.Items { + binding := &bindings.Items[i] + base := binding.DeepCopy() + condition(&binding.Status.Conditions, "CapacityAvailable", !rejected[string(binding.UID)], "ShardCapacity", binding.Generation, now) + if !jsonEqualStatus(base.Status, binding.Status) { + if err := p.Client.Status().Patch(ctx, binding, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } } if err := snapshot.Validate(); err != nil { return err @@ -251,19 +264,10 @@ func (p *Planner) Step(ctx context.Context) error { return err } payload, _ := json.Marshal(env) - maxBytes := c.MaxSnapshotBytes - if maxBytes <= 0 { - maxBytes = 512 * 1024 - } if len(payload) > maxBytes { return fmt.Errorf("shard serving envelope exceeds configured transport budget: %d > %d", len(payload), maxBytes) } - deps := make([]string, 0, len(dependencies)) - for name := range dependencies { - deps = append(deps, name) - } - sort.Strings(deps) - out := &dnsv1alpha1.DNSTransportOutbox{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: c.Namespace, Labels: map[string]string{shardOwnerLabel: owner.Name, "internal-dns.miloapis.com/region": model.SafeToken(c.Region), "internal-dns.miloapis.com/shard": model.SafeToken(c.Shard)}}, Spec: dnsv1alpha1.DNSTransportOutboxSpec{Subject: model.ServingSubject(c.Region, c.Shard), ResourceUID: owner.UID, WriterEpoch: int64(state.Epoch), Revision: int64(revision), PayloadHash: model.Hash(payload), Payload: payload, Activation: true, DependsOn: deps}} + out := &dnsv1alpha1.DNSTransportOutbox{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: c.Namespace, Labels: map[string]string{shardOwnerLabel: owner.Name, "internal-dns.miloapis.com/region": model.SafeToken(c.Region), "internal-dns.miloapis.com/shard": model.SafeToken(c.Shard)}}, Spec: dnsv1alpha1.DNSTransportOutboxSpec{Subject: model.ServingSubject(c.Region, c.Shard), ResourceUID: owner.UID, WriterEpoch: int64(state.Epoch), Revision: int64(revision), PayloadHash: model.Hash(payload), Payload: payload, Activation: true, DependsOn: nil}} if err := p.Client.Create(ctx, out); apierrors.IsAlreadyExists(err) { var existing dnsv1alpha1.DNSTransportOutbox if err := p.Client.Get(ctx, client.ObjectKeyFromObject(out), &existing); err != nil { diff --git a/internal/internaldns/platform/platform_test.go b/internal/internaldns/platform/platform_test.go index 1adaec72..cd0ec66c 100644 --- a/internal/internaldns/platform/platform_test.go +++ b/internal/internaldns/platform/platform_test.go @@ -162,6 +162,17 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { if pending.Bindings[0].Authorization.Revision != uint64(originalAuthorization.Sequence) { t.Fatal("pending configuration renewed the original authorization") } + changing.Spec.Configuration.ZoneApices = map[string]string{"zone-extra": "apps.internal"} + if err := cl.Update(context.Background(), &changing); err != nil { + t.Fatal(err) + } + if err := p.Step(context.Background()); err != nil { + t.Fatal(err) + } + isolated := latestPlannerSnapshot(t, &p) + if isolated.Bindings[0].ConfigurationPending || len(isolated.Bindings[0].Zones) != 2 || !isolated.Bindings[0].Zones[1].Pending { + t.Fatalf("a known pending apex suppressed the ready zone: %#v", isolated.Bindings[0]) + } for _, obj := range []client.Object{ &dnsv1alpha1.DNSPublicationOwnership{ObjectMeta: metav1.ObjectMeta{Name: "owner-" + model.OpaqueToken("zone-extra")[:20], Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationOwnershipSpec{ZoneUID: "zone-extra", ActiveManifestName: "manifest-extra"}}, &dnsv1alpha1.DNSPublicationManifest{ObjectMeta: metav1.ObjectMeta{Name: "manifest-extra", Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationManifestSpec{ZoneRef: dnsv1alpha1.DNSObjectReference{UID: "zone-extra"}, ZoneApex: "apps.internal", WriterEpoch: 1, Revision: 1, ServingTargets: []dnsv1alpha1.DNSPublicationTarget{{Region: region, Shard: shard}}}}, @@ -189,7 +200,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { t.Fatal(err) } _ = cl.List(context.Background(), &out) - if len(out.Items) != 4 { + if len(out.Items) != 5 { t.Fatal("regional export removal did not change the shared snapshot") } var updatedOwner corev1.ConfigMap @@ -517,3 +528,30 @@ func TestZeroTargetTombstoneNeedsNoReplicaACK(t *testing.T) { t.Fatal("prior-target withdrawal must await its ACKs") } } + +func TestServingAvailabilityDoesNotRequireEveryReplica(t *testing.T) { + now := time.Now().UTC() + until := metav1.NewTime(now.Add(time.Minute)) + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{UID: "binding"}} + binding.Spec.Configuration.Generation = 1 + binding.Spec.Configuration.Revision = 1 + binding.Spec.Authorization = dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 1, Sequence: 1} + for _, member := range []string{"node-a", "regional-a"} { + binding.Status.MemberAcknowledgements = append(binding.Status.MemberAcknowledgements, dnsv1alpha1.DNSApplyAcknowledgement{MemberID: member, WriterEpoch: 1, SnapshotRevision: 1, Revision: 1, AuthorizationIssuerEpoch: 1, AuthorizationRevision: 1, Phase: string(model.AckVerified), ValidUntil: &until}) + } + sink := AckSink{Members: []model.Member{{MemberID: "node-a", Role: "node"}, {MemberID: "node-b", Role: "node"}, {MemberID: "regional-a", Role: "regional"}, {MemberID: "regional-b", Role: "regional"}}} + envelope := &model.Envelope{Epoch: 1, Revision: 1} + snapshot := &model.ServingSnapshot{Bindings: []model.Binding{{BindingUID: "binding", BindingGeneration: 1, ConfigurationRevision: 1, Authorization: model.Authorization{IssuerEpoch: 1, Revision: 1}}}} + if !sink.bindingVerified(binding, envelope, snapshot, now) { + t.Fatal("a missing replica report suppressed healthy capacity") + } + sink.bindingConditions(binding, envelope, snapshot, now, true) + if apimeta.IsStatusConditionTrue(binding.Status.Conditions, "FullyReplicated") { + t.Fatal("degraded replication was reported complete") + } + binding.Status.ObservedConfigurationRevision = 1 + sink.bindingConditions(binding, envelope, snapshot, now.Add(2*time.Minute), false) + if c := apimeta.FindStatusCondition(binding.Status.Conditions, "Serving"); c == nil || c.Status != metav1.ConditionUnknown { + t.Fatal("stale observations claimed a serving outage") + } +} diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 182773bb..dd7ae9e3 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -193,6 +193,8 @@ type Agent struct { tx FileTransaction publicationVerifier PublicationVerifier mu sync.Mutex + healthMu sync.Mutex + healthEnabled bool state checkpoint needsActivation bool lastServingAck map[string]servingAckState @@ -267,7 +269,11 @@ func (a *Agent) Start(ctx context.Context) error { // still serves it, so force one validated apply before renewing the lease. a.mu.Lock() a.needsActivation = true + if err := a.restoreActive(ctx); err != nil { + a.report(err) + } a.mu.Unlock() + go a.runActiveHealth(ctx) for { err := a.bootstrapAndStep(ctx) if err == nil { @@ -320,6 +326,7 @@ func (a *Agent) Start(ctx context.Context) error { } func (a *Agent) failClosed(ctx context.Context) error { + a.suspendActiveHealth() a.needsActivation = true // Invalidate independent watchdog readiness immediately, even if the daemon // gate command itself fails. It must never inherit a prior healthy lease. @@ -476,7 +483,7 @@ func (a *Agent) Handle(ctx context.Context, env model.Envelope) error { case model.KindPublicationManifest: if err := a.handleManifest(ctx, env); err != nil { - _ = a.failClosed(ctx) + a.gateRejectedUpdate(ctx, err) return err } return a.finishEvent(ctx) @@ -847,13 +854,13 @@ func (a *Agent) Step(ctx context.Context) error { } if a.cfg.Mode == ModeResolver { if err := a.retryPendingPublications(ctx); err != nil { - _ = a.failClosed(ctx) + a.gateRejectedUpdate(ctx, err) return err } } if a.cfg.Mode == ModeCombined { if err := a.retryPendingPublications(ctx); err != nil { - _ = a.failClosed(ctx) + a.gateRejectedUpdate(ctx, err) return err } if a.needsActivation || len(a.installedPublications) == 0 { @@ -893,7 +900,7 @@ func (a *Agent) reconcile(ctx context.Context) error { filtered := *a.state.Snapshot filtered.Bindings = nil for _, b := range a.state.Snapshot.Bindings { - eligible := !b.Tombstone && a.publicationsReady(b) + eligible := !b.Tombstone && !b.ConfigurationPending if eligible { active[b.BindingUID] = true } @@ -903,8 +910,17 @@ func (a *Agent) reconcile(ctx context.Context) error { } renderConfig := a.cfg.Render renderConfig.ReadyMembers = map[string]map[string]bool{} + renderConfig.ReadyZoneMembers = map[string]map[string]map[string]bool{} for _, binding := range filtered.Bindings { - renderConfig.ReadyMembers[binding.BindingUID] = a.eligibleRegionalMembers(binding) + public := binding + public.Zones = nil + renderConfig.ReadyMembers[binding.BindingUID] = a.eligibleRegionalMembers(public) + renderConfig.ReadyZoneMembers[binding.BindingUID] = map[string]map[string]bool{} + for _, zone := range binding.Zones { + private := binding + private.Zones = []model.ZoneAttachment{zone} + renderConfig.ReadyZoneMembers[binding.BindingUID][zone.ZoneUID] = a.eligibleRegionalMembers(private) + } } rendered, err := renderPublications(renderConfig, filtered, active, a.state.Publications) if err != nil { @@ -912,6 +928,13 @@ func (a *Agent) reconcile(ctx context.Context) error { } hash := hashFiles(rendered.Files) if hash != a.state.RenderedHash || a.needsActivation { + if !a.canRetainActive() { + a.suspendActiveHealth() + a.state.Active = nil + if err := a.store.Save(a.state); err != nil { + return err + } + } var err error if a.needsActivation { err = a.tx.ApplyForce(ctx, rendered.Files) @@ -919,7 +942,7 @@ func (a *Agent) reconcile(ctx context.Context) error { err = a.tx.Apply(ctx, rendered.Files) } if err != nil { - _ = a.failClosed(ctx) + a.gateRejectedUpdate(ctx, err) return err } a.state.RenderedHash = hash @@ -959,11 +982,15 @@ func (a *Agent) reconcile(ctx context.Context) error { return err } a.state.RenderedHash = gatedHash + rendered = gated if err := a.store.Save(a.state); err != nil { return err } } } + if err := a.approveConfiguration(rendered.Files); err != nil { + return err + } // Local DNS proof renews the watchdog independently of broker acknowledgements. if a.cfg.Mode == ModeResolver { if err := a.renewSafeWatchdogLease(ctx); err != nil { @@ -975,7 +1002,7 @@ func (a *Agent) reconcile(ctx context.Context) error { continue } p := model.AckVerified - if !active[b.BindingUID] { + if !active[b.BindingUID] || !a.publicationsReady(b) { p = model.AckApplied } if err := a.emitServingAck(ctx, b, p); err != nil { @@ -1002,6 +1029,7 @@ func (a *Agent) verifyResolverBindings(ctx context.Context, bindings []model.Bin eligible := make([]model.Binding, 0, len(bindings)) for _, binding := range bindings { if active[binding.BindingUID] { + binding.Zones = a.readyZones(binding) eligible = append(eligible, binding) } } @@ -1080,7 +1108,7 @@ func (a *Agent) publicationsReady(b model.Binding) bool { if a.cfg.Mode == ModeCombined { for _, z := range b.Zones { p, ok := a.state.Publications[z.ZoneUID] - if !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + if z.Pending || !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { return false } } @@ -1109,7 +1137,7 @@ func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { ready := true for _, z := range b.Zones { p, ok := a.state.Publications[z.ZoneUID] - if !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + if z.Pending || !ok || !p.Verified || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { ready = false break } @@ -1131,6 +1159,18 @@ func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { return out } +func (a *Agent) readyZones(b model.Binding) []model.ZoneAttachment { + var ready []model.ZoneAttachment + for _, zone := range b.Zones { + candidate := b + candidate.Zones = []model.ZoneAttachment{zone} + if len(a.eligibleRegionalMembers(candidate)) > 0 { + ready = append(ready, zone) + } + } + return ready +} + func (a *Agent) verifyPublication(ctx context.Context, p publicationState) error { if p.Manifest.Tombstone { return nil @@ -1162,7 +1202,7 @@ func (a *Agent) publicationViewsAt(p publicationState) []PublicationView { continue } for _, z := range b.Zones { - if z.ZoneUID == p.Plan.ZoneUID && model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) >= 0 { + if !z.Pending && z.ZoneUID == p.Plan.ZoneUID && model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) >= 0 { out = append(out, PublicationView{View: b.ViewName(), DestinationAddress: b.ClusterAddress, DestinationPort: b.Port, Serial: publicationSerial(p), FingerprintMailbox: publicationMailbox(p)}) } } @@ -1450,6 +1490,10 @@ func (a *Agent) renewWatchdogLease(ctx context.Context) error { if a.cfg.WatchdogLeasePath == "" { return nil } + return a.writeWatchdogLease() +} + +func (a *Agent) writeWatchdogLease() error { lease := a.cfg.WatchdogLease if lease <= 0 { lease = 5 * time.Second diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go new file mode 100644 index 00000000..4d32b24a --- /dev/null +++ b/internal/internaldns/serving/continuity.go @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "strconv" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +type activeConfiguration struct { + Snapshot model.ServingSnapshot `json:"snapshot"` + Publications map[string]publicationState `json:"publications"` + Files map[string][]byte `json:"files"` +} + +func (a *Agent) gateRejectedUpdate(ctx context.Context, err error) { + var preserved preservedTransactionError + if errors.As(err, &preserved) && a.canRetainActive() { + return + } + _ = a.failClosed(ctx) +} + +func (a *Agent) canRetainActive() bool { + old := a.state.Active + if old == nil || a.state.Snapshot == nil { + return false + } + next := map[string]model.Binding{} + for _, binding := range a.state.Snapshot.Bindings { + next[binding.BindingUID] = binding + } + for _, binding := range old.Snapshot.Bindings { + candidate, ok := next[binding.BindingUID] + if !ok || candidate.Tombstone || candidate.ConfigurationPending || bindingIdentity(binding) != bindingIdentity(candidate) || binding.Authorization != candidate.Authorization { + return false + } + for _, zone := range binding.Zones { + present := false + for _, attached := range candidate.Zones { + if attached.ZoneUID == zone.ZoneUID && attached.Apex == zone.Apex && !attached.Pending { + present = true + } + } + if !present && !zone.Pending { + return false + } + } + } + for uid, publication := range old.Publications { + if publication.Manifest.Tombstone { + continue + } + candidate, ok := a.state.Publications[uid] + if !ok || candidate.Manifest.Tombstone || candidate.Plan.Apex != publication.Plan.Apex { + return false + } + records := map[string]bool{} + for _, rrset := range candidate.Plan.RRSets { + for _, record := range rrset.Records { + if record.Eligible { + records[rrset.Name+"\x00"+rrset.Type+"\x00"+record.Content] = true + } + } + } + for _, rrset := range publication.Plan.RRSets { + for _, record := range rrset.Records { + if record.Eligible && !records[rrset.Name+"\x00"+rrset.Type+"\x00"+record.Content] { + return false + } + } + } + } + return true +} + +func (a *Agent) approveConfiguration(files map[string][]byte) error { + candidate := activeConfiguration{Snapshot: *a.state.Snapshot, Publications: a.state.Publications, Files: files} + data, err := json.Marshal(candidate) + if err != nil { + return err + } + var active activeConfiguration + if err := json.Unmarshal(data, &active); err != nil { + return err + } + if a.state.Active != nil && equalJSON(*a.state.Active, active) { + return nil + } + a.state.Active = &active + if err := a.store.Save(a.state); err != nil { + return err + } + a.healthMu.Lock() + a.healthEnabled = true + a.healthMu.Unlock() + return nil +} + +func (a *Agent) suspendActiveHealth() { + a.healthMu.Lock() + defer a.healthMu.Unlock() + a.healthEnabled = false + if a.cfg.WatchdogLeasePath != "" { + _ = os.Remove(a.cfg.WatchdogLeasePath) + } +} + +func (a *Agent) restoreActive(ctx context.Context) error { + if !a.canRetainActive() { + return nil + } + if err := a.tx.ApplyForce(ctx, a.state.Active.Files); err != nil { + return err + } + a.healthMu.Lock() + a.healthEnabled = true + a.healthMu.Unlock() + return nil +} + +func (a *Agent) runActiveHealth(ctx context.Context) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := a.refreshActiveHealth(ctx); err != nil { + a.report(err) + } + } + } +} + +func (a *Agent) refreshActiveHealth(ctx context.Context) error { + a.healthMu.Lock() + enabled := a.healthEnabled + a.healthMu.Unlock() + if !enabled || a.cfg.WatchdogLeasePath == "" { + return nil + } + probeCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + if err := a.probeLocalDaemons(probeCtx); err != nil { + return err + } + a.healthMu.Lock() + defer a.healthMu.Unlock() + if !a.healthEnabled { + return nil + } + return a.writeWatchdogLease() +} + +func (a *Agent) probeLocalDaemons(ctx context.Context) error { + if a.cfg.UnsafeAllowNoDNSVerify { + return nil + } + role := a.cfg.Render.Role + if role == "" { + role = RenderBoth + } + var binds []BINDRenderConfig + var dists []DNSDistRenderConfig + if role == RenderNode || role == RenderBoth { + binds = append(binds, a.cfg.Render.NodeBIND) + dists = append(dists, a.cfg.Render.NodeDNSDist) + } + if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { + binds = append(binds, a.cfg.Render.ClusterBIND) + } + if role == RenderCluster || role == RenderBoth || role == RenderRegionalDNSDist { + dists = append(dists, a.cfg.Render.ClusterDNSDist) + } + for _, bind := range binds { + server := net.JoinHostPort(bind.ListenAddress, strconv.Itoa(int(bind.Port))) + view := PublicationView{DestinationAddress: bind.ListenAddress, DestinationPort: bind.Port} + for _, network := range []string{networkUDP, networkTCP} { + answer, err := exchangePublication(ctx, network, server, ".", view, 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local BIND health %s: %w", server, errors.Join(err, errors.New("DNS response required"))) + } + } + } + for _, dist := range dists { + for _, network := range []string{networkUDP, networkTCP} { + answer, err := exchangeResolverSOA(ctx, network, dist.ListenAddress, ".", 200*time.Millisecond) + if err != nil || answer == nil || !answer.Response { + return fmt.Errorf("local dnsdist health %s: %w", dist.ListenAddress, errors.Join(err, errors.New("DNS response required"))) + } + } + } + return nil +} diff --git a/internal/internaldns/serving/continuity_test.go b/internal/internaldns/serving/continuity_test.go new file mode 100644 index 00000000..a7b4c381 --- /dev/null +++ b/internal/internaldns/serving/continuity_test.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serving + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + "go.miloapis.com/dns-operator/internal/internaldns/model" +) + +func TestRejectedExpansionPreservesServingAndRestarts(t *testing.T) { + now := time.Now().UTC() + runner := &recordingRunner{} + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + activeHash := agent.state.RenderedHash + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0].ConfigurationRevision++ + candidate.Bindings[0].Zones = append(candidate.Bindings[0].Zones, model.ZoneAttachment{ZoneUID: "pending-zone", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + runner.failAt = len(runner.calls) + 1 + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)); err == nil { + t.Fatal("invalid expansion was accepted") + } + if agent.state.RenderedHash != activeHash || agent.state.Active == nil || !agent.healthEnabled { + t.Fatal("invalid expansion interrupted the active configuration") + } + now = now.Add(time.Hour) + if err := agent.refreshActiveHealth(context.Background()); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(cfg.WatchdogLeasePath) + if err != nil { + t.Fatal(err) + } + var lease watchdogLease + if err := json.Unmarshal(data, &lease); err != nil || !lease.ValidUntil.After(now) { + t.Fatalf("active health stopped during rejected update: %v", err) + } + restarted, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + if err := restarted.restoreActive(context.Background()); err != nil || !restarted.healthEnabled { + t.Fatalf("restart could not recover active configuration: %v", err) + } + withdrawn := candidate + withdrawn.ConfigurationRevision++ + withdrawn.Bindings = nil + runner.failAt = len(runner.calls) + 1 + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", withdrawn.ConfigurationEpoch, withdrawn.ConfigurationRevision, withdrawn)); err == nil { + t.Fatal("failed withdrawal was accepted") + } + if agent.state.Active != nil || agent.healthEnabled { + t.Fatal("failed withdrawal retained revoked access") + } +} + +func TestPendingZoneIsIsolatedFromReadyZone(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + binding := agent.state.Snapshot.Bindings[0] + binding.Zones = append(binding.Zones, model.ZoneAttachment{ZoneUID: "pending", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + if zones := agent.readyZones(binding); len(zones) != 1 || zones[0].ZoneUID != "zone-a" { + t.Fatalf("pending zone suppressed healthy zone: %#v", zones) + } + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0] = binding + candidate.Bindings[0].ConfigurationRevision++ + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)); err != nil { + t.Fatal(err) + } + if agent.state.Active == nil || agent.publicationsReady(binding) { + t.Fatal("partial serving state was not distinguished from full readiness") + } + if zone := currentZone(t, agent); zone == "" { + t.Fatal("ready zone was not materialized") + } +} diff --git a/internal/internaldns/serving/render.go b/internal/internaldns/serving/render.go index 972934d0..a41783ba 100644 --- a/internal/internaldns/serving/render.go +++ b/internal/internaldns/serving/render.go @@ -47,12 +47,13 @@ const ( ) type RenderConfig struct { - ReadyMembers map[string]map[string]bool `json:"-"` - Role RenderRole `json:"role"` - NodeDNSDist DNSDistRenderConfig `json:"nodeDNSDist,omitempty"` - ClusterDNSDist DNSDistRenderConfig `json:"clusterDNSDist,omitempty"` - NodeBIND BINDRenderConfig `json:"nodeBIND,omitempty"` - ClusterBIND BINDRenderConfig `json:"clusterBIND,omitempty"` + ReadyZoneMembers map[string]map[string]map[string]bool `json:"-"` + ReadyMembers map[string]map[string]bool `json:"-"` + Role RenderRole `json:"role"` + NodeDNSDist DNSDistRenderConfig `json:"nodeDNSDist,omitempty"` + ClusterDNSDist DNSDistRenderConfig `json:"clusterDNSDist,omitempty"` + NodeBIND BINDRenderConfig `json:"nodeBIND,omitempty"` + ClusterBIND BINDRenderConfig `json:"clusterBIND,omitempty"` } type Rendered struct{ Files map[string][]byte } @@ -90,11 +91,11 @@ func renderPublications(c RenderConfig, snapshot model.ServingSnapshot, active m role = RenderBoth } if role == RenderNode || role == RenderBoth { - files[c.NodeDNSDist.Path] = renderDNSDist(c.NodeDNSDist, nodeBackends, bindings, active, false, nil) + files[c.NodeDNSDist.Path] = renderDNSDist(c.NodeDNSDist, nodeBackends, bindings, active, false, nil, c.ReadyZoneMembers) files[c.NodeBIND.Path] = renderNodeBIND(c.NodeBIND, bindings) } if role == RenderCluster || role == RenderBoth || role == RenderRegionalDNSDist { - files[c.ClusterDNSDist.Path] = renderDNSDist(c.ClusterDNSDist, clusterBackends, bindings, active, true, c.ReadyMembers) + files[c.ClusterDNSDist.Path] = renderDNSDist(c.ClusterDNSDist, clusterBackends, bindings, active, true, c.ReadyMembers, c.ReadyZoneMembers) } if role == RenderCluster || role == RenderBoth || role == RenderRegionalBIND { cluster, err := renderClusterBIND(c.ClusterBIND, bindings, publications, files) @@ -218,7 +219,7 @@ func collectBackends(bindings []model.Binding, get func(model.Binding) []model.B return out } -func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []model.Binding, active map[string]bool, cluster bool, eligible map[string]map[string]bool) []byte { +func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []model.Binding, active map[string]bool, cluster bool, eligible map[string]map[string]bool, zones map[string]map[string]map[string]bool) []byte { var out bytes.Buffer fmt.Fprintf(&out, "setLocal(%q)\n", c.ListenAddress) fmt.Fprint(&out, "setACL({") @@ -233,6 +234,11 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m fmt.Fprintf(&out, "local server = newServer({address=%q, name=%q, pool=%q, useProxyProtocol=true, checkName=\".\", checkType=\"SOA\", mustResolve=false})\n", net.JoinHostPort(b.Address, strconv.Itoa(int(b.Port))), "b_"+model.OpaqueToken(b.MemberID), c.PoolName) if cluster { for _, binding := range bindings { + for _, zone := range binding.Zones { + if zones != nil && active[binding.BindingUID] && zones[binding.BindingUID][zone.ZoneUID][b.MemberID] { + fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()+"_"+model.OpaqueToken(zone.ZoneUID)) + } + } if !binding.ConfigurationPending && active[binding.BindingUID] && (eligible == nil || eligible[binding.BindingUID][b.MemberID]) { fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()) } @@ -250,6 +256,19 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m fmt.Fprint(&out, "}\n\nfunction classifyVPC(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"vpc-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") fmt.Fprint(&out, "addAction(AllRule(), LuaAction(classifyVPC))\n") fmt.Fprint(&out, "addAction(TagRule(\"context-unready\", \"true\"), RCodeAction(DNSRCode.SERVFAIL))\n") + for _, binding := range bindings { + attached := append([]model.ZoneAttachment(nil), binding.Zones...) + sort.Slice(attached, func(i, j int) bool { return len(attached[i].Apex) > len(attached[j].Apex) }) + for _, zone := range attached { + rule := fmt.Sprintf("AndRule({TagRule(\"vpc-context\",%q),QNameSuffixRule(%q)})", binding.ViewName(), model.AbsoluteName(zone.Apex)) + if zone.Pending || zones != nil && len(zones[binding.BindingUID][zone.ZoneUID]) == 0 { + fmt.Fprintf(&out, "addAction(%s,RCodeAction(DNSRCode.SERVFAIL))\n", rule) + } else if cluster && zones != nil { + pool := binding.ViewName() + "_" + model.OpaqueToken(zone.ZoneUID) + fmt.Fprintf(&out, "getPool(%q)\naddAction(AndRule({%s,NotRule(PoolAvailableRule(%q))}),RCodeAction(DNSRCode.SERVFAIL))\naddAction(%s,PoolAction(%q))\n", pool, rule, pool, rule, pool) + } + } + } if cluster { for _, binding := range bindings { fmt.Fprintf(&out, "getPool(%q)\naddAction(AndRule({TagRule(\"vpc-context\",%q),NotRule(PoolAvailableRule(%q))}), RCodeAction(DNSRCode.SERVFAIL))\naddAction(TagRule(\"vpc-context\",%q),PoolAction(%q))\n", binding.ViewName(), binding.ViewName(), binding.ViewName(), binding.ViewName(), binding.ViewName()) @@ -315,7 +334,7 @@ func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publication renderViewStart(&out, b, b.ClusterAddress, c.DefaultCacheSize, maxNegativeCacheTTL(c)) for _, z := range b.Zones { p, ok := publications[z.ZoneUID] - if !ok || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { + if z.Pending || !ok || p.Manifest.Tombstone || model.Compare(p.Fence.Epoch, p.Fence.Revision, z.RequiredPublicationEpoch, z.RequiredPublicationRevision) < 0 { // Never recurse toward the public namespace when a private apex is missing. // The front gate returns immediate SERVFAIL; direct BIND queries fail too. fmt.Fprintf(&out, " zone %q { type forward; forward only; forwarders { 127.0.0.1 port 9; }; };\n", strings.TrimSuffix(model.AbsoluteName(z.Apex), ".")) diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go index bf432057..ead95cfe 100644 --- a/internal/internaldns/serving/state.go +++ b/internal/internaldns/serving/state.go @@ -36,6 +36,7 @@ type replicaAck struct { const checkpointFormatVersion = 5 type checkpoint struct { + Active *activeConfiguration `json:"active,omitempty"` FormatVersion int `json:"formatVersion"` ServingFence fence `json:"servingFence"` Snapshot *model.ServingSnapshot `json:"snapshot,omitempty"` diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index 3043e7e9..a1ba1cd9 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -44,6 +44,11 @@ type FileTransaction struct { Config TransactionConfig Runner Runner } + +type preservedTransactionError struct{ error } + +func (e preservedTransactionError) Unwrap() error { return e.error } + type oldFile struct { data []byte mode os.FileMode @@ -147,7 +152,7 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for commandChanges = nil } if err := t.commandsForChanges(ctx, t.Config.Validate, map[string]string{"{stageDir}": commandStage}, commandChanges); err != nil { - return fmt.Errorf("validate staged serving configuration: %w", err) + return preservedTransactionError{fmt.Errorf("validate staged serving configuration: %w", err)} } installed := make([]string, 0, len(paths)) for _, path := range paths { @@ -176,6 +181,9 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for if err := t.commandsForChanges(ctx, t.Config.Reload, nil, commandChanges); err != nil { rollbackErr := restoreFiles(paths, old) reloadErr := t.commandsForChanges(ctx, t.Config.Reload, nil, commandChanges) + if rollbackErr == nil && reloadErr == nil { + return preservedTransactionError{fmt.Errorf("reload serving configuration: %w", err)} + } return errors.Join(fmt.Errorf("reload serving configuration: %w", err), rollbackErr, reloadErr) } return nil diff --git a/test/internaldns/README.md b/test/internaldns/README.md index 637aa9a7..c6d84d64 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -111,5 +111,7 @@ into the separate DNS service VPC. Kind cannot establish physical fabric, microVM attachment, host-loss durability, regional independence, or production capacity. -The suite requires internal wire version `v1alpha4`, fresh format 5 checkpoints, -and matching controller and fleet versions. +The suite uses wire version `v1alpha4` and persistent format 5 checkpoints. +Compatible fleet upgrades retain those checkpoints and transport state. Roll out +serving support before enabling new controller output; see the +[rollout contract](../../config/internal-dns/README.md#rollout-and-recovery). From c7866d03cda1942b5186aeb43f6fccd1ada4882f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:54:33 -0500 Subject: [PATCH 03/15] test: Qualify DNS serving beyond control plane observation deadlines --- test/internaldns/kubernetes/qualify/scenarios.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index afb1ed07..6f4557f1 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -352,7 +352,7 @@ func (s *suite) offlineServing(a, b *project) (resultErr error) { return err } // The outage spans multiple DNS cache and fleet acknowledgement intervals. - until := time.Now().Add(2 * time.Minute) + until := time.Now().Add(3 * time.Minute) for time.Now().Before(until) { for _, tcp := range []bool{false, true} { for _, tenant := range []struct { From 10265167882bbc0f9df70a4dffdd106d1aacce0f Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:57:06 -0500 Subject: [PATCH 04/15] fix: Persist withdrawal fences before cache cleanup --- internal/internaldns/serving/agent.go | 20 +++++++++++++------- internal/internaldns/serving/continuity.go | 3 +++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index dd7ae9e3..ac4427d1 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -532,14 +532,20 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving if err != nil { return err } - if err := a.flushRemovedBindingZones(ctx, snapshot); err != nil { - _ = a.failClosed(ctx) - return err - } + previous := a.state.Snapshot a.state.BindingFences = bindingFences a.state.ServingFence = fence{snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision} a.state.Snapshot = &snapshot + if !a.canRetainActive() { + a.state.Active = nil + a.suspendActiveHealth() + } if err := a.store.Save(a.state); err != nil { + _ = a.failClosed(ctx) + return err + } + if err := a.flushRemovedBindingZones(ctx, previous, snapshot); err != nil { + _ = a.failClosed(ctx) return err } } @@ -552,8 +558,8 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving // A configuration change can detach a zone without changing its publication. // Clear its old cache tree before accepting the mapping, including when the // context is temporarily pending. The cache hook must flush descendants too. -func (a *Agent) flushRemovedBindingZones(ctx context.Context, snapshot model.ServingSnapshot) error { - if a.state.Snapshot == nil { +func (a *Agent) flushRemovedBindingZones(ctx context.Context, previous *model.ServingSnapshot, snapshot model.ServingSnapshot) error { + if previous == nil { return nil } attached := map[string]bool{} @@ -563,7 +569,7 @@ func (a *Agent) flushRemovedBindingZones(ctx context.Context, snapshot model.Ser } } flushed := map[string]bool{} - for _, binding := range a.state.Snapshot.Bindings { + for _, binding := range previous.Bindings { for _, zone := range binding.Zones { if !attached[binding.BindingUID+"/"+zone.ZoneUID+"/"+zone.Apex] && !flushed[zone.Apex] { if err := a.flushName(ctx, zone.Apex); err != nil { diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go index 4d32b24a..6603dfb1 100644 --- a/internal/internaldns/serving/continuity.go +++ b/internal/internaldns/serving/continuity.go @@ -93,6 +93,9 @@ func (a *Agent) approveConfiguration(files map[string][]byte) error { return err } if a.state.Active != nil && equalJSON(*a.state.Active, active) { + a.healthMu.Lock() + a.healthEnabled = true + a.healthMu.Unlock() return nil } a.state.Active = &active From 06843699ce5e5a3839d43d05b1dbe21b656ff29d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 18:57:52 -0500 Subject: [PATCH 05/15] refactor: Separate DNS capacity status projection --- internal/internaldns/platform/acks.go | 8 +++++--- internal/internaldns/platform/capacity.go | 17 +++++++++++++++++ internal/internaldns/platform/planner.go | 11 ++--------- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/internal/internaldns/platform/acks.go b/internal/internaldns/platform/acks.go index 53acd839..1c35e910 100644 --- a/internal/internaldns/platform/acks.go +++ b/internal/internaldns/platform/acks.go @@ -20,6 +20,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const dnsValueNode = "node" + // AckSink relies on NATS subject ACLs to bind each authenticated serving member // to its own AckSubject. It also verifies membership, region, resource lifetime, // revision and freshness before changing project-facing publication status. @@ -93,7 +95,7 @@ func (s *AckSink) Apply(ctx context.Context, subject string, env model.Envelope) } return s.publication(ctx, a, row, now) case model.KindServingSnapshot: - if member.Role != "node" && member.Role != dnsValueCluster && member.Role != dnsValueResolver && member.Role != dnsValueRegional { + if member.Role != dnsValueNode && member.Role != dnsValueCluster && member.Role != dnsValueResolver && member.Role != dnsValueRegional { return fmt.Errorf("serving ack is not from a resolver member") } return s.serving(ctx, a, row, now) @@ -339,7 +341,7 @@ func (s *AckSink) verified(rows []dnsv1alpha1.DNSApplyAcknowledgement, now time. for _, member := range s.Members { match := member.Role == role if role == dnsValueResolver { - match = member.Role == "node" || member.Role == dnsValueCluster || member.Role == dnsValueResolver || member.Role == dnsValueRegional + match = member.Role == dnsValueNode || member.Role == dnsValueCluster || member.Role == dnsValueResolver || member.Role == dnsValueRegional } if !match { continue @@ -556,7 +558,7 @@ func (s *AckSink) bindingProof(b *dnsv1alpha1.DNSResolverBinding, envelope *mode if role == dnsValueCluster { role = dnsValueRegional } - if role == "node" || role == dnsValueResolver || role == dnsValueRegional { + if role == dnsValueNode || role == dnsValueResolver || role == dnsValueRegional { tiers[role] = append(tiers[role], member) } } diff --git a/internal/internaldns/platform/capacity.go b/internal/internaldns/platform/capacity.go index 6dc5e7fc..1e1d9da5 100644 --- a/internal/internaldns/platform/capacity.go +++ b/internal/internaldns/platform/capacity.go @@ -3,11 +3,14 @@ package platform import ( + "context" "encoding/json" "sort" "time" + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" "go.miloapis.com/dns-operator/internal/internaldns/model" + "sigs.k8s.io/controller-runtime/pkg/client" ) func fitSnapshot(snapshot model.ServingSnapshot, prior model.ServingSnapshot, maxBindings, maxBytes int) (model.ServingSnapshot, map[string]bool) { @@ -78,3 +81,17 @@ func safeExpansion(old, candidate model.Binding) bool { } return true } + +func (p *Planner) reportCapacity(ctx context.Context, bindings []dnsv1alpha1.DNSResolverBinding, rejected map[string]bool, now time.Time) error { + for i := range bindings { + binding := &bindings[i] + base := binding.DeepCopy() + condition(&binding.Status.Conditions, "CapacityAvailable", !rejected[string(binding.UID)], "ShardCapacity", binding.Generation, now) + if !jsonEqualStatus(base.Status, binding.Status) { + if err := p.Client.Status().Patch(ctx, binding, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/internaldns/platform/planner.go b/internal/internaldns/platform/planner.go index 666d3075..2a8197d5 100644 --- a/internal/internaldns/platform/planner.go +++ b/internal/internaldns/platform/planner.go @@ -216,15 +216,8 @@ func (p *Planner) Step(ctx context.Context) error { } var rejected map[string]bool snapshot, rejected = fitSnapshot(snapshot, prior, maxBindings, maxBytes) - for i := range bindings.Items { - binding := &bindings.Items[i] - base := binding.DeepCopy() - condition(&binding.Status.Conditions, "CapacityAvailable", !rejected[string(binding.UID)], "ShardCapacity", binding.Generation, now) - if !jsonEqualStatus(base.Status, binding.Status) { - if err := p.Client.Status().Patch(ctx, binding, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { - return err - } - } + if err := p.reportCapacity(ctx, bindings.Items, rejected, now); err != nil { + return err } if err := snapshot.Validate(); err != nil { return err From d28d68dfa2726d7c6f2dd1aff579c9ff238f9304 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:00:52 -0500 Subject: [PATCH 06/15] fix: Retain activated DNS state when validation rejects an update --- internal/internaldns/serving/bind_test.go | 4 +- internal/internaldns/serving/continuity.go | 3 +- .../internaldns/serving/continuity_test.go | 60 +++++++++++++++++++ internal/internaldns/serving/transaction.go | 22 ++++++- 4 files changed, 85 insertions(+), 4 deletions(-) diff --git a/internal/internaldns/serving/bind_test.go b/internal/internaldns/serving/bind_test.go index d0dfb6b7..4926b1e4 100644 --- a/internal/internaldns/serving/bind_test.go +++ b/internal/internaldns/serving/bind_test.go @@ -151,8 +151,8 @@ func TestBINDActivationFailurePersistsObservationFence(t *testing.T) { if saved.Publications["zone-a"].Verified { t.Fatal("failed publication acknowledged") } - if runner.calls[len(runner.calls)-1].Path != "gate" { - t.Fatal("failed activation did not fail closed") + if runner.calls[len(runner.calls)-1].Path == "gate" || saved.Active == nil { + t.Fatal("rejected expansion discarded the active configuration") } runner.failAt = 0 if err := a.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, m)); err != nil { diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go index 6603dfb1..e1c917bc 100644 --- a/internal/internaldns/serving/continuity.go +++ b/internal/internaldns/serving/continuity.go @@ -23,7 +23,8 @@ type activeConfiguration struct { func (a *Agent) gateRejectedUpdate(ctx context.Context, err error) { var preserved preservedTransactionError - if errors.As(err, &preserved) && a.canRetainActive() { + var uncertain uncertainTransactionError + if !errors.As(err, &uncertain) && a.canRetainActive() && (errors.As(err, &preserved) || !a.needsActivation && a.state.RenderedHash == hashFiles(a.state.Active.Files)) { return } _ = a.failClosed(ctx) diff --git a/internal/internaldns/serving/continuity_test.go b/internal/internaldns/serving/continuity_test.go index a7b4c381..e84f1b59 100644 --- a/internal/internaldns/serving/continuity_test.go +++ b/internal/internaldns/serving/continuity_test.go @@ -91,3 +91,63 @@ func TestPendingZoneIsIsolatedFromReadyZone(t *testing.T) { t.Fatal("ready zone was not materialized") } } + +type blockedValidationRunner struct { + recordingRunner + blocked bool + started chan struct{} + release chan struct{} +} + +func (r *blockedValidationRunner) Run(ctx context.Context, command Command) ([]byte, error) { + if command.Path == "validate" && r.blocked { + close(r.started) + select { + case <-r.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return r.recordingRunner.Run(ctx, command) +} + +func TestActiveHealthDoesNotWaitForCandidateValidation(t *testing.T) { + now := time.Now().UTC() + runner := &blockedValidationRunner{started: make(chan struct{}), release: make(chan struct{})} + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: runner, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + runner.blocked = true + candidate := *agent.state.Snapshot + candidate.ConfigurationRevision++ + candidate.Bindings[0].ConfigurationRevision++ + candidate.Bindings[0].Zones = append(candidate.Bindings[0].Zones, model.ZoneAttachment{ZoneUID: "pending", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + done <- agent.Handle(ctx, envelope(t, model.KindServingSnapshot, "shard", candidate.ConfigurationEpoch, candidate.ConfigurationRevision, candidate)) + }() + select { + case <-runner.started: + case <-ctx.Done(): + t.Fatal("candidate validation did not start") + } + healthDone := make(chan error, 1) + go func() { healthDone <- agent.refreshActiveHealth(ctx) }() + select { + case err := <-healthDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("active health waited for the candidate transaction") + } + close(runner.release) + if err := <-done; err != nil { + t.Fatal(err) + } +} diff --git a/internal/internaldns/serving/transaction.go b/internal/internaldns/serving/transaction.go index a1ba1cd9..467ee010 100644 --- a/internal/internaldns/serving/transaction.go +++ b/internal/internaldns/serving/transaction.go @@ -49,6 +49,10 @@ type preservedTransactionError struct{ error } func (e preservedTransactionError) Unwrap() error { return e.error } +type uncertainTransactionError struct{ error } + +func (e uncertainTransactionError) Unwrap() error { return e.error } + type oldFile struct { data []byte mode os.FileMode @@ -66,7 +70,22 @@ func (t FileTransaction) ApplyForce(ctx context.Context, files map[string][]byte return t.apply(ctx, files, true) } -func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, force bool) error { +func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, force bool) (resultErr error) { + mutating := false + defer func() { + if resultErr == nil { + return + } + var preserved preservedTransactionError + if errors.As(resultErr, &preserved) { + return + } + if mutating { + resultErr = uncertainTransactionError{resultErr} + } else { + resultErr = preservedTransactionError{resultErr} + } + }() if t.Runner == nil { t.Runner = ExecRunner{} } @@ -155,6 +174,7 @@ func (t FileTransaction) apply(ctx context.Context, files map[string][]byte, for return preservedTransactionError{fmt.Errorf("validate staged serving configuration: %w", err)} } installed := make([]string, 0, len(paths)) + mutating = true for _, path := range paths { if !changed[path] { continue From 628b04f3070197fd715f9a310005817370d8b4c7 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:01:24 -0500 Subject: [PATCH 07/15] test: Account for independently pending zone snapshots --- internal/internaldns/platform/platform_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/internaldns/platform/platform_test.go b/internal/internaldns/platform/platform_test.go index cd0ec66c..62816583 100644 --- a/internal/internaldns/platform/platform_test.go +++ b/internal/internaldns/platform/platform_test.go @@ -223,7 +223,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { t.Fatal(err) } _ = cl.List(context.Background(), &out) - if len(out.Items) != 4 { + if len(out.Items) != 5 { t.Fatal("another holder published during a current lease") } } From 44564194766de52641b5417c50b9001971bcc3f2 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:09:37 -0500 Subject: [PATCH 08/15] docs: Align fleet configuration with local health probes --- config/internal-dns/README.md | 15 +++++++++------ config/internal-dns/fleet.example.yaml | 8 ++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index b2ba8aa5..3c35d83f 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -79,15 +79,18 @@ Regional publication proof connects directly to that member's BIND listener, supplies the exact context destination in a protected PROXYv2 header over both UDP and TCP, and verifies the installed SOA serial and fingerprint. Restrict `proxyPeers` to regional dnsdist and the member's local publication agent -network. +network. Local health probes also require the BIND listener address in +`proxyPeers` and loopback in dnsdist ACLs. These peers must remain inside the +protected serving network. ## Transactions, health, and outage recovery -An attached zone that has no active publication gates its context until the -configuration is complete. Pending contexts deny queries, recursion, and cache -answers while retaining the original access authorization. Configuration -completion can restore service without renewing authorization; an actual access -withdrawal still requires a newer source authorization to reactivate. +Each attached private apex has independent readiness. A pending zone returns +SERVFAIL while published zones continue serving. Its names cannot fall through +to public recursion or cached answers. If the private apex is unknown, the +context stays gated until its configuration is complete. Completing publication +restores service without renewing authorization; explicit access withdrawal +requires a newer source authorization to reactivate. The agent writes a complete candidate set into its stage directory. Regional zone files are siblings of `cluster-bind.conf`; validation rewrites file diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml index b5a43141..0c919476 100644 --- a/config/internal-dns/fleet.example.yaml +++ b/config/internal-dns/fleet.example.yaml @@ -11,7 +11,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", - "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64","::1/128"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20","fd70:10::10"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, @@ -28,7 +28,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", - "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, + "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64","::1/128"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}]}, "reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, @@ -45,7 +45,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", - "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30","fd70:20::10"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} }, @@ -65,7 +65,7 @@ data: "stream":{"name":"DNS_PRIVATE","snapshotsBucket":"DNS_PRIVATE_SNAPSHOTS"}, "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", - "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, + "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31","fd70:20::11"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"reconcileInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} }, From ae533d9f68b00b877ac2a4b45c7be1fa41525c7d Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:11:12 -0500 Subject: [PATCH 09/15] fix: Report resolver access independently of pending zones --- config/internal-dns/README.md | 3 ++- internal/internaldns/serving/agent.go | 2 +- internal/internaldns/serving/continuity_test.go | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 3c35d83f..bd9e3eec 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -89,7 +89,8 @@ Each attached private apex has independent readiness. A pending zone returns SERVFAIL while published zones continue serving. Its names cannot fall through to public recursion or cached answers. If the private apex is unknown, the context stays gated until its configuration is complete. Completing publication -restores service without renewing authorization; explicit access withdrawal +restores service without renewing authorization. Resolver access readiness +reports the applied path independently of each zone's publication status. Explicit access withdrawal requires a newer source authorization to reactivate. The agent writes a complete candidate set into its stage directory. Regional diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index ac4427d1..27601480 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -1008,7 +1008,7 @@ func (a *Agent) reconcile(ctx context.Context) error { continue } p := model.AckVerified - if !active[b.BindingUID] || !a.publicationsReady(b) { + if !active[b.BindingUID] { p = model.AckApplied } if err := a.emitServingAck(ctx, b, p); err != nil { diff --git a/internal/internaldns/serving/continuity_test.go b/internal/internaldns/serving/continuity_test.go index e84f1b59..dd54db1d 100644 --- a/internal/internaldns/serving/continuity_test.go +++ b/internal/internaldns/serving/continuity_test.go @@ -72,6 +72,9 @@ func TestPendingZoneIsIsolatedFromReadyZone(t *testing.T) { t.Fatal(err) } installTestPublication(t, agent, now) + pub := &fakePublisher{} + agent.deps.AckPublisher = pub + now = now.Add(time.Minute) binding := agent.state.Snapshot.Bindings[0] binding.Zones = append(binding.Zones, model.ZoneAttachment{ZoneUID: "pending", Apex: "pending.internal", Pending: true, RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}) if zones := agent.readyZones(binding); len(zones) != 1 || zones[0].ZoneUID != "zone-a" { @@ -90,6 +93,19 @@ func TestPendingZoneIsIsolatedFromReadyZone(t *testing.T) { if zone := currentZone(t, agent); zone == "" { t.Fatal("ready zone was not materialized") } + verified := false + for _, env := range pub.envs { + var ack model.MemberAck + if err := json.Unmarshal(env.Payload, &ack); err != nil { + t.Fatal(err) + } + if ack.Kind == model.KindServingSnapshot && ack.Revision == candidate.ConfigurationRevision && ack.Phase == model.AckVerified { + verified = true + } + } + if !verified { + t.Fatal("pending zone prevented resolver access from becoming ready") + } } type blockedValidationRunner struct { From 29e36093efe6204be1e09c6f801241e45a386096 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:12:09 -0500 Subject: [PATCH 10/15] fix: Keep capacity observations scoped to their shard --- internal/internaldns/platform/capacity.go | 3 +++ .../internaldns/platform/capacity_test.go | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/internal/internaldns/platform/capacity.go b/internal/internaldns/platform/capacity.go index 1e1d9da5..632fcc87 100644 --- a/internal/internaldns/platform/capacity.go +++ b/internal/internaldns/platform/capacity.go @@ -85,6 +85,9 @@ func safeExpansion(old, candidate model.Binding) bool { func (p *Planner) reportCapacity(ctx context.Context, bindings []dnsv1alpha1.DNSResolverBinding, rejected map[string]bool, now time.Time) error { for i := range bindings { binding := &bindings[i] + if binding.Spec.Placement.Region != p.Config.Region || binding.Spec.Placement.Shard != p.Config.Shard { + continue + } base := binding.DeepCopy() condition(&binding.Status.Conditions, "CapacityAvailable", !rejected[string(binding.UID)], "ShardCapacity", binding.Generation, now) if !jsonEqualStatus(base.Status, binding.Status) { diff --git a/internal/internaldns/platform/capacity_test.go b/internal/internaldns/platform/capacity_test.go index bb59bb75..1ab8ef18 100644 --- a/internal/internaldns/platform/capacity_test.go +++ b/internal/internaldns/platform/capacity_test.go @@ -3,11 +3,15 @@ package platform import ( + "context" "fmt" "testing" "time" + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" "go.miloapis.com/dns-operator/internal/internaldns/model" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) func capacityBinding(index int) model.Binding { @@ -47,3 +51,20 @@ func TestByteOverflowPreservesExistingContext(t *testing.T) { t.Fatalf("overflow interrupted an established context: %#v", bounded) } } + +func TestCapacityReportDoesNotOverwriteAnotherShard(t *testing.T) { + ctx := context.Background() + binding := &dnsv1alpha1.DNSResolverBinding{ObjectMeta: metav1.ObjectMeta{Name: "binding", Namespace: "platform", UID: "binding-uid"}, Spec: dnsv1alpha1.DNSResolverBindingSpec{Placement: dnsv1alpha1.DNSResolverBindingPlacement{Region: "west", Shard: "b"}}, Status: dnsv1alpha1.DNSResolverBindingStatus{Conditions: []metav1.Condition{{Type: "CapacityAvailable", Status: metav1.ConditionFalse, Reason: "ShardCapacity", LastTransitionTime: metav1.Now()}}}} + cl := testClient(t, binding) + planner := Planner{Client: cl, Config: PlannerConfig{Region: "east", Shard: "a"}} + if err := planner.reportCapacity(ctx, []dnsv1alpha1.DNSResolverBinding{*binding}, nil, time.Now()); err != nil { + t.Fatal(err) + } + var updated dnsv1alpha1.DNSResolverBinding + if err := cl.Get(ctx, client.ObjectKeyFromObject(binding), &updated); err != nil { + t.Fatal(err) + } + if len(updated.Status.Conditions) != 1 || updated.Status.Conditions[0].Status != metav1.ConditionFalse { + t.Fatal("capacity from another shard was overwritten") + } +} From 28caec714be42ac55b4279a7b130269c3b94cd2e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:29:03 -0500 Subject: [PATCH 11/15] fix: Keep withdrawal activation within the watchdog deadline --- internal/internaldns/serving/continuity.go | 5 +-- .../internaldns/serving/continuity_test.go | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/internal/internaldns/serving/continuity.go b/internal/internaldns/serving/continuity.go index e1c917bc..53478e6d 100644 --- a/internal/internaldns/serving/continuity.go +++ b/internal/internaldns/serving/continuity.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "net" - "os" "strconv" "time" @@ -113,9 +112,7 @@ func (a *Agent) suspendActiveHealth() { a.healthMu.Lock() defer a.healthMu.Unlock() a.healthEnabled = false - if a.cfg.WatchdogLeasePath != "" { - _ = os.Remove(a.cfg.WatchdogLeasePath) - } + } func (a *Agent) restoreActive(ctx context.Context) error { diff --git a/internal/internaldns/serving/continuity_test.go b/internal/internaldns/serving/continuity_test.go index dd54db1d..a99fc60b 100644 --- a/internal/internaldns/serving/continuity_test.go +++ b/internal/internaldns/serving/continuity_test.go @@ -3,6 +3,7 @@ package serving import ( + "bytes" "context" "encoding/json" "os" @@ -167,3 +168,36 @@ func TestActiveHealthDoesNotWaitForCandidateValidation(t *testing.T) { t.Fatal(err) } } + +func TestWithdrawalTransactionRetainsOnlyExistingWatchdogDeadline(t *testing.T) { + now := time.Now().UTC() + cfg := regionalTestConfig(t) + agent, err := NewAgent(cfg, Dependencies{Runner: &recordingRunner{}, Clock: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + installTestPublication(t, agent, now) + before, err := os.ReadFile(cfg.WatchdogLeasePath) + if err != nil { + t.Fatal(err) + } + agent.suspendActiveHealth() + now = now.Add(time.Hour) + if err := agent.refreshActiveHealth(context.Background()); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(cfg.WatchdogLeasePath) + if err != nil || !bytes.Equal(before, after) { + t.Fatal("transaction changed the existing watchdog deadline") + } + var lease watchdogLease + if err := json.Unmarshal(after, &lease); err != nil || lease.ValidUntil.After(now) { + t.Fatal("suspended health extended serving authority") + } + if err := agent.failClosed(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(cfg.WatchdogLeasePath); !os.IsNotExist(err) { + t.Fatal("failed withdrawal retained watchdog readiness") + } +} From 96d80b9b0c6d7e6f20e660ca708c305b1158cb55 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 19:56:16 -0500 Subject: [PATCH 12/15] test: compare outage DNS response addresses --- test/internaldns/kubernetes/qualify/scenarios.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index 6f4557f1..d78235d3 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -363,7 +363,7 @@ func (s *suite) offlineServing(a, b *project) (resultErr error) { if err != nil { return err } - if answer.Error != "" || answer.Rcode != dns.RcodeSuccess || len(answer.Answers) != 1 || answer.Answers[0] != tenant.address { + if answer.Error != "" || answer.Rcode != dns.RcodeSuccess || len(answer.Addresses) != 1 || answer.Addresses[0] != tenant.address { return fmt.Errorf("outage query for %s TCP=%t: %#v", tenant.project.Name, tcp, answer) } } From 00a24bc8ea5b48b9304d0d7952079b810db452bd Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 08:17:52 -0500 Subject: [PATCH 13/15] fix: Keep DNS reference identities internal --- api/v1alpha1/internaldns_project_types.go | 37 ++-- .../internaldns_validation_envtest_test.go | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 16 +- ...ing.miloapis.com_dnsmanagednamespaces.yaml | 63 +------ ...orking.miloapis.com_dnsnamingpolicies.yaml | 74 +------- ...working.miloapis.com_dnsregistrations.yaml | 19 +- ...iloapis.com_dnsresolveraccessbindings.yaml | 22 --- ...king.miloapis.com_dnsresolvercontexts.yaml | 15 +- ...king.miloapis.com_dnszoneassociations.yaml | 74 +------- config/internal-dns/rbac.yaml | 8 +- internal/internaldns/admission/handler.go | 9 +- .../internaldns/admission/handler_test.go | 12 +- .../access_recovery_envtest_test.go | 2 +- internal/internaldns/controlplane/compiler.go | 29 +-- .../internaldns/controlplane/compiler_test.go | 11 +- .../controlplane/context_reconciler.go | 115 +++++++----- .../controlplane/context_reconciler_test.go | 23 ++- .../controlplane/managed_domain_test.go | 27 ++- .../controlplane/named_reference_api_test.go | 8 +- .../controlplane/named_reference_test.go | 24 ++- internal/internaldns/controlplane/naming.go | 44 +++-- .../controlplane/naming_selector_test.go | 24 ++- .../internaldns/controlplane/naming_test.go | 14 +- .../controlplane/publication_reconciler.go | 33 ++++ .../publication_reconciler_test.go | 14 +- .../publication_regression_test.go | 4 +- .../internaldns/controlplane/reconciler.go | 2 +- .../controlplane/reference_state.go | 168 ++++++++++++++++++ .../controlplane/reference_state_test.go | 82 +++++++++ internal/internaldns/platform/acks.go | 2 +- .../internaldns/platform/platform_test.go | 2 +- .../runtime/deployment_permissions_test.go | 12 +- test/internaldns/kubernetes/qualify/main.go | 6 +- .../kubernetes/qualify/scenarios.go | 10 +- 34 files changed, 554 insertions(+), 453 deletions(-) create mode 100644 internal/internaldns/controlplane/reference_state.go create mode 100644 internal/internaldns/controlplane/reference_state_test.go diff --git a/api/v1alpha1/internaldns_project_types.go b/api/v1alpha1/internaldns_project_types.go index 297a1b07..319a331b 100644 --- a/api/v1alpha1/internaldns_project_types.go +++ b/api/v1alpha1/internaldns_project_types.go @@ -36,11 +36,9 @@ type DNSZoneAssociationSpec struct { } type DNSZoneAssociationStatus struct { - ResolvedDNSZoneRef DNSObjectReference `json:"resolvedDNSZoneRef,omitempty"` - ResolvedVPCRef DNSObjectReference `json:"resolvedVPCRef,omitempty"` - ResolvedResolverContextRef DNSObjectReference `json:"resolvedResolverContextRef,omitempty"` - BindingRef *DNSObjectReference `json:"bindingRef,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` + ResolvedDNSZoneRef DNSNameReference `json:"resolvedDNSZoneRef,omitempty"` + ResolvedResolverContextRef DNSNameReference `json:"resolvedResolverContextRef,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` } // DNSResolverContext is the DNS-owned serving and namespace boundary created by @@ -65,8 +63,8 @@ type DNSResolverContextManagedNamespace struct { } type DNSResolverContextManagedNamespaceStatus struct { - DNSZoneRef DNSObjectReference `json:"dnsZoneRef,omitempty"` - Suffix string `json:"suffix,omitempty"` + DNSZoneRef DNSNameReference `json:"dnsZoneRef,omitempty"` + Suffix string `json:"suffix,omitempty"` } type DNSResolverContextStatus struct { @@ -139,9 +137,8 @@ type DNSResolverAccessBindingSpec struct { } type DNSResolverAccessBindingStatus struct { - ObservedSequence int64 `json:"observedSequence,omitempty"` - BindingRef *DNSObjectReference `json:"bindingRef,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` + ObservedSequence int64 `json:"observedSequence,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` } // +kubebuilder:object:root=true @@ -188,9 +185,9 @@ const ( const DNSRegistrationMaxTTLSeconds int32 = 30 -// +kubebuilder:validation:XValidation:rule="has(self.dnsZoneRef.uid) && self.dnsZoneRef.uid.size() > 0",message="dnsZoneRef must pin the zone UID" +// +kubebuilder:validation:XValidation:rule="self.dnsZoneRef.name.size() > 0",message="dnsZoneRef requires a name" type DNSRegistrationSpec struct { - DNSZoneRef DNSObjectReference `json:"dnsZoneRef"` + DNSZoneRef DNSNameReference `json:"dnsZoneRef"` Name string `json:"name"` RecordTypes []RRType `json:"recordTypes"` PublicationPolicy DNSPublicationPolicy `json:"publicationPolicy"` @@ -361,16 +358,14 @@ func (s *DNSRegistrationSelector) AsLabelSelector() *metav1.LabelSelector { } type DNSNamingPolicyStatus struct { - ResolvedVPCRef DNSObjectReference `json:"resolvedVPCRef,omitempty"` - ResolvedResolverContextRef DNSObjectReference `json:"resolvedResolverContextRef,omitempty"` - ResolvedDNSZoneRef DNSObjectReference `json:"resolvedDNSZoneRef,omitempty"` + ResolvedResolverContextRef DNSNameReference `json:"resolvedResolverContextRef,omitempty"` ResolvedAdditionalNames []DNSResolvedAdditionalNameRule `json:"resolvedAdditionalNames,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty"` } type DNSResolvedAdditionalNameRule struct { Selector *metav1.LabelSelector `json:"selector,omitempty"` - DNSZoneRef DNSObjectReference `json:"dnsZoneRef"` + DNSZoneRef DNSNameReference `json:"dnsZoneRef"` NamePrefix string `json:"namePrefix"` } @@ -394,15 +389,13 @@ type DNSNamingPolicyList struct { // canonical private namespace of a VPC. Product controllers read its status; // only the DNS platform controller writes the object. type DNSManagedNamespaceSpec struct { - ProjectUID types.UID `json:"projectUID"` - VPCRef DNSObjectReference `json:"vpcRef"` - DomainSuffix string `json:"domainSuffix"` - DNSZoneClassName string `json:"dnsZoneClassName"` + ResolverContextRef DNSNameReference `json:"resolverContextRef"` + DomainSuffix string `json:"domainSuffix"` + DNSZoneClassName string `json:"dnsZoneClassName"` } type DNSManagedNamespaceStatus struct { - DNSZoneRef DNSObjectReference `json:"dnsZoneRef,omitempty"` - AssociationRef DNSObjectReference `json:"associationRef,omitempty"` + DNSZoneRef DNSNameReference `json:"dnsZoneRef,omitempty"` CanonicalSuffix string `json:"canonicalSuffix,omitempty"` Conditions []metav1.Condition `json:"conditions,omitempty"` } diff --git a/api/v1alpha1/internaldns_validation_envtest_test.go b/api/v1alpha1/internaldns_validation_envtest_test.go index 73b1c0f3..5f71b9d6 100644 --- a/api/v1alpha1/internaldns_validation_envtest_test.go +++ b/api/v1alpha1/internaldns_validation_envtest_test.go @@ -126,7 +126,7 @@ func TestPrivateDNSValidationAtAPIServer(t *testing.T) { }) for _, ttl := range []int32{0, 31, 2147483647} { t.Run("ttl", func(t *testing.T) { - a := &DNSRegistration{ObjectMeta: metav1.ObjectMeta{GenerateName: "registration-", Namespace: "validation"}, Spec: DNSRegistrationSpec{DNSZoneRef: DNSObjectReference{Name: "zone", UID: "zone-uid"}, Name: "web", RecordTypes: []RRType{RRTypeAAAA}, PublicationPolicy: DNSPublicationPolicyEligibleContributions, TTLSeconds: ttl}} + a := &DNSRegistration{ObjectMeta: metav1.ObjectMeta{GenerateName: "registration-", Namespace: "validation"}, Spec: DNSRegistrationSpec{DNSZoneRef: DNSNameReference{Name: "zone"}, Name: "web", RecordTypes: []RRType{RRTypeAAAA}, PublicationPolicy: DNSPublicationPolicyEligibleContributions, TTLSeconds: ttl}} requireInvalid(t, cl.Create(ctx, a)) }) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 55998ec0..fc8cb9ec 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -336,7 +336,7 @@ func (in *DNSManagedNamespaceList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSManagedNamespaceSpec) DeepCopyInto(out *DNSManagedNamespaceSpec) { *out = *in - out.VPCRef = in.VPCRef + out.ResolverContextRef = in.ResolverContextRef } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DNSManagedNamespaceSpec. @@ -353,7 +353,6 @@ func (in *DNSManagedNamespaceSpec) DeepCopy() *DNSManagedNamespaceSpec { func (in *DNSManagedNamespaceStatus) DeepCopyInto(out *DNSManagedNamespaceStatus) { *out = *in out.DNSZoneRef = in.DNSZoneRef - out.AssociationRef = in.AssociationRef if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -475,9 +474,7 @@ func (in *DNSNamingPolicySpec) DeepCopy() *DNSNamingPolicySpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSNamingPolicyStatus) DeepCopyInto(out *DNSNamingPolicyStatus) { *out = *in - out.ResolvedVPCRef = in.ResolvedVPCRef out.ResolvedResolverContextRef = in.ResolvedResolverContextRef - out.ResolvedDNSZoneRef = in.ResolvedDNSZoneRef if in.ResolvedAdditionalNames != nil { in, out := &in.ResolvedAdditionalNames, &out.ResolvedAdditionalNames *out = make([]DNSResolvedAdditionalNameRule, len(*in)) @@ -1358,11 +1355,6 @@ func (in *DNSResolverAccessBindingSpec) DeepCopy() *DNSResolverAccessBindingSpec // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DNSResolverAccessBindingStatus) DeepCopyInto(out *DNSResolverAccessBindingStatus) { *out = *in - if in.BindingRef != nil { - in, out := &in.BindingRef, &out.BindingRef - *out = new(DNSObjectReference) - **out = **in - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -1963,13 +1955,7 @@ func (in *DNSZoneAssociationSpec) DeepCopy() *DNSZoneAssociationSpec { func (in *DNSZoneAssociationStatus) DeepCopyInto(out *DNSZoneAssociationStatus) { *out = *in out.ResolvedDNSZoneRef = in.ResolvedDNSZoneRef - out.ResolvedVPCRef = in.ResolvedVPCRef out.ResolvedResolverContextRef = in.ResolvedResolverContextRef - if in.BindingRef != nil { - in, out := &in.BindingRef, &out.BindingRef - *out = new(DNSObjectReference) - **out = **in - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml index 5b4c9ecc..d39c312d 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml @@ -45,64 +45,24 @@ spec: type: string domainSuffix: type: string - projectUID: + resolverContextRef: description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - type: string - vpcRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object required: - dnsZoneClassName - domainSuffix - - projectUID - - vpcRef + - resolverContextRef type: object status: properties: - associationRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object canonicalSuffix: type: string conditions: @@ -163,23 +123,12 @@ spec: type: array dnsZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml index 6c0d2294..93c5e519 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml @@ -216,23 +216,12 @@ spec: properties: dnsZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object @@ -292,69 +281,14 @@ spec: - namePrefix type: object type: array - resolvedDNSZoneRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object resolvedResolverContextRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object - resolvedVPCRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsregistrations.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsregistrations.yaml index 701b0e8a..3fbeb8c3 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsregistrations.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsregistrations.yaml @@ -40,23 +40,12 @@ spec: properties: dnsZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object @@ -105,8 +94,8 @@ spec: - ttlSeconds type: object x-kubernetes-validations: - - message: dnsZoneRef must pin the zone UID - rule: has(self.dnsZoneRef.uid) && self.dnsZoneRef.uid.size() > 0 + - message: dnsZoneRef requires a name + rule: self.dnsZoneRef.name.size() > 0 status: properties: canonicalFQDN: diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml index f743b3d3..c4ac5037 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml @@ -131,28 +131,6 @@ spec: > oldSelf.authorization.sequence status: properties: - bindingRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object conditions: items: description: Condition contains details for one aspect of the current diff --git a/config/crd/bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml b/config/crd/bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml index 15b51f51..cb730aa9 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml @@ -137,23 +137,12 @@ spec: properties: dnsZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object diff --git a/config/crd/bases/dns.networking.miloapis.com_dnszoneassociations.yaml b/config/crd/bases/dns.networking.miloapis.com_dnszoneassociations.yaml index 15161d08..9fef021b 100644 --- a/config/crd/bases/dns.networking.miloapis.com_dnszoneassociations.yaml +++ b/config/crd/bases/dns.networking.miloapis.com_dnszoneassociations.yaml @@ -85,28 +85,6 @@ spec: rule: self.dnsZoneRef.name.size() > 0 status: properties: - bindingRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object conditions: items: description: Condition contains details for one aspect of the current @@ -165,67 +143,23 @@ spec: type: array resolvedDNSZoneRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object resolvedResolverContextRef: description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. - properties: - generation: - format: int64 - minimum: 0 - type: integer - name: - maxLength: 253 - type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string - required: - - name - type: object - resolvedVPCRef: - description: |- - DNSObjectReference pins a namespaced object by API-assigned UID. Generation - is used where a writer must also be fenced against policy changes. + DNSNameReference selects a resource in the same project namespace. + DNS records its resolved lifetime in controller-owned status. properties: - generation: - format: int64 - minimum: 0 - type: integer name: maxLength: 253 type: string - uid: - description: |- - UID is a type that holds unique ID values, including UUIDs. Because we - don't ONLY use UUIDs, this is an alias to string. Being a type captures - intent and helps make sure that UIDs and names do not get conflated. - maxLength: 128 - type: string required: - name type: object diff --git a/config/internal-dns/rbac.yaml b/config/internal-dns/rbac.yaml index eb4695c1..feec5ab2 100644 --- a/config/internal-dns/rbac.yaml +++ b/config/internal-dns/rbac.yaml @@ -22,7 +22,7 @@ rules: resources: [dnsresolvercontexts, dnsresolveraccessbindings] verbs: [get, list, watch, create, update, patch, delete] - apiGroups: [dns.networking.miloapis.com] - resources: [dnsresolvercontexts/status, dnsresolveraccessbindings/status, dnsmanagednamespaces, dnsmanagednamespaces/status] + resources: [dnsresolvercontexts/status, dnsresolveraccessbindings/status] verbs: [get, list, watch] --- # Project products write declarations and their own freshness observations. @@ -37,7 +37,7 @@ rules: resources: [dnsregistrations, dnsrecordcontributions] verbs: [get, list, watch, create, update, patch, delete] - apiGroups: [dns.networking.miloapis.com] - resources: [dnscontributiongrants, dnsresolvercontexts, dnsmanagednamespaces, dnsnamingpolicies] + resources: [dnscontributiongrants, dnsresolvercontexts, dnsnamingpolicies] verbs: [get, list, watch] - apiGroups: [dns.networking.miloapis.com] resources: [dnsrecordcontributions/status] @@ -54,7 +54,7 @@ rules: resources: [dnszones, dnsrecordsets, dnszoneassociations, dnsnamingpolicies, dnsregistrations] verbs: [get, list, watch, create, update, patch, delete] - apiGroups: [dns.networking.miloapis.com] - resources: [dnsresolvercontexts, dnsmanagednamespaces] + resources: [dnsresolvercontexts] verbs: [get, list, watch] --- # Bind only to a trusted issuer that applies the product's registration policy. @@ -122,7 +122,7 @@ rules: verbs: [get, list, watch, create, update, patch] - apiGroups: [""] resources: [configmaps] - verbs: [get, list, watch, create, update, patch] + verbs: [get, list, watch, create, update, patch, delete] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/internaldns/admission/handler.go b/internal/internaldns/admission/handler.go index 575a5e02..4425ca3e 100644 --- a/internal/internaldns/admission/handler.go +++ b/internal/internaldns/admission/handler.go @@ -339,16 +339,15 @@ func (h *Handler) registration(ctx context.Context, cl client.Client, req kubead return nil } -// Zone references cannot silently follow a replacement with the same name. -func currentPrivateZone(ctx context.Context, cl client.Client, namespace string, ref dnsv1alpha1.DNSObjectReference) (*dnsv1alpha1.DNSZone, error) { - if ref.Name == "" || ref.UID == "" { - return nil, fmt.Errorf("zone reference must pin name and UID") +func currentPrivateZone(ctx context.Context, cl client.Client, namespace string, ref dnsv1alpha1.DNSNameReference) (*dnsv1alpha1.DNSZone, error) { + if ref.Name == "" { + return nil, fmt.Errorf("zone reference requires a name") } var zone dnsv1alpha1.DNSZone if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ref.Name}, &zone); err != nil { return nil, fmt.Errorf("private zone lookup: %w", err) } - if zone.UID != ref.UID || !zone.DeletionTimestamp.IsZero() || (ref.Generation != 0 && ref.Generation != zone.Generation) { + if !zone.DeletionTimestamp.IsZero() { return nil, fmt.Errorf("zone lifetime is not current") } if zone.Spec.Visibility != dnsv1alpha1.DNSZoneVisibilityPrivate { diff --git a/internal/internaldns/admission/handler_test.go b/internal/internaldns/admission/handler_test.go index f49d8e79..a8b1ef0c 100644 --- a/internal/internaldns/admission/handler_test.go +++ b/internal/internaldns/admission/handler_test.go @@ -24,7 +24,7 @@ func TestContributionAuthenticatedWriter(t *testing.T) { producer := "system:serviceaccount:compute-system:dns-publisher" registration := &dnsv1alpha1.DNSRegistration{ ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "project", UID: "reg-uid", Generation: 3}, - Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "private"}, RecordTypes: []dnsv1alpha1.RRType{"AAAA"}}, + Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "private"}, RecordTypes: []dnsv1alpha1.RRType{"AAAA"}}, } grant := &dnsv1alpha1.DNSContributionGrant{ ObjectMeta: metav1.ObjectMeta{Name: "compute", Namespace: "project", UID: "grant-uid"}, @@ -259,7 +259,7 @@ func TestPrivateRegistrationRejectsAuthorityOverrides(t *testing.T) { rt dnsv1alpha1.RRType allowed bool }{{"api", dnsv1alpha1.RRTypeA, true}, {"api", dnsv1alpha1.RRTypeCNAME, true}, {"@", dnsv1alpha1.RRTypeCNAME, false}, {"CORP.INTERNAL.", dnsv1alpha1.RRTypeCNAME, false}, {"@", dnsv1alpha1.RRTypeSOA, false}} { - registration := dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, TTLSeconds: 5, Name: tt.owner, RecordTypes: []dnsv1alpha1.RRType{tt.rt}}} + registration := dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, TTLSeconds: 5, Name: tt.owner, RecordTypes: []dnsv1alpha1.RRType{tt.rt}}} response := h.Handle(context.Background(), request("dnsregistrations", "tenant", registration, nil)) if response.Allowed != tt.allowed { t.Fatalf("owner %s type %s allowed=%v expected=%v", tt.owner, tt.rt, response.Allowed, tt.allowed) @@ -275,15 +275,15 @@ func TestRegistrationRequiresCurrentZone(t *testing.T) { zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", Namespace: "project", UID: "current"}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} h := Handler{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(zone).Build()} for _, tt := range []struct { - uid types.UID + name string ttl int32 allowed bool }{ - {"current", 1, true}, {"current", 30, true}, {"", 5, false}, {"retired", 5, false}, + {zone.Name, 1, true}, {zone.Name, 30, true}, {"", 5, false}, {"missing", 5, false}, } { - reg := dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: tt.uid}, TTLSeconds: tt.ttl, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}}} + reg := dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: tt.name}, TTLSeconds: tt.ttl, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}}} if got := h.Handle(context.Background(), request("dnsregistrations", "tenant", reg, nil)); got.Allowed != tt.allowed { - t.Errorf("UID=%q TTL=%d: allowed=%v: %v", tt.uid, tt.ttl, got.Allowed, got.Result) + t.Errorf("name=%q TTL=%d: allowed=%v: %v", tt.name, tt.ttl, got.Allowed, got.Result) } } } diff --git a/internal/internaldns/controlplane/access_recovery_envtest_test.go b/internal/internaldns/controlplane/access_recovery_envtest_test.go index 3586d3ab..eacfce64 100644 --- a/internal/internaldns/controlplane/access_recovery_envtest_test.go +++ b/internal/internaldns/controlplane/access_recovery_envtest_test.go @@ -37,7 +37,7 @@ func TestAccessRecoveryPreservesSourceFencesWithCEL(t *testing.T) { } }) scheme := runtime.NewScheme() - requireAccessTestSuccess(t, dnsv1alpha1.AddToScheme(scheme)) + requireAccessTestSuccess(t, addReferenceStateToScheme(scheme)) requireAccessTestSuccess(t, corev1.AddToScheme(scheme)) cl, err := client.New(cfg, client.Options{Scheme: scheme}) if err != nil { diff --git a/internal/internaldns/controlplane/compiler.go b/internal/internaldns/controlplane/compiler.go index 30d95e7d..6c3317f7 100644 --- a/internal/internaldns/controlplane/compiler.go +++ b/internal/internaldns/controlplane/compiler.go @@ -53,14 +53,16 @@ type RegistrationFence struct { } type CompileInput struct { - Zone *dnsv1alpha1.DNSZone - Associations []dnsv1alpha1.DNSZoneAssociation - Registrations []dnsv1alpha1.DNSRegistration - Grants []dnsv1alpha1.DNSContributionGrant - Contributions []dnsv1alpha1.DNSRecordContribution - StaticRecords []dnsv1alpha1.DNSRecordSet - Previous *PublicationPlan - Now time.Time + Zone *dnsv1alpha1.DNSZone + Associations []dnsv1alpha1.DNSZoneAssociation + AssociationReferences map[types.UID]associationReferences + RegistrationZoneReferences map[types.UID]dnsv1alpha1.DNSObjectReference + Registrations []dnsv1alpha1.DNSRegistration + Grants []dnsv1alpha1.DNSContributionGrant + Contributions []dnsv1alpha1.DNSRecordContribution + StaticRecords []dnsv1alpha1.DNSRecordSet + Previous *PublicationPlan + Now time.Time } type CompileResult struct { @@ -86,7 +88,7 @@ func Compile(in CompileInput) (CompileResult, error) { result.Plan.ZoneUID = in.Zone.UID result.Plan.ZoneApex = canonicalName(in.Zone.Spec.DomainName) - result.Plan.ContextUIDs = associatedContextUIDs(in.Zone, in.Associations) + result.Plan.ContextUIDs = associatedContextUIDs(in.Zone, in.Associations, in.AssociationReferences) claims := map[ownershipKey]types.UID{} staticClaim := map[ownershipKey]types.UID{} @@ -106,7 +108,7 @@ func Compile(in CompileInput) (CompileResult, error) { result.Reasons[reg.UID] = "InvalidTTL" continue } - if !refMatches(reg.Spec.DNSZoneRef, in.Zone.Name, in.Zone.UID, in.Zone.Generation) { + if reg.Spec.DNSZoneRef.Name != in.Zone.Name || !refMatches(in.RegistrationZoneReferences[reg.UID], in.Zone.Name, in.Zone.UID, in.Zone.Generation) { continue } if reg.UID == "" { @@ -484,12 +486,13 @@ func sortAggregate(records []dnsv1alpha1.RecordEntry, uids []types.UID) { } } -func associatedContextUIDs(zone *dnsv1alpha1.DNSZone, associations []dnsv1alpha1.DNSZoneAssociation) []types.UID { +func associatedContextUIDs(zone *dnsv1alpha1.DNSZone, associations []dnsv1alpha1.DNSZoneAssociation, references map[types.UID]associationReferences) []types.UID { vpcs := map[types.UID]struct{}{} for i := range associations { a := &associations[i] - consumerUID := a.Status.ResolvedResolverContextRef.UID - if associationMatchesZone(a, zone) && consumerUID != "" { + refs := references[a.UID] + consumerUID := refs.Context.UID + if associationMatchesZone(a, zone, refs) && consumerUID != "" { vpcs[consumerUID] = struct{}{} } } diff --git a/internal/internaldns/controlplane/compiler_test.go b/internal/internaldns/controlplane/compiler_test.go index fe5b6c80..7ae21f78 100644 --- a/internal/internaldns/controlplane/compiler_test.go +++ b/internal/internaldns/controlplane/compiler_test.go @@ -187,12 +187,16 @@ func TestCompileSRVTargetUsesAbsoluteZoneIdentity(t *testing.T) { func fixture(now time.Time) CompileInput { zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", UID: "zone-a", Generation: 2}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} assoc := func(name string, vpc types.UID) dnsv1alpha1.DNSZoneAssociation { - return dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name)}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: name}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: name, UID: vpc}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} + return dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name)}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: name}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: name}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} } - reg := dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: "registration-a", Generation: 3}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 30}} + reg := dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: "registration-a", Generation: 3}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 30}} grant := dnsv1alpha1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: "compute", UID: "grant-a", Generation: 2}, Spec: dnsv1alpha1.DNSContributionGrantSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, ProducerID: "compute-east", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}}, Status: dnsv1alpha1.DNSContributionGrantStatus{ActiveWriterEpoch: 7, ObservedGrantGeneration: 2, ObservedRegistrationGeneration: 3}} c := dnsv1alpha1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint-a", UID: "contribution-a", Generation: 4}, Spec: dnsv1alpha1.DNSRecordContributionSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: grant.Name, UID: grant.UID}, RecordSets: []dnsv1alpha1.DNSContributionRecordSet{{RecordType: dnsv1alpha1.RRTypeAAAA, Records: []dnsv1alpha1.RecordEntry{{Name: "api.corp.internal", AAAA: &dnsv1alpha1.AAAARecordSpec{Content: "fd00::1"}}}}}}, Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 4, WriterEpoch: 7, Sequence: 18, Eligible: true}} - return CompileInput{Zone: zone, Associations: []dnsv1alpha1.DNSZoneAssociation{assoc("b", "vpc-b"), assoc("a", "vpc-a")}, Registrations: []dnsv1alpha1.DNSRegistration{reg}, Grants: []dnsv1alpha1.DNSContributionGrant{grant}, Contributions: []dnsv1alpha1.DNSRecordContribution{c}, Now: now} + zoneRef := dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation} + return CompileInput{Zone: zone, Associations: []dnsv1alpha1.DNSZoneAssociation{assoc("b", "vpc-b"), assoc("a", "vpc-a")}, AssociationReferences: map[types.UID]associationReferences{ + "a": {Zone: zoneRef, Context: dnsv1alpha1.DNSObjectReference{Name: "a", UID: "vpc-a"}}, + "b": {Zone: zoneRef, Context: dnsv1alpha1.DNSObjectReference{Name: "b", UID: "vpc-b"}}, + }, RegistrationZoneReferences: map[types.UID]dnsv1alpha1.DNSObjectReference{reg.UID: zoneRef}, Registrations: []dnsv1alpha1.DNSRegistration{reg}, Grants: []dnsv1alpha1.DNSContributionGrant{grant}, Contributions: []dnsv1alpha1.DNSRecordContribution{c}, Now: now} } func structLocal(name string) corev1.LocalObjectReference { @@ -217,6 +221,7 @@ func TestCompileRejectsPrivateAuthorityOverridesWithoutBlockingHealthyRecords(t bad.Spec.Name = tt.entry.Name bad.Spec.RecordTypes = []dnsv1alpha1.RRType{tt.recordType} in.Registrations = append(in.Registrations, *bad) + in.RegistrationZoneReferences[bad.UID] = in.RegistrationZoneReferences[in.Registrations[0].UID] got, err := Compile(in) if err != nil { t.Fatal(err) diff --git a/internal/internaldns/controlplane/context_reconciler.go b/internal/internaldns/controlplane/context_reconciler.go index 4c7f27f9..b0ca4d94 100644 --- a/internal/internaldns/controlplane/context_reconciler.go +++ b/internal/internaldns/controlplane/context_reconciler.go @@ -43,37 +43,43 @@ func pinNamedReference(name string, previous, current dnsv1alpha1.DNSObjectRefer } func (r *Reconciler) reconcileNamingPolicies(ctx context.Context, items []dnsv1alpha1.DNSNamingPolicy, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + associationPins, err := r.associationReferences(ctx, assocs) + if err != nil { + return err + } for i := range items { p := &items[i] base := p.DeepCopy() var resolverContext dnsv1alpha1.DNSResolverContext err := r.Get(ctx, client.ObjectKey{Namespace: p.Namespace, Name: p.Spec.ResolverContextRef.Name}, &resolverContext) - current := dnsv1alpha1.DNSObjectReference{} + if err != nil && !apierrors.IsNotFound(err) { + return err + } + current := dnsv1alpha1.DNSObjectReference{Name: p.Spec.ResolverContextRef.Name} if err == nil && resolverContext.DeletionTimestamp.IsZero() { current = dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID} } - contextPin, contextCurrent := pinNamedReference(p.Spec.ResolverContextRef.Name, p.Status.ResolvedResolverContextRef, current) - ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") - valid := contextCurrent && p.DeletionTimestamp.IsZero() && ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == resolverContext.Generation - p.Status.ResolvedResolverContextRef = contextPin - if contextCurrent { - p.Status.ResolvedVPCRef = dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: types.UID(resolverContext.Spec.ConsumerID)} + selected := map[string]dnsv1alpha1.DNSObjectReference{"context": current} + for _, rule := range p.Spec.AdditionalNames { + ref := dnsv1alpha1.DNSObjectReference{Name: rule.DNSZoneRef.Name} + if zone := zones[rule.DNSZoneRef.Name]; zone != nil && zone.DeletionTimestamp.IsZero() && zone.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate { + ref.UID, ref.Generation = zone.UID, zone.Generation + } + selected["zone/"+rule.DNSZoneRef.Name] = ref } - - previousZones := map[string]dnsv1alpha1.DNSObjectReference{} - for _, rule := range base.Status.ResolvedAdditionalNames { - previousZones[rule.DNSZoneRef.Name] = rule.DNSZoneRef + pins, contextCurrent, err := r.pinReferences(ctx, p, selected) + if err != nil { + return err } + contextPin := pins["context"] + ready := apimeta.FindStatusCondition(resolverContext.Status.Conditions, "Ready") + valid := contextCurrent && p.DeletionTimestamp.IsZero() && ready != nil && ready.Status == metav1.ConditionTrue && ready.ObservedGeneration == resolverContext.Generation + p.Status.ResolvedResolverContextRef = dnsv1alpha1.DNSNameReference{Name: contextPin.Name} resolved := make([]dnsv1alpha1.DNSResolvedAdditionalNameRule, 0, len(p.Spec.AdditionalNames)) seen := map[string]bool{} for _, rule := range p.Spec.AdditionalNames { - z := zones[rule.DNSZoneRef.Name] - currentZone := dnsv1alpha1.DNSObjectReference{} - if z != nil && z.DeletionTimestamp.IsZero() && z.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate { - currentZone = dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation} - } - zonePin, zoneCurrent := pinNamedReference(rule.DNSZoneRef.Name, previousZones[rule.DNSZoneRef.Name], currentZone) - if !zoneCurrent || !contextCurrent || !associatedWithContext(assocs, zonePin.UID, contextPin.UID) { + zonePin := pins["zone/"+rule.DNSZoneRef.Name] + if !contextCurrent || !associatedWithContext(assocs, associationPins, zonePin.UID, contextPin.UID) { valid = false } selector, selectorErr := namingSelector(rule.Selector.AsLabelSelector()) @@ -88,7 +94,7 @@ func (r *Reconciler) reconcileNamingPolicies(ctx context.Context, items []dnsv1a valid = false } seen[key] = true - resolved = append(resolved, dnsv1alpha1.DNSResolvedAdditionalNameRule{Selector: rule.Selector.AsLabelSelector().DeepCopy(), DNSZoneRef: zonePin, NamePrefix: canonicalName(rule.NamePrefix)}) + resolved = append(resolved, dnsv1alpha1.DNSResolvedAdditionalNameRule{Selector: rule.Selector.AsLabelSelector().DeepCopy(), DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zonePin.Name}, NamePrefix: canonicalName(rule.NamePrefix)}) } p.Status.ResolvedAdditionalNames = resolved status := metav1.ConditionTrue @@ -113,13 +119,14 @@ func associationAccepted(a *dnsv1alpha1.DNSZoneAssociation) bool { return a.DeletionTimestamp.IsZero() && accepted != nil && accepted.Status == metav1.ConditionTrue && accepted.ObservedGeneration == a.Generation } -func associationMatchesZone(a *dnsv1alpha1.DNSZoneAssociation, z *dnsv1alpha1.DNSZone) bool { - return z != nil && associationAccepted(a) && refMatches(a.Status.ResolvedDNSZoneRef, z.Name, z.UID, z.Generation) +func associationMatchesZone(a *dnsv1alpha1.DNSZoneAssociation, z *dnsv1alpha1.DNSZone, refs associationReferences) bool { + return z != nil && associationAccepted(a) && refMatches(refs.Zone, z.Name, z.UID, z.Generation) } -func associatedWithContext(assocs []dnsv1alpha1.DNSZoneAssociation, zoneUID, contextUID types.UID) bool { +func associatedWithContext(assocs []dnsv1alpha1.DNSZoneAssociation, pins map[types.UID]associationReferences, zoneUID, contextUID types.UID) bool { for _, a := range assocs { - if associationAccepted(&a) && a.Status.ResolvedDNSZoneRef.UID == zoneUID && a.Status.ResolvedResolverContextRef.UID == contextUID { + refs := pins[a.UID] + if associationAccepted(&a) && refs.Zone.UID == zoneUID && refs.Context.UID == contextUID { return true } } @@ -134,13 +141,17 @@ func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, } for i := range list.Items { m := &list.Items[i] - if m.Spec.ProjectUID != r.Options.ProjectUID || m.Spec.VPCRef.UID == "" { - if err := r.patchCondition(ctx, m, dnsValueAccepted, metav1.ConditionFalse, "InvalidIdentity", "project and pinned VPC UID are required", now); err != nil { + var contextObject dnsv1alpha1.DNSResolverContext + if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: m.Spec.ResolverContextRef.Name}, &contextObject); err != nil { + return err + } + if !metav1.IsControlledBy(m, &contextObject) || !contextObject.DeletionTimestamp.IsZero() { + if err := r.patchCondition(ctx, m, dnsValueAccepted, metav1.ConditionFalse, "InvalidIdentity", "managed namespace must be owned by its resolver context", now); err != nil { return err } continue } - lifetimeToken := model.OpaqueToken(m.Labels["internal-dns.miloapis.com/context-uid"]) + lifetimeToken := model.OpaqueToken(string(contextObject.UID)) suffix := canonicalName(m.Spec.DomainSuffix) zoneName := "managed-" + lifetimeToken[:12] @@ -159,10 +170,6 @@ func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, assocName := zoneName + "-vpc" assocSpec := dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Managed: true} - var contextObject dnsv1alpha1.DNSResolverContext - if err := r.Get(ctx, client.ObjectKey{Namespace: ns, Name: m.Spec.VPCRef.Name}, &contextObject); err != nil { - return err - } assocSpec.ResolverContextRef = dnsv1alpha1.DNSNameReference{Name: contextObject.Name} assoc := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: assocName, Namespace: ns, Labels: map[string]string{"dns.networking.miloapis.com/managed": "true"}}, Spec: assocSpec} @@ -179,15 +186,17 @@ func (r *Reconciler) reconcileManagedNamespaces(ctx context.Context, ns string, } base := m.DeepCopy() accepted := apimeta.FindStatusCondition(assoc.Status.Conditions, dnsValueAccepted) - ready := zone.Spec.DomainName == suffix && metav1.IsControlledBy(zone, m) && accepted != nil && accepted.Status == metav1.ConditionTrue && assoc.Status.ResolvedDNSZoneRef.UID == zone.UID + refs, err := r.readReferences(ctx, assoc) + if err != nil { + return err + } + ready := zone.Spec.DomainName == suffix && metav1.IsControlledBy(zone, m) && accepted != nil && accepted.Status == metav1.ConditionTrue && accepted.ObservedGeneration == assoc.Generation && refMatches(refs["zone"], zone.Name, zone.UID, zone.Generation) if ready { - m.Status.DNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID} - m.Status.AssociationRef = dnsv1alpha1.DNSObjectReference{Name: assoc.Name, UID: assoc.UID} + m.Status.DNSZoneRef = dnsv1alpha1.DNSNameReference{Name: zone.Name} m.Status.CanonicalSuffix = suffix setCondition(&m.Status.Conditions, dnsValueAccepted, metav1.ConditionTrue, "Ready", "managed namespace is allocated", m.Generation, now) } else { - m.Status.DNSZoneRef = dnsv1alpha1.DNSObjectReference{} - m.Status.AssociationRef = dnsv1alpha1.DNSObjectReference{} + m.Status.DNSZoneRef = dnsv1alpha1.DNSNameReference{} m.Status.CanonicalSuffix = "" setCondition(&m.Status.Conditions, dnsValueAccepted, metav1.ConditionFalse, "PendingAssociation", "managed namespace association is not accepted", m.Generation, now) } @@ -237,7 +246,7 @@ func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, n c.Status.ManagedNamespace = dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{} if c.Spec.ManagedNamespace.Enabled && valid { name := "context-" + model.OpaqueToken(string(c.UID))[:20] - m := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{"internal-dns.miloapis.com/platform-owned": "true", "internal-dns.miloapis.com/context-uid": string(c.UID)}}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ProjectUID: r.Options.ProjectUID, VPCRef: dnsv1alpha1.DNSObjectReference{Name: c.Name, UID: types.UID(c.Spec.ConsumerID)}, DomainSuffix: c.Spec.ManagedNamespace.DomainSuffix, DNSZoneClassName: r.Options.PrivateZoneClassName}} + m := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{"internal-dns.miloapis.com/platform-owned": "true"}}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: c.Name}, DomainSuffix: c.Spec.ManagedNamespace.DomainSuffix, DNSZoneClassName: r.Options.PrivateZoneClassName}} if r.Scheme != nil { if err := controllerutil.SetControllerReference(c, m, r.Scheme); err != nil { return err @@ -249,7 +258,7 @@ func (r *Reconciler) reconcileResolverContexts(ctx context.Context, ns string, n if err := r.Get(ctx, client.ObjectKeyFromObject(m), m); err != nil { return err } - if condition := apimeta.FindStatusCondition(m.Status.Conditions, dnsValueAccepted); condition != nil && condition.Status == metav1.ConditionTrue && condition.ObservedGeneration == m.Generation && m.Status.DNSZoneRef.UID != "" && metav1.IsControlledBy(m, c) && m.Spec.DomainSuffix == c.Spec.ManagedNamespace.DomainSuffix && m.Status.CanonicalSuffix == c.Spec.ManagedNamespace.DomainSuffix { + if condition := apimeta.FindStatusCondition(m.Status.Conditions, dnsValueAccepted); condition != nil && condition.Status == metav1.ConditionTrue && condition.ObservedGeneration == m.Generation && m.Status.DNSZoneRef.Name != "" && metav1.IsControlledBy(m, c) && m.Spec.DomainSuffix == c.Spec.ManagedNamespace.DomainSuffix && m.Status.CanonicalSuffix == c.Spec.ManagedNamespace.DomainSuffix { c.Status.ManagedNamespace.DNSZoneRef = m.Status.DNSZoneRef c.Status.ManagedNamespace.Suffix = m.Status.CanonicalSuffix ready = true @@ -289,24 +298,29 @@ func (r *Reconciler) reconcileAssociations(ctx context.Context, items []dnsv1alp a := &items[i] base := a.DeepCopy() z := zones[a.Spec.DNSZoneRef.Name] - currentZone := dnsv1alpha1.DNSObjectReference{} + currentZone := dnsv1alpha1.DNSObjectReference{Name: a.Spec.DNSZoneRef.Name} if z != nil && z.DeletionTimestamp.IsZero() && z.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate { currentZone = dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation} } - zonePin, zoneCurrent := pinNamedReference(a.Spec.DNSZoneRef.Name, a.Status.ResolvedDNSZoneRef, currentZone) var resolverContext dnsv1alpha1.DNSResolverContext lookupErr := r.Get(ctx, client.ObjectKey{Namespace: a.Namespace, Name: a.Spec.ResolverContextRef.Name}, &resolverContext) - currentContext := dnsv1alpha1.DNSObjectReference{} + if lookupErr != nil && !apierrors.IsNotFound(lookupErr) { + return lookupErr + } + currentContext := dnsv1alpha1.DNSObjectReference{Name: a.Spec.ResolverContextRef.Name} if lookupErr == nil && resolverContext.DeletionTimestamp.IsZero() { currentContext = dnsv1alpha1.DNSObjectReference{Name: resolverContext.Name, UID: resolverContext.UID} } - contextPin, contextCurrent := pinNamedReference(a.Spec.ResolverContextRef.Name, a.Status.ResolvedResolverContextRef, currentContext) - a.Status.ResolvedDNSZoneRef = zonePin - a.Status.ResolvedResolverContextRef = contextPin - a.Status.ResolvedVPCRef = dnsv1alpha1.DNSObjectReference{} + pins, referencesCurrent, err := r.pinReferences(ctx, a, map[string]dnsv1alpha1.DNSObjectReference{"zone": currentZone, "context": currentContext}) + if err != nil { + return err + } + zonePin, contextPin := pins["zone"], pins["context"] + a.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSNameReference{Name: zonePin.Name} + a.Status.ResolvedResolverContextRef = dnsv1alpha1.DNSNameReference{Name: contextPin.Name} status := metav1.ConditionTrue reason, msg := dnsValueAccepted, "association accepted" - if !a.DeletionTimestamp.IsZero() || !zoneCurrent || !contextCurrent { + if !a.DeletionTimestamp.IsZero() || !referencesCurrent { status = metav1.ConditionFalse reason = "InvalidReference" msg = "private zone and resolver context must match their resolved lifetimes; recreate the association to reattach replacements" @@ -332,6 +346,10 @@ func (r *Reconciler) reconcileBindings(ctx context.Context, ns string, assocs [] } func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, assocs []dnsv1alpha1.DNSZoneAssociation, zones map[string]*dnsv1alpha1.DNSZone, now time.Time) error { + associationPins, err := r.associationReferences(ctx, assocs) + if err != nil { + return err + } var accesses dnsv1alpha1.DNSResolverAccessBindingList var contexts dnsv1alpha1.DNSResolverContextList var existing dnsv1alpha1.DNSResolverBindingList @@ -358,8 +376,9 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as zonesByContext := map[types.UID][]types.UID{} for _, a := range assocs { z := zones[a.Spec.DNSZoneRef.Name] - if associationMatchesZone(&a, z) { - zonesByContext[a.Status.ResolvedResolverContextRef.UID] = append(zonesByContext[a.Status.ResolvedResolverContextRef.UID], z.UID) + refs := associationPins[a.UID] + if associationMatchesZone(&a, z, refs) { + zonesByContext[refs.Context.UID] = append(zonesByContext[refs.Context.UID], z.UID) } } sort.Slice(accesses.Items, func(i, j int) bool { return accesses.Items[i].Name < accesses.Items[j].Name }) @@ -401,7 +420,6 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as } setCondition(&a.Status.Conditions, dnsValueAccepted, status, reason, msg, a.Generation, now) if !valid { - a.Status.BindingRef = nil setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "AccessRejected", "access binding is not accepted", a.Generation, now) if !jsonEqual(base.Status, a.Status) { if err := r.Status().Patch(ctx, a, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { @@ -461,7 +479,6 @@ func (r *Reconciler) reconcileContextBindings(ctx context.Context, ns string, as binding = &old } a.Status.ObservedSequence = spec.Authorization.Sequence - a.Status.BindingRef = &dnsv1alpha1.DNSObjectReference{Name: binding.Name, UID: binding.UID} setCondition(&a.Status.Conditions, "Ready", metav1.ConditionFalse, "PendingReplicaVerification", "serving fleet acknowledgement is pending", a.Generation, now) if c := apimeta.FindStatusCondition(binding.Status.Conditions, "Serving"); c != nil && c.Status == metav1.ConditionTrue && binding.Status.ObservedConfigurationRevision == binding.Spec.Configuration.Revision && bindingAckCurrent(binding, now) { setCondition(&a.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "serving capacity applied this access generation", a.Generation, now) diff --git a/internal/internaldns/controlplane/context_reconciler_test.go b/internal/internaldns/controlplane/context_reconciler_test.go index c019ab4a..3ed1ff18 100644 --- a/internal/internaldns/controlplane/context_reconciler_test.go +++ b/internal/internaldns/controlplane/context_reconciler_test.go @@ -57,17 +57,18 @@ func TestContextAccessPersistsUntilExplicitWithdrawal(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } contextObject := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}, Status: dnsv1alpha1.DNSResolverContextStatus{AccessWriterEpoch: 4}} setCondition(&contextObject.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 1, now) zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", Namespace: "project", UID: "zone-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "assoc", Namespace: "project", UID: "assoc-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "zone"}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: "ctx"}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "zone", UID: "zone-uid", Generation: 1}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}}} + association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "assoc", Namespace: "project", UID: "assoc-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "zone"}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: "ctx"}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "zone"}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: "ctx"}}} setCondition(&association.Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) access := &dnsv1alpha1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "access", Namespace: "project", UID: "access-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverAccessBindingSpec{ContextRef: dnsv1alpha1.DNSObjectReference{Name: "ctx", UID: "context-uid"}, Region: "central", QueryIdentity: dnsv1alpha1.DNSResolverQueryIdentity{Type: dnsv1alpha1.DNSResolverQueryIdentityDestinationAddress, Value: "fd70:100::10"}, Port: 53, Transports: []string{"UDP", "TCP"}, Authorization: dnsv1alpha1.DNSResolverAccessAuthorization{WriterEpoch: 4, Sequence: 12}}} c := fake.NewClientBuilder().WithScheme(s).WithObjects(contextObject, zone, association, access).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSResolverBinding{}).Build() r := &Reconciler{Client: c, Scheme: s, Options: ReconcilerOptions{ProjectUID: "project-uid", PlatformNamespace: "platform", Region: "central", Shard: "shared", AddressAllocator: testAllocator{}}} + seedReferenceState(t, r, association, map[string]dnsv1alpha1.DNSObjectReference{"zone": {Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, "context": {Name: contextObject.Name, UID: contextObject.UID}}) if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association}, map[string]*dnsv1alpha1.DNSZone{"zone": zone}, now); err != nil { t.Fatal(err) } @@ -97,7 +98,9 @@ func TestContextAccessPersistsUntilExplicitWithdrawal(t *testing.T) { extraAssociation := association.DeepCopy() extraAssociation.Name = "extra-assoc" extraAssociation.Spec.DNSZoneRef = dnsv1alpha1.DNSNameReference{Name: extraZone.Name} - extraAssociation.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: extraZone.Name, UID: extraZone.UID, Generation: extraZone.Generation} + extraAssociation.UID = "extra-association-uid" + extraAssociation.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSNameReference{Name: extraZone.Name} + seedReferenceState(t, r, extraAssociation, map[string]dnsv1alpha1.DNSObjectReference{"zone": {Name: extraZone.Name, UID: extraZone.UID, Generation: extraZone.Generation}, "context": {Name: contextObject.Name, UID: contextObject.UID}}) if err := r.reconcileContextBindings(ctx, "project", []dnsv1alpha1.DNSZoneAssociation{*association, *extraAssociation}, map[string]*dnsv1alpha1.DNSZone{"zone": zone, "extra": extraZone}, now); err != nil { t.Fatal(err) } @@ -224,21 +227,29 @@ func TestContextNamingPolicyResolvesMultipleZonesWithoutNetworking(t *testing.T) ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } resolverContext := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "ctx", Namespace: "project", UID: "ctx-uid", Generation: 2}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-uid"}} setCondition(&resolverContext.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 2, now) z1 := dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project", UID: "zone-1", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} z2 := dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "services", Namespace: "project", UID: "zone-2", Generation: 3}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - associations := []dnsv1alpha1.DNSZoneAssociation{{Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: z1.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolverContext.UID}}}, {Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: z2.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolverContext.UID}}}} + associations := []dnsv1alpha1.DNSZoneAssociation{{Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{}}}, {Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{}}}} for i := range associations { + zone := []dnsv1alpha1.DNSZone{z1, z2}[i] + associations[i].Name = zone.Name + "-association" + associations[i].Namespace = "project" + associations[i].UID = types.UID(associations[i].Name) associations[i].Generation = 1 setCondition(&associations[i].Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) } policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "names", Namespace: "project", UID: "policy-uid", Generation: 1}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolverContext.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: z1.Name}, NamePrefix: "instances"}, {DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: z2.Name}, NamePrefix: "services"}}}} c := fake.NewClientBuilder().WithScheme(s).WithObjects(resolverContext, policy).WithStatusSubresource(&dnsv1alpha1.DNSResolverContext{}, &dnsv1alpha1.DNSResolverAccessBinding{}, &dnsv1alpha1.DNSNamingPolicy{}).Build() r := &Reconciler{Client: c, Options: ReconcilerOptions{}} + for i := range associations { + zone := []dnsv1alpha1.DNSZone{z1, z2}[i] + seedReferenceState(t, r, &associations[i], map[string]dnsv1alpha1.DNSObjectReference{"zone": {Name: zone.Name, UID: zone.UID, Generation: zone.Generation}, "context": {Name: resolverContext.Name, UID: resolverContext.UID}}) + } if err := r.reconcileNamingPolicies(ctx, []dnsv1alpha1.DNSNamingPolicy{*policy}, associations, map[string]*dnsv1alpha1.DNSZone{z1.Name: &z1, z2.Name: &z2}, now); err != nil { t.Fatal(err) } @@ -249,7 +260,7 @@ func TestContextNamingPolicyResolvesMultipleZonesWithoutNetworking(t *testing.T) if accepted == nil || accepted.Status != metav1.ConditionTrue || len(policy.Status.ResolvedAdditionalNames) != 2 { t.Fatalf("context naming policy not resolved: %#v", policy.Status) } - if policy.Status.ResolvedResolverContextRef.UID != resolverContext.UID || policy.Status.ResolvedVPCRef.UID != "consumer-uid" { + if policy.Status.ResolvedResolverContextRef.Name != resolverContext.Name { t.Fatalf("identity projection incorrect: %#v", policy.Status) } } diff --git a/internal/internaldns/controlplane/managed_domain_test.go b/internal/internaldns/controlplane/managed_domain_test.go index deed10d8..4f75b12f 100644 --- a/internal/internaldns/controlplane/managed_domain_test.go +++ b/internal/internaldns/controlplane/managed_domain_test.go @@ -18,7 +18,7 @@ import ( func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } domains := map[string]string{"vpc-a": "datum.internal", "vpc-b": "datum.internal", "custom": "private.example.net"} @@ -52,17 +52,12 @@ func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { if err := c.List(ctx, &associations); err != nil { t.Fatal(err) } - for i := range associations.Items { - a := &associations.Items[i] - for _, z := range zones.Items { - if z.Name == a.Spec.DNSZoneRef.Name { - a.Status.ResolvedDNSZoneRef = dnsv1alpha1.DNSObjectReference{Name: z.Name, UID: z.UID, Generation: z.Generation} - } - } - setCondition(&a.Status.Conditions, "Accepted", metav1.ConditionTrue, "Ready", "", a.Generation, now) - if err := c.Status().Update(ctx, a); err != nil { - t.Fatal(err) - } + zoneByName := map[string]*dnsv1alpha1.DNSZone{} + for i := range zones.Items { + zoneByName[zones.Items[i].Name] = &zones.Items[i] + } + if err := r.reconcileAssociations(ctx, associations.Items, zoneByName, now); err != nil { + t.Fatal(err) } if err := r.reconcileManagedNamespaces(ctx, "project", now); err != nil { t.Fatal(err) @@ -75,7 +70,7 @@ func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { t.Fatal(err) } for _, resolver := range contexts.Items { - if resolver.Status.ManagedNamespace.Suffix != domains[resolver.Name] || resolver.Status.ManagedNamespace.DNSZoneRef.UID == "" { + if resolver.Status.ManagedNamespace.Suffix != domains[resolver.Name] || resolver.Status.ManagedNamespace.DNSZoneRef.Name == "" { t.Fatalf("unallocated context: %#v", resolver.Status) } var zone dnsv1alpha1.DNSZone @@ -86,7 +81,7 @@ func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { t.Fatalf("requested %q, allocated %q", domains[resolver.Name], zone.Spec.DomainName) } } - if contexts.Items[0].Status.ManagedNamespace.DNSZoneRef.UID == contexts.Items[1].Status.ManagedNamespace.DNSZoneRef.UID { + if contexts.Items[0].Status.ManagedNamespace.DNSZoneRef.Name == contexts.Items[1].Status.ManagedNamespace.DNSZoneRef.Name { t.Fatal("contexts share a zone lifetime") } zone := &zones.Items[0] @@ -104,7 +99,7 @@ func TestManagedDomainIsSharedWithSeparateContextZones(t *testing.T) { t.Fatal(err) } for _, resolver := range contexts.Items { - if resolver.Status.ManagedNamespace.DNSZoneRef.UID == zone.UID { + if resolver.Status.ManagedNamespace.DNSZoneRef.Name == zone.Name { t.Fatal("a mismatched zone domain remained allocated") } } @@ -114,7 +109,7 @@ func TestManagedNamespaceRequiresRequestedDomain(t *testing.T) { for _, domain := range []string{"", "bad..internal", "*.internal"} { t.Run(domain, func(t *testing.T) { s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "consumer", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "opaque-consumer", ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespace{Enabled: true, DomainSuffix: domain}}} diff --git a/internal/internaldns/controlplane/named_reference_api_test.go b/internal/internaldns/controlplane/named_reference_api_test.go index bda1cb8f..5a8e3e92 100644 --- a/internal/internaldns/controlplane/named_reference_api_test.go +++ b/internal/internaldns/controlplane/named_reference_api_test.go @@ -27,7 +27,7 @@ func verifyNamedReferencesWithAPI(t *testing.T, cl client.Client) { policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "named-policy", Namespace: "project"}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{Selector: &dnsv1alpha1.DNSRegistrationSelector{MatchLabels: map[string]string{"app": "storefront"}}, DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, NamePrefix: "hosts"}}}} requireAccessTestSuccess(t, cl.Create(ctx, association)) requireAccessTestSuccess(t, cl.Create(ctx, policy)) - r := &Reconciler{Client: cl} + r := &Reconciler{Client: cl, Options: Options{PlatformNamespace: "platform"}} reconcile := func(want metav1.ConditionStatus) { t.Helper() requireAccessTestSuccess(t, cl.Get(ctx, client.ObjectKeyFromObject(association), association)) @@ -56,7 +56,11 @@ func verifyNamedReferencesWithAPI(t *testing.T, cl client.Client) { t.Fatal("replacement did not receive a new API-assigned UID") } reconcile(metav1.ConditionFalse) - if association.Status.ResolvedDNSZoneRef.UID != originalUID || policy.Status.ResolvedAdditionalNames[0].DNSZoneRef.UID != originalUID { + associationPins, err := r.readReferences(ctx, association) + requireAccessTestSuccess(t, err) + policyPins, err := r.readReferences(ctx, policy) + requireAccessTestSuccess(t, err) + if associationPins["zone"].UID != originalUID || policyPins["zone/"+zone.Name].UID != originalUID { t.Fatal("API-backed replacement transferred lifetime pins") } } diff --git a/internal/internaldns/controlplane/named_reference_test.go b/internal/internaldns/controlplane/named_reference_test.go index c030a356..df665a99 100644 --- a/internal/internaldns/controlplane/named_reference_test.go +++ b/internal/internaldns/controlplane/named_reference_test.go @@ -21,16 +21,24 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { ctx := context.Background() now := time.Now().UTC() scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "application", Namespace: "project", UID: "context-original", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer"}} setCondition(&resolver.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", resolver.Generation, now) zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", UID: "zone-original", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "prod.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}}} - policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", Generation: 1}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, NamePrefix: "hosts"}}}} + association := &dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", UID: "source-original", Generation: 1}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}}} + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "production", Namespace: "project", UID: "source-original", Generation: 1}, Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, NamePrefix: "hosts"}}}} cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver, zone, association, policy).WithStatusSubresource(association, policy).Build() r := &Reconciler{Client: cl} + references := func(source client.Object) map[string]dnsv1alpha1.DNSObjectReference { + t.Helper() + refs, err := r.readReferences(ctx, source) + if err != nil { + t.Fatal(err) + } + return refs + } zones := map[string]*dnsv1alpha1.DNSZone{zone.Name: zone} reconcile := func() { t.Helper() @@ -65,13 +73,13 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { } reconcile() assertAccepted(true) - if association.Status.ResolvedDNSZoneRef.UID != zone.UID || policy.Status.ResolvedResolverContextRef.UID != resolver.UID { + if references(association)["zone"].UID != zone.UID || references(policy)["context"].UID != resolver.UID { t.Fatal("name-only references did not pin lifetimes") } zone.Generation++ reconcile() assertAccepted(true) - if association.Status.ResolvedDNSZoneRef.Generation != zone.Generation || policy.Status.ResolvedAdditionalNames[0].DNSZoneRef.Generation != zone.Generation { + if references(association)["zone"].Generation != zone.Generation || references(policy)["zone/"+zone.Name].Generation != zone.Generation { t.Fatal("resolved generations were not persisted for the same zone lifetime") } originalZoneUID := zone.UID @@ -88,7 +96,7 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { } reconcile() assertAccepted(false) - if association.Status.ResolvedDNSZoneRef.UID != originalZoneUID || policy.Status.ResolvedResolverContextRef.UID != originalContextUID { + if references(association)["zone"].UID != originalZoneUID || references(policy)["context"].UID != originalContextUID { t.Fatal("deletion cleared lifetime pins") } replaced.SetResourceVersion("") @@ -101,7 +109,7 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { } reconcile() assertAccepted(false) - if association.Status.ResolvedDNSZoneRef.UID != originalZoneUID || policy.Status.ResolvedAdditionalNames[0].DNSZoneRef.UID != originalZoneUID || policy.Status.ResolvedResolverContextRef.UID != originalContextUID { + if references(association)["zone"].UID != originalZoneUID || references(policy)["zone/"+zone.Name].UID != originalZoneUID || references(policy)["context"].UID != originalContextUID { t.Fatal("replacement inherited lifetime pins") } // Recreate both customer resources to explicitly attach the replacement. @@ -109,6 +117,7 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { t.Fatal(err) } association.ResourceVersion = "" + association.UID = "association-recreated" association.Status = dnsv1alpha1.DNSZoneAssociationStatus{} if err := cl.Create(ctx, association); err != nil { t.Fatal(err) @@ -121,6 +130,7 @@ func TestNamedReferencesPreserveLifetimesAcrossReplacement(t *testing.T) { t.Fatal(err) } policy.ResourceVersion = "" + policy.UID = "policy-recreated" policy.Status = dnsv1alpha1.DNSNamingPolicyStatus{} if err := cl.Create(ctx, policy); err != nil { t.Fatal(err) diff --git a/internal/internaldns/controlplane/naming.go b/internal/internaldns/controlplane/naming.go index af6aae30..72e15a99 100644 --- a/internal/internaldns/controlplane/naming.go +++ b/internal/internaldns/controlplane/naming.go @@ -26,27 +26,24 @@ type ResolvedName struct { // additive policy names after it. It never selects an arbitrary associated // customer zone. // Selectors match DNSRegistration metadata labels supplied by the publisher. -func ResolveNaming(ctx context.Context, c client.Client, namespace string, projectUID, vpcUID types.UID, registrationLabels map[string]string, allocatedName string) ([]ResolvedName, error) { +func ResolveNaming(ctx context.Context, c client.Client, namespace string, projectUID types.UID, contextName string, registrationLabels map[string]string, allocatedName string) ([]ResolvedName, error) { label := dnsLabel(allocatedName) if label == "" { return nil, fmt.Errorf("allocated name %q has no DNS-safe label", allocatedName) } - var managed dnsv1alpha1.DNSManagedNamespaceList - if err := c.List(ctx, &managed, client.InNamespace(namespace)); err != nil { + var resolver dnsv1alpha1.DNSResolverContext + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: contextName}, &resolver); err != nil { return nil, err } - var out []ResolvedName - for i := range managed.Items { - m := &managed.Items[i] - accepted := apimeta.FindStatusCondition(m.Status.Conditions, dnsValueAccepted) - if m.Spec.ProjectUID == projectUID && m.Spec.VPCRef.UID == vpcUID && accepted != nil && accepted.Status == metav1.ConditionTrue { - out = append(out, ResolvedName{DNSZoneRef: m.Status.DNSZoneRef, Name: label, Canonical: true}) - break - } + if !resolver.DeletionTimestamp.IsZero() || !apimeta.IsStatusConditionTrue(resolver.Status.Conditions, "Ready") { + return nil, fmt.Errorf("managed DNS namespace for context %s is not ready", contextName) } - if len(out) == 0 { - return nil, fmt.Errorf("managed DNS namespace for VPC %s is not ready", vpcUID) + var canonical dnsv1alpha1.DNSZone + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: resolver.Status.ManagedNamespace.DNSZoneRef.Name}, &canonical); err != nil { + return nil, err } + out := []ResolvedName{{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: canonical.Name, UID: canonical.UID}, Name: label, Canonical: true}} + r := &Reconciler{Client: c, Options: Options{ProjectUID: projectUID}} seen := map[string]bool{string(out[0].DNSZoneRef.UID) + "\x00" + out[0].Name: true} var policies dnsv1alpha1.DNSNamingPolicyList if err := c.List(ctx, &policies, client.InNamespace(namespace)); err != nil { @@ -61,7 +58,14 @@ func ResolveNaming(ctx context.Context, c client.Client, namespace string, proje for i := range policies.Items { p := &policies.Items[i] accepted := apimeta.FindStatusCondition(p.Status.Conditions, dnsValueAccepted) - if p.Status.ResolvedVPCRef.UID != vpcUID || !p.DeletionTimestamp.IsZero() || accepted == nil || accepted.Status != metav1.ConditionTrue || accepted.ObservedGeneration != p.Generation { + if p.Status.ResolvedResolverContextRef.Name != contextName || !p.DeletionTimestamp.IsZero() || accepted == nil || accepted.Status != metav1.ConditionTrue || accepted.ObservedGeneration != p.Generation { + continue + } + pins, err := r.readReferences(ctx, p) + if err != nil { + return nil, err + } + if pins["context"].UID != resolver.UID { continue } for _, rule := range p.Status.ResolvedAdditionalNames { @@ -70,9 +74,17 @@ func ResolveNaming(ctx context.Context, c client.Client, namespace string, proje return nil, fmt.Errorf("naming policy %s has an invalid selector: %w", p.Name, err) } name := label + "." + canonicalName(rule.NamePrefix) - key := string(rule.DNSZoneRef.UID) + "\x00" + name + zoneRef := pins["zone/"+rule.DNSZoneRef.Name] + var zone dnsv1alpha1.DNSZone + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: zoneRef.Name}, &zone); err != nil { + return nil, err + } + if !refMatches(zoneRef, zone.Name, zone.UID, zone.Generation) { + continue + } + key := string(zoneRef.UID) + "\x00" + name if selector.Matches(labels.Set(registrationLabels)) && !seen[key] { - out = append(out, ResolvedName{DNSZoneRef: rule.DNSZoneRef, Name: name}) + out = append(out, ResolvedName{DNSZoneRef: zoneRef, Name: name}) seen[key] = true } } diff --git a/internal/internaldns/controlplane/naming_selector_test.go b/internal/internaldns/controlplane/naming_selector_test.go index d1e46bdc..b8df2b13 100644 --- a/internal/internaldns/controlplane/naming_selector_test.go +++ b/internal/internaldns/controlplane/naming_selector_test.go @@ -18,13 +18,15 @@ import ( func TestNamingSelectorsMatchRegistrationLabelsWithinContext(t *testing.T) { scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } - managed := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project"}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ProjectUID: "project-a", VPCRef: dnsv1alpha1.DNSObjectReference{UID: "consumer-a"}}, Status: dnsv1alpha1.DNSManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "canonical", UID: "canonical-zone"}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} - zone := dnsv1alpha1.DNSObjectReference{Name: "custom", UID: "custom-zone"} + resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "consumer-a", Namespace: "project", UID: "context-a"}, Status: dnsv1alpha1.DNSResolverContextStatus{ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "canonical"}}, Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}}}} + canonical := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "canonical", Namespace: "project", UID: "canonical-zone"}} + custom := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project", UID: "custom-zone"}} + zone := dnsv1alpha1.DNSNameReference{Name: custom.Name} storefront := &metav1.LabelSelector{MatchLabels: map[string]string{"app": "storefront"}} - policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "names", Namespace: "project", Generation: 1}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedVPCRef: dnsv1alpha1.DNSObjectReference{UID: "consumer-a"}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue, ObservedGeneration: 1}}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{ + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "names", Namespace: "project", UID: "names-uid", Generation: 1}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue, ObservedGeneration: 1}}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{ {DNSZoneRef: zone, NamePrefix: "all"}, {Selector: &metav1.LabelSelector{}, DNSZoneRef: zone, NamePrefix: "empty"}, {Selector: storefront, DNSZoneRef: zone, NamePrefix: "storefront"}, @@ -33,7 +35,7 @@ func TestNamingSelectorsMatchRegistrationLabelsWithinContext(t *testing.T) { }}} other := policy.DeepCopy() other.Name = "other-context" - other.Status.ResolvedVPCRef.UID = "consumer-b" + other.Status.ResolvedResolverContextRef.Name = "consumer-b" other.Status.ResolvedAdditionalNames = []dnsv1alpha1.DNSResolvedAdditionalNameRule{{DNSZoneRef: zone, NamePrefix: "other-context"}} stale := policy.DeepCopy() stale.Name = "stale-policy" @@ -43,7 +45,9 @@ func TestNamingSelectorsMatchRegistrationLabelsWithinContext(t *testing.T) { foreign.Name = "other-namespace" foreign.Namespace = "other-project" foreign.Status.ResolvedAdditionalNames = []dnsv1alpha1.DNSResolvedAdditionalNameRule{{DNSZoneRef: zone, NamePrefix: "foreign"}} - cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(managed, policy, other, stale, foreign).Build() + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver, canonical, custom, policy, other, stale, foreign).Build() + r := &Reconciler{Client: cl, Options: Options{ProjectUID: "project-a"}} + seedReferenceState(t, r, policy, map[string]dnsv1alpha1.DNSObjectReference{"context": {Name: resolver.Name, UID: resolver.UID}, "zone/custom": {Name: custom.Name, UID: custom.UID}}) for _, tc := range []struct { name string labels map[string]string @@ -92,13 +96,13 @@ func TestNamingPolicyValidatesSelectorsAndDuplicateRules(t *testing.T) { ctx := context.Background() now := time.Now() scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "context", Namespace: "project", UID: "context-uid", Generation: 1}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer"}} setCondition(&resolver.Status.Conditions, "Ready", metav1.ConditionTrue, "Ready", "", 1, now) zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", UID: "zone-uid", Generation: 1}, Spec: dnsv1alpha1.DNSZoneSpec{Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - association := dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Generation: 1}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSObjectReference{UID: zone.UID}, ResolvedResolverContextRef: dnsv1alpha1.DNSObjectReference{UID: resolver.UID}}} + association := dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: "association", Namespace: "project", UID: "association-uid", Generation: 1}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}}} setCondition(&association.Status.Conditions, "Accepted", metav1.ConditionTrue, "Accepted", "", 1, now) for _, tc := range []struct { name string @@ -121,6 +125,7 @@ func TestNamingPolicyValidatesSelectorsAndDuplicateRules(t *testing.T) { } cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(resolver, policy).WithStatusSubresource(policy).Build() r := &Reconciler{Client: cl} + seedReferenceState(t, r, &association, map[string]dnsv1alpha1.DNSObjectReference{"context": {Name: resolver.Name, UID: resolver.UID}, "zone": {Name: zone.Name, UID: zone.UID, Generation: zone.Generation}}) if err := r.reconcileNamingPolicies(ctx, []dnsv1alpha1.DNSNamingPolicy{*policy}, []dnsv1alpha1.DNSZoneAssociation{association}, map[string]*dnsv1alpha1.DNSZone{zone.Name: zone}, now); err != nil { t.Fatal(err) } @@ -131,7 +136,8 @@ func TestNamingPolicyValidatesSelectorsAndDuplicateRules(t *testing.T) { if condition == nil || (condition.Status == metav1.ConditionTrue) != tc.valid { t.Fatalf("accepted=%v want %v", condition, tc.valid) } - if policy.Status.ResolvedAdditionalNames[0].DNSZoneRef.UID != zone.UID { + pins, err := r.readReferences(ctx, policy) + if err != nil || pins["zone/"+zone.Name].UID != zone.UID { t.Fatal("selector validation lost the zone lifetime pin") } }) diff --git a/internal/internaldns/controlplane/naming_test.go b/internal/internaldns/controlplane/naming_test.go index 1e00f3e1..5cb1aef8 100644 --- a/internal/internaldns/controlplane/naming_test.go +++ b/internal/internaldns/controlplane/naming_test.go @@ -12,13 +12,17 @@ import ( func TestResolveNamingReturnsCanonicalThenAuthorizedAliases(t *testing.T) { s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } - managed := &dnsv1alpha1.DNSManagedNamespace{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "project"}, Spec: dnsv1alpha1.DNSManagedNamespaceSpec{ProjectUID: "project-a", VPCRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}}, Status: dnsv1alpha1.DNSManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "canonical", UID: "zone-c"}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} - policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project"}, Spec: dnsv1alpha1.DNSNamingPolicySpec{Priority: 10}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedVPCRef: dnsv1alpha1.DNSObjectReference{Name: "prod", UID: "vpc-a"}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: "custom", UID: "zone-x"}, NamePrefix: "instances"}}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} - c := fake.NewClientBuilder().WithScheme(s).WithObjects(managed, policy).Build() - names, err := ResolveNaming(context.Background(), c, "project", "project-a", "vpc-a", nil, "Web 01") + resolver := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "project", UID: "context-a"}, Status: dnsv1alpha1.DNSResolverContextStatus{ManagedNamespace: dnsv1alpha1.DNSResolverContextManagedNamespaceStatus{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "canonical"}}, Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue}}}} + canonical := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "canonical", Namespace: "project", UID: "zone-c"}} + custom := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project", UID: "zone-x"}} + policy := &dnsv1alpha1.DNSNamingPolicy{ObjectMeta: metav1.ObjectMeta{Name: "custom", Namespace: "project", UID: "policy-a"}, Spec: dnsv1alpha1.DNSNamingPolicySpec{Priority: 10}, Status: dnsv1alpha1.DNSNamingPolicyStatus{ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: resolver.Name}, ResolvedAdditionalNames: []dnsv1alpha1.DNSResolvedAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: "custom"}, NamePrefix: "instances"}}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(resolver, canonical, custom, policy).Build() + r := &Reconciler{Client: c, Options: Options{ProjectUID: "project-a"}} + seedReferenceState(t, r, policy, map[string]dnsv1alpha1.DNSObjectReference{"context": {Name: resolver.Name, UID: resolver.UID}, "zone/custom": {Name: custom.Name, UID: custom.UID}}) + names, err := ResolveNaming(context.Background(), c, "project", "project-a", resolver.Name, nil, "Web 01") if err != nil { t.Fatal(err) } diff --git a/internal/internaldns/controlplane/publication_reconciler.go b/internal/internaldns/controlplane/publication_reconciler.go index 7214859c..06b8c344 100644 --- a/internal/internaldns/controlplane/publication_reconciler.go +++ b/internal/internaldns/controlplane/publication_reconciler.go @@ -57,6 +57,27 @@ func (r *Reconciler) reconcileProject(ctx context.Context, ns string) (ctrl.Resu if err := r.reconcileNamingPolicies(ctx, policies.Items, assocs.Items, zoneByName, now); err != nil { return ctrl.Result{}, err } + var sources []client.Object + for i := range assocs.Items { + sources = append(sources, &assocs.Items[i]) + } + for i := range policies.Items { + sources = append(sources, &policies.Items[i]) + } + for i := range regs.Items { + registration := ®s.Items[i] + ref := dnsv1alpha1.DNSObjectReference{Name: registration.Spec.DNSZoneRef.Name} + if zone := zoneByName[ref.Name]; zone != nil && zone.DeletionTimestamp.IsZero() && zone.Spec.Visibility == dnsv1alpha1.DNSZoneVisibilityPrivate { + ref.UID, ref.Generation = zone.UID, zone.Generation + } + if _, _, err := r.pinReferences(ctx, registration, map[string]dnsv1alpha1.DNSObjectReference{"zone": ref}); err != nil { + return ctrl.Result{}, err + } + sources = append(sources, registration) + } + if err := r.collectReferenceStates(ctx, sources); err != nil { + return ctrl.Result{}, err + } if changed, err := r.issueGrants(ctx, grants.Items, regs.Items, now); err != nil || changed { return ctrl.Result{Requeue: changed}, err } @@ -266,6 +287,18 @@ func (r *Reconciler) compileZone(ctx context.Context, z *dnsv1alpha1.DNSZone, as return false, time.Time{}, err } input := CompileInput{Zone: z, Associations: assocs, Registrations: regs, Grants: grants, Contributions: cs, StaticRecords: records, Previous: previous, Now: now} + input.AssociationReferences, err = r.associationReferences(ctx, assocs) + if err != nil { + return false, time.Time{}, err + } + input.RegistrationZoneReferences = map[types.UID]dnsv1alpha1.DNSObjectReference{} + for i := range regs { + refs, err := r.readReferences(ctx, ®s[i]) + if err != nil { + return false, time.Time{}, err + } + input.RegistrationZoneReferences[regs[i].UID] = refs["zone"] + } tombstone := !z.DeletionTimestamp.IsZero() var payload []byte var ownPlan PublicationPlan diff --git a/internal/internaldns/controlplane/publication_reconciler_test.go b/internal/internaldns/controlplane/publication_reconciler_test.go index 596da265..bccc40c8 100644 --- a/internal/internaldns/controlplane/publication_reconciler_test.go +++ b/internal/internaldns/controlplane/publication_reconciler_test.go @@ -23,7 +23,7 @@ func TestExpiredOwnershipTakeoverFencesResumedWriter(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "zone", Namespace: "project", UID: "zone-a"}} @@ -73,7 +73,7 @@ func TestQuiescentSurvivorRenewsOwnershipBeyondLeaseWindow(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 7, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } network := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "project", UID: "vpc-a"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-a"}} @@ -153,7 +153,7 @@ func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } payload, err := json.Marshal(model.PublicationPlan{ZoneUID: "zone-a", Apex: "corp.internal", ObservationFences: []model.ObservationFence{{ContributionUID: "contribution-a", GrantUID: "grant-a", WriterEpoch: 7, Sequence: 19}}, RRSets: []model.RRSet{}}) @@ -210,7 +210,7 @@ func TestOwnershipTakeoverLoadsOldEpochHighWater(t *testing.T) { func TestLoadPreviousFailsWhenActiveChunkIsMissing(t *testing.T) { ctx := context.Background() s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } manifest := &dnsv1alpha1.DNSPublicationManifest{ @@ -231,7 +231,7 @@ func TestLoadPreviousFailsWhenActiveChunkIsMissing(t *testing.T) { func TestLoadPreviousRejectsHashValidMalformedPayload(t *testing.T) { ctx := context.Background() s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } payload := []byte(`{"zoneUID":`) @@ -250,7 +250,7 @@ func TestLoadPreviousRejectsHashValidMalformedPayload(t *testing.T) { func TestEmptyPublicationUsesJSONArraysForRequiredSlices(t *testing.T) { ctx := context.Background() s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } c := fake.NewClientBuilder().WithScheme(s).Build() @@ -276,7 +276,7 @@ func TestReconcilerPersistsFencedMultiRegionOutbox(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) s := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(s); err != nil { + if err := addReferenceStateToScheme(s); err != nil { t.Fatal(err) } network := &dnsv1alpha1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: "project", UID: "vpc-a"}, Spec: dnsv1alpha1.DNSResolverContextSpec{ConsumerID: "consumer-a"}} diff --git a/internal/internaldns/controlplane/publication_regression_test.go b/internal/internaldns/controlplane/publication_regression_test.go index fc013e93..819f6bc5 100644 --- a/internal/internaldns/controlplane/publication_regression_test.go +++ b/internal/internaldns/controlplane/publication_regression_test.go @@ -57,7 +57,7 @@ func TestCommittedUntargetedTombstoneCompletes(t *testing.T) { ctx := context.Background() now := time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } cl := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&dnsv1alpha1.DNSPublicationManifest{}).Build() @@ -90,7 +90,7 @@ func TestCommittedUntargetedTombstoneCompletes(t *testing.T) { func TestTerminatingAuthorityDoesNotIssueOrBindWriterEpochs(t *testing.T) { now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) scheme := runtime.NewScheme() - if err := dnsv1alpha1.AddToScheme(scheme); err != nil { + if err := addReferenceStateToScheme(scheme); err != nil { t.Fatal(err) } for _, terminating := range []string{"registration", "grant"} { diff --git a/internal/internaldns/controlplane/reconciler.go b/internal/internaldns/controlplane/reconciler.go index 02cfdc39..6d621f2a 100644 --- a/internal/internaldns/controlplane/reconciler.go +++ b/internal/internaldns/controlplane/reconciler.go @@ -48,7 +48,7 @@ type AddressAllocator interface { func (o *ReconcilerOptions) defaults() { if o.PlatformNamespace == "" { - o.PlatformNamespace = "internal-dns-system" + o.PlatformNamespace = defaultPlatformNamespace } if o.Region == "" { o.Region = "default" diff --git a/internal/internaldns/controlplane/reference_state.go b/internal/internaldns/controlplane/reference_state.go new file mode 100644 index 00000000..6d66d24f --- /dev/null +++ b/internal/internaldns/controlplane/reference_state.go @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/model" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const referenceStateLabel = "internal-dns.miloapis.com/reference-state" +const referenceProjectLabel = "internal-dns.miloapis.com/project-uid" +const referenceStateDataKey = "references.json" +const defaultPlatformNamespace = "internal-dns-system" + +type referenceState struct { + SourceNamespace string `json:"sourceNamespace"` + SourceKind string `json:"sourceKind"` + Source dnsv1.DNSObjectReference `json:"source"` + References map[string]dnsv1.DNSObjectReference `json:"references"` +} + +func referenceKind(source client.Object) string { + switch source.(type) { + case *dnsv1.DNSZoneAssociation: + return "DNSZoneAssociation" + case *dnsv1.DNSNamingPolicy: + return "DNSNamingPolicy" + case *dnsv1.DNSRegistration: + return "DNSRegistration" + default: + return "" + } +} + +func (r *Reconciler) referenceStateObject(source client.Object) *corev1.ConfigMap { + namespace := r.Options.PlatformNamespace + if namespace == "" { + namespace = defaultPlatformNamespace + } + identity := fmt.Sprintf("%s/%s/%s/%s/%s", r.Options.ProjectUID, source.GetNamespace(), referenceKind(source), source.GetName(), source.GetUID()) + return &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, Name: "references-" + model.OpaqueToken(identity)[:32], + Labels: map[string]string{referenceStateLabel: "true", referenceProjectLabel: string(r.Options.ProjectUID)}, + }} +} + +func (r *Reconciler) readReferences(ctx context.Context, source client.Object) (map[string]dnsv1.DNSObjectReference, error) { + _, state, err := r.loadReferenceState(ctx, source) + return state.References, err +} + +func (r *Reconciler) loadReferenceState(ctx context.Context, source client.Object) (*corev1.ConfigMap, referenceState, error) { + object := r.referenceStateObject(source) + state := referenceState{SourceKind: referenceKind(source), SourceNamespace: source.GetNamespace(), Source: dnsv1.DNSObjectReference{Name: source.GetName(), UID: source.GetUID()}, References: map[string]dnsv1.DNSObjectReference{}} + if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + return object, state, client.IgnoreNotFound(err) + } + if err := json.Unmarshal([]byte(object.Data[referenceStateDataKey]), &state); err != nil { + return object, state, fmt.Errorf("decode DNS reference state: %w", err) + } + if state.SourceKind != referenceKind(source) || state.SourceNamespace != source.GetNamespace() || state.Source.Name != source.GetName() || state.Source.UID != source.GetUID() { + return object, state, fmt.Errorf("DNS reference state source does not match") + } + return object, state, nil +} + +func (r *Reconciler) pinReferences(ctx context.Context, source client.Object, selected map[string]dnsv1.DNSObjectReference) (map[string]dnsv1.DNSObjectReference, bool, error) { + object, state, err := r.loadReferenceState(ctx, source) + if err != nil { + return nil, false, err + } + current := true + resolved := map[string]dnsv1.DNSObjectReference{} + for key, ref := range selected { + pin, valid := pinNamedReference(ref.Name, state.References[key], ref) + resolved[key] = pin + current = current && valid + } + if reflect.DeepEqual(state.References, resolved) { + return resolved, current, nil + } + state.References = resolved + payload, err := json.Marshal(state) + if err != nil { + return nil, false, err + } + if object.ResourceVersion == "" { + object.Data = map[string]string{referenceStateDataKey: string(payload)} + err = r.Create(ctx, object) + } else { + base := object.DeepCopy() + object.Data[referenceStateDataKey] = string(payload) + err = r.Patch(ctx, object, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) + } + return resolved, current, err +} + +func (r *Reconciler) collectReferenceStates(ctx context.Context, sources []client.Object) error { + live := map[string]bool{} + for _, source := range sources { + live[r.referenceStateObject(source).Name] = true + } + var states corev1.ConfigMapList + namespace := r.Options.PlatformNamespace + if namespace == "" { + namespace = defaultPlatformNamespace + } + if err := r.List(ctx, &states, client.InNamespace(namespace), client.MatchingLabels{referenceStateLabel: "true", referenceProjectLabel: string(r.Options.ProjectUID)}); err != nil { + return err + } + for i := range states.Items { + if !live[states.Items[i].Name] { + var state referenceState + if err := json.Unmarshal([]byte(states.Items[i].Data[referenceStateDataKey]), &state); err != nil { + return err + } + var source client.Object + switch state.SourceKind { + case "DNSZoneAssociation": + source = &dnsv1.DNSZoneAssociation{} + case "DNSNamingPolicy": + source = &dnsv1.DNSNamingPolicy{} + case "DNSRegistration": + source = &dnsv1.DNSRegistration{} + default: + return fmt.Errorf("unknown DNS reference source kind %q", state.SourceKind) + } + err := r.Get(ctx, client.ObjectKey{Namespace: state.SourceNamespace, Name: state.Source.Name}, source) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && source.GetUID() == state.Source.UID { + continue + } + if err := r.Delete(ctx, &states.Items[i], client.Preconditions{UID: &states.Items[i].UID}); client.IgnoreNotFound(err) != nil { + return err + } + } + } + return nil +} + +type associationReferences struct { + Zone dnsv1.DNSObjectReference + Context dnsv1.DNSObjectReference +} + +func (r *Reconciler) associationReferences(ctx context.Context, associations []dnsv1.DNSZoneAssociation) (map[types.UID]associationReferences, error) { + result := map[types.UID]associationReferences{} + for i := range associations { + refs, err := r.readReferences(ctx, &associations[i]) + if err != nil { + return nil, err + } + result[associations[i].UID] = associationReferences{Zone: refs["zone"], Context: refs["context"]} + } + return result, nil +} diff --git a/internal/internaldns/controlplane/reference_state_test.go b/internal/internaldns/controlplane/reference_state_test.go new file mode 100644 index 00000000..0709106d --- /dev/null +++ b/internal/internaldns/controlplane/reference_state_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controlplane + +import ( + "context" + "testing" + + dnsv1alpha1 "go.miloapis.com/dns-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func addReferenceStateToScheme(scheme *runtime.Scheme) error { + if err := corev1.AddToScheme(scheme); err != nil { + return err + } + return dnsv1alpha1.AddToScheme(scheme) +} + +func seedReferenceState(t *testing.T, r *Reconciler, source client.Object, selected map[string]dnsv1alpha1.DNSObjectReference) { + t.Helper() + if _, valid, err := r.pinReferences(context.Background(), source, selected); err != nil || !valid { + t.Fatalf("reference state: valid=%t err=%v", valid, err) + } +} + +func TestReferenceStateIsPrivateScopedAndCollected(t *testing.T) { + scheme := runtime.NewScheme() + if err := addReferenceStateToScheme(scheme); err != nil { + t.Fatal(err) + } + ctx := context.Background() + cl := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &Reconciler{Client: cl, Options: Options{ProjectUID: "project-a", PlatformNamespace: "platform"}} + other := &Reconciler{Client: cl, Options: Options{ProjectUID: "project-b", PlatformNamespace: "platform"}} + source := &dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "project", UID: "registration-original"}} + selected := map[string]dnsv1alpha1.DNSObjectReference{"zone": {Name: "private", UID: "zone-original", Generation: 1}} + if _, valid, err := r.pinReferences(ctx, source, selected); err != nil || !valid { + t.Fatalf("initial reference: valid=%t err=%v", valid, err) + } + if _, valid, err := other.pinReferences(ctx, source, selected); err != nil || !valid { + t.Fatalf("other project reference: valid=%t err=%v", valid, err) + } + selected["zone"] = dnsv1alpha1.DNSObjectReference{Name: "private", UID: "zone-replacement", Generation: 1} + pins, valid, err := r.pinReferences(ctx, source, selected) + if err != nil || valid || pins["zone"].UID != "zone-original" { + t.Fatalf("replacement transferred a resolved lifetime: pins=%v valid=%t err=%v", pins, valid, err) + } + var public corev1.ConfigMapList + if err := cl.List(ctx, &public, client.InNamespace(source.Namespace)); err != nil || len(public.Items) != 0 { + t.Fatalf("reference state appeared in the project: count=%d err=%v", len(public.Items), err) + } + if err := r.collectReferenceStates(ctx, nil); err != nil { + t.Fatal(err) + } + var states corev1.ConfigMapList + if err := cl.List(ctx, &states, client.InNamespace("platform")); err != nil || len(states.Items) != 1 || states.Items[0].Labels[referenceProjectLabel] != "project-b" { + t.Fatalf("cleanup removed another project's state: states=%v err=%v", states.Items, err) + } + if err := cl.Create(ctx, source); err != nil { + t.Fatal(err) + } + if err := other.collectReferenceStates(ctx, nil); err != nil { + t.Fatal(err) + } + if err := cl.List(ctx, &states, client.InNamespace("platform")); err != nil || len(states.Items) != 1 { + t.Fatalf("collector removed a live source absent from the list snapshot: count=%d err=%v", len(states.Items), err) + } + if err := cl.Delete(ctx, source); err != nil { + t.Fatal(err) + } + if err := other.collectReferenceStates(ctx, nil); err != nil { + t.Fatal(err) + } + if err := cl.List(ctx, &states, client.InNamespace("platform")); err != nil || len(states.Items) != 0 { + t.Fatalf("deleted source retained private state: count=%d err=%v", len(states.Items), err) + } +} diff --git a/internal/internaldns/platform/acks.go b/internal/internaldns/platform/acks.go index 1c35e910..f7fdb945 100644 --- a/internal/internaldns/platform/acks.go +++ b/internal/internaldns/platform/acks.go @@ -265,7 +265,7 @@ func (s *AckSink) projectPublicationForScope(ctx context.Context, project Projec if registrationGenerations[string(r.UID)] != uint64(r.Generation) { continue } - if r.Spec.DNSZoneRef.Name != z.Name || r.Spec.DNSZoneRef.UID != "" && r.Spec.DNSZoneRef.UID != z.UID { + if r.Spec.DNSZoneRef.Name != z.Name { continue } available := false diff --git a/internal/internaldns/platform/platform_test.go b/internal/internaldns/platform/platform_test.go index 62816583..3471f181 100644 --- a/internal/internaldns/platform/platform_test.go +++ b/internal/internaldns/platform/platform_test.go @@ -52,7 +52,7 @@ func TestProjectPublicationContinuesAfterSourceFailure(t *testing.T) { t.Fatal(err) } zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "healthy", Namespace: "healthy-project", UID: "zone-healthy", Generation: 4}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "healthy.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - registration := &dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: zone.Namespace, UID: "registration-healthy", Generation: 2}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID}, Name: "api"}} + registration := &dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: zone.Namespace, UID: "registration-healthy", Generation: 2}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api"}} healthy := fake.NewClientBuilder().WithScheme(scheme).WithObjects(zone, registration).WithStatusSubresource(&dnsv1alpha1.DNSZone{}, &dnsv1alpha1.DNSRegistration{}).Build() blocked := &blockingListClient{Client: healthy, started: make(chan struct{})} sink := AckSink{Client: testClient(t, chunk), Namespace: "platform", Projects: []ProjectScope{{Client: blocked, Namespace: "offline-project", RequestTimeout: 200 * time.Millisecond}, {Client: healthy, Namespace: zone.Namespace}}} diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index 1c1cefc8..9f89ecb7 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -110,8 +110,8 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { {"internal-dns-control-plane", "dnspublicationmanifests", "", "create", true}, {"product", "dnsresolvercontexts", "", "get", true}, {"product", "dnsresolvercontexts", "", "list", true}, - {"product", "dnsmanagednamespaces", "", "get", true}, - {"product", "dnsmanagednamespaces", "", "list", true}, + {"product", "dnsmanagednamespaces", "", "get", false}, + {"product", "dnsmanagednamespaces", "", "list", false}, {"product", "dnsmanagednamespaces", "", "patch", false}, {"product", "dnsmanagednamespaces", "status", "patch", false}, {"product", "dnsnamingpolicies", "", "get", true}, @@ -130,7 +130,7 @@ func TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries(t *testing.T) { {"customer", "dnszoneassociations", "", "create", true}, {"customer", "dnsnamingpolicies", "", "create", true}, {"customer", "dnsresolvercontexts", "", "get", true}, - {"customer", "dnsmanagednamespaces", "", "get", true}, + {"customer", "dnsmanagednamespaces", "", "get", false}, {"customer", "dnsresolvercontexts", "", "patch", false}, {"customer", "dnscontributiongrants", "", "create", false}, {"customer", "dnscontributiongrants", "status", "patch", false}, @@ -282,9 +282,9 @@ func verifyProjectReferenceSchemas(t *testing.T, ctx context.Context, cl client. object client.Object valid bool }{ - {"registration-valid", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, true}, - {"registration-unpinned", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSObjectReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, - {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: ref, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, + {"registration-name-only", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, true}, + {"registration-empty-zone", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 30, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, + {"registration-ttl31", &dnsv1alpha1.DNSRegistration{Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeA}, TTLSeconds: 31, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions}}, false}, {"association-valid", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}}}, true}, {"association-empty-zone", &dnsv1alpha1.DNSZoneAssociation{Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}}}, false}, {"naming-valid", &dnsv1alpha1.DNSNamingPolicy{Spec: dnsv1alpha1.DNSNamingPolicySpec{ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: contextRef.Name}, AdditionalNames: []dnsv1alpha1.DNSAdditionalNameRule{{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: ref.Name}, NamePrefix: "services"}}}}, true}, diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go index 1f6a5431..2c2a8951 100644 --- a/test/internaldns/kubernetes/qualify/main.go +++ b/test/internaldns/kubernetes/qualify/main.go @@ -288,7 +288,7 @@ func (s *suite) run(path string) error { } } a, b := &s.env.Projects[0], &s.env.Projects[1] - if a.resolverContext.UID == b.resolverContext.UID || a.zone.UID == b.zone.UID || a.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID == b.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID { + if a.resolverContext.UID == b.resolverContext.UID || a.zone.UID == b.zone.UID || a.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name == b.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name { return errors.New("source projects lost their distinct context, zone, or managed namespace identity") } for _, p := range []*project{a, b} { @@ -335,7 +335,7 @@ func (s *suite) prepare(p *project) error { } if err := s.wait(p.Name+" managed namespace", 120*time.Second, func() (bool, error) { err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&p.resolverContext), &p.resolverContext) - return condition(p.resolverContext.Status.Conditions, "Ready", p.resolverContext.Generation) && p.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID != "", err + return condition(p.resolverContext.Status.Conditions, "Ready", p.resolverContext.Generation) && p.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name != "", err }); err != nil { return err } @@ -394,7 +394,7 @@ func recordSets(name, address, ipv6 string) []dnsv1.DNSContributionRecordSet { return sets } -func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string) (*publication, error) { +func (s *suite) publish(p *project, zone dnsv1.DNSNameReference, prefix, name, address, ipv6 string) (*publication, error) { pub := &publication{registration: dnsv1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-registration", Namespace: p.Namespace}, Spec: dnsv1.DNSRegistrationSpec{DNSZoneRef: zone, Name: name, RecordTypes: []dnsv1.RRType{dnsv1.RRTypeA}, PublicationPolicy: dnsv1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 2}}} if ipv6 != "" { pub.registration.Spec.RecordTypes = append(pub.registration.Spec.RecordTypes, dnsv1.RRTypeAAAA) diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index d78235d3..538eb80f 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -24,11 +24,11 @@ func (s *suite) scenarios(a, b *project) error { return err } } - pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10") + pa, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "common", "common", "10.42.0.10", "fd42::10") if err != nil { return err } - pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20") + pb, err := s.publish(b, dnsv1.DNSNameReference{Name: b.zone.Name}, "common", "common", "10.42.0.20", "fd42::20") if err != nil { return err } @@ -75,7 +75,7 @@ func (s *suite) scenarios(a, b *project) error { return err } s.pass("an unbound destination cannot resolve either tenant's private records", nil) - if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", ""); err != nil { + if _, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "only-a", "only-a", "10.42.0.20", ""); err != nil { return err } if err := s.expect(a, "only-a.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { @@ -114,7 +114,7 @@ func (s *suite) scenarios(a, b *project) error { if err := s.associate(a, apps, "apps"); err != nil { return err } - if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", ""); err != nil { + if _, err := s.publish(a, dnsv1.DNSNameReference{Name: apps.Name}, "apps", "api", "10.42.0.12", ""); err != nil { return err } if err := s.expect(a, "api.apps.internal", "A", true, dns.RcodeSuccess, "10.42.0.12"); err != nil { @@ -165,7 +165,7 @@ func (s *suite) scenarios(a, b *project) error { if err := s.observe(b, pb, true); err != nil { return err } - deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "") + deleted, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "deleted", "deleted", "10.42.0.14", "") if err != nil { return err } From fc04a811c362c7978c1e95dab006ec4c6aaff803 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 08:23:23 -0500 Subject: [PATCH 14/15] fix: Validate resolved registration readiness --- internal/internaldns/controlplane/compiler.go | 6 +++++- internal/internaldns/controlplane/compiler_test.go | 12 ++++++++++++ .../controlplane/publication_reconciler.go | 10 ++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/internal/internaldns/controlplane/compiler.go b/internal/internaldns/controlplane/compiler.go index 6c3317f7..b7dde702 100644 --- a/internal/internaldns/controlplane/compiler.go +++ b/internal/internaldns/controlplane/compiler.go @@ -108,13 +108,17 @@ func Compile(in CompileInput) (CompileResult, error) { result.Reasons[reg.UID] = "InvalidTTL" continue } - if reg.Spec.DNSZoneRef.Name != in.Zone.Name || !refMatches(in.RegistrationZoneReferences[reg.UID], in.Zone.Name, in.Zone.UID, in.Zone.Generation) { + if reg.Spec.DNSZoneRef.Name != in.Zone.Name { continue } if reg.UID == "" { result.Reasons[reg.UID] = "MissingUID" continue } + if !refMatches(in.RegistrationZoneReferences[reg.UID], in.Zone.Name, in.Zone.UID, in.Zone.Generation) { + result.Reasons[reg.UID] = "InvalidReference" + continue + } name := zoneOwner(reg.Spec.Name, result.Plan.ZoneApex) valid := true for _, rt := range reg.Spec.RecordTypes { diff --git a/internal/internaldns/controlplane/compiler_test.go b/internal/internaldns/controlplane/compiler_test.go index 7ae21f78..2d403d14 100644 --- a/internal/internaldns/controlplane/compiler_test.go +++ b/internal/internaldns/controlplane/compiler_test.go @@ -203,6 +203,18 @@ func structLocal(name string) corev1.LocalObjectReference { return corev1.LocalObjectReference{Name: name} } +func TestCompileRejectsReplacementBehindANameOnlyReference(t *testing.T) { + in := fixture(time.Now().UTC()) + in.Zone.UID = "replacement-zone" + result, err := Compile(in) + if err != nil { + t.Fatal(err) + } + if result.Accepted[in.Registrations[0].UID] || result.Reasons[in.Registrations[0].UID] != "InvalidReference" || len(result.Plan.RRsets) != 0 { + t.Fatalf("replacement reused a registration's private reference: %#v", result) + } +} + func TestCompileRejectsPrivateAuthorityOverridesWithoutBlockingHealthyRecords(t *testing.T) { for _, tt := range []struct { name string diff --git a/internal/internaldns/controlplane/publication_reconciler.go b/internal/internaldns/controlplane/publication_reconciler.go index 06b8c344..1e947eb4 100644 --- a/internal/internaldns/controlplane/publication_reconciler.go +++ b/internal/internaldns/controlplane/publication_reconciler.go @@ -167,7 +167,7 @@ func (r *Reconciler) aggregatePublicationStatus(ctx context.Context, z *dnsv1alp } for i := range regs { reg := ®s[i] - if !refMatches(reg.Spec.DNSZoneRef, z.Name, z.UID, z.Generation) || registrationFences[reg.UID] != reg.Generation { + if reg.Spec.DNSZoneRef.Name != z.Name || registrationFences[reg.UID] != reg.Generation { continue } regUIDs[reg.UID] = true @@ -590,7 +590,7 @@ func (r *Reconciler) patchInputStatuses(ctx context.Context, zone *dnsv1alpha1.D relevant := map[types.UID]bool{} for i := range regs { reg := ®s[i] - if !refMatches(reg.Spec.DNSZoneRef, zone.Name, zone.UID, zone.Generation) { + if reg.Spec.DNSZoneRef.Name != zone.Name { continue } relevant[reg.UID] = true @@ -601,6 +601,12 @@ func (r *Reconciler) patchInputStatuses(ctx context.Context, zone *dnsv1alpha1.D } base := reg.DeepCopy() reg.Status.CanonicalFQDN = absoluteOwner(reg.Spec.Name, result.Plan.ZoneApex) + if !accepted { + reg.Status.CanonicalFQDN = "" + reg.Status.PublicationWriterEpoch = 0 + reg.Status.PublicationRevision = 0 + projectstatus.Published(®.Status.Conditions, false, reg.Generation, now) + } reg.Status.ObservedGeneration = reg.Generation setCondition(®.Status.Conditions, "Accepted", boolCondition(accepted), reason, reason, reg.Generation, now) if err := r.Status().Patch(ctx, reg, client.MergeFrom(base)); err != nil && !apierrors.IsNotFound(err) { From dabfc3fd9692b79b29372af29dea13f6165f76cf Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 08:29:19 -0500 Subject: [PATCH 15/15] fix: Use bounded DNS reference record names --- config/internal-dns/README.md | 6 +++++- internal/internaldns/controlplane/compiler_test.go | 4 ++-- internal/internaldns/controlplane/reference_state.go | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index bd9e3eec..3008d094 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -168,8 +168,12 @@ Publication-result updates cannot change a publisher's observation. Resource permissions do not depend on webhook principal lists. ## Rollout and recovery +Project APIs select zones by name and report names and readiness. DNS stores +resolved ownership identities in reference ConfigMaps in the service project. +The collector checks source resources before removing those records. + The supported envelope version is `v1alpha4` and checkpoint format is 5. Keep -transport state and persistent checkpoints across compatible upgrades. Deploy +reference records, transport state, and persistent checkpoints across compatible upgrades. Deploy serving agents before controllers that emit per-zone pending state. Enable the controller change after every serving member supports that contract; a matching wire version alone does not establish feature compatibility. diff --git a/internal/internaldns/controlplane/compiler_test.go b/internal/internaldns/controlplane/compiler_test.go index 2d403d14..278103d7 100644 --- a/internal/internaldns/controlplane/compiler_test.go +++ b/internal/internaldns/controlplane/compiler_test.go @@ -186,14 +186,14 @@ func TestCompileSRVTargetUsesAbsoluteZoneIdentity(t *testing.T) { func fixture(now time.Time) CompileInput { zone := &dnsv1alpha1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "private", UID: "zone-a", Generation: 2}, Spec: dnsv1alpha1.DNSZoneSpec{DomainName: "corp.internal", Visibility: dnsv1alpha1.DNSZoneVisibilityPrivate}} - assoc := func(name string, vpc types.UID) dnsv1alpha1.DNSZoneAssociation { + assoc := func(name string) dnsv1alpha1.DNSZoneAssociation { return dnsv1alpha1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name)}, Spec: dnsv1alpha1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1alpha1.DNSNameReference{Name: name}}, Status: dnsv1alpha1.DNSZoneAssociationStatus{ResolvedDNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, ResolvedResolverContextRef: dnsv1alpha1.DNSNameReference{Name: name}, Conditions: []metav1.Condition{{Type: "Accepted", Status: metav1.ConditionTrue}}}} } reg := dnsv1alpha1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: "api", UID: "registration-a", Generation: 3}, Spec: dnsv1alpha1.DNSRegistrationSpec{DNSZoneRef: dnsv1alpha1.DNSNameReference{Name: zone.Name}, Name: "api", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}, PublicationPolicy: dnsv1alpha1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 30}} grant := dnsv1alpha1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: "compute", UID: "grant-a", Generation: 2}, Spec: dnsv1alpha1.DNSContributionGrantSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, ProducerID: "compute-east", RecordTypes: []dnsv1alpha1.RRType{dnsv1alpha1.RRTypeAAAA}}, Status: dnsv1alpha1.DNSContributionGrantStatus{ActiveWriterEpoch: 7, ObservedGrantGeneration: 2, ObservedRegistrationGeneration: 3}} c := dnsv1alpha1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint-a", UID: "contribution-a", Generation: 4}, Spec: dnsv1alpha1.DNSRecordContributionSpec{RegistrationRef: dnsv1alpha1.DNSObjectReference{Name: reg.Name, UID: reg.UID, Generation: reg.Generation}, GrantRef: dnsv1alpha1.DNSObjectReference{Name: grant.Name, UID: grant.UID}, RecordSets: []dnsv1alpha1.DNSContributionRecordSet{{RecordType: dnsv1alpha1.RRTypeAAAA, Records: []dnsv1alpha1.RecordEntry{{Name: "api.corp.internal", AAAA: &dnsv1alpha1.AAAARecordSpec{Content: "fd00::1"}}}}}}, Status: dnsv1alpha1.DNSRecordContributionStatus{ObservedGeneration: 4, WriterEpoch: 7, Sequence: 18, Eligible: true}} zoneRef := dnsv1alpha1.DNSObjectReference{Name: zone.Name, UID: zone.UID, Generation: zone.Generation} - return CompileInput{Zone: zone, Associations: []dnsv1alpha1.DNSZoneAssociation{assoc("b", "vpc-b"), assoc("a", "vpc-a")}, AssociationReferences: map[types.UID]associationReferences{ + return CompileInput{Zone: zone, Associations: []dnsv1alpha1.DNSZoneAssociation{assoc("b"), assoc("a")}, AssociationReferences: map[types.UID]associationReferences{ "a": {Zone: zoneRef, Context: dnsv1alpha1.DNSObjectReference{Name: "a", UID: "vpc-a"}}, "b": {Zone: zoneRef, Context: dnsv1alpha1.DNSObjectReference{Name: "b", UID: "vpc-b"}}, }, RegistrationZoneReferences: map[types.UID]dnsv1alpha1.DNSObjectReference{reg.UID: zoneRef}, Registrations: []dnsv1alpha1.DNSRegistration{reg}, Grants: []dnsv1alpha1.DNSContributionGrant{grant}, Contributions: []dnsv1alpha1.DNSRecordContribution{c}, Now: now} diff --git a/internal/internaldns/controlplane/reference_state.go b/internal/internaldns/controlplane/reference_state.go index 6d66d24f..56a44ba0 100644 --- a/internal/internaldns/controlplane/reference_state.go +++ b/internal/internaldns/controlplane/reference_state.go @@ -49,7 +49,7 @@ func (r *Reconciler) referenceStateObject(source client.Object) *corev1.ConfigMa } identity := fmt.Sprintf("%s/%s/%s/%s/%s", r.Options.ProjectUID, source.GetNamespace(), referenceKind(source), source.GetName(), source.GetUID()) return &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, Name: "references-" + model.OpaqueToken(identity)[:32], + Namespace: namespace, Name: "references-" + model.OpaqueToken(identity), Labels: map[string]string{referenceStateLabel: "true", referenceProjectLabel: string(r.Options.ProjectUID)}, }} }